Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support vlen r/w with simple parent types #124

Merged
merged 13 commits into from
May 28, 2024
Merged

Support vlen r/w with simple parent types #124

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
c85a390
Sync with master
mattjala Mar 21, 2024
7d76dc2
Committing clang-format changes
github-actions[bot] Mar 21, 2024
938f080
Sync with master
mattjala Apr 22, 2024
4f340e7
Cleanup
mattjala Apr 23, 2024
9b5540b
Use H5Treclaim for vlen cleanup
mattjala May 9, 2024
1729430
Fix incorrect buffer size return from RV_pack_vlen_data
mattjala May 21, 2024
7d398c4
Test vlen point I/O
mattjala May 21, 2024
ce6107c
Test vlen hyperslab I/O
mattjala May 22, 2024
6e9c541
Remove unused RV_json_values_to_binary callback
mattjala May 22, 2024
5dd5e97
Remove unused RV_vlen_data_to_json
mattjala May 22, 2024
84b4520
Avoid unsafe pointer cast
mattjala May 22, 2024
8f39213
Do selection to binary conv in separate func
mattjala May 22, 2024
ee35ca4
Fix requests version in workflow
mattjala May 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 14 additions & 4 deletions .github/workflows/main.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,12 @@ jobs:
sudo apt update
sudo apt install valgrind
working-directory: ${{ github.workspace }}


# Requests 2.32.0 breaks requests-unixsocket, used by HSDS for socket connections
- name: Fix requests version
run: |
pip install requests==2.31.0

- name: Start HSDS
if: ${{ matrix.endpoint != 'http://127.0.0.1:5101'}}
run: |
Expand All @@ -133,7 +138,7 @@ jobs:
ROOT_DIR=${{github.workspace}}/hsdadata ./runall.sh --no-docker 1 &
sleep 10
working-directory: ${{github.workspace}}/hsds

- name: Test HSDS
if: ${{matrix.endpoint != 'http://127.0.0.1:5101'}}
run: |
Expand Down Expand Up @@ -240,7 +245,12 @@ jobs:
sudo apt update
sudo apt install valgrind
working-directory: ${{ github.workspace }}


# Requests 2.32.0 breaks requests-unixsocket, used by HSDS for socket connections
- name: Fix requests version
run: |
pip install requests==2.31.0

- name: Start HSDS
if: ${{ matrix.endpoint != 'http://127.0.0.1:5101'}}
run: |
Expand All @@ -254,7 +264,7 @@ jobs:
ROOT_DIR=${{github.workspace}}/hsdadata ./runall.sh --no-docker 1 &
sleep 10
working-directory: ${{github.workspace}}/hsds

- name: Test HSDS
if: ${{matrix.endpoint != 'http://127.0.0.1:5101'}}
run: |
Expand Down
75 changes: 71 additions & 4 deletions src/rest_vol.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3656,6 +3656,7 @@ RV_curl_multi_perform(CURL *curl_multi_handle, dataset_transfer_info *transfer_i
int maxfd = -1;
long timeout_ms = 0;
struct timeval timeout;
hid_t vlen_buf_space = H5I_INVALID_HID;

if ((failed_handles_to_retry = calloc(count, sizeof(CURL *))) == NULL)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL,
Expand Down Expand Up @@ -3757,6 +3758,8 @@ RV_curl_multi_perform(CURL *curl_multi_handle, dataset_transfer_info *transfer_i
fail_count++;
}
else if (response_code == 200) {
H5T_class_t dtype_class = H5T_NO_CLASS;

num_finished++;
succeed_count++;

Expand Down Expand Up @@ -3794,11 +3797,71 @@ RV_curl_multi_perform(CURL *curl_multi_handle, dataset_transfer_info *transfer_i
transfer_info[handle_index].curl_easy_handle = NULL;

if (transfer_info[handle_index].transfer_type == WRITE) {
RV_free(transfer_info[handle_index].u.write_info.write_body);
transfer_info[handle_index].u.write_info.write_body = NULL;
if (transfer_info[handle_index].tconv_buf) {
htri_t has_vlen = FALSE;

if ((has_vlen =
H5Tdetect_class(transfer_info[handle_index].mem_type_id, H5T_VLEN)) < 0)
FUNC_DONE_ERROR(H5E_DATASET, H5E_CANTGET, FAIL,
"can't check if dtype contains vlen");

/* Clean up memory allocated by type conversion of vlen types */
if (has_vlen > 0) {
/* Buffer was gathered before type conversion, so we can manually free vlen
* memory by iteration */
hssize_t num_elems = 0;
if ((num_elems = H5Sget_select_npoints(
transfer_info[handle_index].mem_space_id)) <= 0)
FUNC_DONE_ERROR(H5E_DATASET, H5E_CANTGET, FAIL,
"can't get number of elements in dataspace");

/* Vlen buffer is packed, so generate a 1D dataspace to describe its layout */
if ((vlen_buf_space = H5Screate_simple(1, &num_elems, NULL)) < 0)
FUNC_DONE_ERROR(H5E_DATASPACE, H5E_CANTCREATE, FAIL,
"can't create dataspace for vlen buffer");

if ((H5Treclaim(transfer_info[handle_index].mem_type_id, vlen_buf_space,
H5P_DEFAULT, transfer_info[handle_index].tconv_buf)) < 0)
FUNC_DONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL,
"can't free vlen data from buffer");

if (H5Sclose(vlen_buf_space) < 0)
FUNC_DONE_ERROR(H5E_DATASPACE, H5E_CANTCLOSEOBJ, FAIL,
"can't close dataspace for vlen buffer");

vlen_buf_space = H5I_INVALID_HID;
}
}

if (transfer_info[handle_index].u.write_info.gather_buf) {
RV_free(transfer_info[handle_index].u.write_info.gather_buf);
transfer_info[handle_index].u.write_info.gather_buf = NULL;
}

if (transfer_info[handle_index].u.write_info.serialize_buf) {
RV_free(transfer_info[handle_index].u.write_info.serialize_buf);
transfer_info[handle_index].u.write_info.serialize_buf = NULL;
}

if (transfer_info[handle_index].u.write_info.base64_encoded_values) {
RV_free(transfer_info[handle_index].u.write_info.base64_encoded_values);
transfer_info[handle_index].u.write_info.base64_encoded_values = NULL;
}

if (transfer_info[handle_index].u.write_info.point_sel_buf) {
RV_free(transfer_info[handle_index].u.write_info.point_sel_buf);
transfer_info[handle_index].u.write_info.point_sel_buf = NULL;
}
}

RV_free(transfer_info[handle_index].u.write_info.base64_encoded_values);
transfer_info[handle_index].u.write_info.base64_encoded_values = NULL;
if (transfer_info[handle_index].tconv_buf) {
RV_free(transfer_info[handle_index].tconv_buf);
transfer_info[handle_index].tconv_buf = NULL;
}

if (transfer_info[handle_index].bkg_buf) {
RV_free(transfer_info[handle_index].bkg_buf);
transfer_info[handle_index].bkg_buf = NULL;
}

RV_free(transfer_info[handle_index].request_url);
Expand Down Expand Up @@ -3838,6 +3901,10 @@ RV_curl_multi_perform(CURL *curl_multi_handle, dataset_transfer_info *transfer_i
done:
RV_free(failed_handles_to_retry);

if (vlen_buf_space != H5I_INVALID_HID)
if (H5Sclose(vlen_buf_space) < 0)
FUNC_DONE_ERROR(H5E_DATASPACE, H5E_CANTCLOSEOBJ, FAIL, "can't close dataspace for vlen buffer");

return ret_value;
}

Expand Down
10 changes: 7 additions & 3 deletions src/rest_vol.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -586,10 +586,14 @@ struct RV_object_t {

/* Structures to hold information for cURL requests to read/write to datasets */
typedef struct dataset_write_info {
char *write_body;
char *base64_encoded_values;
curl_off_t write_len;
/* Dynamically allocated buffers for each step of the write pipeline */
void *gather_buf;
void *base64_encoded_values;
void *serialize_buf;
void *point_sel_buf;
void *vlen_buf;
upload_info uinfo;
/* Pointer to user-provided write buffer */
const void *buf;

/* If writing using compound subsetting, this is a packed version of the
Expand Down
Loading
Loading