Merge pull request #65 from hasindu2008/lazymt

Lazymt

.github/workflows/c-cpp.yml

@@ -2,9 +2,9 @@ name: C/C++ CI
 
 on:
   push:
-    branches: [ master, dev, cleanup ]
+    branches: [ '*' ]
   pull_request:
-    branches: [ master, dev, cleanup ]
+    branches: [ '*' ]
 
 jobs:
   ubuntu_14:

.github/workflows/python.yml

@@ -2,9 +2,9 @@ name: Python CI
 
 on:
   push:
-    branches: [ master, dev, cleanup ]
+    branches: [ '*' ]
   pull_request:
-    branches: [ master, dev, cleanup ]
+    branches: [ '*' ]
 
 jobs:
   ubuntu_14:

Makefile

@@ -19,6 +19,11 @@ else
 CFLAGS		+= -DSLOW5_USE_ZSTD
 CPPFLAGS 	+= -I $(zstd_local)
 endif
+ifeq ($(slow5_mt),1)
+CFLAGS		+= -DSLOW5_ENABLE_MT
+LDFLAGS		+= -lpthread
+endif
+
 BUILD_DIR	= lib
 
 STATICLIB	= $(BUILD_DIR)/libslow5.a
@@ -28,6 +33,7 @@ OBJ = $(BUILD_DIR)/slow5.o \
 		$(BUILD_DIR)/slow5_idx.o \
 		$(BUILD_DIR)/slow5_misc.o \
 		$(BUILD_DIR)/slow5_press.o \
+		$(BUILD_DIR)/slow5_mt.o \
 
 PREFIX = /usr/local
 VERSION = `git describe --tags`
@@ -61,6 +67,9 @@ $(BUILD_DIR)/slow5_misc.o: src/slow5_misc.c src/slow5_misc.h include/slow5/slow5
 $(BUILD_DIR)/slow5_press.o: src/slow5_press.c include/slow5/slow5_press.h src/slow5_misc.h include/slow5/slow5_error.h
 	$(CC) $(CFLAGS) $(CPPFLAGS) $< -c -fpic -o $@
 
+$(BUILD_DIR)/slow5_mt.o: src/slow5_mt.c include/slow5/slow5_mt.h $(SLOW5_H)
+	$(CC) $(CFLAGS) $(CPPFLAGS) $< -c -fpic -o $@
+
 clean:
 	rm -rf $(OBJ) $(STATICLIB) $(SHAREDLIB) $(SHAREDLIBV)
 	make -C $(SVB) clean

docs/pyslow5_api/pyslow5.md

@@ -311,31 +311,67 @@ Returns an ordered list of auxiliary attribute types (same order as get_aux_name
 
 This can mostly be ignored, but will be used in error tracing in the future, as auxiliary field requests have multiple types, each with their own calls, and not all are used. It could be the case a call for an auxiliary filed fails, and knowing which type the field is requesting is very helpful in understanding which function in C is being called, that could be causing the error.
 
+#### `get_aux_enum_labels(label)`:
+
+Returns an ordered list representing the values in the enum struct in the type header.
+
+The value in the read can then be used to access the labels as an index to the list.
+
+Example:
+
+```python
+s5 = slow5.Open(file,'w')
+end_reason_labels = s5.get_aux_enum_labels('end_reason')
+print(end_reason_labels)
+
+> ['unknown', 'partial', 'mux_change', 'unblock_mux_change', 'signal_positive', 'signal_negative']
+
+readID = "r1"
+read = s5.get_read(readID, aux='all')
+er_index = read['end_reason']
+er = end_reason_labels[er_index]
+
+print("{}: {}".format(er_index, er))
+
+> 4: signal_positive
+```
+
 ### Writing a file
 
 To write a file, `mode` in `Open()` must be set to `'w'` and when appending, `'a'`
 
-#### `get_empty_header()`:
+#### `get_empty_header(aux=False)`:
 
 Returns a dictionary containing all known header attributes with their values set to `None`.
 
 User can modify each value, and add or remove attributes to be used has header items.
 All values end up stored as strings, and anything left as `None` will be skipped.
 To write header, see `write_header()`
 
+If `aux=True`, an ordered list of strings for the enum `end_reason` will be returned.
+This can be modified depending on the end reason.
+
 Example:
 
 ```python
 s5 = slow5.Open(file,'w')
 header = s5.get_empty_header()
 ```
 
-#### `write_header(header, read_group=0)`:
+`end_reason` enum example
+
+```python
+s5 = slow5.Open(file, w)
+header, end_reason_labels = s5.get_empty_header(aux=True)
+```
+
+#### `write_header(header, read_group=0, end_reason_labels=None)`:
 
 Write header to file
 
 + `header` = populated dictionary from `get_empty_header()`
 + read_group = read group integer for when multiple runs are written to the same slow5 file
++ end_reason_labels = ordered list used for end_reason enum
 + returns 0 on success, <0 on error with error code
 
 You must write `read_group=0` (default) first before writing any other read_groups, and it is advised to write read_groups in sequential order.
@@ -365,6 +401,23 @@ ret = s5.write_header(header2, read_group=1)
 print("ret: write_header(): {}".format(ret))
 ```
 
+`end_reason` example:
+
+```python
+# Get some empty headers
+header, end_reason_labels = s5.get_empty_header(aux=True)
+
+# Populate headers with some test data
+counter = 0
+for i in header:
+    header[i] = "test_{}".format(counter)
+    counter += 1
+
+# Write first read group
+ret = s5.write_header(header, end_reason_labels=end_reason_labels)
+print("ret: write_header(): {}".format(ret))
+```
+
 #### `get_empty_record(aux=False)`:
 
 Get empty read record for populating with data. Use with `write_record()`

docs/slow5_api/low_level_api/slow5_aux_add_enum.md

@@ -0,0 +1,148 @@
+# slow5_aux_add_enum
+
+## NAME
+
+slow5_aux_add_enum -  adds an auxiliary field of type enum to a SLOW5 header
+
+## SYNOPSYS
+
+`int slow5_aux_add_enum(const char *field, const char **enum_labels, uint8_t num_labels, slow5_hdr_t *header)`
+
+## DESCRIPTION
+`slow5_aux_add_enum()` adds an auxiliary field named *field* of the datatype *enum* whose enum labels are pointed by *enum_labels* to a SLOW5 file header pointed by *header*.
+
+The number of enum labels in should be in *num_labels*.
+
+The argument *header* points to a SLOW5 header of type *slow5_hdr_t* and typically this is the *s5p->header* member inside the *slow5_file_t \*s5p* returned by  `slow5_open()`.
+
+## RETURN VALUE
+Upon successful completion, `slow5_aux_add_enum()` returns a non negative integer (>=0). Otherwise, a negative value is returned.
+
+## ERRORS
+
+A negative value is returned when an error occurs and can be due to following occasions (not an exhaustive list):
+
+- input parameter is NULL
+- enum value is invalid
+- other error
+
+## NOTES
+
+In the future `slow5_errno` will be set to indicate the error.
+
+## EXAMPLES
+
+```
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <slow5/slow5.h>
+
+#define FILE_PATH "test.blow5"
+
+int main(){
+
+    //open the SLOW5 file for writing
+    slow5_file_t *sp = slow5_open(FILE_PATH, "w");
+    if(sp==NULL){
+        fprintf(stderr,"Error opening file!\n");
+        exit(EXIT_FAILURE);
+    }
+
+    /*********************** Header ******************/
+    /*
+    @run_id	run_id_0
+    */
+    slow5_hdr_t *header=sp->header; //pointer to the SLOW5 header
+
+    //add a header group attribute called run_id
+    if (slow5_hdr_add("run_id", header) < 0){
+        fprintf(stderr,"Error adding run_id attribute\n");
+        exit(EXIT_FAILURE);
+    }
+
+    //set the run_id attribute to "run_0" for read group 0
+    if (slow5_hdr_set("run_id", "run_0", 0, header) < 0){
+        fprintf(stderr,"Error setting run_id attribute in read group 0\n");
+        exit(EXIT_FAILURE);
+    }
+
+    /*
+    enum{unknown,partial,mux_change,unblock_mux_change,signal_positive,signal_negative}
+    end_reason
+    */
+    const char *enum_labels[] = {"unknown", "partial", "mux_change", "unblock_mux_change", "signal_positive", "signal_negative"};
+    uint8_t num_labels = 6;
+    if (slow5_aux_add_enum("end_reason", enum_labels, num_labels, sp->header) < 0){
+        fprintf(stderr,"Error adding end_reason auxilliary field\n");
+        exit(EXIT_FAILURE);
+    }
+
+    if(slow5_hdr_write(sp) < 0){
+        fprintf(stderr,"Error writing header!\n");
+        exit(EXIT_FAILURE);
+    }
+
+
+    /******************* A SLOW5 record ************************/
+    slow5_rec_t *slow5_record = slow5_rec_init();
+    if(slow5_record == NULL){
+        fprintf(stderr,"Could not allocate space for a slow5 record.");
+        exit(EXIT_FAILURE);
+    }
+
+    /* primary fields
+    #read_id	read_group	digitisation	offset	range	sampling_rate	len_raw_signal	raw_signal
+    read_0	    0	        4096	        3	    10	    4000	        10	            0,1,2,3,4,5,6,7,8,9
+    */
+    slow5_record -> read_id = strdup("read_0");
+    if(slow5_record->read_id == NULL){
+        fprintf(stderr,"Could not do strdup.");
+        exit(EXIT_FAILURE);
+    }
+    slow5_record-> read_id_len = strlen(slow5_record -> read_id);
+    slow5_record -> read_group = 0;
+    slow5_record -> digitisation = 4096.0;
+    slow5_record -> offset = 3.0;
+    slow5_record -> range = 10.0;
+    slow5_record -> sampling_rate = 4000.0;
+    slow5_record -> len_raw_signal = 10;
+    slow5_record -> raw_signal = (int16_t *)malloc(sizeof(int16_t) * slow5_record->len_raw_signal);
+    if(slow5_record->raw_signal == NULL){
+        fprintf(stderr,"Could not allocate space for raw signal.");
+        exit(EXIT_FAILURE);
+    }
+    for(int i=0; i<slow5_record->len_raw_signal; i++){
+        slow5_record->raw_signal[i] = i;
+    }
+
+    /* auxiliary fileds
+    end_reason
+    1
+    */
+    uint8_t end_reason = 1;
+    if(slow5_aux_set(slow5_record, "end_reason", &end_reason, sp->header) < 0){
+        fprintf(stderr,"Error setting end_reason auxilliary field\n");
+        exit(EXIT_FAILURE);
+    }
+
+    //write to file
+    if(slow5_write(slow5_record, sp) < 0){
+        fprintf(stderr,"Error writing record!\n");
+        exit(EXIT_FAILURE);
+    }
+
+    //free the slow5 record
+    slow5_rec_free(slow5_record);
+
+    //close the SLOW5 file
+    slow5_close(sp);
+
+    return 0;
+
+}
+
+```
+
+## SEE ALSO
+[`slow5_aux_set()`](../slow5_aux_set.md).

docs/slow5_api/slow5_low_level_api.md

@@ -33,6 +33,8 @@ Low-level API allows much more efficient access to BLOW5 files compared to the h
 
 ### Writing and editing
 
+* [slow5_aux_add_enum](low_level_api/slow5_aux_add_enum.md)<br/>
+  &nbsp;&nbsp;&nbsp;&nbsp;adds an auxiliary field of type enum to a SLOW5 header
 * [slow5_encode](low_level_api/slow5_encode.md)<br/>
   &nbsp;&nbsp;&nbsp;&nbsp;encodes a SLOW5 record
 * [slow5_write_bytes](low_level_api/slow5_write_bytes.md)<br/>

examples/.gitignore

@@ -17,6 +17,10 @@ example_write_aux.blow5
 write
 append
 adv/auxiliary_field_enum
+adv/auxiliary_field_enum_write
 adv/sequential_read_pthreads
 adv/sequential_read_openmp
 adv/get_all_read_ids
+mt/mt
+mt/lazymt
+

examples/adv/README.md

@@ -5,6 +5,7 @@ This directory contains following advanced examples that uses low-level API.
 - *sequential_read_pthreads.c* demonstrates how to sequentially read raw SLOW5 records from a slow5/blow5 file using a single thread and then decode those in parallel using *pthreads*.
 - *sequential_read_openmp.c* demonstrates how to sequentially read raw SLOW5 records from a slow5/blow5 file using a single thread and then decode those in parallel using *openMP*.
 - *get_all_read_ids.c* demonstrates how to get the list of all read IDs from a slow5/blow5 file.
+- *auxiliary_field_enum_write.c* demonstrates how to write a slow5/blow5 file containing an auxiliary field of type enum.
 
 You can invoke [build.sh](build.sh) from slow5lib directory as `examples/adv/build.sh` to compile the example programmes. Have a look at the script to see the commands used for compiling and linking. Also make sure you get familiar with the basic examples first, before trying these advanced examples.