MacOS support, basic os abstraction layer, README example syntax

LemonGraph/cffi_stubs.py

@@ -217,7 +217,6 @@
 typedef ... DIR;
 
 // custom extras
-void watch_parent(int sig);
 void free(void *);
 DIR *opendir(const char *name);
 char *_readdir(DIR *dirp);
@@ -236,10 +235,15 @@
 size_t graph_get_mapsize(graph_t g);
 size_t graph_get_disksize(graph_t g);
 db_snapshot_t graph_snapshot_new(graph_t g, int compact);
+
+typedef int HANDLE;
+int osal_fdatasync(HANDLE fd);
+void osal_set_pdeathsig(int sig);
+
 '''
 
 C_KEYWORDS = dict(
-    sources=['deps/lmdb/libraries/liblmdb/mdb.c', 'deps/lmdb/libraries/liblmdb/midl.c', 'lib/lemongraph.c', 'lib/db.c'],
+    sources=['deps/lmdb/libraries/liblmdb/mdb.c', 'deps/lmdb/libraries/liblmdb/midl.c', 'lib/lemongraph.c', 'lib/db.c', 'lib/osal.c'],
     include_dirs=['lib','deps/lmdb/libraries/liblmdb'],
     libraries=['z'],
 )

LemonGraph/collection.py

@@ -11,6 +11,7 @@
 from time import sleep, time
 import uuid
 
+
 from lazy import lazy
 from pysigset import suspended_signals
 
@@ -474,7 +475,7 @@ def daemon(self, poll=250, maxopen=1000):
             # Otherwise, we might have to mmap the whole region and msync it - maybe?
             # Opening it via LemonGraph adds overhead, burns double the file descriptors, and
             # currently explodes if I try to set RLIMIT_NOFILE > 2050. I know not why.
-            # We also assume that fdatasync() is good, which it is for Linux >= 3.6
+            # fdatasync() is used for Linux and assumed good v >= 3.6. The more portable fsync() used for Mac & Win should work, it's effectively fdatasync + additional guarantee to update the file's modification time - hence slight perf hit. But, testing wouldn't hurt.
             count = 0
             backlog = True
             while backlog:
@@ -495,7 +496,7 @@ def daemon(self, poll=250, maxopen=1000):
                             count += len(uuids)
                             backlog = len(ctx.updatedDB)
                         for fd in todo:
-                            os.fdatasync(fd)
+                            lib.osal_fdatasync(fd)
                     finally:
                         for fd in todo:
                             os.close(fd)

LemonGraph/httpd.py

@@ -212,21 +212,48 @@ def chunkify(self, gen):
 # because every multiprocessing.Process().start() very helpfully
 # does a waitpid(WNOHANG) across all known children, and I want
 # to use os.wait() to catch exiting children
+# TODO: replace mon-linux hackiness with cross-platform C osal_fdatasync
 class Process(object):
     def __init__(self, func):
         sys.stdout.flush()
         sys.stderr.flush()
-        self.pid = os.fork()
-        if self.pid == 0:
-            lib.watch_parent(signal.SIGTERM)
-            code = 1
-            try:
-                func()
-                code = 0
-            finally:
-                sys.stdout.flush()
-                sys.stderr.flush()
-                os._exit(code)
+
+        if sys.platform == "Linux":
+            self.pid = os.fork()
+            if self.pid == 0:
+                lib.osal_fdatasync(signal.SIGTERM)
+                code = 1
+                try:
+                    func()
+                    code = 0
+                finally:
+                    sys.stdout.flush()
+                    sys.stderr.flush()
+                    os._exit(code)
+        else:
+            (r, w) = os.pipe()
+            self.pid = os.fork()
+            if self.pid == 0:
+                os.close(w) # close write end of pipe
+                os.setpgid(0, 0) # prevent ^C in parent from stopping this process
+                child = os.fork()
+                if child == 0:
+                    os.close(r) # close read end of pipe (don't need it here)
+                    # os.execl(args[0], *args)
+                    code = 1
+                    try:
+                        func()
+                        code = 0
+                    finally:
+                        sys.stdout.flush()
+                        sys.stderr.flush()
+                        os._exit(code)
+                    #os._exit(1)
+                os.read(r, 1)
+                os.kill(child, 9)
+                os._exit(1)
+        os.close(r)
+
 
     def terminate(self, sig=signal.SIGTERM):
         os.kill(self.pid, sig)

Makefile

@@ -1,3 +1,4 @@
+
 PYTHON=python
 PYTHON_CFLAGS=-O3 -Wall
 CC+=-pipe
@@ -8,14 +9,15 @@ SNAPSHOT:=lg-$(shell date +%Y%m%d)
 
 default: build
 
-liblemongraph.a:  mdb.o midl.o lemongraph.o db.o
-liblemongraph.so: mdb.o midl.o lemongraph.o db.o
+liblemongraph.a:  mdb.o midl.o lemongraph.o db.o osal.o
+liblemongraph.so: mdb.o midl.o lemongraph.o db.o osal.o
 liblemongraph.so: LDFLAGS=-pthread
 liblemongraph.so: LDLIBS=-lz
 
 clean:
-	@echo $(wildcard *.a *.so *.o *.pyc LemonGraph/*.pyc LemonGraph/*/*.pyc LemonGraph/*.so MANIFEST) | xargs --no-run-if-empty rm -v
-	@echo $(wildcard .eggs build dist LemonGraph/__pycache__ LemonGraph/*/__pycache__ LemonGraph.egg-info)  | xargs --no-run-if-empty rm -rv
+	hash gxargs 2>/dev/null && args=gxargs || args=xargs; \
+	echo $(wildcard *.a *.so *.o *.pyc LemonGraph/*.pyc LemonGraph/*/*.pyc  LemonGraph/*.so MANIFEST) | $$args --no-run-if-empty rm -v && \
+	echo $(wildcard .eggs build dist LemonGraph/__pycache__ LemonGraph/*/__pycache__ LemonGraph.egg-info)  | $$args --no-run-if-empty rm -rv
 
 distclean: clean
 	@find deps -mindepth 2 -maxdepth 2 -exec rm -rv {} \;

README.md

@@ -80,6 +80,7 @@ Python 3 should work now though - replace `python`/`pypy`/`easy_install` and pac
 	* `pypy -mensurepip`
 
 ## LemonGraph installation
+
 * `python setup.py install` (or you know, `pypy`)
 
 Or to run without proper installation, you must manually install dependencies:
@@ -99,6 +100,7 @@ fd, path = tempfile.mkstemp()
 os.close(fd)
 
 # open graph object
+print("graph save to ", path)
 g = LemonGraph.Graph(path)
 
 # enter a write transaction
@@ -121,10 +123,10 @@ with g.transaction(write=True) as txn:
 
     # print out nodes and edges
     for n in txn.nodes():
-        print n
+        print(n)
 
     for e in txn.edges():
-        print e
+        print(e)
 
     b4_delete = txn.lastID
 
@@ -133,29 +135,29 @@ with g.transaction(write=True) as txn:
 
     # delete a node - cascades to edges and properties
     node2.delete()
-    print
+    print()
 
 with g.transaction(write=False) as txn:
     # run an ad-hoc query before delete
-    print "ad-hoc query: nodes before deletions"
+    print("ad-hoc query: nodes before deletions")
     for chain in txn.query('n()', stop=b4_delete):
-        print chain
-    print
+        print(chain)
+    print()
 
     # run an ad-hoc query
-    print "ad-hoc query: nodes after deletions"
+    print("ad-hoc query: nodes after deletions")
     for chain in txn.query('n()'):
-        print chain
-    print
+        print(chain)
+    print()
 
     # run streaming queries
-    print "streaming query: nodes/edges"
+    print("streaming query: nodes/edges")
     for q, chain in txn.mquery(['n(prop3)','e()','n()->n()'], start=1):
-        print q, "=>", chain
-    print
+        print(q, "=>", chain)
+    print()
 
     # dump the internal graph log to stdout
-    print "dump:"
+    print("dump:")
     txn.dump()
 
 # delete graph artifacts from disk

lib/db.c

@@ -15,6 +15,7 @@
 
 #include"lmdb.h"
 #include"db.h"
+#include"osal.h"
 
 #include"static_assert.h"
 
@@ -310,9 +311,9 @@ int db_txn_init(txn_t txn, db_t db, txn_t parent, int flags){
 int db_sync(db_t db, int force){
 	int r = mdb_env_sync((MDB_env *)db->env, force);
 	// lmdb refuses to sync envs opened with mdb_readonly
-	// I am not bothering with figuring out if fdatasync is broken on your platform
+	// I've begun bothering with figuring out cross-platform fdatasync
 	if(EACCES == r)
-		r = fdatasync(db->fd);
+		r = osal_fdatasync(db->fd);
 	return r;
 }
 
@@ -648,7 +649,7 @@ int txn_iter_init(iter_t iter, txn_t txn, int dbi, void *pfx, const unsigned int
 
 static INLINE int _iter_next(iter_t iter, const int data){
 	// set/advance key
-	iter->r = cursor_get((cursor_t)iter, &(iter->key), &(iter->data), (MDB_cursor_op)iter->op);
+	iter->r = cursor_get((cursor_t)iter, &(iter->key), &(iter->data), iter->op);
 	if(DB_NEXT != iter->op)
 		iter->op = DB_NEXT;
 	if(DB_SUCCESS != iter->r)

lib/db.h

@@ -92,7 +92,7 @@ typedef struct buffer_t {
 } buffer_t;
 
 // base database object
-struct db_t{
+struct db_t {
 	void *env;
 	db_dbi *handles;
 	pthread_mutex_t mutex;

lib/lemongraph-cffi.c

@@ -2,20 +2,10 @@
 #include<inttypes.h>
 #include<fcntl.h>
 #include<signal.h>
-#include<dirent.h>
-#include<sys/prctl.h>
 
 #include<db.h>
 #include<lemongraph.h>
-
-void watch_parent(int sig){
-	prctl(PR_SET_PDEATHSIG, sig);
-}
-
-char *_readdir(DIR *dirp){
-    struct dirent *de = readdir(dirp);
-    return de ? de->d_name : NULL;
-}
+#include<osal.h>
 
 db_snapshot_t graph_snapshot_new(graph_t g, int compact){
     return db_snapshot_new((db_t)g, compact);

lib/lemongraph.h

@@ -42,7 +42,7 @@ typedef struct kv_iter_t * kv_iter_t;
 // For deletions, the 'next' field points to the top-level entry that was the target of the delete.
 // As a deletion may cascade to multiple children, I don't think it makes any sense to reserve it for pointing to a future entry.
 
-struct entry_t{
+struct entry_t {
 	logID_t id;
 	uint8_t is_new:1;
 	uint8_t rectype:7;

lib/osal.c

@@ -0,0 +1,35 @@
+/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer*/
+
+#include <unistd.h>
+
+#if defined(__linux__)
+#include <prctl.h>
+#endif
+
+#include"osal.h"
+
+int
+osal_fdatasync(HANDLE fd)
+{
+#if defined(__linux__)
+    return fdatasync(fd);
+#else
+    return fsync(fd);
+#endif
+}
+
+void
+osal_set_pdeathsig(int sig)
+{
+#if defined(__linux__)
+	prctl(PR_SET_PDEATHSIG, sig);
+#else
+    // TODO:
+#endif
+}
+
+char *
+_readdir(DIR *dirp){
+    struct dirent *de = readdir(dirp);
+    return de ? de->d_name : NULL;
+}

lib/osal.h

@@ -0,0 +1,25 @@
+/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */
+
+// Inspired by MDBX's abstraction setup, this is just a first step towards platform-specific encapsulation with enough needed to extend support to MacOS. With a bit of copy-paste, LMDB source provides necessary details to add Windows, but current linux binary should run atop WSL.
+
+#ifndef _OSAL_H
+#define _OSAL_H
+
+#ifndef _BSD_SOURCE
+#define _BSD_SOURCE
+#endif
+
+#include<dirent.h>
+
+/*  HANDLE
+    An abstraction for a file handle.
+    On POSIX systems file handles are small integers. On Windows
+    they're opaque pointers.
+*/
+#define	HANDLE int
+
+int osal_fdatasync(HANDLE fd);
+void osal_set_pdeathsig(int sig);
+char *_readdir(DIR *dirp);
+
+#endif