From 18128a7e693f9eb7b4a7771d8d86ccb6811ce193 Mon Sep 17 00:00:00 2001 From: D4 Date: Tue, 12 Mar 2019 01:03:08 -0700 Subject: [PATCH 1/2] MacOS support, basic os abstraction layer, README example syntax completeness --- LemonGraph/cffi_stubs.py | 8 +++++-- LemonGraph/collection.py | 5 ++-- LemonGraph/httpd.py | 49 +++++++++++++++++++++++++++++++--------- Makefile | 9 ++++---- README.md | 28 ++++++++++++----------- lib/db.c | 7 +++--- lib/db.h | 2 +- lib/lemongraph-cffi.c | 12 +--------- lib/lemongraph.h | 2 +- lib/osal.c | 35 ++++++++++++++++++++++++++++ lib/osal.h | 25 ++++++++++++++++++++ 11 files changed, 134 insertions(+), 48 deletions(-) create mode 100644 lib/osal.c create mode 100644 lib/osal.h diff --git a/LemonGraph/cffi_stubs.py b/LemonGraph/cffi_stubs.py index 7e284fc..dce9c29 100644 --- a/LemonGraph/cffi_stubs.py +++ b/LemonGraph/cffi_stubs.py @@ -217,7 +217,6 @@ typedef ... DIR; // custom extras -void watch_parent(int sig); void free(void *); DIR *opendir(const char *name); char *_readdir(DIR *dirp); @@ -236,10 +235,15 @@ size_t graph_get_mapsize(graph_t g); size_t graph_get_disksize(graph_t g); db_snapshot_t graph_snapshot_new(graph_t g, int compact); + +typedef int HANDLE; +int osal_fdatasync(HANDLE fd); +void osal_set_pdeathsig(int sig); + ''' C_KEYWORDS = dict( - sources=['deps/lmdb/libraries/liblmdb/mdb.c', 'deps/lmdb/libraries/liblmdb/midl.c', 'lib/lemongraph.c', 'lib/db.c'], + sources=['deps/lmdb/libraries/liblmdb/mdb.c', 'deps/lmdb/libraries/liblmdb/midl.c', 'lib/lemongraph.c', 'lib/db.c', 'lib/osal.c'], include_dirs=['lib','deps/lmdb/libraries/liblmdb'], libraries=['z'], ) diff --git a/LemonGraph/collection.py b/LemonGraph/collection.py index 8af4186..01d45d0 100644 --- a/LemonGraph/collection.py +++ b/LemonGraph/collection.py @@ -11,6 +11,7 @@ from time import sleep, time import uuid + from lazy import lazy from pysigset import suspended_signals @@ -474,7 +475,7 @@ def daemon(self, poll=250, maxopen=1000): # Otherwise, we might have to mmap the whole region and msync it - maybe? # Opening it via LemonGraph adds overhead, burns double the file descriptors, and # currently explodes if I try to set RLIMIT_NOFILE > 2050. I know not why. - # We also assume that fdatasync() is good, which it is for Linux >= 3.6 + # fdatasync() is used for Linux and assumed good v >= 3.6. The more portable fsync() used for Mac & Win should work, it's effectively fdatasync + additional guarantee to update the file's modification time - hence slight perf hit. But, testing wouldn't hurt. count = 0 backlog = True while backlog: @@ -495,7 +496,7 @@ def daemon(self, poll=250, maxopen=1000): count += len(uuids) backlog = len(ctx.updatedDB) for fd in todo: - os.fdatasync(fd) + lib.osal_fdatasync(fd) finally: for fd in todo: os.close(fd) diff --git a/LemonGraph/httpd.py b/LemonGraph/httpd.py index c0f68a7..d2be19a 100644 --- a/LemonGraph/httpd.py +++ b/LemonGraph/httpd.py @@ -212,21 +212,48 @@ def chunkify(self, gen): # because every multiprocessing.Process().start() very helpfully # does a waitpid(WNOHANG) across all known children, and I want # to use os.wait() to catch exiting children +# TODO: replace mon-linux hackiness with cross-platform C osal_fdatasync class Process(object): def __init__(self, func): sys.stdout.flush() sys.stderr.flush() - self.pid = os.fork() - if self.pid == 0: - lib.watch_parent(signal.SIGTERM) - code = 1 - try: - func() - code = 0 - finally: - sys.stdout.flush() - sys.stderr.flush() - os._exit(code) + + if sys.platform == "Linux": + self.pid = os.fork() + if self.pid == 0: + lib.osal_fdatasync(signal.SIGTERM) + code = 1 + try: + func() + code = 0 + finally: + sys.stdout.flush() + sys.stderr.flush() + os._exit(code) + else: + (r, w) = os.pipe() + self.pid = os.fork() + if self.pid == 0: + os.close(w) # close write end of pipe + os.setpgid(0, 0) # prevent ^C in parent from stopping this process + child = os.fork() + if child == 0: + os.close(r) # close read end of pipe (don't need it here) + # os.execl(args[0], *args) + code = 1 + try: + func() + code = 0 + finally: + sys.stdout.flush() + sys.stderr.flush() + os._exit(code) + #os._exit(1) + os.read(r, 1) + os.kill(child, 9) + os._exit(1) + os.close(r) + def terminate(self, sig=signal.SIGTERM): os.kill(self.pid, sig) diff --git a/Makefile b/Makefile index 1dbad91..ed07a00 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,4 @@ + PYTHON=python PYTHON_CFLAGS=-O3 -Wall CC+=-pipe @@ -8,14 +9,14 @@ SNAPSHOT:=lg-$(shell date +%Y%m%d) default: build -liblemongraph.a: mdb.o midl.o lemongraph.o db.o -liblemongraph.so: mdb.o midl.o lemongraph.o db.o +liblemongraph.a: mdb.o midl.o lemongraph.o db.o osal.o +liblemongraph.so: mdb.o midl.o lemongraph.o db.o osal.o liblemongraph.so: LDFLAGS=-pthread liblemongraph.so: LDLIBS=-lz clean: - @echo $(wildcard *.a *.so *.o *.pyc LemonGraph/*.pyc LemonGraph/*/*.pyc LemonGraph/*.so MANIFEST) | xargs --no-run-if-empty rm -v - @echo $(wildcard .eggs build dist LemonGraph/__pycache__ LemonGraph/*/__pycache__ LemonGraph.egg-info) | xargs --no-run-if-empty rm -rv + @echo $(wildcard *.a *.so *.o *.pyc LemonGraph/*.pyc LemonGraph/*/*.pyc LemonGraph/*.so MANIFEST) | gxargs --no-run-if-empty rm -v + @echo $(wildcard .eggs build dist LemonGraph/__pycache__ LemonGraph/*/__pycache__ LemonGraph.egg-info) | gxargs --no-run-if-empty rm -rv distclean: clean @find deps -mindepth 2 -maxdepth 2 -exec rm -rv {} \; diff --git a/README.md b/README.md index 79cde52..d66aeab 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,7 @@ Python 3 should work now though - replace `python`/`pypy`/`easy_install` and pac * `pypy -mensurepip` ## LemonGraph installation + * `python setup.py install` (or you know, `pypy`) Or to run without proper installation, you must manually install dependencies: @@ -99,6 +100,7 @@ fd, path = tempfile.mkstemp() os.close(fd) # open graph object +print("graph save to ", path) g = LemonGraph.Graph(path) # enter a write transaction @@ -121,10 +123,10 @@ with g.transaction(write=True) as txn: # print out nodes and edges for n in txn.nodes(): - print n + print(n) for e in txn.edges(): - print e + print(e) b4_delete = txn.lastID @@ -133,29 +135,29 @@ with g.transaction(write=True) as txn: # delete a node - cascades to edges and properties node2.delete() - print + print() with g.transaction(write=False) as txn: # run an ad-hoc query before delete - print "ad-hoc query: nodes before deletions" + print("ad-hoc query: nodes before deletions") for chain in txn.query('n()', stop=b4_delete): - print chain - print + print(chain) + print() # run an ad-hoc query - print "ad-hoc query: nodes after deletions" + print("ad-hoc query: nodes after deletions") for chain in txn.query('n()'): - print chain - print + print(chain) + print() # run streaming queries - print "streaming query: nodes/edges" + print("streaming query: nodes/edges") for q, chain in txn.mquery(['n(prop3)','e()','n()->n()'], start=1): - print q, "=>", chain - print + print(q, "=>", chain) + print() # dump the internal graph log to stdout - print "dump:" + print("dump:") txn.dump() # delete graph artifacts from disk diff --git a/lib/db.c b/lib/db.c index 72b4fc9..a653393 100644 --- a/lib/db.c +++ b/lib/db.c @@ -15,6 +15,7 @@ #include"lmdb.h" #include"db.h" +#include"osal.h" #include"static_assert.h" @@ -310,9 +311,9 @@ int db_txn_init(txn_t txn, db_t db, txn_t parent, int flags){ int db_sync(db_t db, int force){ int r = mdb_env_sync((MDB_env *)db->env, force); // lmdb refuses to sync envs opened with mdb_readonly - // I am not bothering with figuring out if fdatasync is broken on your platform + // I've begun bothering with figuring out cross-platform fdatasync if(EACCES == r) - r = fdatasync(db->fd); + r = osal_fdatasync(db->fd); return r; } @@ -648,7 +649,7 @@ int txn_iter_init(iter_t iter, txn_t txn, int dbi, void *pfx, const unsigned int static INLINE int _iter_next(iter_t iter, const int data){ // set/advance key - iter->r = cursor_get((cursor_t)iter, &(iter->key), &(iter->data), (MDB_cursor_op)iter->op); + iter->r = cursor_get((cursor_t)iter, &(iter->key), &(iter->data), iter->op); if(DB_NEXT != iter->op) iter->op = DB_NEXT; if(DB_SUCCESS != iter->r) diff --git a/lib/db.h b/lib/db.h index 4b6dcd3..659b3a3 100644 --- a/lib/db.h +++ b/lib/db.h @@ -92,7 +92,7 @@ typedef struct buffer_t { } buffer_t; // base database object -struct db_t{ +struct db_t { void *env; db_dbi *handles; pthread_mutex_t mutex; diff --git a/lib/lemongraph-cffi.c b/lib/lemongraph-cffi.c index d593793..2967054 100644 --- a/lib/lemongraph-cffi.c +++ b/lib/lemongraph-cffi.c @@ -2,20 +2,10 @@ #include #include #include -#include -#include #include #include - -void watch_parent(int sig){ - prctl(PR_SET_PDEATHSIG, sig); -} - -char *_readdir(DIR *dirp){ - struct dirent *de = readdir(dirp); - return de ? de->d_name : NULL; -} +#include db_snapshot_t graph_snapshot_new(graph_t g, int compact){ return db_snapshot_new((db_t)g, compact); diff --git a/lib/lemongraph.h b/lib/lemongraph.h index dec64f8..9293563 100644 --- a/lib/lemongraph.h +++ b/lib/lemongraph.h @@ -42,7 +42,7 @@ typedef struct kv_iter_t * kv_iter_t; // For deletions, the 'next' field points to the top-level entry that was the target of the delete. // As a deletion may cascade to multiple children, I don't think it makes any sense to reserve it for pointing to a future entry. -struct entry_t{ +struct entry_t { logID_t id; uint8_t is_new:1; uint8_t rectype:7; diff --git a/lib/osal.c b/lib/osal.c new file mode 100644 index 0000000..b07d5e1 --- /dev/null +++ b/lib/osal.c @@ -0,0 +1,35 @@ +/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer*/ + +#include + +#if defined(__linux__) +#include +#endif + +#include"osal.h" + +int +osal_fdatasync(HANDLE fd) +{ +#if defined(__linux__) + return fdatasync(fd); +#else + return fsync(fd); +#endif +} + +void +osal_set_pdeathsig(int sig) +{ +#if defined(__linux__) + prctl(PR_SET_PDEATHSIG, sig); +#else + // TODO: +#endif +} + +char * +_readdir(DIR *dirp){ + struct dirent *de = readdir(dirp); + return de ? de->d_name : NULL; +} \ No newline at end of file diff --git a/lib/osal.h b/lib/osal.h new file mode 100644 index 0000000..9cc27c6 --- /dev/null +++ b/lib/osal.h @@ -0,0 +1,25 @@ +/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */ + +// Inspired by MDBX's abstraction setup, this is just a first step towards platform-specific encapsulation with enough needed to extend support to MacOS. With a bit of copy-paste, LMDB source provides necessary details to add Windows, but current linux binary should run atop WSL. + +#ifndef _OSAL_H +#define _OSAL_H + +#ifndef _BSD_SOURCE +#define _BSD_SOURCE +#endif + +#include + +/* HANDLE + An abstraction for a file handle. + On POSIX systems file handles are small integers. On Windows + they're opaque pointers. +*/ +#define HANDLE int + +int osal_fdatasync(HANDLE fd); +void osal_set_pdeathsig(int sig); +char *_readdir(DIR *dirp); + +#endif \ No newline at end of file From 430724814b7a91d7124bcca18618d529558ad01e Mon Sep 17 00:00:00 2001 From: D4 Date: Tue, 12 Mar 2019 04:06:06 -0700 Subject: [PATCH 2/2] Makefile: clean recipe w/ conditional use of gxargs (MacOS) --- Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index ed07a00..6e78a12 100644 --- a/Makefile +++ b/Makefile @@ -15,8 +15,9 @@ liblemongraph.so: LDFLAGS=-pthread liblemongraph.so: LDLIBS=-lz clean: - @echo $(wildcard *.a *.so *.o *.pyc LemonGraph/*.pyc LemonGraph/*/*.pyc LemonGraph/*.so MANIFEST) | gxargs --no-run-if-empty rm -v - @echo $(wildcard .eggs build dist LemonGraph/__pycache__ LemonGraph/*/__pycache__ LemonGraph.egg-info) | gxargs --no-run-if-empty rm -rv + hash gxargs 2>/dev/null && args=gxargs || args=xargs; \ + echo $(wildcard *.a *.so *.o *.pyc LemonGraph/*.pyc LemonGraph/*/*.pyc LemonGraph/*.so MANIFEST) | $$args --no-run-if-empty rm -v && \ + echo $(wildcard .eggs build dist LemonGraph/__pycache__ LemonGraph/*/__pycache__ LemonGraph.egg-info) | $$args --no-run-if-empty rm -rv distclean: clean @find deps -mindepth 2 -maxdepth 2 -exec rm -rv {} \;