-
Notifications
You must be signed in to change notification settings - Fork 1
Introduce GeoSpatial Index To CouchDB
Paul J. Davis edited this page May 9, 2017
·
4 revisions
Let us describe the experience of introducing GeoSpatial Index to CouchDB.
a. Download mirror of Apache CouchDB
git clone [email protected]:apache/couchdb.git
b. Modify rebar.config.script in order to load hastings and easton dependent repositories
DepDescs = [
%% must be compiled first as it has a custom behavior
{couch_epi, "couch-epi", "f6ad55d804ac741b59fe37dd092787113847661c"},
...
{easton, "easton", "99a3609ea98feacc0d9956cc158b024dde62f479"},
...
{hastings , "hastings", "fb49c4263a4f2b7879975bf09a1ec286de79acaa"},
...
],
MakeDep = fun
({AppName, {url, Url}, Version}) ->
{AppName, ".*", {git, Url, Version}};
({AppName, {url, Url}, Version, Options}) ->
{AppName, ".*", {git, Url, Version}, Options};
({AppName, RepoName, Version}) when AppName == meck ->
MeckUrl = "https://git-wip-us.apache.org/repos/asf/couchdb-meck.git",
{AppName, ".*", {git, MeckUrl, Version}};
({AppName, RepoName, Version}) when AppName == hastings; AppName == easton ->
Url = "[email protected]:cloudant-labs/" ++ RepoName ++ ".git",
{AppName, ".*", {git, Url, Version}};
({AppName, RepoName, Version}) ->
Url = BaseUrl ++ "couchdb-" ++ RepoName ++ ".git",
{AppName, ".*", {git, Url, Version}};
({AppName, RepoName, Version, Options}) ->
Url = BaseUrl ++ "couchdb-" ++ RepoName ++ ".git",
{AppName, ".*", {git, Url, Version}, Options}
c. Configure CouchDB and its dependencies
./configure --disable-fauxton --disable-docs
a. add hastings_epi plugin to couch_epi.config under rel/apps/ directory
{plugins, [
couch_db_epi,
chttpd_epi,
couch_index_epi,
global_changes_epi,
hastings_epi,
mango_epi,
mem3_epi,
setup_epi
]}.
b. Add hastings to reltool.config under rel directory
{sys, [
{rel, "couchdb", "2.0.0a", [
...
global_changes,
hastings,
ibrowse,
...
]},
%% couchdb
{app, global_changes, [{incl_cond, include}]},
{app, hastings, [{incl_cond, include}]},
{app, ibrowse, [{incl_cond, include}]},
]}
a. Create spatial.js under share/server directory using below content
var Spatial = (function() {
var index_results = []; // holds temporary emitted values during index
function handleIndexError(err, doc) {
if (err == "fatal_error") {
throw(["error", "map_runtime_error", "function raised 'fatal_error'"]);
} else if (err[0] == "fatal") {
throw(err);
}
var message = "function raised exception " + err.toSource();
if (doc) message += " with doc._id " + doc._id;
log(message);
};
return {
index: function(value, options) {
index_results.push([value, options || {}]);
},
indexDoc: function(doc) {
Couch.recursivelySeal(doc);
var buf = [];
for each (fun in State.funs) {
index_results = [];
try {
fun(doc);
buf.push(index_results);
} catch (err) {
handleIndexError(err, doc);
buf.push([]);
}
}
print(JSON.stringify(buf));
}
}
})();
b. add "st_index_doc": Spatial.indexDoc branch to main.js, main-coffe.js and loop.js under share/server directory
var Loop = function() {
var line, cmd, cmdkey, dispatch = {
"ddoc" : DDoc.ddoc,
// "view" : Views.handler,
"reset" : State.reset,
"add_fun" : State.addFun,
"add_lib" : State.addLib,
"map_doc" : Views.mapDoc,
"reduce" : Views.reduce,
"rereduce" : Views.rereduce,
"st_index_doc": Spatial.indexDoc
};
c. add share/server/spatial.js to build_js.escript under support directory
main([]) ->
JsFiles = ["share/server/json2.js",
"share/server/filter.js",
"share/server/mimeparse.js",
"share/server/render.js",
"share/server/state.js",
"share/server/util.js",
"share/server/validate.js",
"share/server/views.js",
"share/server/spatial.js",
"share/server/loop.js"],
CoffeeFiles = ["share/server/json2.js",
"share/server/filter.js",
"share/server/mimeparse.js",
"share/server/render.js",
"share/server/state.js",
"share/server/util.js",
"share/server/validate.js",
"share/server/views.js",
"share/server/spatial.js",
"share/server/coffee-script.js",
"share/server/loop.js"],
a. replace {user, cloudant_util:customer_name(Db)}, with supported function in CouchDB
Case 1: noproc error when triggering Geo indexing
{"error":"{noproc,\n {gen_server,call,\n [hastings_index_manager,\n {get_index,<<\"shards/e0000000-ffffffff/reltest.1477017301\">>,\n {h_idx,undefined,undefined,<<\"_design/geodd\">>,<<\"geoidx\">>,\n <<\"function(doc) {if (doc.geometry && doc.geometry.coordinates) {st_index(doc.geometry);}}\">>,\n <<\"javascript\">>,<<\"rtree\">>,2,4326,0,\n <<\"e66df316792ab411705e2741bba44371\">>}},\n infinity]}}","reason":"[{gen_server,call,3,[{file,\"gen_server.erl\"},{line,212}]},\n {hastings_rpc,get_index_pid,3,[{file,\"src/hastings_rpc.erl\"},{line,64}]},\n {hastings_rpc,search,4,[{file,\"src/hastings_rpc.erl\"},{line,22}]},\n {rexi_server,init_p,3,[{file,\"src/rexi_server.erl\"},{line,139}]}]"}
Solution: Need to check log file and see whether hastings application has been started and see whether there is entry about startup of hastings server. If not, review section add hastings to application list to be started
[info] 2016-10-21T10:01:51.919228Z [email protected] <0.7.0> -------- Application couch_replicator started on node '[email protected]'
[info] 2016-10-21T10:01:51.919320Z [email protected] <0.7.0> -------- Application bear started on node '[email protected]'
[info] 2016-10-21T10:01:51.919397Z [email protected] <0.7.0> -------- Application easton started on node '[email protected]'
[info] 2016-10-21T10:01:51.921395Z [email protected] <0.7.0> -------- Application hastings started on node '[email protected]'
[info] 2016-10-21T10:01:51.922600Z [email protected] <0.7.0> -------- Application global_changes started on node '[email protected]'
[info] 2016-10-21T10:01:51.922632Z [email protected] <0.7.0> -------- Application couch_plugins started on node '[email protected]'
[info] 2016-10-21T10:01:51.955120Z [email protected] <0.7.0> -------- Application runtime_tools started on node '[email protected]'
[info] 2016-10-21T10:01:51.955571Z [email protected] <0.7.0> -------- Application ddoc_cache started on node '[email protected]'
[info] 2016-10-21T10:01:51.955855Z [email protected] <0.7.0> -------- Application couch_index started on node '[email protected]'
Case 2: unknown_command error when triggering Geo indexing
{
"error": "{nocatch,{unknown_command,<<\"unknown command 'st_index_doc'\">>}}",
"reason": "[{couch_os_process,prompt,2,[{file,\"src/couch_os_process.erl\"},{line,59}]},\n {couch_query_servers,proc_prompt,2,\n [{file,\"src/couch_query_servers.erl\"},{line,427}]},\n {hastings_index_updater,load_docs,2,\n [{file,\"src/hastings_index_updater.erl\"},{line,100}]},\n {couch_db,conv_to_full_doc_info,2,[{file,\"src/couch_db.erl\"},{line,1533}]},\n {couch_bt_engine,drop_reductions,4,\n [{file,\"src/couch_bt_engine.erl\"},{line,828}]},\n {couch_btree,stream_kv_node2,8,[{file,\"src/couch_btree.erl\"},{line,783}]},\n {couch_btree,fold,4,[{file,\"src/couch_btree.erl\"},{line,220}]},\n {couch_bt_engine,fold_changes,5,\n [{file,\"src/couch_bt_engine.erl\"},{line,431}]}]"
}
Solution: Need to reivew section enable spatial.js for hastings and decide whether all steps are taken.
jiangphs-mbp:geotest jiangph$ curl -u foo:bar -X GET 'http://localhost:15984/reltest1/_design/geodd/_geo/geoidx?bbox=-180,-90,180,90&limit=2' | python -m json.tool
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 301 100 301 0 0 97 0 0:00:03 0:00:03 --:--:-- 97
{
"bookmark": "g2wAAAABaANkAA9ub2RlMkAxMjcuMC4wLjFsAAAAAmIgAAAAYj____9qaAJtAAAAIDc5ZjE0YjY0YzU3NDYxNTg0YjE1MjEyM2UzOGE2NDQ5RkBUsmENOjDEag",
"rows": [
{
"geometry": {
"coordinates": [
-71.13687953,
42.34690635
],
"type": "Point"
},
"id": "79f14b64c57461584b152123e38a6449",
"rev": "1-13051c832df60ec3e9869cdc801a2104"
}
]
}