From 5a2edf90dd7933b29f477d2b0d2403d43957f496 Mon Sep 17 00:00:00 2001 From: Kelly McLaughlin Date: Fri, 24 Sep 2010 08:27:56 -0600 Subject: [PATCH 1/3] Count function with selectors working, but requires lots of cleanup. --- Makefile | 5 ++++- emongo.rel | 7 +++++++ priv/msg.config | 10 ++++++++++ src/emongo.erl | 44 +++++++++++++++++++++++++++++++++++++++---- src/emongo_bson.erl | 36 ++++++++++++++++++++++++++++++----- src/emongo_packet.erl | 21 +++++++++++++++------ 6 files changed, 107 insertions(+), 16 deletions(-) create mode 100644 emongo.rel create mode 100644 priv/msg.config diff --git a/Makefile b/Makefile index 4486b38..8f7c70f 100644 --- a/Makefile +++ b/Makefile @@ -7,4 +7,7 @@ test: emake prove t/*.t clean: - rm -rf $(wildcard ebin/*.beam) erl_crash.dump \ No newline at end of file + rm -rf $(wildcard ebin/*.beam) erl_crash.dump + +boot: + erl -pa ./ebin/ -noshell -run systools make_script emongo -run init stop diff --git a/emongo.rel b/emongo.rel new file mode 100644 index 0000000..d17744c --- /dev/null +++ b/emongo.rel @@ -0,0 +1,7 @@ +{release, {"emongo", "emongo-0.2"}, {erts, "5.8"}, + [{kernel, "2.13.5"}, + {stdlib, "1.16.5"}, + {sasl, "2.1.9"}, + {emongo, "0.2"} + ]}. + diff --git a/priv/msg.config b/priv/msg.config new file mode 100644 index 0000000..24b686c --- /dev/null +++ b/priv/msg.config @@ -0,0 +1,10 @@ +[{emongo, [ + {pools, [ + {message_pool, [ + {size, 1}, + {host, "localhost"}, + {port, 27017}, + {database, "msg_store_db"} + ]} + ]} +]}]. diff --git a/src/emongo.erl b/src/emongo.erl index 5775125..09587ed 100644 --- a/src/emongo.erl +++ b/src/emongo.erl @@ -30,7 +30,7 @@ auth/3, find/2, find/3, find/4, find_all/2, find_all/3, find_all/4, get_more/4, get_more/5, find_one/3, find_one/4, kill_cursors/2, insert/3, update/4, update/5, update_sync/4, update_sync/5, - delete/2, delete/3, ensure_index/3, count/2, dec2hex/1, + delete/2, delete/3, ensure_index/3, count/2, count/4, dec2hex/1, hex2dec/1]). -include("emongo.hrl"). @@ -292,6 +292,32 @@ count(PoolId, Collection) -> undefined end. +count(PoolId, Collection, [], []) -> + count(PoolId, Collection); +count(PoolId, Collection, Selector, Options) -> + {Pid, Pool} = gen_server:call(?MODULE, {pid, PoolId}, infinity), + %Query1 = create_query(Options, Selector), + %Query = create_query(Options, #emo_query{}, transform_selector(Selector), [{<<"count">>, Collection}, {<<"ns">>, Pool#pool.database}]), + io:format("Options: ~p~n", [Options]), + Query = case Options of + [] -> + create_query([{limit, 1}, {fields, []}], #emo_query{}, transform_selector(Selector), [{<<"count">>, Collection}]); + _ -> + create_query(Options, #emo_query{}, transform_selector(Selector), [{<<"count">>, Collection}]) + end, + %[TypeSelector] = Query1#emo_query.q, + %%[FieldSelector] = Query1#emo_query.field_selector, + %CountQ = lists:flatten([Query1#emo_query.q, {<<"count">>, Collection}, {<<"ns">>, Pool#pool.database}]), + %Query = #emo_query{q=[{<<"count">>, Collection}, {<<"ns">>, Pool#pool.database}] , field_selector=[TypeSelector]}, + %Query = #emo_query{q=CountQ}, + io:format("Query: ~p~n", [Query]), + Packet = emongo_packet:do_query(Pool#pool.database, "$cmd", Pool#pool.req_id, Query), + case emongo_conn:send_recv(Pid, Pool#pool.req_id, Packet, ?TIMEOUT) of + #response{documents=[[{<<"n">>,Count}|_]]} -> + round(Count); + _ -> + undefined + end. %drop_collection(PoolId, Collection) when is_atom(PoolId), is_list(Collection) -> %%==================================================================== @@ -507,7 +533,10 @@ create_query([], QueryRec, [], OptDoc) -> QueryRec#emo_query{q=OptDoc}; create_query([], QueryRec, QueryDoc, OptDoc) -> - QueryRec#emo_query{q=(OptDoc ++ [{<<"query">>, QueryDoc}])}; + io:format("create_query query: ~p~n", [OptDoc]), + Q1 = QueryRec#emo_query{q=(OptDoc ++ [{<<"query">>, QueryDoc}] ++ [{<<"fields">>, []}])}, + io:format("create_query query: ~p~n", [Q1]), + Q1; create_query([{limit, Limit}|Options], QueryRec, QueryDoc, OptDoc) -> QueryRec1 = QueryRec#emo_query{limit=Limit}, @@ -523,8 +552,15 @@ create_query([{orderby, Orderby}|Options], QueryRec, QueryDoc, OptDoc) -> create_query(Options, QueryRec, QueryDoc, OptDoc1); create_query([{fields, Fields}|Options], QueryRec, QueryDoc, OptDoc) -> - QueryRec1 = QueryRec#emo_query{field_selector=[{Field, 1} || Field <- Fields]}, - create_query(Options, QueryRec1, QueryDoc, OptDoc); + OptDoc1 = case Fields of + [] -> + io:format("create_query fields~n"), + OptDoc ++ [{<<"fields">>, <<0,5,0,0,0,0,0>>}]; + _ -> + QueryFields = QueryRec#emo_query{field_selector=[{Field, 1} || Field <- Fields]}, + OptDoc ++ [{<<"fields">>, QueryFields}] + end, + create_query(Options, QueryRec, QueryDoc, OptDoc); create_query([_|Options], QueryRec, QueryDoc, OptDoc) -> create_query(Options, QueryRec, QueryDoc, OptDoc). diff --git a/src/emongo_bson.erl b/src/emongo_bson.erl index a908c1b..12a9b59 100644 --- a/src/emongo_bson.erl +++ b/src/emongo_bson.erl @@ -28,8 +28,24 @@ encode([]) -> <<5,0,0,0,0>>; encode([{_,_}|_]=List) when is_list(List) -> + T = fun({Key, Val}) -> + io:format("EKV: ~p~n", [encode_key_value(Key, Val)]) + end, + [T(X) || X <- List], Bin = iolist_to_binary([encode_key_value(Key, Val) || {Key, Val} <- List]), - <<(size(Bin)+5):32/little-signed, Bin/binary, 0:8>>. + io:format("Bin: ~p~n", [Bin]), + io:format("Bin size: ~p~n", [size(Bin)]), + <<(size(Bin)+4):32/little-signed, Bin/binary>>. + +encode_document([{_,_}|_]=List) when is_list(List) -> + T = fun({Key, Val}) -> + io:format("EKV: ~p~n", [encode_key_value(Key, Val)]) + end, + [T(X) || X <- List], + Bin = iolist_to_binary([encode_key_value(Key, Val) || {Key, Val} <- List]), + io:format("Bin: ~p~n", [Bin]), + io:format("Bin size: ~p~n", [size(Bin)]), + <<(size(Bin)+5):32/little-signed, Bin/binary>>. %% FLOAT encode_key_value(Key, Val) when is_float(Val) -> @@ -39,20 +55,30 @@ encode_key_value(Key, Val) when is_float(Val) -> %% STRING encode_key_value(Key, Val) when is_binary(Val) orelse Val == [] orelse (is_list(Val) andalso length(Val) > 0 andalso is_integer(hd(Val))) -> Key1 = encode_key(Key), + io:format("Key: ~p Value: ~p~n", [Key1, Val]), case unicode:characters_to_binary(Val) of {error, Bin, RestData} -> exit({cannot_convert_chars_to_binary, Val, Bin, RestData}); {incomplete, Bin1, Bin2} -> exit({cannot_convert_chars_to_binary, Val, Bin1, Bin2}); Val1 -> - <<2, Key1/binary, 0, (byte_size(Val1)+1):32/little-signed, Val1/binary, 0:8>> + case Key1 of + <<"fields">> -> + FieldsVal = <<0,0,0,0>>, + <<3, Key1/binary, 0, (byte_size(FieldsVal)+1):8/signed, FieldsVal/binary, 0:8>>; + _ -> + io:format("MEOW~n"), + <<2, Key1/binary, 0, (byte_size(Val1)+1):32/little-signed, Val1/binary, 0:8>> + end end; %% NESTED OBJECT encode_key_value(Key, [{_,_}|_]=Val) -> Key1 = encode_key(Key), - Val1 = encode(Val), - <<3, Key1/binary, 0, Val1/binary>>; + io:format("ABC: ~p~n",[Val]), + Val1 = encode_document(Val), + io:format("Key1: ~p Val1 ~p~n",[Key1, Val1]), + <<3, Key1/binary, 0, Val1/binary, 0:8>>; %% DATA ARRAY encode_key_value(Key, {array, Val}) when is_list(Val) -> @@ -230,4 +256,4 @@ decode_value(18, <>) -> {Int, Tail}; decode_value(_, _) -> - exit(oh_fuck). \ No newline at end of file + exit(oh_fuck). diff --git a/src/emongo_packet.erl b/src/emongo_packet.erl index 8b58fa4..d1bbb87 100644 --- a/src/emongo_packet.erl +++ b/src/emongo_packet.erl @@ -47,19 +47,28 @@ insert(Database, Collection, ReqID, Documents) -> do_query(Database, Collection, ReqID, Query) when is_record(Query, emo_query) -> OptsSum = lists:foldl(fun(X, Acc) -> Acc+X end, 0, Query#emo_query.opts), FullName = unicode:characters_to_binary([Database, ".", Collection]), + io:format("Query: ~p~n", [Query#emo_query.q]), EncodedDocument = if is_binary(Query#emo_query.q) -> Query#emo_query.q; true -> emongo_bson:encode(Query#emo_query.q) end, - EncodedFieldSelector = if - Query#emo_query.field_selector == [] -> <<>>; - true -> emongo_bson:encode(Query#emo_query.field_selector) - end, + %EncodedFieldSelector = if + %Query#emo_query.field_selector == [] -> <<"fields", 0,5,0,0,0,0,0>>; + %true -> emongo_bson:encode(Query#emo_query.field_selector) + %end, + io:format("ED: ~p~n", [EncodedDocument]), + %io:format("EFS: ~p~n", [EncodedFieldSelector]), + %Message = <>, Message = <>, + EncodedDocument/binary>>, Length = byte_size(Message), + io:format("MESSAGE: ~p~n", [Message]), + io:format("Length: ~p~n", [Length]), <<(Length+16):32/little-signed, ReqID:32/little-signed, 0:32, ?OP_QUERY:32/little-signed, Message/binary>>. get_more(Database, Collection, ReqID, NumToReturn, CursorID) -> @@ -127,4 +136,4 @@ index_name([{Key, Val}|Tail], Bin) -> is_integer(Val) -> list_to_binary(integer_to_list(Val)); true -> <<>> end, - index_name(Tail, <>). \ No newline at end of file + index_name(Tail, <>). From f3936ac5f56b832fab1064944c499dcf83836e73 Mon Sep 17 00:00:00 2001 From: Kelly McLaughlin Date: Fri, 24 Sep 2010 15:50:23 -0600 Subject: [PATCH 2/3] find/4 and count/4 functions working properly --- src/emongo.erl | 45 +++++++++++++++---------------------------- src/emongo_bson.erl | 18 +---------------- src/emongo_packet.erl | 25 ++++++++---------------- 3 files changed, 25 insertions(+), 63 deletions(-) diff --git a/src/emongo.erl b/src/emongo.erl index 09587ed..adbd6b2 100644 --- a/src/emongo.erl +++ b/src/emongo.erl @@ -158,7 +158,13 @@ find(PoolId, Collection, Query) when is_record(Query, emo_query) -> %% Result = documents() | response() find(PoolId, Collection, Selector, Options) when ?IS_DOCUMENT(Selector), is_list(Options) -> {Pid, Pool} = gen_server:call(?MODULE, {pid, PoolId}, infinity), - Query = create_query(Options, Selector), + Query = case proplists:is_defined(fields, Options) of + true -> + Query1 = create_query(proplists:delete(fields,Options), Selector), + Query1#emo_query{field_selector=[{Field, 1} || Field <- proplists:get_value(fields, Options)]}; + false -> + create_query(Options, Selector) + end, Packet = emongo_packet:do_query(Pool#pool.database, Collection, Pool#pool.req_id, Query), Resp = emongo_conn:send_recv(Pid, Pool#pool.req_id, Packet, proplists:get_value(timeout, Options, ?TIMEOUT)), case lists:member(response_options, Options) of @@ -296,21 +302,7 @@ count(PoolId, Collection, [], []) -> count(PoolId, Collection); count(PoolId, Collection, Selector, Options) -> {Pid, Pool} = gen_server:call(?MODULE, {pid, PoolId}, infinity), - %Query1 = create_query(Options, Selector), - %Query = create_query(Options, #emo_query{}, transform_selector(Selector), [{<<"count">>, Collection}, {<<"ns">>, Pool#pool.database}]), - io:format("Options: ~p~n", [Options]), - Query = case Options of - [] -> - create_query([{limit, 1}, {fields, []}], #emo_query{}, transform_selector(Selector), [{<<"count">>, Collection}]); - _ -> - create_query(Options, #emo_query{}, transform_selector(Selector), [{<<"count">>, Collection}]) - end, - %[TypeSelector] = Query1#emo_query.q, - %%[FieldSelector] = Query1#emo_query.field_selector, - %CountQ = lists:flatten([Query1#emo_query.q, {<<"count">>, Collection}, {<<"ns">>, Pool#pool.database}]), - %Query = #emo_query{q=[{<<"count">>, Collection}, {<<"ns">>, Pool#pool.database}] , field_selector=[TypeSelector]}, - %Query = #emo_query{q=CountQ}, - io:format("Query: ~p~n", [Query]), + Query = create_query(Options ++ [{fields, []}, {limit, 1}], #emo_query{}, transform_selector(Selector), [{<<"count">>, Collection}]), Packet = emongo_packet:do_query(Pool#pool.database, "$cmd", Pool#pool.req_id, Query), case emongo_conn:send_recv(Pid, Pool#pool.req_id, Packet, ?TIMEOUT) of #response{documents=[[{<<"n">>,Count}|_]]} -> @@ -533,10 +525,11 @@ create_query([], QueryRec, [], OptDoc) -> QueryRec#emo_query{q=OptDoc}; create_query([], QueryRec, QueryDoc, OptDoc) -> - io:format("create_query query: ~p~n", [OptDoc]), - Q1 = QueryRec#emo_query{q=(OptDoc ++ [{<<"query">>, QueryDoc}] ++ [{<<"fields">>, []}])}, - io:format("create_query query: ~p~n", [Q1]), - Q1; + QueryRec#emo_query{q=(OptDoc ++ [{<<"query">>, QueryDoc}])}; + +create_query([{fields, Fields}], QueryRec, QueryDoc, OptDoc) -> + QueryFields = [{Field, 1} || Field <- Fields], + QueryRec#emo_query{q=(OptDoc ++ [{<<"query">>, QueryDoc}] ++ [{<<"fields">>, QueryFields}])}; create_query([{limit, Limit}|Options], QueryRec, QueryDoc, OptDoc) -> QueryRec1 = QueryRec#emo_query{limit=Limit}, @@ -552,15 +545,9 @@ create_query([{orderby, Orderby}|Options], QueryRec, QueryDoc, OptDoc) -> create_query(Options, QueryRec, QueryDoc, OptDoc1); create_query([{fields, Fields}|Options], QueryRec, QueryDoc, OptDoc) -> - OptDoc1 = case Fields of - [] -> - io:format("create_query fields~n"), - OptDoc ++ [{<<"fields">>, <<0,5,0,0,0,0,0>>}]; - _ -> - QueryFields = QueryRec#emo_query{field_selector=[{Field, 1} || Field <- Fields]}, - OptDoc ++ [{<<"fields">>, QueryFields}] - end, - create_query(Options, QueryRec, QueryDoc, OptDoc); + %% The fields document needs to be last + %% so it requires some special handling + create_query(proplists:delete(fields, Options) ++ [{fields, Fields}], QueryRec, QueryDoc, OptDoc); create_query([_|Options], QueryRec, QueryDoc, OptDoc) -> create_query(Options, QueryRec, QueryDoc, OptDoc). diff --git a/src/emongo_bson.erl b/src/emongo_bson.erl index 12a9b59..e0f35ff 100644 --- a/src/emongo_bson.erl +++ b/src/emongo_bson.erl @@ -28,23 +28,11 @@ encode([]) -> <<5,0,0,0,0>>; encode([{_,_}|_]=List) when is_list(List) -> - T = fun({Key, Val}) -> - io:format("EKV: ~p~n", [encode_key_value(Key, Val)]) - end, - [T(X) || X <- List], Bin = iolist_to_binary([encode_key_value(Key, Val) || {Key, Val} <- List]), - io:format("Bin: ~p~n", [Bin]), - io:format("Bin size: ~p~n", [size(Bin)]), - <<(size(Bin)+4):32/little-signed, Bin/binary>>. + <<(size(Bin)+5):32/little-signed, Bin/binary, 0:8>>. encode_document([{_,_}|_]=List) when is_list(List) -> - T = fun({Key, Val}) -> - io:format("EKV: ~p~n", [encode_key_value(Key, Val)]) - end, - [T(X) || X <- List], Bin = iolist_to_binary([encode_key_value(Key, Val) || {Key, Val} <- List]), - io:format("Bin: ~p~n", [Bin]), - io:format("Bin size: ~p~n", [size(Bin)]), <<(size(Bin)+5):32/little-signed, Bin/binary>>. %% FLOAT @@ -55,7 +43,6 @@ encode_key_value(Key, Val) when is_float(Val) -> %% STRING encode_key_value(Key, Val) when is_binary(Val) orelse Val == [] orelse (is_list(Val) andalso length(Val) > 0 andalso is_integer(hd(Val))) -> Key1 = encode_key(Key), - io:format("Key: ~p Value: ~p~n", [Key1, Val]), case unicode:characters_to_binary(Val) of {error, Bin, RestData} -> exit({cannot_convert_chars_to_binary, Val, Bin, RestData}); @@ -67,7 +54,6 @@ encode_key_value(Key, Val) when is_binary(Val) orelse Val == [] orelse (is_list( FieldsVal = <<0,0,0,0>>, <<3, Key1/binary, 0, (byte_size(FieldsVal)+1):8/signed, FieldsVal/binary, 0:8>>; _ -> - io:format("MEOW~n"), <<2, Key1/binary, 0, (byte_size(Val1)+1):32/little-signed, Val1/binary, 0:8>> end end; @@ -75,9 +61,7 @@ encode_key_value(Key, Val) when is_binary(Val) orelse Val == [] orelse (is_list( %% NESTED OBJECT encode_key_value(Key, [{_,_}|_]=Val) -> Key1 = encode_key(Key), - io:format("ABC: ~p~n",[Val]), Val1 = encode_document(Val), - io:format("Key1: ~p Val1 ~p~n",[Key1, Val1]), <<3, Key1/binary, 0, Val1/binary, 0:8>>; %% DATA ARRAY diff --git a/src/emongo_packet.erl b/src/emongo_packet.erl index d1bbb87..694a7cf 100644 --- a/src/emongo_packet.erl +++ b/src/emongo_packet.erl @@ -47,28 +47,19 @@ insert(Database, Collection, ReqID, Documents) -> do_query(Database, Collection, ReqID, Query) when is_record(Query, emo_query) -> OptsSum = lists:foldl(fun(X, Acc) -> Acc+X end, 0, Query#emo_query.opts), FullName = unicode:characters_to_binary([Database, ".", Collection]), - io:format("Query: ~p~n", [Query#emo_query.q]), EncodedDocument = if is_binary(Query#emo_query.q) -> Query#emo_query.q; true -> emongo_bson:encode(Query#emo_query.q) end, - %EncodedFieldSelector = if - %Query#emo_query.field_selector == [] -> <<"fields", 0,5,0,0,0,0,0>>; - %true -> emongo_bson:encode(Query#emo_query.field_selector) - %end, - io:format("ED: ~p~n", [EncodedDocument]), - %io:format("EFS: ~p~n", [EncodedFieldSelector]), - %Message = <>, - Message = <>, + EncodedFieldSelector = if + Query#emo_query.field_selector == [] -> <<>>; + true -> emongo_bson:encode(Query#emo_query.field_selector) + end, + Message = <>, Length = byte_size(Message), - io:format("MESSAGE: ~p~n", [Message]), - io:format("Length: ~p~n", [Length]), <<(Length+16):32/little-signed, ReqID:32/little-signed, 0:32, ?OP_QUERY:32/little-signed, Message/binary>>. get_more(Database, Collection, ReqID, NumToReturn, CursorID) -> From 6fd6399edd590434a6df7652f2af540da0054f7a Mon Sep 17 00:00:00 2001 From: Kelly McLaughlin Date: Tue, 28 Sep 2010 07:47:49 -0600 Subject: [PATCH 3/3] Remove test config file and release file and rollback Makefile edits. --- Makefile | 3 --- emongo.rel | 7 ------- priv/msg.config | 10 ---------- 3 files changed, 20 deletions(-) delete mode 100644 emongo.rel delete mode 100644 priv/msg.config diff --git a/Makefile b/Makefile index 8f7c70f..f2ad018 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,3 @@ test: emake clean: rm -rf $(wildcard ebin/*.beam) erl_crash.dump - -boot: - erl -pa ./ebin/ -noshell -run systools make_script emongo -run init stop diff --git a/emongo.rel b/emongo.rel deleted file mode 100644 index d17744c..0000000 --- a/emongo.rel +++ /dev/null @@ -1,7 +0,0 @@ -{release, {"emongo", "emongo-0.2"}, {erts, "5.8"}, - [{kernel, "2.13.5"}, - {stdlib, "1.16.5"}, - {sasl, "2.1.9"}, - {emongo, "0.2"} - ]}. - diff --git a/priv/msg.config b/priv/msg.config deleted file mode 100644 index 24b686c..0000000 --- a/priv/msg.config +++ /dev/null @@ -1,10 +0,0 @@ -[{emongo, [ - {pools, [ - {message_pool, [ - {size, 1}, - {host, "localhost"}, - {port, 27017}, - {database, "msg_store_db"} - ]} - ]} -]}].