Dperl graph3

priv/dev/static/index.html

@@ -24,6 +24,8 @@
                     <li><a id="btn-about">About</a></li>
                     <li><a id="btn-disconnect">Disconnect</a></li>
                     <li><a id="btn-restart">Restart Application</a></li>
+                    <li><a id="btn-office-365">Register Office 365</a></li>
+                    <li><a id="btn-oura-ring">Register Oura Ring</a></li>
                 </ul>
             </li>
             <li><a id="connect-button">New Connection</a></li>

priv/dev/static/index.js

@@ -8,7 +8,8 @@ import {loginAjax} from "./scripts/login";
 import {alert_jq} from './dialogs/dialogs';
 import {dderlState, show_qry_files, register_fido2_key,
         change_password, show_about_dlg} from "./scripts/dderl";
-import {new_connection_tab, logout, restart} from "./scripts/login";
+import {new_connection_tab, logout, restart,
+        authorize_office, authorize_oura} from "./scripts/login";
 import {disconnect_tab, close_tab} from "./scripts/connect";
 import {newSqlEditor} from "./scripts/dderl.sql";
 import {patch_jquery_ui} from "./jquery-ui-helper/helper.js";
@@ -94,3 +95,5 @@ addClick('btn-restart',         () => { restart();              });
 addClick('connect-button',      () => { new_connection_tab();   });
 addClick('newsql-button',       () => { newSqlEditor();         });
 addClick('btn-logout',          () => { logout();               });
+addClick('btn-office-365',      () => { authorize_office();     });
+addClick('btn-oura-ring',       () => { authorize_oura();       });

priv/dev/static/scripts/login.js

@@ -23,6 +23,30 @@ export function loginAjax(data = {}) {
 
 window.loginCb = loginCb;
 
+// oauth2 callback handler
+const url = new URL(window.location.href);
+const code = url.searchParams.get('code');
+const error = url.searchParams.get('error');
+const state = JSON.parse(url.searchParams.get('state'));
+if (code) {
+    dderlState.xsrfToken = state.xsrfToken;
+    const body = { 'oauth2_callback': { 'code': code, 'state': state } };
+    ajaxCall(null, 'oauth2_callback', body, 'oauth2_callback',
+        function () {
+            window.close();
+        },
+        function (error) {
+            alert('Error fetching access token : ' + error);
+            console.log('Error fetching access token', error);
+            window.close();
+        });
+} else if (error) {
+    const errorDesc = url.searchParams.get('error_description');
+    alert('Login error : ' + error + ' : ' + errorDesc);
+    window.close();
+}
+// oauth2 callback handler end
+
 function loginCb(resp) {
     try {
         if (window.opener && window.opener.isScreensaver && window.opener.loginCb && $.isFunction(window.opener.loginCb)) {
@@ -417,9 +441,18 @@ export function change_login_password(loggedInUser, shouldConnect) {
     });
 }
 
+    ajaxCall(null, 'office_365_auth_config', {}, 'office_365_auth_config', function (auth_config) {
+        const params = 'scrollbars=no,resizable=no,status=no,location=no,toolbar=no,menubar=no,width=500,height=600,left=100,top=100';
+        window.open(auth_config.url, 'Office 365 login', params);
+    });
+}
+
+export function authorize_oura() {
+    ajaxCall(null, 'oura_ring_auth_config', {}, 'oura_ring_auth_config', function(auth_config) {
+        const params = 'scrollbars=no,resizable=no,status=no,location=no,toolbar=no,menubar=no,width=500,height=600,left=100,top=100';
+        window.open(auth_config.url, 'Oura Ring login', params);
+    });
 // fido2 helper functions
-function _arrayBufferToString(buffer) {
-    var binary = '';
     var bytes = new Uint8Array(buffer);
     var len = bytes.byteLength;
     for (var i = 0; i < len; i++) {

src/dderl.hrl

@@ -1,6 +1,7 @@
 -ifndef(DDERL_HRL).
 -define(DDERL_HRL, true).
 
+-include_lib("imem/src/imem_config.hrl").
 -include_lib("imem/src/imem_meta.hrl").
 -include_lib("imem/src/imem_exports.hrl").
 
@@ -343,4 +344,10 @@
          end)()
 ).
 
+% OAUTH 
+
+-define(SYNC_OURARING, dpjob_ouraring_crawl).
+
+-define(SYNC_OFFICE365, dpjob_office_365).
+
 -endif.

src/dderl_dal.erl

@@ -45,6 +45,9 @@
         ,get_d3_templates_path/1
         ,get_host_app/0
         ,is_proxy/2
+        ,create_check_avatar_channel/1
+        ,write_to_avatar_channel/3
+        ,read_from_avatar_channel/2
         ,fido2_register_config/4
         ]).
 
@@ -61,6 +64,12 @@
 -define(USE_CONN(__ConnId), {dderl, conn, {conn, __ConnId}, use}).
 -define(USE_LOCAL_CONN, {dderl, conn, local, use}).
 
+-define(GET_AVATAR_CHANNEL_PREFIX,
+    ?GET_CONFIG(avatarChannelPrefix,[], <<"ph">>,"PRIVACY_HUB - binary string prefix for avatar channel name")).
+
+-define(GET_AVATAR_CHANNEL_OPTIONS,
+    ?GET_CONFIG(avatarChannelOptions,[], [encrypted,audit,history],"PRIVACY_HUB - table options for avatar channel")).
+
 %% Validate this permission.
 -define(USE_ADAPTER, {dderl, adapter, {id, __AdaptId}, use}).
 
@@ -76,13 +85,13 @@ del_conn(Sess, UserId, ConId) ->
     HasAll = (erlimem_session:run_cmd(Sess, have_permission, [[?MANAGE_CONNS]]) == true),
     if HasAll ->
         ok = erlimem_session:run_cmd(Sess, delete, [ddConn, ConId]),
-        ?Info("user ~p deleted connection ~p", [UserId, ConId]),
+        %?Info("user ~p deleted connection ~p", [UserId, ConId]),
         ok;
     true ->
         case erlimem_session:run_cmd(Sess, select, [ddConn, [{#ddConn{id=ConId,owner=UserId,_='_'},[],['$_']}]]) of
             {[_|_], true} ->
                 ok = erlimem_session:run_cmd(Sess, delete, [ddConn, ConId]),
-                ?Info("user ~p deleted connection ~p", [UserId, ConId]),
+                %?Info("user ~p deleted connection ~p", [UserId, ConId]),
                 ok;
             _ ->
                 ?Error("user ~p doesn't have permission to delete connection ~p", [UserId, ConId]),
@@ -599,7 +608,7 @@ is_local_query(Qry) ->
 can_connect_locally(Sess) ->
     erlimem_session:run_cmd(Sess, have_permission, [[?USE_LOCAL_CONN]]) == true.
 
--spec conn_permission({atom(), pid()}, ddEntityId(), #ddConn{}) -> boolean().
+-spec conn_permission({atom(), pid()}, integer()|atom(), #ddConn{}) -> boolean().
 conn_permission(_Sess, UserId, #ddConn{owner=UserId}) -> true; %% If it is the owner always allow usage.
 conn_permission(Sess, _UserId, #ddConn{id=ConnId, owner=system}) ->
     erlimem_session:run_cmd(Sess, have_permission, [?USE_SYS_CONNS]) orelse  %% If it can use system connections.
@@ -894,6 +903,28 @@ is_proxy(AppId, NetCtx) ->
         _ -> false
     end.
 
+avatar_channel_name(AccountId) when is_atom(AccountId) ->
+    avatar_channel_name(atom_to_binary(AccountId, utf8));
+avatar_channel_name(AccountId) when is_list(AccountId) ->
+    avatar_channel_name(list_to_binary(AccountId));
+avatar_channel_name(AccountId) when is_binary(AccountId) ->
+    Prefix = ?GET_AVATAR_CHANNEL_PREFIX,
+    <<Prefix/binary, AccountId/binary>>.
+
+% -spec create_check_avatar_channel(ddEntityId()) -> binary().
+create_check_avatar_channel(AccountId) ->
+    imem_dal_skvh:create_check_channel(avatar_channel_name(AccountId), ?GET_AVATAR_CHANNEL_OPTIONS).
+
+write_to_avatar_channel(AccountId, Key, Value) ->
+    imem_dal_skvh:write(AccountId, avatar_channel_name(AccountId), Key, Value).
+
+read_from_avatar_channel(AccountId, Key) ->
+    case imem_dal_skvh:read(AccountId, avatar_channel_name(AccountId), [Key]) of
+        [#{cvalue := CValue}] when is_binary(CValue) -> imem_json:decode(CValue, [return_maps]);
+        [#{cvalue := CValue}] when is_map(CValue)->     CValue;
+        [] ->                                           []
+    end.
+
 -spec exec_is_proxy_fun(reference(), map()) -> boolean().
 exec_is_proxy_fun(Fun, NetCtx) ->
     case catch Fun(NetCtx) of

src/dderl_diff.erl

@@ -10,7 +10,9 @@
 
 -spec term_diff(term(), pid(), atom(), binary(), atom(), binary()) -> binary().
 term_diff(Sess, SessPid, LeftType, LeftValue, RightType, RightValue) ->
-    case Sess:run_cmd(term_diff, [LeftType, LeftValue, RightType, RightValue, [ignore_whitespace,ignore_casing,ignore_dquotes]]) of
+    %?Info("term_diff called with args: ~p ~p ~p ~p", [LeftType, LeftValue, RightType, RightValue]),
+    %?Info("term_diff called with Sess ~p SessPid ~p", [Sess, SessPid]),
+    case erlimem_session:run_cmd(Sess, term_diff, [LeftType, LeftValue, RightType, RightValue, [ignore_whitespace,ignore_casing,ignore_dquotes]]) of
         {error, {{Ex, M}, _Stacktrace} = Error} ->
             ?Error("Error on term_diff ~p: ~p", [{LeftType, LeftValue, RightType, RightValue}, Error], _Stacktrace),
             Err = list_to_binary(atom_to_list(Ex) ++ ": " ++
@@ -26,7 +28,7 @@ term_diff(Sess, SessPid, LeftType, LeftValue, RightType, RightValue) ->
             Err = list_to_binary(lists:flatten(io_lib:format("~p", [Reason]))),
             #{<<"error">> => Err};
         DiffResult ->
-            ?Debug("The diff result ~p", [DiffResult]),
+            %?Info("The diff result ~p", [DiffResult]),
             FsmCtx = get_fsmctx(DiffResult),
             StmtFsm = dderl_fsm:start(FsmCtx, SessPid),
             Columns = gen_adapter:build_column_json(lists:reverse(get_columns())),
@@ -42,8 +44,9 @@ term_diff(Sess, SessPid, LeftType, LeftValue, RightType, RightValue) ->
 get_fsmctx(Result) ->
     % <<Id:32>> = crypto:strong_rand_bytes(4),
     RowCols = get_columns(),
+    Pid = self(),
     FullMap = build_full_map(RowCols),
-    #fsmctxs{stmtRefs       = [self()]
+    #fsmctxs{stmtRefs       = [Pid]
             ,stmtTables     = [<<"term_diff">>]
             ,rowCols        = RowCols
             ,rowFun         = get_rowfun()
@@ -56,8 +59,7 @@ get_fsmctx(Result) ->
                 [fun(_Opts, _Count) ->
                     Rows = [{{}, {RowId, Left, Cmp, Right}} || {ddTermDiff, RowId, Left, Cmp, Right} <- Result],
                     % This seems hackish but we don't want to keep a process here.
-                    % TODO: Revisit after tuple calls have been removed.
-                    dderl_fsm:rows({self(), {Rows, true}}, {dderl_fsm, self()})
+                    dderl_fsm:rows(self(), {Pid, {Rows, true}})
                 end]
             ,fetch_close_funs = [fun() -> ok end]
             ,stmt_close_funs  = [fun() -> ok end]

src/dderl_fsm.erl

@@ -333,7 +333,7 @@ rows(Pid, {StmtRef, {Rows,Completed}}) when is_list(Rows) ->  % from erlimem/ime
     %?Info("dderl_fsm:rows from ~p ~p~n~p", [StmtRef, length(Rows), Rows]),
     gen_statem:cast(Pid, {rows, {StmtRef,Rows,Completed}});
 rows(Pid, {Rows,Completed}) when is_list(Rows) ->
-    %?Info("dderl_fsm:rows ~p ~p", [length(Rows), Completed]),
+    ?Info("dderl_fsm:rows ~p ~p", [length(Rows), Completed]),
     gen_statem:cast(Pid, {rows, {self(),Rows,Completed}});
 rows(Pid, {StmtRef, Error}) ->   % from erlimem/imem_server
     %?Info("dderl_fsm:rows from ~p ~p", [StmtRef, Error]),
@@ -1858,13 +1858,11 @@ sort_distinct_count_rows([_ | XT], [_ | YT]) -> sort_distinct_count_rows(XT, YT)
 %% --------------------------------------------------------------------
 handle_info({StmtRef,{error,Reason}}, SN, State) ->
     %?Info("dderl_fsm:handle_info from ~p ~p",[StmtRef, {error,Reason}]),
-    Fsm = {?MODULE,self()},
-    Fsm:rows({StmtRef,{error,Reason}}),
+    dderl_fsm:rows(self(), {StmtRef,{error,Reason}}),
     {next_state, SN, State};
 handle_info({StmtRef,{Rows,Completed}}, SN, State) ->
     %?Info("dderl_fsm:handle_info from ~p Rows ~p completed ~p",[StmtRef, length(Rows), Completed]),
-    Fsm = {?MODULE,self()},
-    Fsm:rows({StmtRef,Rows,Completed}),
+    dderl_fsm:rows(self(), {StmtRef,{Rows,Completed}}),
     {next_state, SN, State};
 handle_info(cmd_stack_timeout, SN, #state{stack={button, <<"tail">>, RT}}=State)
     when SN =:= tailing; SN =:= passthrough ->

src/dderl_oauth.erl

@@ -0,0 +1,89 @@
+-module(dderl_oauth).
+
+-include("dderl.hrl").
+
+-define(TOKEN_KEYPART, "#token#").
+
+-export([ get_authorize_url/3
+        , get_access_token/4
+        , get_token_info/3
+        , refresh_access_token/3
+        ]).
+
+get_token_info(AccountId, TokenPrefix, _SyncType) ->
+    dderl_dal:read_from_avatar_channel(AccountId, TokenPrefix ++ [?TOKEN_KEYPART]).
+
+set_token_info(AccountId, TokenPrefix, TokenInfo, SyncMod) when is_map(TokenInfo) ->
+    set_token_info(AccountId, TokenPrefix, imem_json:encode(TokenInfo), SyncMod);
+set_token_info(AccountId, TokenPrefix, TokenInfo, SyncMod) when is_list(TokenInfo) ->
+    set_token_info(AccountId, TokenPrefix, list_to_binary(TokenInfo), SyncMod);
+set_token_info(AccountId, TokenPrefix, TokenInfo, _SyncType) when is_binary(TokenInfo) ->
+    %?Info("set_token_info using ~p",[imem_enc_mnesia:get_enc_hash()]),
+    dderl_dal:write_to_avatar_channel(AccountId, TokenPrefix ++ [?TOKEN_KEYPART], TokenInfo).
+
+get_authorize_url(XSRFToken, AuthConfig, SyncMod) ->
+    State = #{xsrfToken => XSRFToken, type => SyncMod},
+    #{auth_url:=Url, client_id:=ClientId, redirect_uri:=RedirectURI, scope:=Scope} = AuthConfig, 
+    UrlParams = dperl_dal:url_enc_params(
+        #{"client_id" => ClientId, "redirect_uri" => {enc, RedirectURI}
+         ,"scope" => {enc, Scope}, "state" => {enc, imem_json:encode(State)}}),
+    erlang:iolist_to_binary([Url, "&", UrlParams]).
+
+
+%% get token info for a web service using the configuration from callback module
+%% store it in the avatar table of AccountId under the key TokenPrefix || "#token#"  
+-spec get_access_token(ddEntityId(), list(), string(), module()) -> ok | {error, term()}.
+get_access_token(AccountId, TokenPrefix, Code, SyncMod) ->
+    AuthConfig = try 
+        SyncMod:get_auth_config() % ToDo: AuthConfig may depend on JobName or TokenPrefix
+    catch 
+        _:E:S ->
+            ?Error("Finding AuthConfig : ~p~n~p", [E,S]),
+            {error, E}
+    end,
+    %?Info("get_access_token AuthConfig: ~p",[AuthConfig]),
+    #{token_url:=TUrl, client_id:=ClientId, redirect_uri:=RedirectURI
+     ,client_secret:=Secret, grant_type:=GrantType
+     ,scope:=Scope} = AuthConfig,
+    Body = dperl_dal:url_enc_params(
+        #{ "client_id"=>ClientId, "scope"=>{enc, Scope}, "code"=>Code
+         , "redirect_uri"=>{enc, RedirectURI}, "grant_type"=>GrantType
+         , "client_secret"=>{enc, Secret}}),
+    ContentType = "application/x-www-form-urlencoded",
+    case httpc:request(post, {TUrl, "", ContentType, Body}, [], []) of
+        {ok, {{_, 200, "OK"}, _, TokenInfo}} ->
+            set_token_info(AccountId, TokenPrefix, TokenInfo, SyncMod),
+            ok;
+        {ok, {{_, Code, _}, _, Error}} ->
+            {error, Error};
+        {error, Error} ->
+            ?Error("Fetching access token : ~p", [Error]),
+            {error, Error}
+    end.
+
+%% refresh access token from web service using the configuration from callback module
+%% store it in the avatar table of AccountId under the key TokenPrefix || "#token#"  
+-spec refresh_access_token(ddEntityId(), list(), module()) -> {ok, binary()} | {error, term()}.
+refresh_access_token(AccountId, TokenPrefix, SyncMod) ->
+    #{token_url:=TUrl, client_id:=ClientId  % , redirect_uri:=RedirectURI
+     ,client_secret:=Secret                 % , grant_type:=GrantType
+     ,scope:=Scope} = SyncMod:get_auth_config(),
+    %?Info("refresh_access_token ~p ~p ~p",[AccountId, TokenPrefix, SyncMod]),
+    #{<<"refresh_token">>:=RefreshToken} = get_token_info(AccountId, TokenPrefix, SyncMod),
+    Body = dperl_dal:url_enc_params(#{ "client_id"=>ClientId, "client_secret"=>{enc, Secret}
+                                     , "scope"=>{enc, Scope}, "refresh_token"=>RefreshToken
+                                     , "grant_type"=>"refresh_token"}),
+    ContentType = "application/x-www-form-urlencoded",
+    %?Info("refresh_access_token TUrl=~p",[TUrl]),
+    %?Info("refresh_access_token ContentType=~p",[ContentType]),
+    %?Info("refresh_access_token Body=~p",[Body]),
+    %?Info("refresh_access_token RefreshToken=~p",[RefreshToken]),
+    case httpc:request(post, {TUrl, "", ContentType, Body}, [], []) of
+        {ok, {{_, 200, "OK"}, _, TokenBody}} ->
+            TokenInfo = imem_json:decode(list_to_binary(TokenBody), [return_maps]),
+            set_token_info(AccountId, TokenPrefix, TokenInfo, SyncMod),
+            #{<<"access_token">> := Token} = TokenInfo,
+            {ok, Token};
+        Error ->
+            {error, Error}
+    end.

src/dderl_resource.erl

@@ -16,7 +16,7 @@
 -record(state, {sessionToken}).
 
 init(Req0, []) ->
-    Req = ?COW_REQ_SET_META(reqTime, os:timestamp(), Req0),
+    Req = ?COW_REQ_SET_META(reqTime, imem_meta:time(), Req0),
     Req1 = ?COW_REQ_SET_META(accessLog, #{}, Req),
     ?Debug("Request : ~p", [Req1]),
     case cowboy_req:has_body(Req1) of
@@ -166,7 +166,7 @@ terminate(_Reason, Req, _State) ->
     RespSize = ?COW_REQ_GET_META(respSize, Req, 0),
     ReqSize = cowboy_req:body_length(Req),
     Size = ReqSize + RespSize,
-    ProcessingTimeMicroS = timer:now_diff(os:timestamp(), ReqTime),
+    ProcessingTimeMicroS = imem_datatype:musec_diff(ReqTime),
     catch dderl_access_logger:log(Log#{bytes => Size,
                                        time => ProcessingTimeMicroS}).
 

src/dderl_saml_handler.erl

@@ -20,7 +20,7 @@ init(Req0, Args) ->
                 #{app := Application} -> Application;
                 _ -> dderl
             end,
-            Req = ?COW_REQ_SET_META(App, reqTime, os:timestamp(), Req0),
+            Req = ?COW_REQ_SET_META(App, reqTime, imem_meta:time(), Req0),
             Req1 = ?COW_REQ_SET_META(App, accessLog, #{}, Req),
             HostUrl = iolist_to_binary(cowboy_req:uri(Req1, #{path => undefined, qs => undefined})),
             Url = iolist_to_binary(cowboy_req:uri(Req1)),