From 572384a9202951416563e134ba3c55105d7aad22 Mon Sep 17 00:00:00 2001 From: sreepuramsudheer Date: Thu, 30 May 2024 19:31:15 +0530 Subject: [PATCH] Initial draft of telemetry. Signed-off-by: sreepuramsudheer --- src/oc_erchef/apps/chef_telemetry/.gitignore | 15 ++ src/oc_erchef/apps/chef_telemetry/Makefile | 30 ++++ src/oc_erchef/apps/chef_telemetry/README.md | 3 + .../apps/chef_telemetry/priv/.gitkeep | 0 .../apps/chef_telemetry/rebar.config | 30 ++++ .../chef_telemetry/src/chef_telemetry.app.src | 32 +++++ .../chef_telemetry/src/chef_telemetry.erl | 52 +++++++ .../chef_telemetry/src/chef_telemetry_app.erl | 34 +++++ .../chef_telemetry/src/chef_telemetry_sup.erl | 37 +++++ .../src/chef_telemetry_worker.erl | 133 ++++++++++++++++++ .../src/chef_teleterty_http.erl | 96 +++++++++++++ 11 files changed, 462 insertions(+) create mode 100644 src/oc_erchef/apps/chef_telemetry/.gitignore create mode 100644 src/oc_erchef/apps/chef_telemetry/Makefile create mode 100644 src/oc_erchef/apps/chef_telemetry/README.md create mode 100644 src/oc_erchef/apps/chef_telemetry/priv/.gitkeep create mode 100644 src/oc_erchef/apps/chef_telemetry/rebar.config create mode 100644 src/oc_erchef/apps/chef_telemetry/src/chef_telemetry.app.src create mode 100644 src/oc_erchef/apps/chef_telemetry/src/chef_telemetry.erl create mode 100644 src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_app.erl create mode 100644 src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_sup.erl create mode 100644 src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl create mode 100644 src/oc_erchef/apps/chef_telemetry/src/chef_teleterty_http.erl diff --git a/src/oc_erchef/apps/chef_telemetry/.gitignore b/src/oc_erchef/apps/chef_telemetry/.gitignore new file mode 100644 index 0000000000..6a325fcd24 --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/.gitignore @@ -0,0 +1,15 @@ +.eunit +deps/ +ebin/ +ebin_dialyzer/ +TAGS +.DS_Store +doc/*.html +*.beam +/doc/edoc-info +/doc/erlang.png +/doc/stylesheet.css +/deps.plt +test/*.out +.rebar +log/ diff --git a/src/oc_erchef/apps/chef_telemetry/Makefile b/src/oc_erchef/apps/chef_telemetry/Makefile new file mode 100644 index 0000000000..410366ea1c --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/Makefile @@ -0,0 +1,30 @@ +REBAR3_URL=https://s3.amazonaws.com/rebar3/rebar3 + +# If there is a rebar in the current directory, use it +ifeq ($(wildcard rebar3),rebar3) +REBAR3 = $(CURDIR)/rebar3 +endif + +# Fallback to rebar on PATH +REBAR3 ?= $(shell which rebar3) + +# And finally, prep to download rebar if all else fails +ifeq ($(REBAR3),) +REBAR3 = rebar3 +endif + +all: $(REBAR3) + @$(REBAR3) do clean, compile, eunit, dialyzer + +rel: all + @$(REBAR3) release + +distclean: + @rm -rf _build + +$(REBAR3): + curl -Lo rebar3 $(REBAR3_URL) || wget $(REBAR3_URL) + chmod a+x rebar3 + +install: $(REBAR3) distclean + $(REBAR3) update diff --git a/src/oc_erchef/apps/chef_telemetry/README.md b/src/oc_erchef/apps/chef_telemetry/README.md new file mode 100644 index 0000000000..a08073773c --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/README.md @@ -0,0 +1,3 @@ +# Telemetry + +Telemetry is an HTTP exporter of Chef Server node stats for external services. diff --git a/src/oc_erchef/apps/chef_telemetry/priv/.gitkeep b/src/oc_erchef/apps/chef_telemetry/priv/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/oc_erchef/apps/chef_telemetry/rebar.config b/src/oc_erchef/apps/chef_telemetry/rebar.config new file mode 100644 index 0000000000..a84b5bc0c3 --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/rebar.config @@ -0,0 +1,30 @@ +%% -*- mode: erlang -*- +%% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 ft=erlang et + +{deps, + [ + %% lager has to come first since we use its parse transform + {lager, ".*", + {git, "https://github.com/erlang-lager/lager", {branch, "master"}}}, + {opscoderl_httpc, ".*", + {git, "https://github.com/chef/opscoderl_httpc", {branch, "main"}}}, + {pooler, ".*", + {git, "https://github.com/chef/pooler", {branch, "master"}}} + ] +}. + +{profiles, [{ + test, [ + {deps, [meck]}, + {erl_opts, [export_all]} + ] +}]}. + +{erl_opts, [ + warnings_as_errors, + {parse_transform, lager_transform}, + debug_info +]}. + +{cover_enabled, true}. diff --git a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry.app.src b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry.app.src new file mode 100644 index 0000000000..54d495ed2e --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry.app.src @@ -0,0 +1,32 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil; fill-column: 92 -*- +%% ex: ts=4 sw=4 et +%% +%% +%% Copyright 2016 Chef Software, Inc. All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +{application, chef_telemetry, [ + {description, "Chef Server Telemetry"}, + {vsn, {cmd,"cat ../../VERSION | awk '{print $0}'"}}, + {registered, []}, + {applications, [ + lager, + chef_secrets, + opscoderl_httpc + ]}, + {mod, {chef_telemetry_app, []}} +]}. diff --git a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry.erl b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry.erl new file mode 100644 index 0000000000..180a3726be --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry.erl @@ -0,0 +1,52 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil; fill-column: 92 -*- +%% ex: ts=4 sw=4 et +%% +%% +%% Copyright 2016 Chef Software, Inc. All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(chef_telemetry). +-export([ + ping/0, + is_enabled/0, + token/0 + ]). + +-spec ping() -> pong | pang. +ping() -> + case chef_telemetry_http:get("/") of + ok -> pong; + _ -> pang + end. + +-spec is_enabled() -> boolean(). +is_enabled() -> + case application:get_env(chef_telemetry, root_url) of + {ok, _Value} -> + true; + undefined -> + false + end. + +-spec token() -> string() | atom(). +token() -> + case chef_secrets:get(<<"data_collector">>, <<"token">>) of + {ok, Token} -> + erlang:binary_to_list(Token); + {error, not_found} -> + undefined + end. diff --git a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_app.erl b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_app.erl new file mode 100644 index 0000000000..cacbb18a06 --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_app.erl @@ -0,0 +1,34 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil; fill-column: 92 -*- +%% ex: ts=4 sw=4 et +%% +%% Copyright 2016 Chef Software, Inc. All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(chef_telemetry_app). + +-behaviour(application). + +%% API +-export([start/2, + stop/1 + ]). + +start(_StartType, _StartArgs) -> + chef_telemetry_sup:start_link(). + +stop(_State) -> + ok. diff --git a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_sup.erl b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_sup.erl new file mode 100644 index 0000000000..94b1c02463 --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_sup.erl @@ -0,0 +1,37 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil; fill-column: 92 -*- +%% ex: ts=4 sw=4 et +%% +%% +%% Copyright 2016 Chef Software, Inc. All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(chef_telemetry_sup). + +-behaviour(supervisor). + +-export([init/1, + start_link/0 + ]). + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +init([]) -> + Worker = {chef_telemetry_worker, + {chef_telemetry_worker, start_link, []}, + permanent, 5000, supervisor, [chef_telemetry_worker]}, + {ok, {{one_for_one, 10, 10}, [Worker]}}. diff --git a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl new file mode 100644 index 0000000000..bdea7abd90 --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl @@ -0,0 +1,133 @@ +-module(chef_telemetry_worker). + +-behaviour(gen_server). + +-export([ + start_link/0, + solr_search/1 +]). + +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + code_change/2, + terminate/2 +]). + +-record(state, { + http_client, + timer_ref, + report_time +}). + +-record(oc_chef_organization, { + server_api_version, + id, + authz_id, + name, + full_name, + assigned_at, + last_updated_by, + created_at, + updated_at + }). + +-define(DEFAULT_DAYS, 30). + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +init(_Config) -> + % RootUrl = proplists:get_value(root_url, Config), + % Options = proplists:get_value(ibrowse_options, Config, []), + % HttpClient = oc_httpc_worker:start_link(RootUrl, Options, []), + State = #state{ + report_time = {12, 00}}, + gen_server:cast(self(), init_timer), + {ok, State}. + +handle_call(_Message, _From, State) -> + {noreply, State}. + +handle_cast(send_data, State) -> + ReqId = base64:encode(term_to_binary(make_ref())), + EpochNow = erlang:system_time(seconds), + TimeDuration = ?DEFAULT_DAYS * 86400, + Epoch30DaysOld = EpochNow - TimeDuration, + QueryString = lists:flatten(io_lib:format("ohai_time:{~p TO ~p}", [Epoch30DaysOld, EpochNow])), + Query1 = chef_index:query_from_params("node", QueryString, undefined, undefined), + DbContext = chef_db:make_context("1.0", ReqId, false), + _Count = + case chef_db:count_nodes(DbContext) of + Count1 when is_integer(Count1) -> Count1; + Error -> throw({db_error, Error}) + end, + Orgs = + case chef_db:list(#oc_chef_organization{}, DbContext) of + Orgs1 when is_list(Orgs1) -> Orgs1; + Error1 -> throw({db_error, Error1}) + end, + _Stats = [ {Org, get_org_nodes(Org, Query1, ReqId, DbContext)} || Org <- Orgs ], + % sending data logic hear + gen_server:cast(self(), init_timer), + {noreply, State}; + +handle_cast(init_timer, State) -> + {_Date, {Hour, Min, _Sec}} = erlang:universaltime(), + {RHour, RMin} = State#state.report_time, + CurrentDaySeconds = Hour * 3600 + Min * 60, + ReportingSeconds = RHour * 3600 + RMin * 60, + Diff = ReportingSeconds - CurrentDaySeconds, + if + Diff == 0 -> + gen_server:cast(self(), send_data); + Diff > 0 -> + timer:apply_after(Diff * 1000, gen_server, cast, [self(), send_data]); + Diff < 0 -> + timer:apply_after((Diff + 86400) * 1000, gen_server, cast, [self(), send_data]) + end, + {noreply, State}; + +handle_cast(_Message, State) -> + {noreply, State}. + +handle_info(_Message, State) -> + {noreply, State}. + +code_change(_OldVsn, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +get_org_nodes(OrgName, Query1, ReqId, DbContext) -> + {Guid1, _AuthzId1} = + case chef_db:fetch_org_metadata(DbContext, OrgName) of + not_found -> throw({org_not_found, OrgName}); + {Guid, AuthzId} -> {Guid, AuthzId} + end, + Query = chef_index:add_org_guid_to_query(Query1, Guid1), + case search(Query, ReqId) of + {ok, _Start0, _SolrNumFound, Ids} -> + erlang:length(Ids); + {error, {solr_400, _}=Why} -> + io:format("error while getting statestics ~p~n", Why); + {error, {solr_500, _}=Why} -> + io:format("error while getting statestics ~p~n", Why) + end. + +search(Query, ReqId) -> + stats_hero:ctime(ReqId, {chef_solr, search}, + fun() -> + solr_search(Query) + end). + +solr_search(Query) -> + try + chef_index:search(Query) + catch + Error:Reason -> + {Error, Reason} + end. \ No newline at end of file diff --git a/src/oc_erchef/apps/chef_telemetry/src/chef_teleterty_http.erl b/src/oc_erchef/apps/chef_telemetry/src/chef_teleterty_http.erl new file mode 100644 index 0000000000..f6eed554a4 --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/src/chef_teleterty_http.erl @@ -0,0 +1,96 @@ +-module(chef_telemetry_http). + +-export([ + request/3, + request/4, + get/1, + get/2, + get/3, + post/2, + post/3, + delete/2, + delete/3, + create_pool/0, + delete_pool/0 + ]). + +-define(DEFAULT_HEADERS, [{"Content-Type", "application/json"}]). + +default_headers() -> + maybe_add_data_collector_token_header(?DEFAULT_HEADERS). + +maybe_add_data_collector_token_header(Headers) -> + case data_collector:token() of + undefined -> + Headers; + Token when is_list(Token) -> + [{"x-data-collector-token", Token} | Headers] + end. + +request(Path, Method, Body) -> + request(Path, Method, Body, default_headers()). + +request(Path, Method, Body, Headers) -> + {ok, Timeout} = application:get_env(data_collector, timeout), + oc_httpc:request(?MODULE, Path, Headers, Method, Body, Timeout). + +%% +%% Simple helpers for requets when you only +%% care about success or failure. +%% +-spec get(list()) -> ok | {error, term()}. +get(Path) -> + get(Path, [], default_headers()). + +-spec get(list(), iolist() | binary()) -> ok | {error, term()}. +get(Path, Body) -> + get(Path, Body, default_headers()). + +-spec get(list(), iolist() | binary(), list()) -> ok | {error, term()}. +get(Path, Body, Headers) -> + request_with_caught_errors(Path, get, Body, Headers). + +-spec post(list(), iolist() | binary()) -> ok | {error, term()}. +post(Path, Body) -> + post(Path, Body, default_headers()). + +-spec post(list(), iolist() | binary(), list()) -> ok | {error, term()}. +post(Path, Body, Headers) -> + request_with_caught_errors(Path, post, Body, Headers). + +-spec delete(list(), iolist() | binary()) -> ok | {error, term()}. +delete(Path, Body) -> + delete(Path, Body, default_headers()). + +-spec delete(list(), iolist() | binary(), list()) -> ok | {error, term()}. +delete(Path, Body, Headers) -> + request_with_caught_errors(Path, delete, Body, Headers). + +request_with_caught_errors(Path, Method, Body, Headers) when is_list(Body)-> + request_with_caught_errors(Path, Method, iolist_to_binary(Body), Headers); +request_with_caught_errors(Path, Method, Body, Headers) -> + try + case request(Path, Method, Body, Headers) of + {ok, [$2|_], _Head, _RespBody} -> ok; + Error -> {error, Error} + end + catch + How:Why -> + error_logger:error_report({?MODULE, Method, How, Why}), + {error, Why} + end. + +%% +%% Pool management functions +%% +-spec create_pool() -> ok. +create_pool() -> + lager:info("Creating Data Collector HTTP pool"), + oc_httpc:add_pool(?MODULE, application:get_all_env()), + ok. + +-spec delete_pool() -> ok. +delete_pool() -> + lager:info("Removing Data Collector HTTP pool"), + oc_httpc:delete_pool(?MODULE), + ok.