Монитор памяти ETS Изменение конфигурации
Как я могу изменить valid_exchanges в записи конфигурации во время работы? Мы получаем использование памяти от подчиненных узлов, и если оно превышает пороговое значение, мы отключаем обмены. Однако конфигурация по умолчанию не меняется даже после развертывания. Можем ли мы точно настроить пороги памяти во время выполнения, изменив конфигурацию? Я также хочу изменить список действительных бирж.
-module(ebid_mon).
-behaviour(gen_server).
-export([start_link/1]).
-export([
enable_exchanges/1,
disable_exchanges/1,
check_memory/0,
get_monitoring_config/0,
set_config/1
]).
-export([init/1, handle_call/3, handle_cast/2, handle_info/2, code_change/3, terminate/2]).
-include("logger.hrl").
-record(state, {
length_timer = undefined,
config = undefined
}).
-record(ebid_settings, {name, value}).
-record(config,{
master_name = ['ebid@rtb0','ebid@dev6'],
count_threshold = 2,
time_threshold = 20000, % in Miliseconds
memory_threshold = 0.95,
disable_memory_threshold = 0.95,
enable_memory_threshold = 0.88,
inbetween_count_threshold = 4,
strategy = exchange,
emailto = "adx@adx.com",
emailfrom = "adx@adx.com",
valid_exchanges = [adx]
}).
-define(TIME_INTERVAL,timer:seconds(20)).
-define(LOG_INITIALS,"ebid_mon :").
get_monitoring_config() ->
get_default_config().
get_default_config() ->
[{count_threshold,2},{time_threshold,20000},{memory_threshold,0.95},{disable_memory_threshold,0.95},{enable_memory_threshold,0.88},{strategy,exchange},{valid_exchanges,[adx]},{to,"adx@adx.com"},{from,"adx@adx.com"},{master_name,['ebid@rtb0','ebid@dev6']},{inbetween_count_threshold,4}].
set_config(Config) ->
gen_server:call(?MODULE,{set_config,Config}).
start_link(Config) ->
gen_server:start_link({local,?MODULE},?MODULE,[Config],[]).
init([Data]) ->
Config = get_config(Data),
init_process(Config).
init_process(Config) ->
process_flag(trap_exit, true),
try
?log_error("~p Process table to store temporary data ~p",[?LOG_INITIALS, ets:new(?MODULE,[set,public,named_table])])
catch
Exception:Reason ->
?log_error("~p Failed to create the ets table ~p~p",[?LOG_INITIALS,Exception,Reason])
end,
LengthRef = erlang:start_timer(Config#config.time_threshold, self(), check_load),
{ok, #state{length_timer = LengthRef, config = Config}}.
handle_call({set_config,Data},_From,State) ->
case erlang:is_list(Data) of
true ->
%ebid_settings:set(<<"default_ebid_mon_config">>,Data),
Config = get_config(Data),
{reply,success,State#state{config = Config}};
false ->
?log_error("~p Bad configuration passed. Make sure its is a list of tuples",[?LOG_INITIALS]),
{reply,bad_config,State}
end;
handle_call(_Call, _From, State) ->
{reply, undefined, State}.
handle_cast(_Cast, State) ->
{noreply, State}.
handle_info({timeout,_Tref,check_load},State = #state{config = Config}) ->
?log_error("~p Config is ~p",[?LOG_INITIALS,Config]),
case lists:member(erlang:node(),Config#config.master_name) of
true ->
collect_slaves_memory_stats(Config);
false ->
?log_error("~p This node is not part of master node list. Which is ~p",[?LOG_INITIALS,Config#config.master_name])
end,
Tref = erlang:start_timer(Config#config.time_threshold,self(),check_load),
{noreply,State#state{length_timer = Tref}};
handle_info(_Info,State) ->
{noreply,State}.
code_change(_OldVsn,State,_Extra) ->
{ok,State}.
terminate(Reason,State) ->
error_logger:error_msg("~p Process terminating : ~p \t ~p",[?LOG_INITIALS,Reason,State]).
disable_exchanges(Valid_Exchanges) ->
?log_error("~p Disabling Exchanges",[?LOG_INITIALS]),
not lists:any(fun(Exchange) -> disable_exchange(Exchange) end,Valid_Exchanges).
disable_exchange(Exchange) ->
case mnesia:dirty_read(ebid_settings,to_binary(to_list(Exchange) ++ "_enabled")) of
[] ->
true;
[Record] ->
case Record#ebid_settings.value of
false ->
?log_error("~p Exchange ~p is already disabled",[?LOG_INITIALS,Exchange]),
true;
true ->
ebid_utils:set_exchange_enabled(Exchange,false),
false
end
end.
enable_exchanges(Valid_Exchanges) ->
?log_error("~p Enabling Exchanges ~p",[?LOG_INITIALS,Valid_Exchanges]),
not lists:any(fun(Exchange) -> enable_exchange(Exchange) end,Valid_Exchanges).
enable_exchange(Exchange) ->
?log_error("~p checking mnesia ebid_settings",[?LOG_INITIALS]),
case mnesia:dirty_read(ebid_settings,to_binary(to_list(Exchange) ++ "_enabled")) of
[] ->
true;
[Record] ->
case Record#ebid_settings.value of
true ->
?log_error("~p Exchange ~p is already enabled",[?LOG_INITIALS,Exchange]),
true;
false ->
ebid_utils:set_exchange_enabled(Exchange,true),
false
end
end.
%%%%%%%%%Private methods to module goes below this point and public goes above this point%%%%%%%%%%%%%%%%
collect_slaves_memory_stats(Config) ->
Alarm = check(ebid_master_alarm_for_exchanges),
?log_error("~p Alarm Status = ~p~n~n~n",[?LOG_INITIALS,Alarm]),
case Alarm of
undefined ->
?log_error("~p Taking No Alarm Action",[?LOG_INITIALS]),
take_action(no_alarm,Config);
_Alarm ->
%If alarm is set then insert in to ets and if N continous checks are set then
?log_error("~p Taking Alarm action",[?LOG_INITIALS]),
take_action(alarm,Config)
end.
take_action(alarm,Config) ->
?log_error("~p Active Alarm",[?LOG_INITIALS]),
?log_error("~p Node Details are:",[?LOG_INITIALS]),
MemoryUsageAlarmOnAllNodes = lists:any(fun(Node) ->
Memory = rpc:call(Node,ebid_mon,check_memory,[]),
?log_error("~p ~p \t ~p",[?LOG_INITIALS,Node,Memory]),
case Memory of
{badrpc,_} -> false;
_ -> Memory >= Config#config.disable_memory_threshold
end
end,
mnesia:system_info(running_db_nodes) -- [ebid@rtb0]),
case decision_for_exchange(ebid_master_key_to_disable_exchanges,Config#config.count_threshold) of
true ->
case MemoryUsageAlarmOnAllNodes of
true ->
?log_error("~p Disabling Exchanges ~p",[?LOG_INITIALS,Config#config.valid_exchanges]),
case disable_exchanges(Config#config.valid_exchanges) of
true ->
Data = create_email_data(),
send_email(disable,Data,Config);
false ->
?log_error("~p Exchanges are already disabled thus not sending and email",[?LOG_INITIALS])
end,
clear_ets_entry(ebid_master_key_to_enable_exchanges_for_middle),
clear_ets_entry(ebid_master_key_to_disable_exchanges);
false ->
clear_alarm(ebid_master_alarm_for_exchanges),
clear_ets_entry(ebid_master_key_to_disable_exchanges)
end;
false ->
?log_error("~p setting ets entry = ~p",[?LOG_INITIALS,make_ets_entry(ebid_master_key_to_disable_exchanges)])
end;
take_action(no_alarm,Config) ->
?log_error("~p No active Alarm",[?LOG_INITIALS]),
?log_error("~p Node Details are:",[?LOG_INITIALS]),
MemoryUsageAlarmOnAllNodes = lists:foldl(fun(Node,Acc) ->
Memory = rpc:call(Node,ebid_mon,check_memory,[]),
?log_error("~p ~p \t ~p",[?LOG_INITIALS,Node,Memory]),
Result = case Memory of
{badrpc,_} -> false;
_ -> if
Memory < Config#config.enable_memory_threshold ->
false;
Memory >= Config#config.disable_memory_threshold ->
true;
true ->
undefined
end
end,
Acc ++ [Result]
end,
[],
mnesia:system_info(running_db_nodes) -- [ebid@rtb0]),
case lists:member(undefined,MemoryUsageAlarmOnAllNodes) or lists:member(true,MemoryUsageAlarmOnAllNodes) of
true ->
case lists:member(undefined,MemoryUsageAlarmOnAllNodes) and not lists:member(true,MemoryUsageAlarmOnAllNodes) of
false ->
?log_error("~p One of the node is loaded. Thus setting the alarm",[?LOG_INITIALS]),
set_alarm(ebid_master_alarm_for_exchanges),
make_ets_entry(ebid_master_key_to_disable_exchanges);
true ->
?log_error("~p In middle of both threshold. Thus enable after 5 count",[?LOG_INITIALS]),
key_based_enable_decision(ebid_master_key_to_enable_exchanges_for_middle,Config#config.inbetween_count_threshold,Config)
end;
false ->
key_based_enable_decision(ebid_master_key_to_enable_exchanges,Config#config.count_threshold,Config)
end.
key_based_enable_decision(Key,Count,Config) ->
case decision_for_exchange(Key,Count) of
true ->
?log_error("~p None of the node is loaded. Thus enable exchanges",[?LOG_INITIALS]),
clear_alarm(ebid_master_alarm_for_exchanges),
clear_ets_entry(Key),
clear_ets_entry(ebid_master_key_to_enable_exchanges_for_middle),
case enable_exchanges(Config#config.valid_exchanges) of
true ->
Data = create_email_data(),
send_email(enable,Data,Config);
false ->
?log_error("~p Exchanges are already enabled thus not sending and email",[?LOG_INITIALS])
end;
false ->
?log_error("~p setting ets entry for key ~p = ~p",[?LOG_INITIALS,Key,make_ets_entry(Key)])
end.
create_email_data() ->
Data = lists:foldl(fun(Node,A) -> Memory = rpc:call(Node,ebid_mon,check_memory,[]), Result = case Memory of {badrpc,_} -> 0.0; _ -> Memory end, A ++ ["<tr><td>",erlang:atom_to_list(Node),"</td><td>",erlang:float_to_list(Result*100,[{decimals,5}])," %</td></tr>"] end,[], mnesia:system_info(running_db_nodes) -- [ebid@rtb0]),
?log_error("~p Email Data is : ~p",[?LOG_INITIALS,Data]),
lists:flatten(io_lib:format("<html><body><table> ~s </table></body></html>",[Data])).
clear_ets_entry(Key) ->
ets:delete(?MODULE,Key).
make_ets_entry(Key) ->
ets:update_counter(?MODULE,Key,1,{Key,0}).
decision_for_exchange(Key,ThresholdCount) ->
?log_error("~p checking the decision for ~p",[?LOG_INITIALS,Key]),
Decision = case ets:lookup(?MODULE,Key) of
[] ->
false;
DataObject ->
[{Key,Value}] = DataObject,
?log_error("~p Counter Value for key = ~p is ~p",[?LOG_INITIALS,Key,Value]),
Value >= ThresholdCount
end,
Decision.
check_memory() ->
SystemMemory = memsup:get_system_memory_data(),
((get_value(total_memory,SystemMemory) - (get_value(free_memory,SystemMemory) + get_value(cached_memory,SystemMemory) + get_value(buffered_memory,SystemMemory)))/get_value(total_memory,SystemMemory)).
check_alarm(Alarm) ->
case check(Alarm) of
undefined ->
false;
_Value ->
true
end.
check(Alarm) ->
proplists:get_value(Alarm,alarm_handler:get_alarms()).
set_alarm(Alarm) ->
case check_alarm(Alarm) of
true ->
already_set;
false ->
alarm_handler:set_alarm({Alarm,[]})
end.
clear_alarm(Alarm) ->
alarm_handler:clear_alarm(Alarm).
get_config(Data) ->
#config{
master_name = get_value(master_name,Data),
count_threshold = get_value(count_threshold,Data),
time_threshold = get_value(time_threshold,Data),
memory_threshold = get_value(memory_threshold,Data),
disable_memory_threshold = get_value(disable_memory_threshold,Data),
enable_memory_threshold = get_value(enable_memory_threshold,Data),
valid_exchanges = get_value(valid_exchanges,Data),
emailto = get_value(to,Data),
emailfrom = get_value(from,Data),
strategy = get_value(strategy,Data),
inbetween_count_threshold = get_value(inbetween_count_threshold,Data)
}.
get_value(Key,Data) ->
case proplists:get_value(Key,Data) of
undefined ->
get_value(Key);
V ->
V
end.
get_value(master_name) ->
#config.master_name;
get_value(count_threshold) ->
#config.count_threshold;
get_value(time_threshold) ->
#config.time_threshold;
get_value(memory_threshold) ->
#config.memory_threshold;
get_value(disable_memory_threshold) ->
#config.disable_memory_threshold;
get_value(enable_memory_threshold) ->
#config.enable_memory_threshold;
get_value(valid_exchanges) ->
#config.valid_exchanges;
get_value(to) ->
#config.emailto;
get_value(from) ->
#config.emailfrom;
get_value(inbetween_count_threshold) ->
#config.inbetween_count_threshold;
get_value(strategy) ->
#config.strategy.
to_binary(Value) when is_binary(Value) ->
Value;
to_binary(Value) when is_list(Value) ->
erlang:list_to_binary(Value);
to_binary(Value) when is_atom(Value) ->
erlang:atom_to_binary(Value,utf8).
%to_atom(Value) when is_atom(Value) ->
% Value;
%to_atom(Value) when is_list(Value) ->
% erlang:list_to_atom(Value);
%to_atom(Value) when is_binary(Value) ->
% erlang:binary_to_atom(Value,utf8).
to_list(Value) when is_list(Value) ->
Value;
to_list(Value) when is_binary(Value) ->
erlang:binary_to_list(Value);
to_list(Value) when is_atom(Value) ->
erlang:atom_to_list(Value).
send_email(enable,Data,Config) ->
send_email("After Enabling",Data,Config);
send_email(disable,Data,Config) ->
send_email("After disabling",Data,Config);
send_email(Type,Data,Config) ->
?log_error("Sending email"),
os:cmd("echo \"From:" ++ Config#config.emailfrom ++"\r\nTo:" ++ Config#config.emailto ++ "\r\nSubject: Memory Status : each node : " ++ Type ++ " \r\nMime-Version: 1.1 \r\nContent-Type: text/html \r\n\r\n" ++ Data ++ "\" | /usr/sbin/sendmail -t").