%% ``The contents of this file are subject to the Erlang Public License,
%% Version 1.1, (the "License"); you may not use this file except in
%% compliance with the License. You should have received a copy of the
%% Erlang Public License along with this software. If not, it can be
%% retrieved via the world wide web at http://www.erlang.org/.
%% 
%% Software distributed under the License is distributed on an "AS IS"
%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
%% the License for the specific language governing rights and limitations
%% under the License.
%% 
%% The Initial Developer of the Original Code is Ericsson Utvecklings AB.
%% Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings
%% AB. All Rights Reserved.''
%% 
%%     $Id$
%%

-module(bench).

%% User interface
-export([run/0]).

%% Exported to be used in spawn
-export([measure/4]).

%% Internal constants 
-define(MAX, 999999999999999).
-define(RANGE_MAX, 16#7ffffff).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%% 	Interface 
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%---------------------------------------------------------------------------
%% run() -> _
%%
%% Compiles and runs all benchmarks in the current directory,
%% and creates a report
%%---------------------------------------------------------------------------
run() ->
    run(compiler_options()).


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%% 	Generic Benchmark functions
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%---------------------------------------------------------------------------
%% compiler_options() -> OptionsList
%%	OptionsList = list() - See Erlang/OTP module compile
%%---------------------------------------------------------------------------
compiler_options() ->
    [report_errors, report_warnings].

%%---------------------------------------------------------------------------
%% run(OptionsList) ->
%%	OptionsList = list() - See Erlang/OTP module compile
%%
%% Help function to run/0.
%%---------------------------------------------------------------------------
run(OptionsList) ->
    Bms = compile_benchmarks(OptionsList),
    run_benchmarks(Bms),
    report().

%%---------------------------------------------------------------------------
%% compile_benchmarks(OptionsList) -> [BmInfo| _]
%%	OptionsList = list() - See Erlang/OTP module compile
%%	BmInfo = {Module, Iterations, [BmFunctionName| _]}
%%	Module = atom()
%%	Iterations = integer()
%%	BmFunctionName = atom()
%%  
%% Compiles all benchmark modules in the current directory and
%% returns info about the benchmarks. 
%%---------------------------------------------------------------------------
compile_benchmarks(OptionsList) ->
    {ok, FilesInCurrentDir} = file:list_dir("."),
    ErlFiles = [ErlFile || ErlFile <- lists:sort(FilesInCurrentDir), 
			 lists:suffix(".erl", ErlFile)],
    lists:foldr(fun(File, BmInfoAcc) -> 
			case lists:suffix("_bm.erl", File) of
			    true ->
				BmInfo = bm_compile(File, OptionsList),
				[BmInfo | BmInfoAcc];
			    false ->
				just_compile(File, OptionsList),
				BmInfoAcc
			end
		end, [], ErlFiles).
    
%%---------------------------------------------------------------------------
%%  just_compile(FileName, OptionsList) -> ok
%%	FileName = string() 
%%	OptionsList = list() - See Erlang/OTP module compile
%%  
%% Compiles a support module.
%%---------------------------------------------------------------------------
just_compile(FileName, OptionsList) ->
    io:format("Compiling ~s...\n", [FileName]), % Progress info to user
    case c:c(FileName, OptionsList) of
	{ok, _Mod} ->
	    ok;
	%% If compilation fails there is no point in trying to continue
	error -> 
	    Reason = 
		lists:flatten(
		  io_lib:format("Could not compile file ~s", [FileName])),
	    exit(self(), Reason)
    end.
%%---------------------------------------------------------------------------
%%  bm_compile(FileName, OptionsList) -> BmInfo
%%	FileName = string() 
%%	OptionsList = list() - See Erlang/OTP module compile
%%	BmInfo = {Module, Iterations, [BmFunctionName| _]}
%%	Iterations = integer()
%%	Module = atom()
%%	BmFunctionName = atom()
%%  
%% Compiles the benchmark module implemented in <FileName> and returns
%% information about the benchmark tests. 
%%---------------------------------------------------------------------------
bm_compile(FileName, OptionsList) ->
    io:format("Compiling ~s...\n", [FileName]), % Progress info to user
    case c:c(FileName, OptionsList) of
	{ok, Mod} ->
	    bm_cases(Mod);
	%% If compilation fails there is no point in trying to continue
	error -> 
	    Reason = 
		lists:flatten(
		  io_lib:format("Could not compile file ~s", [FileName])),
	    exit(self(), Reason)
    end.
%%---------------------------------------------------------------------------
%% bm_cases(Module) -> {Module, Iter, [BmFunctionName |_]}
%%	Module = atom() 
%%	Iter = integer()
%%	BmFunctionName = atom()
%%
%% Fetches the number of iterations and the names of the benchmark
%% functions for the module <Module>. 
%%---------------------------------------------------------------------------
bm_cases(Module) ->
    case catch Module:benchmarks() of
	{Iter, BmList} when integer(Iter), list(BmList) ->
	    {Module, Iter, BmList};
	%% The benchmark is incorrect implemented there is no point in
	%% trying to continue
	Other -> 
	    Reason = 
		lists:flatten(
		  io_lib:format("Incorrect return value: ~p " 
				"from ~p:benchmarks()",
				[Other, Module])),
	    exit(self(), Reason)
    end.
%%---------------------------------------------------------------------------
%% run_benchmarks(Bms) ->   
%%	Bms = [{Module, Iter, [BmFunctionName |_]} | _] 
%%	Module = atom() 
%%	Iter = integer()
%%	BmFunctionName = atom()
%%  	    
%% Runs all the benchmark tests described in <Bms>.
%%---------------------------------------------------------------------------
run_benchmarks(Bms) ->
    Ver = erlang:system_info(version),
    Machine = erlang:system_info(machine),
    SysInfo = {Ver,Machine},
   
    Res = [bms_run(Mod, Tests, Iter, SysInfo) || {Mod,Iter,Tests} <- Bms],

    %% Create an intermediate file that is later used to generate a bench
    %% mark report.
    Name = Ver ++ [$.|Machine] ++ ".bmres",
    {ok, IntermediatFile} = file:open(Name, [write]),

    %% Create mark that identifies version of the benchmark modules
    io:format(IntermediatFile, "~p.\n", [erlang:phash(Bms, ?RANGE_MAX)]),
    
    io:format(IntermediatFile, "~p.\n", [Res]),
    file:close(IntermediatFile).

%%---------------------------------------------------------------------------
%% bms_run(Module, BmTests, Iter, Info) ->  
%%	Module = atom(),
%%	BmTests = [BmFunctionName|_],
%%      BmFunctionName = atom()
%%	Iter = integer(),
%%	SysInfo = {Ver, Machine}
%%	Ver = string()
%%	Machine = string()
%%  
%% Runs all benchmark tests in module <Module>. 
%%---------------------------------------------------------------------------
bms_run(Module, BmTests, Iter, SysInfo) ->
    io:format("Running ~s:", [Module]),  % Progress info to user
    Res =
	{Module,{SysInfo,[{Bm, bm_run(Module, Bm, Iter)} || Bm <- BmTests]}},
    io:nl(),
    Res.
%%---------------------------------------------------------------------------
%% bm_run(Module, BmTest, Iter) -> Elapsed
%%	Module = atom(),
%%	BmTest = atom(),
%%	Iter = integer()
%%      Elapsed = integer()  - elapsed time in milliseconds.
%%  
%% Runs the benchmark Module:BmTest(Iter)  
%%---------------------------------------------------------------------------
bm_run(Module, BmTest, Iter) ->
    io:format(" ~s", [BmTest]),  % Progress info to user
    spawn_link(?MODULE, measure, [self(), Module, BmTest, Iter]),
    receive
	{Elapsed, ok} ->
	    Elapsed;
	{_Elapsed, Fault} ->
	    io:nl(),
	    Reason = 
		lists:flatten(
		  io_lib:format("~w", [Fault])),
	    exit(self(), Reason)
    end.
%%---------------------------------------------------------------------------
%% measure(Parent, Module, BmTest, Iter) -> _ 
%%	Parent = pid(),
%%	Module = atom(),
%%	BmTest = atom(),
%%	Iter = integer()
%%
%% Measures the time it take to execute Module:Bm(Iter)  
%% and send the result to <Parent>.
%%---------------------------------------------------------------------------
measure(Parent, Module, BmTest, Iter) ->
    statistics(runtime),
    Res = (catch apply(Module, BmTest, [Iter])), 
    {_TotalRunTime, TimeSinceLastCall} = statistics(runtime),
    Parent ! {TimeSinceLastCall, Res}.
    

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%% 	Report functions
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%---------------------------------------------------------------------------
%% report() -> _
%%  
%% Creates a report of the bench marking test that appeals to a human.
%% Currently this means creating a html-file. (Other formats could be added) 
%%---------------------------------------------------------------------------
report() ->
    {ok, AllFiles} = file:list_dir("."),
    BmResultFiles = [File || File <- AllFiles, lists:suffix(".bmres", File)],

    Results = fetch_bmres_data(BmResultFiles),
    create_report(Results).

%%---------------------------------------------------------------------------
%% fetch_bmres_data(BmResultFiles) -> Results 
%%	BmResultFiles = [FileName | _]
%%	FileName = string()
%%	Results = [[{Bm, Res} | _]]
%%      Bm   =  atom() - Name of benchmark module
%%	Res  = [{VersionInfo, [{Test, Time} | _]}]
%%	VersionInfo = {Ver, Machine}
%%	Ver = string()
%%	Machine = string()
%%	Test = atom()
%%	Time = integer()
%%
%% Reads result data from intermediate files
%%---------------------------------------------------------------------------
fetch_bmres_data(BmResultFiles) ->
    fetch_bmres_data(BmResultFiles, [], undefined).

%%---------------------------------------------------------------------------
%% fetch_bmres_data(BmResultFiles, AccResData, Check) -> Results 
%%	BmResultFiles = [FileName | _]
%%	FileName = string()
%%	AccResData = see Results fetch_bmres_data/1
%%	Check = integer() | undefined (first time)
%%
%% Help function to fetch_bmres_data/1
%%---------------------------------------------------------------------------
fetch_bmres_data([], AccResData, _Check) ->
    AccResData;

fetch_bmres_data([Name | BmResultFiles], AccResData, Check) ->
    {DataList, NewCheck} = read_bmres_file(Name, Check),
    fetch_bmres_data(BmResultFiles, [DataList| AccResData], NewCheck).

%%---------------------------------------------------------------------------
%% read_bmres_file(Name, Check) -> 
%%	Name = string()  
%%	Check = integer() | undefined  
%%
%%  Reads the data from the result files. Checks that all result
%%  files where created with the same set of tests.
%%---------------------------------------------------------------------------
read_bmres_file(Name, Check) ->
    case file:consult(Name) of
	{ok, [Check1, List]} when Check =:= undefined, integer(Check1) ->
	    {List, Check1};
	{ok, [Check, List]} when integer(Check) ->
	    {List, Check};
	{ok, [Check1, _List]} when integer(Check1) ->
	    Reason = 
		lists:flatten(
		  io_lib:format("Different test setup, remove old setup "
				"result by removing *.bmres files and "
				"try again", [])),
	    exit(self(), Reason); 
	{error, Reason} when atom(Reason) ->
	    exit(self(), Reason); 
	{error, Reason} ->
	    exit(self(), file:format(Reason))
    end.

%%---------------------------------------------------------------------------
%% create_report(Results) ->
%%	Results =  see Results fetch_bmres_data/1
%%  
%%  Organizes <Result> so it will be right for create_html_report/1  
%%  i.e. group results for the same benchmark test, run on different versions
%%  of erlang.
%%---------------------------------------------------------------------------
create_report(Results) -> 
    Dictionary = 
	lists:foldl(fun(BmResultList, Dict0) ->
			    lists:foldl(fun({Bm, VerResult}, Dict1) ->
						dict:append(Bm, VerResult, 
							    Dict1) 
					end,Dict0, BmResultList) 
		    end,
		    dict:new(), Results),

    create_html_report(dict:to_list(Dictionary)).
%%---------------------------------------------------------------------------
%%  create_html_report(ResultList) -> _
%%	ResultList = [{Bm, Res} | _]
%%	Bm   =  atom() - Name of benchmark module
%%	Res  = [{VersionInfo, [{Test, Time} | _]} | _] 
%%	VersionInfo = {Ver, Machine}
%%	Ver = string()
%%	Machine = string()
%%	Test = atom()
%%	Time = integer()
%%
%% Writes the result to an html-file 
%%---------------------------------------------------------------------------
create_html_report(ResultList) ->

    {ok, OutputFile} = file:open("index.html", [write]),

    %% Create the begining of the result html-file.
    Head = Title = "Benchmark Results",
    io:put_chars(OutputFile, "<html>\n"),
    io:put_chars(OutputFile, "<head>\n"),
    io:format(OutputFile, "<title>~s</title>\n", [Title]),
    io:put_chars(OutputFile, "</head>\n"),
    io:put_chars(OutputFile, "<body bgcolor=\"#FFFFFF\" text=\"#000000\"" ++
		 " link=\"#0000FF\" vlink=\"#800080\" alink=\"#FF0000\">\n"),
    io:format(OutputFile, "<h1>~s</h1>\n", [Head]),

    %% Add the result tables
    lists:foreach(fun(Element) -> 
			  create_html_table(OutputFile, Element) end,
		  ResultList),
 
    %% Put in the end-html tags
    io:put_chars(OutputFile, "</body>\n"),
    io:put_chars(OutputFile, "</html>\n"),
    
    file:close(OutputFile).

%%---------------------------------------------------------------------------
%% create_html_table(File, {Bm, Res}) -> _
%%	File = file() - html file to write data to. 
%%	Bm  = atom() - Name of benchmark module
%%	Res = [{VersionInfo, [{Test, Time} | _]}]
%%	VersionInfo = {Ver, Machine}
%%	Ver = string()
%%	Machine = string()
%%	Test = atom()
%%	Time = integer()
%%
%% Creates a html table that displays the result of the benchmark <Bm>.
%%---------------------------------------------------------------------------
create_html_table(File, {Bm, Res}) ->

    {MinTime, Order} = min_time_and_sort(Res),  
        
    io:format(File, "<h2>~s</h2>\n" , [Bm]),
    
    %% Fun that calculates relative measure values and puts them in 
    %% a dictionary
    RelativeMesureFun =  fun({TestName, Time}, Dict1) ->
				 dict:append(TestName, Time/MinTime, Dict1) 
			 end,

    %% For all erlang versions that the benchmark tests has been run, 
    %% calculate the relative measure values and put them in a dictionary.
    ResultDict = 
	lists:foldl(fun({_VerInfo, Bms}, Dict0) ->
			    lists:foldl(RelativeMesureFun, Dict0, Bms) end,
		    dict:new(), Res),

    %% Create the table and its headings
    io:put_chars(File, "<table border=0 cellpadding=1><tr>"
		 "<td bgcolor=\"#000000\">\n"),
    io:put_chars(File, "<table cellpadding=3 border=0 cellspacing=1>\n"),
    io:put_chars(File, "<tr bgcolor=white>"),
    io:put_chars(File, "<td>Test</td>"),
    Heads = table_headers(Res),
    lists:foreach(fun({Ver,Machine}) -> io:format(File, "<td>~s<br>~s</td>",
						  [Ver,Machine]) end, Heads),
    io:put_chars(File, "</tr>\n"),
    
    %% Create table rows
    lists:foreach(fun(Name) -> 
			  create_html_row(File, Name, ResultDict) 
		  end, Order),

    %% Tabel end-tags
    io:put_chars(File, "</table></td></tr></table>\n"),
    
    %% Create link to benchmark source code
    io:format(File, "<p><a href=\"~s.erl\">Source for ~s.erl</a>\n",
	      [Bm,Bm]).

%%---------------------------------------------------------------------------
%% create_html_row(File, Name, Dict) -> _
%%	File = file() - html file to write data to. 
%%	Name = atom() - Name of benchmark test 
%%	Dict = dict() - Dictonary where the relative time measures for 
%%      the test can be found.
%%
%% Creates an actual html table-row.
%%---------------------------------------------------------------------------
create_html_row(File, Name, Dict) ->
    ReletiveTimes = dict:fetch(Name, Dict),
    io:put_chars(File, "<tr bgcolor=white>\n"),
    io:format(File, "<td>~s</td>", [Name]),
    lists:foreach(fun(Time) -> 
			  io:format(File, "<td>~-8.2f</td>", [Time]) end, 
		  ReletiveTimes),
    io:put_chars(File, "</tr>\n").

%%---------------------------------------------------------------------------
%% min_time_and_sort(ResultList) -> {MinTime, Order}
%%	ResultList = [{VersionInfo, [{Test, Time} | _]}] 
%%	MinTime = integer() - The execution time of the fastes test 
%%	Order = [BmFunctionName|_] - the order of the testcases in
%%      increasing execution time.
%%	BmFunctionName = atom()   
%%---------------------------------------------------------------------------
min_time_and_sort(ResultList) ->
    
    %% Use the results from the run on the highest version
    %% of Erlang as norm.
    {_, TestRes} = 
	lists:foldl(fun ({{Ver, _}, ResList}, 
			 {CurrentVer, _}) when Ver > CurrentVer ->
			    {Ver, ResList};
			(_, VerAndRes) ->
			    VerAndRes
		    end, {"0", []}, ResultList),
    
    {lists:foldl(fun ({_, Time0}, Min1) when Time0 < Min1 -> 
			 Time0;
		     (_, Min1) -> 
			 Min1
		 end, ?MAX, TestRes),
     [Name || {Name, _} <- lists:keysort(2, TestRes)]}.

%%---------------------------------------------------------------------------
%% table_headers(VerResultList) -> SysInfo
%%	VerResultList = [{{Ver, Machine},[{BmFunctionName, Time}]} | _]
%%	Ver = string()
%%	Machine = string()
%%	BmFunctionName = atom()
%%	Time = integer() 
%%	SysInfo = {Ver, Machine}
%%---------------------------------------------------------------------------
table_headers(VerResultList) ->
    [SysInfo || {SysInfo, _} <- VerResultList].