#!/usr/bin/perl -w

# Tool to summarise a journal file generated by TET 
# or to determine the difference in test results between two
# journals.
#
# (C) Copyright 2001 The Free Standards Group  Inc
#
# 21/5/2001 Chris Yeoh, IBM
#
# This is $Revision: 1.11a $
#
# $Log: tjreport,v $
# Revision 1.11a 2005/02/16 sbenedict@mandrakesoft.com
# very minor mod to "excluding waived" output (Bugzilla #698)
# 
# Revision 1.11  2003/10/31 02:34:37  cyeoh
# Only print test system information if it exists
# Adds verbose flag which causes error information to be printed
#
# Revision 1.10  2003/07/17 07:17:49  cyeoh
# change error about test inconsistency to warning as lsblibchk
# is creating duplicates at the moment
#
# Revision 1.9  2002/01/16 05:14:00  cyeoh
# clean up help information
# remove dependency on DBI when not used
#
# Revision 1.8  2001/12/04 07:39:41  cyeoh
# Add support for waiver files
#
# Revision 1.7  2001/11/05 07:31:19  cyeoh
# Fixes bug that caused the missing of some testcases
# Adds cross check of testcase count
# Removes debug of DBI calls
#
# Revision 1.6  2001/08/22 03:52:32  cyeoh
# Fixes die error reporting
# Adds storage and update of testsuite to test case mapping
#
# Revision 1.5  2001/08/14 08:31:59  cyeoh
# Fix generation of description information so that
# journal prefix ids are removed
#
# Revision 1.4  2001/08/14 04:59:55  cyeoh
# Rework for change in db format.
# Now keeps track of how many results available for each testcase
#
# Revision 1.3  2001/08/14 02:18:07  cyeoh
# Adds ability to add error information for failed tests into database
#
# Revision 1.2  2001/08/13 04:51:04  cyeoh
# Adds ability to add data into SQL database for further analysis
#
# Revision 1.1  2001/08/11 11:46:13  cyeoh
# Initial version
#
#

use strict;
use Getopt::Std;

my(%StateMap) = (
 0 => "PASS",
 1 => "FAIL",
 2 => "UNRESOLVED",
 3 => "NOTINUSE",
 4 => "UNSUPPORTED",
 5 => "UNTESTED",
 6 => "UNINITIATED",
 7 => "UNREPORTED",
 101 => "WARNING",
 102 => "FIP",
 103 => "NOTIMP",
 104 => "UNAPPROVE");

my(%PFMap) = (
  "PASS" => "PASS",
  "FAIL" => "FAIL",
  "UNRESOLVED" => "FAIL",
  "NOTINUSE" => "PASS",
  "UNSUPPORTED" => "PASS",
  "UNTESTED" => "PASS",
  "UNINITIATED" => "FAIL",
  "UNREPORTED" => "FAIL",
  "WARNING" => "PASS",
  "FIP" => "PASS",
  "NOTIMP" => "PASS",
  "UNAPPROVE" => "PASS",
  "MISSING" => "FAIL",
  "UNKNOWN" => "FAIL");
  

my($VerboseSummary) = 0;

# Analyses one journal file to get statistics on pass/failure
#
sub GatherStats($)
{
  my($journalFile) = shift;
  my($stats) = {};
  my($loop);
  my($line);
  local(*JFILE);

  # Initialise
  $stats->{STATE_SUMMARY} = {};
  foreach $loop (keys %StateMap)
  {
    $stats->{STATE_SUMMARY}{$StateMap{$loop}} = 0;
  }
  $stats->{STATE_SUMMARY}{TEST_ERROR} = 0;
  $stats->{STATE_SUMMARY}{UNKNOWN} = 0;
  $stats->{TOTAL_TESTS_PASSED} = 0;
  $stats->{TOTAL_TESTS_FAILED} = 0;

  # Analyse file
  open(JFILE, $journalFile) ||  die "Could not open file: $journalFile\n";
  
  my($testName);
  my($testNum);
  my(@line);
  my($testState);
  my($errorMessage);
  my($tmp);
  while (defined($line=<JFILE>))
  {
    # Look for system info
    if ($line =~ /^0\|(.*)\|/)
    {
      @line = split(/ /, $1);
      $stats->{TEST_DATE} = $line[2];
      $stats->{TEST_TIME} = $line[1];
    }
    elsif ($line=~ /^30\|.*VSX_SYS=(.*)$/)
    {
      $stats->{TEST_SYSTEM} = $1;
    }

    # Look for test results
    @line = split(/ /, $line);
    if ($line[0] =~ /^10/) { $testName = $line[1]; }
    elsif ($line[0] =~ /^200/) 
    { 
      $testNum = $line[1]; 
      $errorMessage = "";
    }
    elsif ($line[0] =~ /^220/)
    {
      # Test state report
      $testState = exists($StateMap{$line[2]}) ? $StateMap{$line[2]} 
        : "UNKNOWN";

      $stats->{STATE_SUMMARY}{$testState}++;

      if (exists($stats->{TESTS}{$testName}{$testNum}))
      {
        print "Dup: $testName $testNum\n";
      }
      $stats->{TESTS}{$testName}{$testNum}{STATE} = $testState;

      if ($PFMap{$testState} eq "PASS")
      {
        $stats->{TOTAL_TESTS_PASSED}++ ;
      }
      else
      {
        $stats->{TOTAL_TESTS_FAILED}++;
        $stats->{TESTS}{$testName}{$testNum}{INFO} = $errorMessage;
      }
    }
    elsif ($line[0] =~ /^520/)
    {
      # Accumulate any error/info messages associated with test
      $line =~ /\|([^\|]*)$/;
      $tmp = $1;
      chomp($tmp);
      $errorMessage .= "$tmp\n";
    }
  }

  close(JFILE);
  return $stats;
}


#----------------------------------------------------------------------
# Find the difference in test results between the two journals
sub DiffJournals($$)
{
  my($j1) = shift;
  my($j2) = shift;
  my($diffStats) = {};
  my($testName);
  my($testNum);

  $diffStats->{TESTS} = {};

  foreach $testName (sort keys %{$j1->{TESTS}})
  {
    foreach $testNum (sort {$a <=> $b} keys %{$j1->{TESTS}{$testName}})
    {
      if (exists($j2->{TESTS}{$testName}) 
	  && exists($j2->{TESTS}{$testName}{$testNum}))
      {
	if ($j1->{TESTS}{$testName}{$testNum} 
	    ne $j2->{TESTS}{$testName}{$testNum})
	{
	  $diffStats->{TESTS}{$testName}{$testNum} = 
	      "$j1->{TESTS}{$testName}{$testNum}{STATE}," .
	      "$j2->{TESTS}{$testName}{$testNum}{STATE}";
	}
      }
      else
      {
	$diffStats->{TESTS}{$testName}{$testNum} = 
	    "$j1->{TESTS}{$testName}{$testNum}{STATE},MISSING";
      }
    }
  }

  # Check reverse
  foreach $testName (sort keys %{$j2->{TESTS}})
  {
    foreach $testNum (sort {$a <=> $b} keys %{$j2->{TESTS}{$testName}})
    {
      if (! (exists($j2->{TESTS}{$testName}) 
	   && exists($j2->{TESTS}{$testName}{$testNum})))
      {
	$diffStats->{TESTS}{$testName}{$testNum} = 
	    "MISSING,$j2->{TESTS}{$testName}{$testNum}{STATE}";
      }
    }
  }

  
  return $diffStats;
}

#----------------------------------------------------------------------
sub PrintDiffSummary($$)
{
  my($diffStats) = shift;
  my($isDetailed)  = shift;

  my($testName);
  my($testNum);
  my($state1, $state2);

  foreach $testName (sort keys %{$diffStats->{TESTS}})
  {
    foreach $testNum (sort {$a <=> $b} keys %{$diffStats->{TESTS}{$testName}})
    {
      ($state1, $state2) = split(/,/, $diffStats->{TESTS}{$testName}{$testNum});
      if (($PFMap{$state1} ne $PFMap{$state2}) || $isDetailed)
      {
	print "$testName $testNum $diffStats->{TESTS}{$testName}{$testNum}\n";
      }
    }
  }
}


#----------------------------------------------------------------------
#
sub PrintSummary($$$)
{
  my($stats) = shift;
  my($isDetailed) = shift;
  my($waivers) = shift;

  my($testState);

  my($testName);
  my($testNum);
  my($numTests);
  my($secondCount) = 0;
  my($waived) = 0;

  foreach $testName (sort keys %{$stats->{TESTS}})
  {
    foreach $testNum (sort {$a <=> $b} keys %{$stats->{TESTS}{$testName}})
    {
      $secondCount++;
#      print "$testName $testNum\n";
      if ( ($PFMap{$stats->{TESTS}{$testName}{$testNum}{STATE}} eq "FAIL")
	   || $isDetailed )
      {
				print "$testName $testNum $stats->{TESTS}{$testName}{$testNum}{STATE}";
        if (defined($waivers) && $waivers->{"$testName-$testNum"})
        {
          print " (WAIVED)";
          $waived++;
        }
        print "\n";
				if ($VerboseSummary)
				{
					print "  $stats->{TESTS}{$testName}{$testNum}{INFO}";
				}
      }
    }
  }

  $numTests = $stats->{TOTAL_TESTS_PASSED} + $stats->{TOTAL_TESTS_FAILED};

  print "Warning: Inconsistency in test count. This is probably due to a bug in this"
      . " program ($numTests, $secondCount).\n"
      unless $numTests==$secondCount;

  print "\n\n";
  print "Test system: $stats->{TEST_SYSTEM}\n" 
			unless (!defined($stats->{TEST_SYSTEM}));
  print "Test was run: $stats->{TEST_DATE} $stats->{TEST_TIME} \n";

  print "Total Tests Passed: $stats->{TOTAL_TESTS_PASSED}\n";
  print "Total Tests Failed (including waived): $stats->{TOTAL_TESTS_FAILED}\n";
  print "Total Tests Failed (excluding waived): " . ($stats->{TOTAL_TESTS_FAILED}-$waived) . "\n";

  print "\nTest Result Breakdown:\n";
  foreach $testState (keys %{$stats->{STATE_SUMMARY}})
  {
    print "$testState: $stats->{STATE_SUMMARY}{$testState}\n";
  }


}

#----------------------------------------------------------------------
#
sub LoadIntoDatabase($$$$)
{
  my($dbh) = shift;
  my($stats) = shift;
  my($vendorName) = shift;
  my($versionName) = shift;
  my($testName);
  my($testNum);
  my($sth);
  my($distroId);

  print "Adding information to DB\n";

  $sth = $dbh->prepare(
    "SELECT DistroId from distributions where VendorName='$vendorName'"
                       . " and VersionName='$versionName'") 
      || die $dbh->errstr;
  $sth->execute() || die $dbh->errstr;
  if ($sth->rows==0)
  {
    # Add distribution information to database
    print "Adding distribution information to database\n";
    $sth = $dbh->prepare(
      "INSERT INTO distributions (VendorName, VersionName)"
      . " Values ('$vendorName', '$versionName')")
        || die $dbh->errstr;
    $sth->execute() || die $dbh->errstr;
    $sth = $dbh->prepare(
      "SELECT DistroId from distributions where VendorName='$vendorName'"
         . " and VersionName='$versionName'") 
        || die $dbh->errstr;
    $sth->execute() || die $dbh->errstr;
  }
  die "found " . $sth->rows . " occurences of distro name\n" 
      unless $sth->rows==1;
  # Get distro id
  $distroId = $sth->fetchrow_hashref->{DistroId};
  print "Distribution id: $distroId\n";

  $sth = $dbh->prepare(
    "INSERT INTO results (Testcase, Distro, State, Description)"
  . "Values (?, ?, ?, ?)") 
      || die $dbh->errstr;

  my($errorMessage);
  my($testcaseId);
  my($id_sth);
  foreach $testName (sort keys %{$stats->{TESTS}})
  {
    foreach $testNum (sort {$a <=> $b} keys %{$stats->{TESTS}{$testName}})
    {
      # Get testcase id
      $id_sth = $dbh->prepare("SELECT TestcaseId from testcases"
                              . " where Name='$testName-$testNum'")
      || die $dbh->errstr;
      $id_sth->execute();
      if ($id_sth->rows==0)
      {
        $id_sth = $dbh->prepare("INSERT INTO testcases (Name)"
                                . " Values ('$testName-$testNum')")
            || die $dbh->errstr;
        $id_sth->execute() || die $dbh->errstr;
        $id_sth = $dbh->prepare("SELECT TestcaseId from testcases"
                                . " where Name='$testName-$testNum'")
            || die $dbh->errstr;
        $id_sth->execute() || die $dbh->errstr;
      }
      $testcaseId = $id_sth->fetchrow_hashref->{TestcaseId};

      if (exists($stats->{TESTS}{$testName}{$testNum}{INFO}))
      {
        $errorMessage = $stats->{TESTS}{$testName}{$testNum}{INFO};
      }
      else
      {
        $errorMessage = "";
      }
      $sth->execute($testcaseId, $distroId,
                    $stats->{TESTS}{$testName}{$testNum}{STATE}, 
                    $errorMessage)
          || die "$dbh->errstr\n$testName\n$testNum";
    }
  }

  # Update test case count
  $sth = $dbh->prepare("REPLACE INTO testcase_count (Testcase, NumberEntries) "
                  ."SELECT Testcase, Count(*) FROM results GROUP BY Testcase")
      || $sth->errstr;
  $sth->execute() || die $sth->errstr;

  # Commit changes
  $dbh->commit || die $dbh->errstr;

}

#----------------------------------------------------------------------
# Set test suite values for each testcase
sub SetTestSuiteForTestcases($)
{
  my($dbh) = shift;
  my($sth);
  my($row);
  my($update);

  $sth = $dbh->prepare("SELECT * from testsuites, testsuite_idstrings "
                       . "where TSItestsuite=TSid")
      || die $dbh->errstr;
  $sth->execute() || die $sth->errstr;
  
  while ($row = $sth->fetchrow_hashref)
  {
    print "$row->{TSname}\n";
#    $dbh->trace(1);
    $update = $dbh->prepare("UPDATE testcases SET testsuiteid=$row->{TSid} "
                            . "WHERE Name LIKE '%/$row->{TSIidstring}/%'")
        || die $dbh->errstr;
    $update->execute() || die $update->errstr;
  }

  # Commit changes
  $dbh->commit() || die $dbh->errstr;
}

#----------------------------------------------------------------------
# Load waiver file 
# Returns hash of waived tests
sub LoadWaiverFile($)
{
  my($waiverFilename) = shift;
  local(*WAIVERFILE);
  my($waived) = {};

  die "Could not open waiver file $waiverFilename" 
      unless open(WAIVERFILE, $waiverFilename);
  my($line);
  while ( defined($line = <WAIVERFILE>) )
  {
    chomp($line);
    # '#' is the comment character for the waiver file
    if ($line!~/^\#/)
    {
      $waived->{$line} = 1;
    }
  }
  return $waived;
}

#----------------------------------------------------------------------
# Main bit

my(%options);

getopts('dhu:p:b:e:r:o:w:v', \%options);

if (exists($options{'h'}) || ($#ARGV!=0 && $#ARGV!=1))
{
  print STDOUT <<"EOM"
Usage: $0 [-h] [-d] [-u username] [-p password] [-b db_name] 
          [-e vendor_name] [-r version] journal [journal2]

    -h               Display Help
    -d               Display a detailed summary of test results. The
                     test result is shown even if the test passed
    -u username      Username for connection to Mysql db
    -p password      Password for connection to Mysql db
    -b db_name       Name of Mysql Database name
    -o host          Host for Mysql databse (defaults to localhost)
    -e vendor_name   Vendor name of distribution
    -r version       Version name for distribution
    -w waiver file   Take into account waived failures for test suite
		-v               Verbose mode (show error messages)

    When one journal file is supplied a summary of the tests
    is output. When two journal files are supplied the difference
    between the two is shown.

EOM
    ;
    exit(0);
}

my($dbh);
if (exists($options{'b'}))
{
  require DBI;
  if (!exists($options{'e'}) || !exists($options{'r'}))
  {
    die "Must specify vendor and version of distribution\n";
  }

  my($data_source) = "DBI:mysql:database=$options{'b'}";
  if (exists($options{'o'}))
  {
    $data_source .= ";host=$options{'o'}";
  }

  print "Using datasource: $data_source\n";
  # DB Init
  $dbh = DBI->connect($data_source, 
                      exists($options{'u'}) ? $options{'u'} : "", 
                      exists($options{'p'}) ? $options{'p'} : "", 
                      { AutoCommit=>0});
  die "Could not connect to database\n" unless defined($dbh);
}

if (exists($options{'v'}))
{
	$VerboseSummary = 1;
}

# Diff or summary
if ($#ARGV==1)
{
  my($stats1);
  my($stats2);
  my($diffStats);
  $stats1 = GatherStats($ARGV[0]);
  $stats2 = GatherStats($ARGV[1]);
  $diffStats = DiffJournals($stats1, $stats2);
  PrintDiffSummary($diffStats, exists($options{'d'}));
}
else
{
  my($stats);
  $stats = GatherStats($ARGV[0]);
  if (exists($options{'b'}))
  {
    LoadIntoDatabase($dbh, $stats, $options{'e'}, $options{'r'});
    SetTestSuiteForTestcases($dbh);
    $dbh->disconnect();
  }
  else
  {
    my($waivers);
    if (exists($options{'w'}))
    {
      $waivers = LoadWaiverFile($options{'w'});
    }
    PrintSummary($stats, exists($options{'d'}), $waivers);
  }
}