#!/usr/bin/perl -w # Tool to summarise a journal file generated by TET # or to determine the difference in test results between two # journals. # # (C) Copyright 2001 The Free Standards Group Inc # # 21/5/2001 Chris Yeoh, IBM # # This is $Revision: 1.11a $ # # $Log: tjreport,v $ # Revision 1.11a 2005/02/16 sbenedict@mandrakesoft.com # very minor mod to "excluding waived" output (Bugzilla #698) # # Revision 1.11 2003/10/31 02:34:37 cyeoh # Only print test system information if it exists # Adds verbose flag which causes error information to be printed # # Revision 1.10 2003/07/17 07:17:49 cyeoh # change error about test inconsistency to warning as lsblibchk # is creating duplicates at the moment # # Revision 1.9 2002/01/16 05:14:00 cyeoh # clean up help information # remove dependency on DBI when not used # # Revision 1.8 2001/12/04 07:39:41 cyeoh # Add support for waiver files # # Revision 1.7 2001/11/05 07:31:19 cyeoh # Fixes bug that caused the missing of some testcases # Adds cross check of testcase count # Removes debug of DBI calls # # Revision 1.6 2001/08/22 03:52:32 cyeoh # Fixes die error reporting # Adds storage and update of testsuite to test case mapping # # Revision 1.5 2001/08/14 08:31:59 cyeoh # Fix generation of description information so that # journal prefix ids are removed # # Revision 1.4 2001/08/14 04:59:55 cyeoh # Rework for change in db format. # Now keeps track of how many results available for each testcase # # Revision 1.3 2001/08/14 02:18:07 cyeoh # Adds ability to add error information for failed tests into database # # Revision 1.2 2001/08/13 04:51:04 cyeoh # Adds ability to add data into SQL database for further analysis # # Revision 1.1 2001/08/11 11:46:13 cyeoh # Initial version # # use strict; use Getopt::Std; my(%StateMap) = ( 0 => "PASS", 1 => "FAIL", 2 => "UNRESOLVED", 3 => "NOTINUSE", 4 => "UNSUPPORTED", 5 => "UNTESTED", 6 => "UNINITIATED", 7 => "UNREPORTED", 101 => "WARNING", 102 => "FIP", 103 => "NOTIMP", 104 => "UNAPPROVE"); my(%PFMap) = ( "PASS" => "PASS", "FAIL" => "FAIL", "UNRESOLVED" => "FAIL", "NOTINUSE" => "PASS", "UNSUPPORTED" => "PASS", "UNTESTED" => "PASS", "UNINITIATED" => "FAIL", "UNREPORTED" => "FAIL", "WARNING" => "PASS", "FIP" => "PASS", "NOTIMP" => "PASS", "UNAPPROVE" => "PASS", "MISSING" => "FAIL", "UNKNOWN" => "FAIL"); my($VerboseSummary) = 0; # Analyses one journal file to get statistics on pass/failure # sub GatherStats($) { my($journalFile) = shift; my($stats) = {}; my($loop); my($line); local(*JFILE); # Initialise $stats->{STATE_SUMMARY} = {}; foreach $loop (keys %StateMap) { $stats->{STATE_SUMMARY}{$StateMap{$loop}} = 0; } $stats->{STATE_SUMMARY}{TEST_ERROR} = 0; $stats->{STATE_SUMMARY}{UNKNOWN} = 0; $stats->{TOTAL_TESTS_PASSED} = 0; $stats->{TOTAL_TESTS_FAILED} = 0; # Analyse file open(JFILE, $journalFile) || die "Could not open file: $journalFile\n"; my($testName); my($testNum); my(@line); my($testState); my($errorMessage); my($tmp); while (defined($line=<JFILE>)) { # Look for system info if ($line =~ /^0\|(.*)\|/) { @line = split(/ /, $1); $stats->{TEST_DATE} = $line[2]; $stats->{TEST_TIME} = $line[1]; } elsif ($line=~ /^30\|.*VSX_SYS=(.*)$/) { $stats->{TEST_SYSTEM} = $1; } # Look for test results @line = split(/ /, $line); if ($line[0] =~ /^10/) { $testName = $line[1]; } elsif ($line[0] =~ /^200/) { $testNum = $line[1]; $errorMessage = ""; } elsif ($line[0] =~ /^220/) { # Test state report $testState = exists($StateMap{$line[2]}) ? $StateMap{$line[2]} : "UNKNOWN"; $stats->{STATE_SUMMARY}{$testState}++; if (exists($stats->{TESTS}{$testName}{$testNum})) { print "Dup: $testName $testNum\n"; } $stats->{TESTS}{$testName}{$testNum}{STATE} = $testState; if ($PFMap{$testState} eq "PASS") { $stats->{TOTAL_TESTS_PASSED}++ ; } else { $stats->{TOTAL_TESTS_FAILED}++; $stats->{TESTS}{$testName}{$testNum}{INFO} = $errorMessage; } } elsif ($line[0] =~ /^520/) { # Accumulate any error/info messages associated with test $line =~ /\|([^\|]*)$/; $tmp = $1; chomp($tmp); $errorMessage .= "$tmp\n"; } } close(JFILE); return $stats; } #---------------------------------------------------------------------- # Find the difference in test results between the two journals sub DiffJournals($$) { my($j1) = shift; my($j2) = shift; my($diffStats) = {}; my($testName); my($testNum); $diffStats->{TESTS} = {}; foreach $testName (sort keys %{$j1->{TESTS}}) { foreach $testNum (sort {$a <=> $b} keys %{$j1->{TESTS}{$testName}}) { if (exists($j2->{TESTS}{$testName}) && exists($j2->{TESTS}{$testName}{$testNum})) { if ($j1->{TESTS}{$testName}{$testNum} ne $j2->{TESTS}{$testName}{$testNum}) { $diffStats->{TESTS}{$testName}{$testNum} = "$j1->{TESTS}{$testName}{$testNum}{STATE}," . "$j2->{TESTS}{$testName}{$testNum}{STATE}"; } } else { $diffStats->{TESTS}{$testName}{$testNum} = "$j1->{TESTS}{$testName}{$testNum}{STATE},MISSING"; } } } # Check reverse foreach $testName (sort keys %{$j2->{TESTS}}) { foreach $testNum (sort {$a <=> $b} keys %{$j2->{TESTS}{$testName}}) { if (! (exists($j2->{TESTS}{$testName}) && exists($j2->{TESTS}{$testName}{$testNum}))) { $diffStats->{TESTS}{$testName}{$testNum} = "MISSING,$j2->{TESTS}{$testName}{$testNum}{STATE}"; } } } return $diffStats; } #---------------------------------------------------------------------- sub PrintDiffSummary($$) { my($diffStats) = shift; my($isDetailed) = shift; my($testName); my($testNum); my($state1, $state2); foreach $testName (sort keys %{$diffStats->{TESTS}}) { foreach $testNum (sort {$a <=> $b} keys %{$diffStats->{TESTS}{$testName}}) { ($state1, $state2) = split(/,/, $diffStats->{TESTS}{$testName}{$testNum}); if (($PFMap{$state1} ne $PFMap{$state2}) || $isDetailed) { print "$testName $testNum $diffStats->{TESTS}{$testName}{$testNum}\n"; } } } } #---------------------------------------------------------------------- # sub PrintSummary($$$) { my($stats) = shift; my($isDetailed) = shift; my($waivers) = shift; my($testState); my($testName); my($testNum); my($numTests); my($secondCount) = 0; my($waived) = 0; foreach $testName (sort keys %{$stats->{TESTS}}) { foreach $testNum (sort {$a <=> $b} keys %{$stats->{TESTS}{$testName}}) { $secondCount++; # print "$testName $testNum\n"; if ( ($PFMap{$stats->{TESTS}{$testName}{$testNum}{STATE}} eq "FAIL") || $isDetailed ) { print "$testName $testNum $stats->{TESTS}{$testName}{$testNum}{STATE}"; if (defined($waivers) && $waivers->{"$testName-$testNum"}) { print " (WAIVED)"; $waived++; } print "\n"; if ($VerboseSummary) { print " $stats->{TESTS}{$testName}{$testNum}{INFO}"; } } } } $numTests = $stats->{TOTAL_TESTS_PASSED} + $stats->{TOTAL_TESTS_FAILED}; print "Warning: Inconsistency in test count. This is probably due to a bug in this" . " program ($numTests, $secondCount).\n" unless $numTests==$secondCount; print "\n\n"; print "Test system: $stats->{TEST_SYSTEM}\n" unless (!defined($stats->{TEST_SYSTEM})); print "Test was run: $stats->{TEST_DATE} $stats->{TEST_TIME} \n"; print "Total Tests Passed: $stats->{TOTAL_TESTS_PASSED}\n"; print "Total Tests Failed (including waived): $stats->{TOTAL_TESTS_FAILED}\n"; print "Total Tests Failed (excluding waived): " . ($stats->{TOTAL_TESTS_FAILED}-$waived) . "\n"; print "\nTest Result Breakdown:\n"; foreach $testState (keys %{$stats->{STATE_SUMMARY}}) { print "$testState: $stats->{STATE_SUMMARY}{$testState}\n"; } } #---------------------------------------------------------------------- # sub LoadIntoDatabase($$$$) { my($dbh) = shift; my($stats) = shift; my($vendorName) = shift; my($versionName) = shift; my($testName); my($testNum); my($sth); my($distroId); print "Adding information to DB\n"; $sth = $dbh->prepare( "SELECT DistroId from distributions where VendorName='$vendorName'" . " and VersionName='$versionName'") || die $dbh->errstr; $sth->execute() || die $dbh->errstr; if ($sth->rows==0) { # Add distribution information to database print "Adding distribution information to database\n"; $sth = $dbh->prepare( "INSERT INTO distributions (VendorName, VersionName)" . " Values ('$vendorName', '$versionName')") || die $dbh->errstr; $sth->execute() || die $dbh->errstr; $sth = $dbh->prepare( "SELECT DistroId from distributions where VendorName='$vendorName'" . " and VersionName='$versionName'") || die $dbh->errstr; $sth->execute() || die $dbh->errstr; } die "found " . $sth->rows . " occurences of distro name\n" unless $sth->rows==1; # Get distro id $distroId = $sth->fetchrow_hashref->{DistroId}; print "Distribution id: $distroId\n"; $sth = $dbh->prepare( "INSERT INTO results (Testcase, Distro, State, Description)" . "Values (?, ?, ?, ?)") || die $dbh->errstr; my($errorMessage); my($testcaseId); my($id_sth); foreach $testName (sort keys %{$stats->{TESTS}}) { foreach $testNum (sort {$a <=> $b} keys %{$stats->{TESTS}{$testName}}) { # Get testcase id $id_sth = $dbh->prepare("SELECT TestcaseId from testcases" . " where Name='$testName-$testNum'") || die $dbh->errstr; $id_sth->execute(); if ($id_sth->rows==0) { $id_sth = $dbh->prepare("INSERT INTO testcases (Name)" . " Values ('$testName-$testNum')") || die $dbh->errstr; $id_sth->execute() || die $dbh->errstr; $id_sth = $dbh->prepare("SELECT TestcaseId from testcases" . " where Name='$testName-$testNum'") || die $dbh->errstr; $id_sth->execute() || die $dbh->errstr; } $testcaseId = $id_sth->fetchrow_hashref->{TestcaseId}; if (exists($stats->{TESTS}{$testName}{$testNum}{INFO})) { $errorMessage = $stats->{TESTS}{$testName}{$testNum}{INFO}; } else { $errorMessage = ""; } $sth->execute($testcaseId, $distroId, $stats->{TESTS}{$testName}{$testNum}{STATE}, $errorMessage) || die "$dbh->errstr\n$testName\n$testNum"; } } # Update test case count $sth = $dbh->prepare("REPLACE INTO testcase_count (Testcase, NumberEntries) " ."SELECT Testcase, Count(*) FROM results GROUP BY Testcase") || $sth->errstr; $sth->execute() || die $sth->errstr; # Commit changes $dbh->commit || die $dbh->errstr; } #---------------------------------------------------------------------- # Set test suite values for each testcase sub SetTestSuiteForTestcases($) { my($dbh) = shift; my($sth); my($row); my($update); $sth = $dbh->prepare("SELECT * from testsuites, testsuite_idstrings " . "where TSItestsuite=TSid") || die $dbh->errstr; $sth->execute() || die $sth->errstr; while ($row = $sth->fetchrow_hashref) { print "$row->{TSname}\n"; # $dbh->trace(1); $update = $dbh->prepare("UPDATE testcases SET testsuiteid=$row->{TSid} " . "WHERE Name LIKE '%/$row->{TSIidstring}/%'") || die $dbh->errstr; $update->execute() || die $update->errstr; } # Commit changes $dbh->commit() || die $dbh->errstr; } #---------------------------------------------------------------------- # Load waiver file # Returns hash of waived tests sub LoadWaiverFile($) { my($waiverFilename) = shift; local(*WAIVERFILE); my($waived) = {}; die "Could not open waiver file $waiverFilename" unless open(WAIVERFILE, $waiverFilename); my($line); while ( defined($line = <WAIVERFILE>) ) { chomp($line); # '#' is the comment character for the waiver file if ($line!~/^\#/) { $waived->{$line} = 1; } } return $waived; } #---------------------------------------------------------------------- # Main bit my(%options); getopts('dhu:p:b:e:r:o:w:v', \%options); if (exists($options{'h'}) || ($#ARGV!=0 && $#ARGV!=1)) { print STDOUT <<"EOM" Usage: $0 [-h] [-d] [-u username] [-p password] [-b db_name] [-e vendor_name] [-r version] journal [journal2] -h Display Help -d Display a detailed summary of test results. The test result is shown even if the test passed -u username Username for connection to Mysql db -p password Password for connection to Mysql db -b db_name Name of Mysql Database name -o host Host for Mysql databse (defaults to localhost) -e vendor_name Vendor name of distribution -r version Version name for distribution -w waiver file Take into account waived failures for test suite -v Verbose mode (show error messages) When one journal file is supplied a summary of the tests is output. When two journal files are supplied the difference between the two is shown. EOM ; exit(0); } my($dbh); if (exists($options{'b'})) { require DBI; if (!exists($options{'e'}) || !exists($options{'r'})) { die "Must specify vendor and version of distribution\n"; } my($data_source) = "DBI:mysql:database=$options{'b'}"; if (exists($options{'o'})) { $data_source .= ";host=$options{'o'}"; } print "Using datasource: $data_source\n"; # DB Init $dbh = DBI->connect($data_source, exists($options{'u'}) ? $options{'u'} : "", exists($options{'p'}) ? $options{'p'} : "", { AutoCommit=>0}); die "Could not connect to database\n" unless defined($dbh); } if (exists($options{'v'})) { $VerboseSummary = 1; } # Diff or summary if ($#ARGV==1) { my($stats1); my($stats2); my($diffStats); $stats1 = GatherStats($ARGV[0]); $stats2 = GatherStats($ARGV[1]); $diffStats = DiffJournals($stats1, $stats2); PrintDiffSummary($diffStats, exists($options{'d'})); } else { my($stats); $stats = GatherStats($ARGV[0]); if (exists($options{'b'})) { LoadIntoDatabase($dbh, $stats, $options{'e'}, $options{'r'}); SetTestSuiteForTestcases($dbh); $dbh->disconnect(); } else { my($waivers); if (exists($options{'w'})) { $waivers = LoadWaiverFile($options{'w'}); } PrintSummary($stats, exists($options{'d'}), $waivers); } }