Implement '--expect-message-count' flag/expect_message_count config option.

Provide some additional safety when messages are ignored or suppressed,
to warn if the number of messages is different from expectation.
(Inspired by yacc 'expect' feature - for similar reasons.)

See man genhtml(1) and lcovrc(5) for more details.

Signed-off-by: Henry Cox <henry.cox@mediatek.com>

Author: henry2cox

README

@@ -1,6 +1,6 @@
 -------------------------------------------------
 - README file for the LTP GCOV extension (LCOV) -
-- Last changes: 2024-03-21
+- Last changes: 2024-09-13
 -------------------------------------------------
 
 Description
@@ -64,9 +64,10 @@ Further README contents
                        to lcov format
   bin/geninfo        - Internal tool (creates LCOV data files)
   bin/genpng         - Internal tool (creates png overviews of source files)
+  lcovrc             - LCOV configuration file
   man                - Directory containing man pages for included tools
   example            - Directory containing an example to demonstrate LCOV
-  lcovrc             - LCOV configuration file
+  tests              - Directory containing lcov regression tests
   Makefile           - Makefile providing 'install' and 'uninstall' targets
 
 
@@ -110,8 +111,8 @@ or if you want to use the perl2lcov utility.
 To view the collected coverage information, point your browser to
 .../lcov_coverage/index.html after running the tests.
 
-Note that the testcases are intended to test LCOV functionality and
-not to be easily readable tutorial examples.
+Note that the testcases are primarily intended to test LCOV functionality
+and not to be easily readable tutorial examples.
 
 3. Dependencies:
 ----------------
@@ -307,13 +308,16 @@ New features and capabilities fall into 7 major categories:
      A generic - but very simple - error handler has been added to the
      lcov tool suite.  The error handler is used to report exceptions,
      and provides a mechanism for the user to ignore the particular
-     message if desired.
+     message if desired.  Note that ignoring certain errors can cause
+     subsequent errors and/or can result in inconsistent or confusing
+     coverage reports.
      See the genhtml/lcov/geninfo man pages for details.
 
      Note that some errors are unrecoverable - and cannot be suppressed or
      ignored.
 
-     Related options:  --ignore-error, --keep-going
+     Related options:
+        --ignore-error, --expect-message-count, --keep-going
 
   c) Navigation and display:
 

lib/lcovutil.pm

@@ -100,6 +100,7 @@ our @EXPORT_OK = qw($tool_name $tool_dir $lcov_version $lcov_url $VERSION
 
 our @ignore;
 our @message_count;
+our @expected_message_count;
 our %message_types;
 our $suppressAfter = 100;    # stop warning after this number of messages
 our %ERROR_ID;
@@ -1047,8 +1048,9 @@ my (@rc_filter, @rc_ignore, @rc_exclude_patterns,
     @rc_erase_patterns, @rc_version_script, @unsupported_config,
     @rc_source_directories, @rc_build_dir, %unsupported_rc,
     $keepGoing, $help, @rc_resolveCallback,
-    @rc_criteria_script, @rc_contextCallback, $rc_no_branch_coverage,
-    $rc_no_func_coverage, $rc_no_checksum, $version);
+    @rc_expected_msg_counts, @rc_criteria_script, @rc_contextCallback,
+    $rc_no_branch_coverage, $rc_no_func_coverage, $rc_no_checksum,
+    $version);
 my $quiet = 0;
 my $tempdirname;
 
@@ -1059,6 +1061,7 @@ our ($lcov_remove,     # If set, removes parts of tracefile
      $lcov_extract);    # If set, extracts parts of tracefile
 our @opt_config_files;
 our @opt_ignore_errors;
+our @opt_expected_message_counts;
 our @opt_filter;
 our @comments;
 
@@ -1101,6 +1104,7 @@ my %rc_common = (
              "lcov_branch_coverage"   => \$lcovutil::br_coverage,
              "ignore_errors"          => \@rc_ignore,
              "max_message_count"      => \$lcovutil::suppressAfter,
+             'expected_message_count' => \@rc_expected_msg_counts,
              'stop_on_error'          => \$lcovutil::stop_on_error,
              'treat_warning_as_error' => \$lcovutil::treat_warning_as_error,
              'warn_once_per_file'     => \$lcovutil::warn_once_per_file,
@@ -1220,17 +1224,18 @@ our %argCommon = ("tempdir=s"         => \$tempdirname,
                       \@ReadCurrentSource::source_directories,
                   'build-directory=s' => \@lcovutil::build_directory,
 
-                  'resolve-script=s'  => \@lcovutil::resolveCallback,
-                  'context-script=s'  => \@lcovutil::contextCallback,
-                  "filter=s"          => \@opt_filter,
-                  "demangle-cpp:s"    => \@lcovutil::cpp_demangle,
-                  "ignore-errors=s"   => \@opt_ignore_errors,
-                  "keep-going"        => \$keepGoing,
-                  "config-file=s"     => \@unsupported_config,
-                  "rc=s%"             => \%unsupported_rc,
-                  "profile:s"         => \$lcovutil::profile,
-                  "exclude=s"         => \@lcovutil::exclude_file_patterns,
-                  "include=s"         => \@lcovutil::include_file_patterns,
+                  'resolve-script=s'       => \@lcovutil::resolveCallback,
+                  'context-script=s'       => \@lcovutil::contextCallback,
+                  "filter=s"               => \@opt_filter,
+                  "demangle-cpp:s"         => \@lcovutil::cpp_demangle,
+                  "ignore-errors=s"        => \@opt_ignore_errors,
+                  "expect-message-count=s" => \@opt_expected_message_counts,
+                  "keep-going"             => \$keepGoing,
+                  "config-file=s"          => \@unsupported_config,
+                  "rc=s%"                  => \%unsupported_rc,
+                  "profile:s"              => \$lcovutil::profile,
+                  "exclude=s"              => \@lcovutil::exclude_file_patterns,
+                  "include=s"              => \@lcovutil::include_file_patterns,
                   "erase-functions=s" => \@lcovutil::exclude_function_patterns,
                   "omit-lines=s"      => \@lcovutil::omit_line_patterns,
                   "substitute=s"      => \@lcovutil::file_subst_patterns,
@@ -1452,6 +1457,7 @@ sub parseOptions
     # apply the RC file settings if no command line arg
     foreach my $rc ([\@opt_filter, \@rc_filter],
                     [\@opt_ignore_errors, \@rc_ignore],
+                    [\@opt_expected_message_counts, \@rc_expected_msg_counts],
                     [\@lcovutil::exclude_file_patterns, \@rc_exclude_patterns],
                     [\@lcovutil::include_file_patterns, \@rc_include_patterns],
                     [\@lcovutil::file_subst_patterns, \@rc_subst_patterns],
@@ -1536,6 +1542,7 @@ sub parseOptions
         lcovutil::init_parallel_params();
         # Determine which errors the user wants us to ignore
         parse_ignore_errors(@opt_ignore_errors);
+        parse_expected_message_counts(@opt_expected_message_counts);
         # Determine what coverpoints the user wants to filter
         push(@opt_filter, 'exception') if $lcovutil::exclude_exception_branch;
         parse_cov_filters(@opt_filter);
@@ -1746,12 +1753,13 @@ sub define_errors()
     my $id = 0;
     foreach my $d (@lcovErrs) {
         my ($k, $ref) = @$d;
-        $$ref               = $id;
-        $lcovErrors{$k}     = $id;
-        $ERROR_ID{$k}       = $id;
-        $ERROR_NAME{$id}    = $k;
-        $ignore[$id]        = 0;
-        $message_count[$id] = 0;
+        $$ref                        = $id;
+        $lcovErrors{$k}              = $id;
+        $ERROR_ID{$k}                = $id;
+        $ERROR_NAME{$id}             = $k;
+        $ignore[$id]                 = 0;
+        $message_count[$id]          = 0;
+        $expected_message_count[$id] = undef;    # no expected count, by default
         ++$id;
     }
 }
@@ -1761,6 +1769,29 @@ sub summarize_messages
     my $silent = shift;
     return if $lcovutil::in_child_process;
 
+    # first check for expected message count constraints
+    for (my $idx = 0; $idx <= $#expected_message_count; ++$idx) {
+        my $expr = $expected_message_count[$idx];
+        next unless defined($expr);
+        my $t = $message_count[$idx];
+        $expr =~ s/%C/$t/g;
+        my $v;
+        eval { $v = eval $expr; };
+        if ($@ || !defined($v)) {
+            # we checked the syntax of the message - so should not be able to fail
+            lcovutil::ignorable_error($lcovutil::ERROR_CALLBACK,
+                                      "evaluation of '$expr' failed: $@");
+            next;
+        }
+        unless ($v) {
+            my $type = $ERROR_NAME{$idx};
+            lcovutil::ignorable_error($lcovutil::ERROR_COUNT,
+                "'$type' constraint '$expr' is not true (see '--expect_message_count' for details)."
+            );
+        }
+    }
+
+    # now summarize
     my %total = ('error'   => 0,
                  'warning' => 0,
                  'ignore'  => 0,);
@@ -1810,6 +1841,56 @@ sub parse_ignore_errors(@)
     }
 }
 
+sub parse_expected_message_counts(@)
+{
+    my @constraints = split($split_char, join($split_char, @_));
+    # parse the list and look for errors..
+    foreach my $c (@constraints) {
+        if ($c =~ /^s*(\S+?)\s*:\s*((\d+)|(.+?))\s*$/) {
+            unless (exists($ERROR_ID{lc($1)})) {
+                lcovutil::ignorable_error($lcovutil::ERROR_USAGE,
+                       "unknown 'expected-message-count' message type \"$1\".");
+                next;
+            }
+
+            my $id = $ERROR_ID{lc($1)};
+            if (defined($expected_message_count[$id])) {
+                my $ignore = $lcovutil::ignore[$lcovutil::ERROR_USAGE];
+                lcovutil::ignorable_error($lcovutil::ERROR_USAGE,
+                                        "duplicate 'expected' constraint '$c'" .
+                                            ($ignore ? ': ignoring.' : ''));
+                next;
+            }
+            # check if syntax look resonable
+            my $expr = $2;
+            if (Scalar::Util::looks_like_number($expr)) {
+                $expected_message_count[$id] = "%C == $expr";
+                next;
+            }
+            lcovutil::ignorable_error($lcovutil::ERROR_USAGE,
+                "expect-message-count constraint '$c' does not appear to depend on message count:  '%C' substitution not found."
+            ) unless ($expr =~ /%C/);
+
+            # now lets try an eval
+            my $v = $expr;
+            $v =~ s/%C/0/g;
+            $v = eval $v;
+            if (defined($v)) {
+                $expected_message_count[$id] = $expr;
+            } else {
+                my $ignore = $lcovutil::ignore[$lcovutil::ERROR_USAGE];
+                lcovutil::ignorable_error($lcovutil::ERROR_USAGE,
+                      "eval error in 'expect-message-count' constraint '$c': $@"
+                          . ($ignore ? ': ignoring.' : ''));
+            }
+        } else {
+            lcovutil::ignorable_error($lcovutil::ERROR_USAGE,
+                "malformed expected-message-count constraint \"$c\". Expected 'msg_type = expr'."
+            );
+        }
+    }
+}
+
 sub message_count($)
 {
     my $code = shift;

man/genhtml.1

@@ -86,6 +86,9 @@ genhtml \- Generate HTML view from LCOV coverage data files
 .RB [ \-\-ignore\-errors
 .IR errors  ]
 .br
+.RB [\-\-expect\-message\-count
+.IR message_type=expr[,message_type=expr..]]
+.br
 .RB [ \-\-keep\-going ]
 .RB [ \-\-config\-file
 .IR config\-file ]
@@ -2862,7 +2865,6 @@ function alias record (
 branch expression (3rd field in the .info file 'BRDA' entry) of merge data does not match
 .PP
 
-
 .PP
 .IP \- 3
 Negative number or zero line number encounted.
@@ -2962,12 +2964,84 @@ a tool called during processing returned an error code (e.g., 'find' encountered
 \-\-version\-script comparison returned non\-zero mismatch indication.  It likely that the version of the file which was used in coverage data extraction is different than the source version which was found.  File annotations may be incorrect.
 .PP
 
+Note that certain error messages are caused by issues that you probably cannot
+fix by yourself - for example, bugs in your tool chain which result in
+.I inconsistent
+coverage DB data (see above).
+In those cases, after reviewing the messages you may want to exclude the offending code or the entire offending
+file, or you may simply ignore the messages - either by converting to warning or suppressing entirely.
+Another alternative is to tell
+.B genhtml
+about the number of messages you expect - so that it can warn you if something changes
+such that the count differs, such that you know to review the messages again.
+See the
+.I "\-\-expect\-message\-count"
+flag, below.
+
 Also see 'man
 .B lcovrc(5)
 ' for a discussion of the 'max_message_count' parameter which can be used to control the number of warnings which are emitted before all subsequent messages are suppressed.  This can be used to reduce log file volume.
 .br
 
 .RE
+.BI "\-\-expect\-message\-count message_type:expr[,message_type:expr]"
+.RS
+Give
+.B genhtml
+a constraint on the number of messages of one or more types which are expected to
+be produced during execution.  Note that the total includes _all_ messages
+of the given type - including those which have been suppressed.
+If the constraint is not true, an
+error of type
+.I "count"
+(see above) is generated.
+.I message_type
+is one of the message mnemonics described above, and
+.I expr
+may be either
+.IP \- 3
+an integer - interpreted to mean that there should be exactly that number
+of messages of the corresponding type, or
+.IP \- 3
+a Perl expression containing the substring
+.B %C
+; %C is replaced with the total number of messages of the corresponding type and
+then evaluated.  The constraint is met if the result is non-zero and is not met
+otherwise.
+
+For example:
+.IP
+\-\-expect\-message\-count inconsistent:5
+.br
+says that we expect exactly 5 messages of type 'inconsistent'.
+.PP
+.IP
+\-\-expect\-message\-count inconsistent:%C==5
+.br
+also says that we expect exactly 5 messages of this type, but specified
+using expression syntax.
+.PP
+.IP
+\-\-expect\-message\-count 'inconsistent : %C > 6 && %C <= 10'
+.br
+says that we expect the number of messages to be in the range (6:10].
+(Note that quoting may be necessary, to protect whitespace from interpretation by
+your shell, if you want to improve expression readability by adding spaces to your expression.)
+.PP
+
+Multiple constraints can be specified using a comma-separated list or
+by using the option multiple times.
+
+This flag is equivalent to the
+.I "expect_message_count"
+configuration option.  See man
+.B lcovrc(5)
+for more details on the expression syntax and how expressions are interpreted.
+The number of messages of the particular type is substituted into the
+expression before it is evaluated.
+
+.RE
+
 .BI "\-\-keep\-going "
 .RS
 Do not stop if error occurs: attempt to generate a result, however flawed.

man/geninfo.1

@@ -55,6 +55,9 @@ geninfo \- Generate tracefiles from GCOV coverage data files
 .RB [ \-\-ignore\-errors
 .IR errors ]
 .br
+.RB [\-\-expect\-message\-count
+.IR message_type=expr[,message_type=expr..]]
+.br
 .RB [ \-\-keep\-going ]
 .br
 .RB [ \-\-preserve ]
@@ -975,8 +978,27 @@ See man
 .B lcovrc(5)
  for a discussion of the 'max_message_count' parameter which can be used to control the number of warnings which are emitted before all subsequent messages are suppressed.  This can be used to reduce log file volume.
 
+.RE
+
+.BI "\-\-expect\-message\-count message_type:expr[,message_type:expr]"
+.RS
+Give
+.B geninfo
+a constraint on the number of messages of one or more types which are expected to
+be produced during execution.  If the constraint is not true, then generate an
+error of type
+.I "count"
+(see above).
 
+See man
+.B genhtml(1)
+for more details about the flag, as well as the
+.I "expect_message_count"
+section in man
+.B lcovrc(5)
+for a description of the equivalent configuration file option.
 .RE
+
 .BI "\-\-keep\-going "
 .RS
 Do not stop if error occurs: attempt to generate a result, however flawed.