From a99deea71a6e9d477d27c3b8dc2de02ecf3076fb Mon Sep 17 00:00:00 2001 From: Mark VanderVoord Date: Mon, 18 May 2026 16:06:36 -0400 Subject: [PATCH] Refactor test reporting for better final summary. --- test/rakefile | 18 ++++---- test/rakefile_helper.rb | 93 +++++++++++++++++++++++++++++++++++++---- vendor/unity | 2 +- 3 files changed, 98 insertions(+), 15 deletions(-) diff --git a/test/rakefile b/test/rakefile index f983c5d..c9ec434 100644 --- a/test/rakefile +++ b/test/rakefile @@ -33,7 +33,7 @@ task :prep_system_tests => SYSTEM_TEST_SUPPORT_DIRS configure_toolchain(DEFAULT_CONFIG_FILE) task :default => [:test] -task :ci => [:no_color, :default, 'test:examples', 'style:check'] +task :ci => [:no_color, :default, 'test:examples', 'style:check', 'test:summary'] task :cruise => :ci desc "Load configuration" @@ -49,9 +49,8 @@ namespace :test do task :all => [:clobber, :prep_system_tests, 'test:units', 'test:c', 'test:system'] desc "Run Unit Tests" - Rake::TestTask.new('units') do |t| - t.pattern = 'unit/*_test.rb' - t.verbose = true + task :units => [:prep_system_tests] do + run_ruby_unit_tests end #individual unit tests @@ -106,6 +105,11 @@ namespace :test do task :profile => [:clobber, :prep_system_tests] do run_system_test_profiles(FileList[SYSTEST_COMPILE_MOCKABLES_PATH + '*.h']) end + + desc "Generate test summary" + task :summary do + report_summary + end end task :no_color do @@ -120,7 +124,7 @@ namespace :style do report "--------------------\n" report "VERIFYING RUBY STYLE\n" report "--------------------\n" - report execute("rubocop ../lib ../examples ../config ../scripts --config ../vendor/unity/test/.rubocop.yml", true) + execute("rubocop ../lib ../examples ../config ../scripts --config ../vendor/unity/test/.rubocop.yml", true) report "Styling Ruby:PASS" end @@ -131,13 +135,13 @@ namespace :style do desc "Attempt to Autocorrect style" task :auto => ['style:clean'] do - report execute("rubocop ../lib ../examples ../config ../scripts --autocorrect --config ../vendor/unity/test/.rubocop.yml", true) + execute("rubocop ../lib ../examples ../config ../scripts --autocorrect --config ../vendor/unity/test/.rubocop.yml", true) report "Autocorrected What We Could." end desc "Update style todo list" task :todo => ['style:clean'] do - report execute("rubocop ../lib ../examples ../config ../scripts --auto-gen-config --config ../vendor/unity/test/.rubocop.yml", true) + execute("rubocop ../lib ../examples ../config ../scripts --auto-gen-config --config ../vendor/unity/test/.rubocop.yml", true) report "Updated Style TODO List." end diff --git a/test/rakefile_helper.rb b/test/rakefile_helper.rb index f68ae0f..15bcf48 100644 --- a/test/rakefile_helper.rb +++ b/test/rakefile_helper.rb @@ -230,6 +230,62 @@ module RakefileHelpers report "Styling C:PASS" end + def save_test_results(test_base, output) + build_root = $proj[:project][:build_root] + test_results = build_root + test_base + test_results += output.match(/OK\s*$/m) ? '.testpass' : '.testfail' + File.open(test_results, 'w') { |f| f.print output } + end + + def collect_test_output(output) + total_tests = 0 + total_failures = 0 + total_ignored = 0 + detail_lines = [] + + output.each_line do |line| + stripped = line.chomp + if stripped =~ /(\d+) Tests (\d+) Failures (\d+) Ignored/ + total_tests += Regexp.last_match(1).to_i + total_failures += Regexp.last_match(2).to_i + total_ignored += Regexp.last_match(3).to_i + elsif stripped =~ /^[^:]+:[^:]+:\w+(?:\([^)]*\))?:(?:PASS|FAIL|IGNORE)/ + detail_lines << stripped + end + end + + synthesized = detail_lines.join("\n") + synthesized += "\n" unless detail_lines.empty? + synthesized += "#{total_tests} Tests #{total_failures} Failures #{total_ignored} Ignored\n" + synthesized += total_failures > 0 ? "FAILED\n" : "OK\n" + synthesized + end + + def run_ruby_unit_tests + report "\n" + report "--------------------\n" + report "RUBY UNIT TEST SUITE\n" + report "--------------------\n" + total_runs = total_failures = total_errors = total_skips = 0 + Dir['unit/*_test.rb'].sort.each do |tst| + report "\nRunning #{tst.to_s}" + output = execute("ruby -I. #{tst}", true, false) + output.each_line do |line| + if line =~ /(\d+) runs, \d+ assertions, (\d+) failures, (\d+) errors, (\d+) skips/ + total_runs += Regexp.last_match(1).to_i + total_failures += Regexp.last_match(2).to_i + total_errors += Regexp.last_match(3).to_i + total_skips += Regexp.last_match(4).to_i + end + end + end + failures = total_failures + total_errors + result = "#{total_runs} Tests #{failures} Failures #{total_skips} Ignored\n" + result += failures > 0 ? "FAILED\n" : "OK\n" + save_test_results('ruby_unit_tests', result) + raise "Ruby unit tests failed." if failures > 0 + end + def report_summary summary = UnityTestSummary.new summary.root = File.expand_path(File.dirname(__FILE__)) + '/' @@ -237,7 +293,7 @@ module RakefileHelpers results_glob.gsub!(/\\/, '/') results = Dir[results_glob] summary.targets = results - summary.run + report summary.run fail_out "FAIL: There were failures" if (summary.failures > 0) end @@ -348,6 +404,11 @@ module RakefileHelpers end report '' + result = failure_messages.join("\n") + result += "\n" unless failure_messages.empty? + result += "#{total_tests} Tests #{total_failures} Failures 0 Ignored\n" + result += total_failures > 0 ? "FAILED\n" : "OK\n" + save_test_results('system_interactions', result) return total_failures end @@ -378,12 +439,16 @@ module RakefileHelpers report "------------------------------------\n" report "SYSTEM TEST MOCK COMPILATION SUMMARY\n" report "------------------------------------\n" + pass_count = 0 mockables.each do |header| mock_filename = 'mock_' + File.basename(header).ext('.c') CMock.new(SYSTEST_COMPILE_MOCKABLES_PATH + 'config.yml').setup_mocks(header) report "Compiling #{mock_filename}..." compile(SYSTEST_GENERATED_FILES_PATH + mock_filename) + pass_count += 1 end + report "#{pass_count} Tests 0 Failures 0 Ignored\nOK\n" + save_test_results('system_compilations', "#{pass_count} Tests 0 Failures 0 Ignored\nOK\n") end def run_system_test_profiles(mockables) @@ -413,6 +478,7 @@ module RakefileHelpers report "----------------\n" ext = $unity_cfg[:extension][:executable] build_root = $proj[:project][:build_root] + combined_output = '' FileList.new("c/*.yml").each do |yaml_file| test = load_yaml(yaml_file) report "\nTesting #{yaml_file.sub('.yml','')}" @@ -422,12 +488,12 @@ module RakefileHelpers link_it('TestCMockC', obj_files) simulator = build_simulator_fields executable = build_root + 'TestCMockC' + ext - if simulator.nil? - execute(executable) - else - execute("#{simulator[:command]} #{simulator[:pre_support]} #{executable} #{simulator[:post_support]}") - end + cmd = simulator.nil? ? executable : "#{simulator[:command]} #{simulator[:pre_support]} #{executable} #{simulator[:post_support]}" + combined_output += execute(cmd, true, false) + "\n" end + result = collect_test_output(combined_output) + save_test_results('c_unit_tests', result) + raise "C unit tests failed." if result =~ /FAILED/ end def run_examples(verbose=false, raise_on_failure=true) @@ -435,12 +501,25 @@ module RakefileHelpers report "-----------------\n" report "VALIDATE EXAMPLES\n" report "-----------------\n" + total_tests = 0 + total_failures = 0 + total_ignored = 0 [ "cd #{File.join("..", "examples", "make_example")} && make clean && make setup && make test", "cd #{File.join("..", "examples", "temp_sensor")} && rake ci" ].each do |cmd| report "Testing '#{cmd}'" - execute(cmd, verbose, raise_on_failure) + execute(cmd, verbose, false).each_line do |line| + if line =~ /(\d+) TOTAL TESTS (\d+) TOTAL FAILURES (\d+) IGNORED/ + total_tests += Regexp.last_match(1).to_i + total_failures += Regexp.last_match(2).to_i + total_ignored += Regexp.last_match(3).to_i + end + end end + result = "#{total_tests} Tests #{total_failures} Failures #{total_ignored} Ignored\n" + result += total_failures > 0 ? "FAILED\n" : "OK\n" + save_test_results('examples', result) + raise "Examples failed." if total_failures > 0 && raise_on_failure end def fail_out(msg) diff --git a/vendor/unity b/vendor/unity index ed99435..c7b0faa 160000 --- a/vendor/unity +++ b/vendor/unity @@ -1 +1 @@ -Subproject commit ed994351776f875fd64f800ec88819470f2c00fb +Subproject commit c7b0faabdcb2af0d9d4d90ccab019ff45836edb3