From a2df1ae3fd83416ab36058009822aa16c8ee2749 Mon Sep 17 00:00:00 2001 From: sure-admin Date: Sun, 24 May 2026 14:33:16 +0000 Subject: [PATCH] test: add CI system test timing reporter --- .github/workflows/ci.yml | 12 +- test/support/ci_system_test_timing_plugin.rb | 161 +++++++++++++++++++ test/test_helper.rb | 2 + 3 files changed, 174 insertions(+), 1 deletion(-) create mode 100644 test/support/ci_system_test_timing_plugin.rb diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0859eca33..4141b33aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -168,7 +168,17 @@ jobs: bin/rails db:seed - name: System tests - run: DISABLE_PARALLELIZATION=true bin/rails test:system + run: DISABLE_PARALLELIZATION=true CI_SYSTEM_TEST_TIMING=true bin/rails test:system + + - name: Upload system test timing artifacts + uses: actions/upload-artifact@v6 + if: always() + with: + name: system-test-timing + path: | + ${{ github.workspace }}/tmp/ci/system_test_timing.json + ${{ github.workspace }}/tmp/ci/system_test_timing.md + if-no-files-found: ignore - name: Keep screenshots from failed system tests uses: actions/upload-artifact@v6 diff --git a/test/support/ci_system_test_timing_plugin.rb b/test/support/ci_system_test_timing_plugin.rb new file mode 100644 index 000000000..1cb594396 --- /dev/null +++ b/test/support/ci_system_test_timing_plugin.rb @@ -0,0 +1,161 @@ +require "json" +require "fileutils" +require "minitest" +require "pathname" +require "time" + +module CiSystemTestTimingPlugin + OUTPUT_DIR = File.expand_path("../../tmp/ci", __dir__) + + THEME_ALIASES = { + "account" => "accounts", + "drag" => "imports", + "import" => "imports", + "transaction" => "transactions", + "setting" => "settings" + }.freeze + + class Reporter < Minitest::StatisticsReporter + attr_reader :entries + + def start + super + @entries = [] + end + + def record(result) + super + + source_location = Array(result.source_location) + file = source_location.first + line = source_location.last + relative_file = file && Pathname.new(file).relative_path_from(Pathname.pwd).to_s + + @entries << { + class_name: result.class_name, + name: result.name, + location: line && relative_file ? "#{relative_file}:#{line}" : relative_file, + file: relative_file, + theme: theme_for(relative_file), + time: result.time.to_f, + assertions: result.assertions, + failures: result.failures.size, + skipped: result.skipped?, + error: result.error? + } + end + + def report + write_reports + super + end + + private + def write_reports + return if entries.empty? + + FileUtils.mkdir_p(OUTPUT_DIR) + + payload = { + generated_at: Time.now.utc.iso8601, + total_time: total_time, + test_count: entries.size, + groups: grouped_summary, + slowest: slowest_entries, + tests: entries.sort_by { |entry| -entry[:time] } + } + + json_path = File.join(OUTPUT_DIR, "system_test_timing.json") + markdown_path = File.join(OUTPUT_DIR, "system_test_timing.md") + + File.write(json_path, JSON.pretty_generate(payload)) + + markdown = build_markdown(payload) + File.write(markdown_path, markdown) + + puts + puts markdown + + append_step_summary(markdown) + end + + def grouped_summary + entries + .group_by { |entry| entry[:theme] } + .map do |theme, theme_entries| + { + theme: theme, + total_time: theme_entries.sum { |entry| entry[:time] }, + test_count: theme_entries.size, + slowest_test: theme_entries.max_by { |entry| entry[:time] } + } + end + .sort_by { |group| -group[:total_time] } + end + + def slowest_entries(limit = 15) + entries.sort_by { |entry| -entry[:time] }.first(limit) + end + + def build_markdown(payload) + lines = [] + lines << "## System test timing summary" + lines << + "Measured #{payload[:test_count]} tests in #{format_seconds(payload[:total_time])}. " \ + "Grouped by likely split theme for future workflow sharding." + lines << "" + lines << "### Slowest themes" + lines << "| Theme | Total | Tests | Slowest test |" + lines << "| --- | ---: | ---: | --- |" + + payload[:groups].each do |group| + slowest_test = group[:slowest_test] + lines << "| #{group[:theme]} | #{format_seconds(group[:total_time])} | #{group[:test_count]} | #{slowest_test[:class_name]}##{slowest_test[:name]} (#{format_seconds(slowest_test[:time])}) |" + end + + lines << "" + lines << "### Slowest individual tests" + lines << "| Test | Theme | Time | Location |" + lines << "| --- | --- | ---: | --- |" + + payload[:slowest].each do |entry| + lines << "| #{entry[:class_name]}##{entry[:name]} | #{entry[:theme]} | #{format_seconds(entry[:time])} | `#{entry[:location]}` |" + end + + lines << "" + lines << "Artifacts: `tmp/ci/system_test_timing.json`, `tmp/ci/system_test_timing.md`" + lines.join("\n") + end + + def append_step_summary(markdown) + summary_path = ENV["GITHUB_STEP_SUMMARY"] + return if summary_path.to_s.empty? + + File.open(summary_path, "a") do |file| + file.puts(markdown) + file.puts + end + end + + def format_seconds(seconds) + format("%.2fs", seconds) + end + + def theme_for(relative_file) + return "unknown" if relative_file.to_s.empty? + + test_path = relative_file.sub(%r{\Atest/system/}, "") + first_segment = test_path.split("/").first.to_s.sub(/_test\.rb\z/, "") + stem = first_segment.split("_").first + + stem = nil if stem.nil? || stem.empty? + THEME_ALIASES.fetch(stem, stem || "unknown") + end + end + + def self.minitest_plugin_init(_options) + Minitest.reporter << Reporter.new($stdout, {}) + end +end + +Minitest.register_plugin(CiSystemTestTimingPlugin) diff --git a/test/test_helper.rb b/test/test_helper.rb index 461ebe3a6..e10792a2f 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -27,6 +27,8 @@ require "rack/test" require "tempfile" require "uri" +require_relative "support/ci_system_test_timing_plugin" if ENV["CI_SYSTEM_TEST_TIMING"] == "true" + VCR.configure do |config| config.cassette_library_dir = "test/vcr_cassettes" config.hook_into :webmock