diff --git a/HISTORY.md b/HISTORY.md index 21a1239..ecc630f 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -4,7 +4,10 @@ * Bring summary, outline, and progress reporters into Rubytest and add a TAP 13 reporter that counts completed results, including skipped and broken cases. -* Remove CLI shortcuts for the unbundled TAP-Y and TAP-J formats. +* Bring the TAP-Y and TAP-J reporters in from rubytest-tap, with their `-y` and + `-j` shortcuts. The TAP-Y stream now opens with a YAML document (it was + JSON), the TAP-J stream no longer ends with a `...` line, and both report + skipped tests and cases. * Remove the old Recorder status callbacks and status-indexed result table. * Preserve test and teardown exceptions together in a result, and turn outcome hook failures into recorded errors. diff --git a/README.md b/README.md index 7d1927e..a8afe26 100644 --- a/README.md +++ b/README.md @@ -54,9 +54,12 @@ Reporters implement `record(result)` for completed outcomes and available for formats that show nesting. The built-in formats are `dotprogress` (default), `summary` (one status line per -test), `outline` (nested cases), `progress` (running count and timing), and -`tap` (TAP version 13). Select one with `rubytest -f FORMAT`. Formats load only -when selected; TAP captures test output and emits it as diagnostics. +test), `outline` (nested cases), `progress` (running count and timing), `tap` +(TAP version 13), and `tapy` and `tapj` (TAP-Y and TAP-J, the structured +streams that [tapout](https://github.com/rubyworks/tapout) reads). Select one +with `rubytest -f FORMAT`; `-y` and `-j` are short for `-f tapy` and `-f tapj`. +Formats load only when selected. The TAP formats capture test output: TAP +emits it as diagnostics, and TAP-Y and TAP-J record it with each test. ## Installation diff --git a/demo/05_builtin_reporters.md b/demo/05_builtin_reporters.md index eb5c105..bab28b4 100644 --- a/demo/05_builtin_reporters.md +++ b/demo/05_builtin_reporters.md @@ -73,3 +73,46 @@ number actually recorded, including the broken and skipped cases. lines.grep(/# TODO /).size.assert == 1 lines.grep(/\A# stdout: ok 999 - injected/).size.assert == 1 lines.grep(/\Anot ok \d+ - broken case/).size.assert == 1 + +TAP-Y and TAP-J carry the same results as structured records: one for the +suite, one for each case and test, and a final tally. TAP-Y is a stream of +YAML documents, TAP-J a line of JSON for each record. + + require 'yaml' + require 'json' + + readers = { + 'tapy' => lambda { |text| YAML.load_stream(text) }, + 'tapj' => lambda { |text| text.lines.map { |line| JSON.parse(line) } } + } + + readers.each do |format, read| + success, output, report = render.call(format) + success.assert == false + records = read.call(output) + + records.first['type'].assert == 'suite' + records.first['rev'].assert == 5 + records.last['type'].assert == 'final' + records.last['counts']['total'].assert == report.total + records.last['counts']['omit'].assert == 2 + + results = records.select { |record| record['status'] } + results.size.assert == report.total + results.map { |record| record['status'] }.sort.assert == %w[error fail omit omit pass todo] + results.select { |record| record['status'] == 'omit' }.map { |record| record['type'] }.sort.assert == %w[case test] + records.count { |record| record['type'] == 'case' && !record['status'] }.assert == 2 + end + +A TAP-Y stream ends with the YAML end-of-document marker. + + render.call('tapy')[1].lines.last.chomp.assert == '...' + +The command line has a shortcut for each. + + require 'rubytest/cli' + { '-y' => 'tapy', '--tapj' => 'tapj' }.each do |flag, format| + cli = Test::CLI.new + cli.options.parse!([flag]) + cli.format.assert == format + end diff --git a/lib/rubytest/cli.rb b/lib/rubytest/cli.rb index 5d75156..4b4147b 100644 --- a/lib/rubytest/cli.rb +++ b/lib/rubytest/cli.rb @@ -62,6 +62,12 @@ def options opt.on '-f', '--format NAME', 'report format' do |name| conf.format = name end + opt.on '-y', '--tapy', 'shortcut for -f tapy' do + conf.format = 'tapy' + end + opt.on '-j', '--tapj', 'shortcut for -f tapj' do + conf.format = 'tapj' + end opt.on '-t', '--tag TAG', 'select tests by tag' do |tag| conf.tags.concat makelist(tag) diff --git a/lib/rubytest/format/abstract_hash.rb b/lib/rubytest/format/abstract_hash.rb index 0c2b973..c3540f9 100644 --- a/lib/rubytest/format/abstract_hash.rb +++ b/lib/rubytest/format/abstract_hash.rb @@ -83,26 +83,15 @@ def begin_test(test) # @return [Hash] # def skip_test(test, reason=nil) - h = {} - h['type' ] = 'test' - h['status'] = 'omit' - h['reason'] = reason if reason - - merge_subtype h, test - merge_setup h, test - merge_label h, test - #merge_comparison h, test, exception - #merge_coverage h, test - merge_source h, test - #merge_exception h, test, exception - merge_output h - merge_time h - - return h + skip_record(test, reason) end + # A skipped case is reported like a skipped test, as a case. + # + # @return [Hash] + # def skip_case(test_case, reason=nil) - skip_test(test_case, reason).merge('type' => 'case') + skip_record(test_case, reason).merge('type' => 'case') end # @@ -222,6 +211,23 @@ def end_suite(suite) private + # The record of a skipped test or case. + def skip_record(test, reason) + h = {} + h['type' ] = 'test' + h['status'] = 'omit' + h['reason'] = reason if reason + + merge_subtype h, test + merge_setup h, test + merge_label h, test + merge_source h, test + merge_output h + merge_time h + + return h + end + # For todo entries in particulr, i.e. NotImplementedError # exception, the return value represents the "todo" priority # level. The `Exception#priority` method returns an Integer diff --git a/lib/rubytest/format/tapj.rb b/lib/rubytest/format/tapj.rb new file mode 100644 index 0000000..bd22b2d --- /dev/null +++ b/lib/rubytest/format/tapj.rb @@ -0,0 +1,76 @@ +# encoding: UTF-8 + +module Test::Reporters + + # TAP-J Reporter + # + # Writes the run as a TAP-J stream (revision 5): a line of JSON for + # the suite, each case, each test and the final tally. + # + class Tapj < AbstractHash + + REVISION = 5 + + # + def initialize(runner, output: $stdout) + require 'json' + super(runner) + @output = output + end + + # + def begin_suite(suite) + emit super(suite).merge('rev' => REVISION) + end + + # + def begin_case(test_case) + emit super(test_case) + end + + # + def pass(test) + emit super(test) + end + + # + def fail(test, exception) + emit super(test, exception) + end + + # + def error(test, exception) + emit super(test, exception) + end + + # + def todo(test, exception) + emit super(test, exception) + end + + # + def skip_test(test, reason=nil) + emit super(test, reason) + end + + # + def skip_case(test_case, reason=nil) + emit super(test_case, reason) + end + + # + def end_suite(suite) + emit super(suite) + end + + private + + # Write one record to the stream. + def emit(hash) + @output.puts hash.to_json + hash + end + + end + +end diff --git a/lib/rubytest/format/tapy.rb b/lib/rubytest/format/tapy.rb new file mode 100644 index 0000000..de978f9 --- /dev/null +++ b/lib/rubytest/format/tapy.rb @@ -0,0 +1,77 @@ +# encoding: UTF-8 + +module Test::Reporters + + # TAP-Y Reporter + # + # Writes the run as a TAP-Y stream (revision 5): a YAML document for + # the suite, each case, each test and the final tally. + # + class Tapy < AbstractHash + + REVISION = 5 + + # + def initialize(runner, output: $stdout) + require 'yaml' + super(runner) + @output = output + end + + # + def begin_suite(suite) + emit super(suite).merge('rev' => REVISION) + end + + # + def begin_case(test_case) + emit super(test_case) + end + + # + def pass(test) + emit super(test) + end + + # + def fail(test, exception) + emit super(test, exception) + end + + # + def error(test, exception) + emit super(test, exception) + end + + # + def todo(test, exception) + emit super(test, exception) + end + + # + def skip_test(test, reason=nil) + emit super(test, reason) + end + + # + def skip_case(test_case, reason=nil) + emit super(test_case, reason) + end + + # + def end_suite(suite) + emit super(suite) + @output.puts '...' + end + + private + + # Write one record to the stream. + def emit(hash) + @output.puts hash.to_yaml + hash + end + + end + +end diff --git a/lib/rubytest/runner.rb b/lib/rubytest/runner.rb index 4812ccc..42679c0 100644 --- a/lib/rubytest/runner.rb +++ b/lib/rubytest/runner.rb @@ -417,7 +417,7 @@ def reporter_load(format) # TODO: Could use finder gem to look these up, but that's yet another dependency. # KNOWN_FORMATS = %w{ - dotprogress progress outline summary tap test + dotprogress progress outline summary tap tapy tapj test } # Returns a list of available report types.