tests: support v -silent in the remaining test runners as well; note it in AGENTS.md (#26590)

Author: spytheman

AGENTS.md

@@ -42,7 +42,7 @@ Get operational from the repo root in three steps:
    * Debug-friendly (recommended): `./v -g -keepc -o ./vnew cmd/v`
 3. Use `./vnew` for everything:
    * Run a file: `./vnew run examples/hello_world.v`
-   * Run tests: `./vnew test vlib/v/`
+   * Run tests: `./vnew -silent test vlib/v/`
    * Format: `./vnew fmt -w path/to/file.v`
 
 Then read Top Rules and Agent Rules before making changes.
@@ -184,7 +184,7 @@ If you read only one section for tests, **read Testing**.
 | Change area | Rebuild? | Minimum tests to run |
 | --- | --- | --- |
 | Docs only (`.md`) | No | `check-md file.md` |
-| Compiler (`vlib/v/`, `cmd/v/`) | Yes | `vlib/v/compiler_errors_test.v`; `test vlib/v/` |
+| Compiler (`vlib/v/`, `cmd/v/`) | Yes | `-silent vlib/v/compiler_errors_test.v`; `test vlib/v/` |
 | Core modules (builtin/strings/os/strconv/time) | Yes | Smallest relevant tests |
 | Parser-only (`vlib/v/parser/`) | Yes | `test vlib/v/parser/` |
 | Checker-only (`vlib/v/checker/`) | Yes | `test vlib/v/checker/` |
@@ -462,7 +462,7 @@ Example: `$if field.typ is $int { ... }`
 ### Comptime changes and testing
 Comptime logic lives in `vlib/v/comptime/` and is exercised by the
 checker, parser, and cgen stages. Changes here require a rebuild of
-`./vnew` and should be tested with `./vnew test vlib/v/tests/` plus
+`./vnew` and should be tested with `./vnew -silent test vlib/v/tests/` plus
 any comptime-specific tests. See the decision table and Testing.
 
 ## Run Programs
@@ -476,11 +476,11 @@ any comptime-specific tests. See the decision table and Testing.
 Run:
 * File (shows test output): `./vnew path/to/file_test.v`.
 * File (test runner report only): `./vnew test path/to/file_test.v`.
-* Dir: `./vnew test path/to/dir/`.
+* Dir: `./vnew -silent test path/to/dir/`.
 * Dir with statistics/metrics: `./vnew -stats test path/to/dir/`.
-* Compiler: `./vnew vlib/v/compiler_errors_test.v`.
+* Compiler: `./vnew -silent vlib/v/compiler_errors_test.v`.
 * Fix outputs (only when intended):
-  `VAUTOFIX=1 ./vnew vlib/v/compiler_errors_test.v`.
+  `VAUTOFIX=1 ./vnew -silent vlib/v/compiler_errors_test.v`.
 * All: `./vnew test-all`.
   Ask before running `./vnew test-all` unless explicitly requested.
 
@@ -494,33 +494,33 @@ When:
 * Run all tests that apply. Start with the smallest targeted tests;
   add slow tests as needed. Order does not matter.
 * Compiler changes (`vlib/v/` or `cmd/v/`):
-  Run `./vnew vlib/v/compiler_errors_test.v`,
-  `./vnew test vlib/v/`.
+  Run `./vnew -silent vlib/v/compiler_errors_test.v`,
+  `./vnew -silent test vlib/v/`.
 * Parser-only changes (`vlib/v/parser/`):
-  Run `./vnew test vlib/v/parser/`.
+  Run `./vnew -silent test vlib/v/parser/`.
 * Checker-only changes (`vlib/v/checker`):
-  Run `./vnew test vlib/v/checker/tests/`.
+  Run `./vnew -silent test vlib/v/checker/`.
 * vlib changes: Run nearest `*_test.v` or
-  `./vnew test vlib/path/`.
+  `./vnew -silent test vlib/path/`.
 * Tool changes (`cmd/tools/`): Run tool-specific tests. If none exist,
   run the smallest relevant `*_test.v` that exercises the tool.
   Note: `cmd/v/` is compiler scope, not tools.
-  Examples: `cmd/tools/vfmt` -> `vlib/v/fmt/fmt_test.v`,
-  `cmd/tools/vdoc` -> `vlib/v/doc/doc_test.v`.
+  Examples: `cmd/tools/vfmt` -> `vlib/v/fmt/fmt_test.v` .
+  `cmd/tools/vdoc` -> `cmd/tools/vdoc/vdoc_test.v`.
 * Diagnostic/output changes:
-  Run `./vnew vlib/v/slow_tests/inout/compiler_test.v`.
-* C codegen changes: Run `./vnew vlib/v/gen/c/coutput_test.v`.
+  Run `./vnew -silent vlib/v/slow_tests/inout/compiler_test.v`.
+* C codegen changes: Run `./vnew -silent vlib/v/gen/c/coutput_test.v`.
   Consider a stricter validation pass:
-  `./vnew -cstrict -cc clang test vlib/v/`.
-* REPL changes: Run `./vnew vlib/v/slow_tests/repl/repl_test.v`.
-* Broad refactors: Run `./vnew test-all`.
+  `./vnew -cstrict -cc clang -silent test vlib/v/`.
+* REPL changes: Run `./vnew -silent vlib/v/slow_tests/repl/repl_test.v`.
+* Broad refactors: Run `./vnew -silent test-all`.
 * Backend-specific changes: run the smallest relevant tests for the
   affected backend. JS/native/WASM backends are incomplete, so avoid
   broad `-b <backend> test vlib/` runs. Prefer targeted `*_test.v`
   files or small test dirs with `-b js|native|wasm`.
 
 If time-constrained, prioritize
-`./vnew vlib/v/compiler_errors_test.v` and the smallest targeted tests.
+`./vnew -silent vlib/v/compiler_errors_test.v` and the smallest targeted tests.
 Run `vlib/v/slow_tests/inout/compiler_test.v`
 and `vlib/v/gen/c/coutput_test.v` when output or codegen changes are
 likely.
@@ -538,9 +538,9 @@ Types:
 * Standard: `*_test.v` files with `test_` functions.
 * Output: `.vv` source + `.out` expected output in
   `vlib/v/slow_tests/inout/`.
-  Example: `./vnew vlib/v/slow_tests/inout/compiler_test.v`.
+  Example: `./vnew -silent vlib/v/slow_tests/inout/compiler_test.v`.
 * `vlib/v/tests/**` may use `.run.out` expectations; run with
-  `./vnew test vlib/v/tests`.
+  `./vnew -silent test vlib/v/tests`.
 
 Docs-only guidance: see Reporting.
 If time-boxed, run at least the smallest relevant test and note
@@ -551,8 +551,7 @@ skipped coverage in the summary.
   Use only when a behavior change is intended.
 * `VTEST_ONLY=glob_pattern` - Run only tests matching pattern.
 * `VTEST_HIDE_OK=1` - Hide successful tests, show only failures.
-* `./vnew -progress test path/to/dir/` - Show only the currently
-  running test.
+* `./vnew -silent test path/to/dir/` - Show only failed tests (if any), and a summary report.
 * `-cc tcc` can speed test builds when TCC is available.
 * Output expectations: update `.out` files only when behavior changes
   are intended; note the rationale in the summary.

vlib/v/compiler_errors_test.v

@@ -42,8 +42,12 @@ const turn_on_normal_test_runner = os.setenv('VTEST_RUNNER', 'normal', true)
 
 const should_autofix = os.getenv('VAUTOFIX') != ''
 
+const is_silent = $if silent ? { true } $else { false }
+
 const github_job = os.getenv('GITHUB_JOB')
 
+const should_show_details = !is_silent && github_job == ''
+
 const v_ci_ubuntu_musl = os.getenv('V_CI_UBUNTU_MUSL').len > 0
 
 const v_ci_musl = os.getenv('V_CI_MUSL').len > 0
@@ -104,24 +108,6 @@ fn test_all() {
 	no_closures_tests := get_tests_in_dir(no_closures_dir, false)
 	checker_with_check_option_tests := get_tests_in_dir(checker_with_check_option_dir,
 		false)
-	mut tasks := Tasks{
-		vexe:  vexe
-		label: 'all tests'
-	}
-	tasks.add('', parser_dir, '', '.out', parser_tests, false)
-	tasks.add('', checker_dir, '', '.out', checker_tests, false)
-	tasks.add('', scanner_dir, '', '.out', scanner_tests, false)
-	tasks.add('', checker_dir, '-enable-globals run', '.run.out', ['globals_error.vv'],
-		false)
-	tasks.add('', global_run_dir, '-enable-globals run', '.run.out', global_run_tests,
-		false)
-	tasks.add('', global_dir, '-enable-globals', '.out', global_tests, false)
-	tasks.add('', module_dir, '-prod run', '.out', module_tests, true)
-	tasks.add('', run_dir, 'run', '.run.out', run_tests, false)
-	tasks.add('', checker_with_check_option_dir, '-check', '.out', checker_with_check_option_tests,
-		false)
-	tasks.add('', no_closures_dir, '-no-closures run', '.out', no_closures_tests, false)
-	tasks.run()
 
 	if os.user_os() == 'linux' {
 		mut su_tasks := Tasks{
@@ -179,6 +165,25 @@ fn test_all() {
 		'custom_comptime_define_if_debug.vv',
 	])
 	ct_tasks.run()
+
+	mut tasks := Tasks{
+		vexe:  vexe
+		label: 'all tests'
+	}
+	tasks.add('', parser_dir, '', '.out', parser_tests, false)
+	tasks.add('', checker_dir, '', '.out', checker_tests, false)
+	tasks.add('', scanner_dir, '', '.out', scanner_tests, false)
+	tasks.add('', checker_dir, '-enable-globals run', '.run.out', ['globals_error.vv'],
+		false)
+	tasks.add('', global_run_dir, '-enable-globals run', '.run.out', global_run_tests,
+		false)
+	tasks.add('', global_dir, '-enable-globals', '.out', global_tests, false)
+	tasks.add('', module_dir, '-prod run', '.out', module_tests, true)
+	tasks.add('', run_dir, 'run', '.run.out', run_tests, false)
+	tasks.add('', checker_with_check_option_dir, '-check', '.out', checker_with_check_option_tests,
+		false)
+	tasks.add('', no_closures_dir, '-no-closures run', '.out', no_closures_tests, false)
+	tasks.run()
 }
 
 fn (mut tasks Tasks) add_checked_run(voptions string, result_extension string, tests []string) {
@@ -267,7 +272,7 @@ fn (mut tasks Tasks) run() {
 	for _ in 0 .. vjobs {
 		spawn work_processor(work, results)
 	}
-	if github_job == '' {
+	if should_show_details {
 		println('')
 	}
 	mut line_can_be_erased := true
@@ -278,8 +283,10 @@ fn (mut tasks Tasks) run() {
 		bench.step()
 		if task.is_skipped {
 			bench.skip()
-			eprintln(bstep_message(mut bench, benchmark.b_skip, task.path, task.took))
-			line_can_be_erased = false
+			if should_show_details {
+				eprintln(bstep_message(mut bench, benchmark.b_skip, task.path, task.took))
+				line_can_be_erased = false
+			}
 			continue
 		}
 		if task.is_error {
@@ -303,20 +310,23 @@ fn (mut tasks Tasks) run() {
 			assert true
 			if tasks.show_cmd {
 				eprintln(bstep_message(mut bench, benchmark.b_ok, '${task.cli_cmd}', task.took))
+				line_can_be_erased = true
 			} else {
-				if github_job == '' {
+				if should_show_details {
 					// local mode:
 					if line_can_be_erased {
 						term.clear_previous_line()
 					}
 					println(bstep_message(mut bench, benchmark.b_ok, task.path, task.took))
+					line_can_be_erased = true
 				}
 			}
-			line_can_be_erased = true
 		}
 	}
 	bench.stop()
-	eprintln(term.h_divider('-'))
+	if should_show_details {
+		eprintln(term.h_divider('-'))
+	}
 	eprintln(bench.total_message(tasks.label))
 	if total_errors != 0 {
 		exit(1)

vlib/v/gen/c/coutput_test.v

@@ -9,7 +9,9 @@ const vexe = @VEXE
 
 const vroot = os.real_path(@VMODROOT)
 
-const testdata_folder = os.join_path(vroot, 'vlib', 'v', 'gen', 'c', 'testdata')
+const local_tdata_path = 'vlib/v/gen/c/testdata'
+
+const testdata_folder = os.real_path(os.join_path(vroot, local_tdata_path))
 
 const show_compilation_output = os.getenv('VTEST_SHOW_COMPILATION_OUTPUT').int() == 1
 
@@ -26,7 +28,6 @@ fn mj(input ...string) string {
 }
 
 fn test_out_files() {
-	println(term.colorize(term.green, '> testing whether .out files match:'))
 	os.chdir(vroot) or {}
 	output_path := os.join_path(os.vtmp_dir(), 'coutput_outs')
 	os.mkdir_all(output_path)!
@@ -39,11 +40,16 @@ fn test_out_files() {
 		eprintln('no `.out` tests found in ${testdata_folder}')
 		return
 	}
-	paths := vtest.filter_vtest_only(tests, basepath: testdata_folder).sorted()
 	mut total_errors := 0
+	mut total_oks := 0
+	mut total_oks_panic := 0
+	mut total_skips := 0
+	paths := vtest.filter_vtest_only(tests, basepath: testdata_folder).sorted()
+	println(term.colorize(term.green, '> testing whether ${paths.len} .out files in ${local_tdata_path} match:'))
 	for out_path in paths {
 		basename, path, relpath, out_relpath := target2paths(out_path, '.out')
 		if should_skip(relpath) {
+			total_skips++
 			continue
 		}
 		pexe := os.join_path(output_path, '${basename}.exe')
@@ -75,7 +81,8 @@ fn test_out_files() {
 			n_expected := normalize_panic_message(expected, vroot)
 			if found.contains('================ V panic ================') {
 				if n_found.starts_with(n_expected) {
-					println('${term.green('OK (panic)')} C:${compile_ms:6}ms, R:${run_ms:2}ms ${label}')
+					vprintln('${term.green('OK (panic)')} C:${compile_ms:6}ms, R:${run_ms:2}ms ${label}')
+					total_oks_panic++
 					continue
 				} else {
 					// Both have panics, but there was a difference...
@@ -100,14 +107,15 @@ fn test_out_files() {
 			println(term.h_divider('-'))
 			total_errors++
 		} else {
-			println('${term.green('OK  ')} C:${compile_ms:6}ms, R:${run_ms:2}ms ${label}')
+			vprintln('${term.green('OK  ')} C:${compile_ms:6}ms, R:${run_ms:2}ms ${label}')
+			total_oks++
 		}
 	}
+	println('>>> Summary for test_out_files: files: ${paths.len}, oks: ${total_oks}, ok panics: ${total_oks_panic}, skipped: ${total_skips}, error: ${total_errors} .')
 	assert total_errors == 0
 }
 
 fn test_c_must_have_files() {
-	println(term.colorize(term.green, '> testing whether all line patterns in `.c.must_have` files match:'))
 	os.chdir(vroot) or {}
 	output_path := os.join_path(os.vtmp_dir(), 'coutput_c_must_haves')
 	os.mkdir_all(output_path)!
@@ -122,10 +130,15 @@ fn test_c_must_have_files() {
 	}
 	paths := vtest.filter_vtest_only(tests, basepath: testdata_folder).sorted()
 	mut total_errors := 0
+	mut total_oks := 0
+	mut total_oks_panic := 0
+	mut total_skips := 0
 	mut failed_descriptions := []string{cap: paths.len}
+	println(term.colorize(term.green, '> testing whether all line patterns in ${paths.len} `.c.must_have` files in ${local_tdata_path} match:'))
 	for must_have_path in paths {
 		basename, path, relpath, must_have_relpath := target2paths(must_have_path, '.c.must_have')
 		if should_skip(relpath) {
+			total_skips++
 			continue
 		}
 		file_options := get_file_options(path)
@@ -159,7 +172,8 @@ fn test_c_must_have_files() {
 			}
 		}
 		if nmatches == expected_lines.len {
-			println('${term.green('OK  ')} C:${compile_ms:5}ms ${description}')
+			vprintln('${term.green('OK  ')} C:${compile_ms:5}ms ${description}')
+			total_oks++
 		} else {
 			if show_compilation_output {
 				eprintln('> ALL lines:')
@@ -179,6 +193,7 @@ fn test_c_must_have_files() {
 		}
 		eprintln('----------------------------------------------------------------------')
 	}
+	println('>>> Summary for test_c_must_have_files: files: ${paths.len}, oks: ${total_oks}, ok panics: ${total_oks_panic}, skipped: ${total_skips}, error: ${total_errors} .')
 	assert total_errors == 0
 }
 
@@ -292,3 +307,8 @@ fn should_skip(relpath string) bool {
 	}
 	return false
 }
+
+@[if !silent ?]
+fn vprintln(msg string) {
+	println(msg)
+}

vlib/v/pref/pref.v

@@ -426,6 +426,8 @@ pub fn parse_args_and_show_errors(known_external_commands []string, args []strin
 			}
 			'-silent' {
 				res.output_mode = .silent
+				res.compile_defines_all << 'silent' // enable `$if silent? {`
+				res.compile_defines << 'silent'
 			}
 			'-skip-running' {
 				res.skip_running = true