tools: support // vtest build: !do_not_test ?, // vtest build: !windows && tinyc to skip files during testing on specific platforms, without having to keep centralised skip lists (#23900)

Author: spytheman

cmd/tools/modules/testing/common.v

@@ -12,6 +12,7 @@ import v.util.vtest
 import runtime
 import rand
 import strings
+import v.build_constraint
 
 pub const max_header_len = get_max_header_len()
 
@@ -98,6 +99,8 @@ pub mut:
 	hash          string // used as part of the name of the temporary directory created for tests, to ease cleanup
 
 	exec_mode ActionMode = .compile // .compile_and_run only for `v test`
+
+	build_environment build_constraint.Environment // see the documentation in v.build_constraint
 }
 
 pub fn (mut ts TestSession) add_failed_cmd(cmd string) {
@@ -443,6 +446,9 @@ pub fn (mut ts TestSession) test() {
 	printing_thread := spawn ts.print_messages()
 	pool_of_test_runners.set_shared_context(ts)
 	ts.reporter.worker_threads_start(remaining_files, mut ts)
+
+	ts.build_environment = get_build_environment()
+
 	// all the testing happens here:
 	pool_of_test_runners.work_on_pointers(unsafe { remaining_files.pointers() })
 
@@ -568,9 +574,23 @@ fn worker_trunner(mut p pool.PoolProcessor, idx int, thread_id int) voidptr {
 	} else {
 		os.quoted_path(generated_binary_fpath)
 	}
+	mut details := get_test_details(file)
+	mut should_be_built := true
+	if details.vbuild != '' {
+		should_be_built = ts.build_environment.eval(details.vbuild) or {
+			eprintln('${file}:${details.vbuild_line}:17: error during parsing the `// v test build` expression `${details.vbuild}`: ${err}')
+			false
+		}
+		$if trace_should_be_built ? {
+			eprintln('${file} has specific build constraint: `${details.vbuild}` => should_be_built: `${should_be_built}`')
+			eprintln('>          env   facts: ${ts.build_environment.facts}')
+			eprintln('>          env defines: ${ts.build_environment.defines}')
+		}
+	}
+
 	ts.benchmark.step()
 	tls_bench.step()
-	if !ts.build_tools && abs_path in ts.skip_files {
+	if !ts.build_tools && (!should_be_built || abs_path in ts.skip_files) {
 		ts.benchmark.skip()
 		tls_bench.skip()
 		if !hide_skips {
@@ -599,7 +619,6 @@ fn worker_trunner(mut p pool.PoolProcessor, idx int, thread_id int) voidptr {
 		ts.append_message_with_duration(.cmd_end, '', cmd_duration, mtc)
 
 		if status != 0 {
-			details := get_test_details(file)
 			os.setenv('VTEST_RETRY_MAX', '${details.retry}', true)
 			for retry := 1; retry <= details.retry; retry++ {
 				if !details.hide_retries {
@@ -686,7 +705,6 @@ fn worker_trunner(mut p pool.PoolProcessor, idx int, thread_id int) voidptr {
 			println(r.output.split_into_lines().filter(it.contains(' assert')).join('\n'))
 		}
 		if r.exit_code != 0 {
-			mut details := get_test_details(file)
 			mut trimmed_output := r.output.trim_space()
 			if trimmed_output.len == 0 {
 				// retry running at least 1 more time, to avoid CI false positives as much as possible
@@ -895,19 +913,25 @@ pub mut:
 	retry int
 	flaky bool // when flaky tests fail, the whole run is still considered successful, unless VTEST_FAIL_FLAKY is 1
 	//
-	hide_retries bool // when true, all retry tries are silent; used by `vlib/v/tests/retry_test.v`
+	hide_retries bool   // when true, all retry tries are silent; used by `vlib/v/tests/retry_test.v`
+	vbuild       string // could be `!(windows && tinyc)`
+	vbuild_line  int    // for more precise error reporting, if the `vbuild` expression is incorrect
 }
 
 pub fn get_test_details(file string) TestDetails {
 	mut res := TestDetails{}
 	lines := os.read_lines(file) or { [] }
-	for line in lines {
+	for idx, line in lines {
 		if line.starts_with('// vtest retry:') {
 			res.retry = line.all_after(':').trim_space().int()
 		}
 		if line.starts_with('// vtest flaky:') {
 			res.flaky = line.all_after(':').trim_space().bool()
 		}
+		if line.starts_with('// vtest build:') {
+			res.vbuild = line.all_after(':').trim_space()
+			res.vbuild_line = idx + 1
+		}
 		if line.starts_with('// vtest hide_retries') {
 			res.hide_retries = true
 		}
@@ -949,3 +973,9 @@ fn get_max_header_len() int {
 	}
 	return cols
 }
+
+fn get_build_environment() &build_constraint.Environment {
+	facts := os.getenv('VBUILD_FACTS').split_any(',')
+	defines := os.getenv('VBUILD_DEFINES').split_any(',')
+	return build_constraint.new_environment(facts, defines)
+}

cmd/v/v.v

@@ -98,6 +98,9 @@ fn main() {
 		exit(1)
 	}
 	timers.show('v parsing CLI args')
+
+	setup_vbuild_env_vars(prefs)
+
 	// Start calling the correct functions/external tools
 	// Note for future contributors: Please add new subcommands in the `match` block below.
 	if command in external_tools {
@@ -206,3 +209,28 @@ fn rebuild(prefs &pref.Preferences) {
 		}
 	}
 }
+
+@[manualfree]
+fn setup_vbuild_env_vars(prefs &pref.Preferences) {
+	mut facts := []string{cap: 10}
+	facts << prefs.os.lower()
+	facts << prefs.ccompiler_type.str()
+	facts << prefs.arch.str()
+	if prefs.is_prod {
+		facts << 'prod'
+	}
+	github_job := os.getenv('GITHUB_JOB')
+	if github_job != '' {
+		facts << github_job
+	}
+	sfacts := facts.join(',')
+	os.setenv('VBUILD_FACTS', sfacts, true)
+
+	sdefines := prefs.compile_defines_all.join(',')
+	os.setenv('VBUILD_DEFINES', sdefines, true)
+
+	unsafe { sdefines.free() }
+	unsafe { sfacts.free() }
+	unsafe { github_job.free() }
+	unsafe { facts.free() }
+}

vlib/math/big/array_ops_test.v

@@ -1,3 +1,4 @@
+// vtest build: !do_not_test ?
 module big
 
 fn test_add_digit_array_01() {

vlib/v/build_constraint/ast.v

@@ -0,0 +1,23 @@
+module build_constraint
+
+// ast:
+struct BExpr {
+	expr BOr
+}
+
+struct BOr {
+	exprs []BAnd
+}
+
+struct BAnd {
+	exprs []BUnary
+}
+
+type BUnary = BNot | BExpr | BFact | BDefine
+
+struct BNot {
+	expr BUnary
+}
+
+type BFact = string
+type BDefine = string

vlib/v/build_constraint/constraint_test.v

@@ -0,0 +1,102 @@
+import v.build_constraint
+
+const benv = build_constraint.new_environment(['linux', 'tinyc'], ['abc', 'def'])
+
+fn test_eval_fact() {
+	assert benv.is_fact('tinyc')
+	assert benv.is_fact('linux')
+	assert !benv.is_fact('macos')
+	assert !benv.is_fact('windows')
+}
+
+fn test_eval_define() {
+	assert benv.is_define('abc')
+	assert benv.is_define('def')
+	assert !benv.is_define('xyz')
+}
+
+fn test_eval_platforms_and_compilers() {
+	assert benv.eval('tinyc')!
+	assert benv.eval(' tinyc')!
+	assert benv.eval('tinyc ')!
+	assert benv.eval(' tinyc ')!
+	assert !benv.eval('gcc')!
+	assert !benv.eval('clang')!
+	assert !benv.eval('msvc')!
+	assert benv.eval('linux')!
+	assert benv.eval(' linux')!
+	assert benv.eval('linux ')!
+	assert benv.eval(' linux ')!
+	assert !benv.eval('windows')!
+	assert !benv.eval('macos')!
+	assert !benv.eval('freebsd')!
+}
+
+fn test_eval_defines() {
+	assert benv.eval('abc?')!
+	assert benv.eval(' abc?')!
+	assert benv.eval('abc? ')!
+	assert benv.eval(' abc? ')!
+	assert benv.eval('abc ?')!
+	assert benv.eval(' abc ?')!
+	assert benv.eval('abc ? ')!
+	assert benv.eval(' abc ? ')!
+	assert benv.eval('def?')!
+}
+
+fn test_eval_not() {
+	assert benv.eval('!gcc')!
+	assert benv.eval('!clang')!
+	assert benv.eval('!msvc')!
+	assert !benv.eval('!tinyc')!
+	assert !benv.eval(' !tinyc')!
+	assert !benv.eval('!tinyc ')!
+	assert !benv.eval(' !tinyc ')!
+	assert benv.eval('!xyz?')!
+}
+
+fn test_eval_and() {
+	assert benv.eval('linux && tinyc')!
+	assert !benv.eval('macos && tinyc')!
+	assert !benv.eval('windows && tinyc')!
+	assert !benv.eval('linux && gcc')!
+	//
+	assert benv.eval('linux && tinyc && abc?')!
+	assert benv.eval('linux && tinyc && def?')!
+	assert !benv.eval('linux && tinyc && xyz?')!
+	//
+	assert benv.eval('linux && !gcc')!
+	assert benv.eval('linux && !clang')!
+	assert benv.eval('!gcc && !windows')!
+	assert !benv.eval('!windows && tcc')!
+	assert !benv.eval('windows && gcc')!
+	assert !benv.eval('gcc && !windows')!
+}
+
+fn test_eval_or() {
+	assert benv.eval('windows||tinyc')!
+	assert benv.eval('windows || macos || tinyc')!
+	assert benv.eval('windows || macos || tinyc')!
+	assert benv.eval('windows || macos || gcc || abc?')!
+	assert benv.eval('!windows||gcc')!
+}
+
+fn test_complex() {
+	assert benv.eval('  (windows || tinyc) &&  linux ')!
+	assert !benv.eval(' (windows || gcc)   &&  linux ')!
+	assert benv.eval('  (windows || tinyc) && !macos ')!
+	assert !benv.eval(' (windows || tinyc) &&  macos ')!
+}
+
+fn test_precedence() {
+	assert benv.eval(' tinyc && !windows ')! == benv.eval(' tinyc && (!windows)')!
+	assert benv.eval(' tinyc && !windows ')! == benv.eval(' (!windows) && tinyc')!
+	assert benv.eval(' !windows && tinyc')! == benv.eval(' (!windows) && tinyc')!
+	assert benv.eval(' !windows || tinyc')! == benv.eval(' (!windows) || tinyc')!
+	assert benv.eval(' !linux   && tinyc')! == benv.eval(' (!linux)   && tinyc')!
+	assert benv.eval(' !linux   || tinyc')! == benv.eval(' (!linux)   || tinyc')!
+	assert benv.eval(' !windows && gcc  ')! == benv.eval(' (!windows) && gcc  ')!
+	assert benv.eval(' !windows || gcc  ')! == benv.eval(' (!windows) || gcc  ')!
+	assert benv.eval(' !linux   && gcc  ')! == benv.eval(' (!linux)   && gcc  ')!
+	assert benv.eval(' !linux   || gcc  ')! == benv.eval(' (!linux)   || gcc  ')!
+}

vlib/v/build_constraint/evaluating.v

@@ -0,0 +1,43 @@
+module build_constraint
+
+// evaluating the AST nodes, in the given environment
+fn (b BExpr) eval(env &Environment) !bool {
+	return b.expr.eval(env)
+}
+
+fn (b BOr) eval(env &Environment) !bool {
+	for e in b.exprs {
+		if e.eval(env)! {
+			return true
+		}
+	}
+	return false
+}
+
+fn (b BAnd) eval(env &Environment) !bool {
+	for e in b.exprs {
+		if !e.eval(env)! {
+			return false
+		}
+	}
+	return true
+}
+
+fn (b BUnary) eval(env &Environment) !bool {
+	match b {
+		BNot, BExpr, BFact, BDefine { return b.eval(env)! }
+	}
+	return false
+}
+
+fn (b BNot) eval(env &Environment) !bool {
+	return !b.expr.eval(env)!
+}
+
+fn (b BFact) eval(env &Environment) !bool {
+	return env.is_fact(b)
+}
+
+fn (b BDefine) eval(env &Environment) !bool {
+	return env.is_define(b)
+}