ast,token,scanner,parser,checker,cgen: add file_idx to token.Pos (#25490)

kbkpbot · web-flow · commit 144b57c4b79e · 2025-10-12T22:56:23.000+03:00
diff --git a/cmd/tools/measure/fmt_speed.v b/cmd/tools/measure/fmt_speed.v
@@ -89,7 +89,7 @@ fn process_files(files []string) ! {
 
 fn new_parser(path string, comments_mode scanner.CommentsMode, table &ast.Table, pref_ &pref.Preferences) &parser.Parser {
 	mut p := &parser.Parser{
-		scanner:  scanner.new_scanner_file(path, comments_mode, pref_) or { panic(err) }
+		scanner:  scanner.new_scanner_file(path, -1, comments_mode, pref_) or { panic(err) }
 		table:    table
 		pref:     pref_
 		scope:    &ast.Scope{
diff --git a/cmd/tools/measure/parser_speed.v b/cmd/tools/measure/parser_speed.v
@@ -87,7 +87,7 @@ fn process_files(files []string) ! {
 
 fn new_parser(path string, comments_mode scanner.CommentsMode, table &ast.Table, pref_ &pref.Preferences) &parser.Parser {
 	mut p := &parser.Parser{
-		scanner:  scanner.new_scanner_file(path, comments_mode, pref_) or { panic(err) }
+		scanner:  scanner.new_scanner_file(path, -1, comments_mode, pref_) or { panic(err) }
 		table:    table
 		pref:     pref_
 		scope:    &ast.Scope{
diff --git a/cmd/tools/measure/scanner_speed.v b/cmd/tools/measure/scanner_speed.v
@@ -55,7 +55,7 @@ fn process_files(files []string) ! {
 		}
 		total_files++
 		sw.restart()
-		s := scanner.new_scanner_file(f, comments_mode, pref_)!
+		s := scanner.new_scanner_file(f, -1, comments_mode, pref_)!
 		f_us := sw.elapsed().microseconds()
 		total_us += f_us
 		total_bytes += s.text.len
diff --git a/vlib/v/ast/ast.v b/vlib/v/ast/ast.v
@@ -2478,7 +2478,7 @@ pub fn (node Node) pos() token.Pos {
 						line_nr:   -1
 						pos:       -1
 						last_line: -1
-						col:       -1
+						col:       0
 					}
 				}
 			}
diff --git a/vlib/v/ast/table.v b/vlib/v/ast/table.v
@@ -102,6 +102,7 @@ pub mut:
 	new_int             bool              // use 64bit/32bit platform dependent `int`
 	new_int_fmt_fix     bool              // vfmt will fix `int` to `i32`
 	export_names        map[string]string // @[export] names
+	filelist            []string          // all files list
 }
 
 pub struct ComptTimeCondResult {
diff --git a/vlib/v/checker/errors.v b/vlib/v/checker/errors.v
@@ -14,7 +14,8 @@ fn (mut c Checker) add_error_detail(s string) {
 }
 
 fn (mut c Checker) add_error_detail_with_pos(msg string, pos token.Pos) {
-	c.add_error_detail(util.formatted_error('details:', msg, c.file.path, pos))
+	file_path := if pos.file_idx < 0 { c.file.path } else { c.table.filelist[pos.file_idx] }
+	c.add_error_detail(util.formatted_error('details:', msg, file_path, pos))
 }
 
 fn (mut c Checker) add_instruction_for_option_type() {
@@ -90,13 +91,14 @@ fn (mut c Checker) note(message string, pos token.Pos) {
 		c.error_details = []
 	}
 	// deduplicate notices for the same line
-	kpos := '${c.file.path}:${pos.line_nr}:${message}'
+	file_path := if pos.file_idx < 0 { c.file.path } else { c.table.filelist[pos.file_idx] }
+	kpos := '${file_path}:${pos.line_nr}:${message}'
 	if kpos !in c.notice_lines {
 		c.notice_lines[kpos] = true
 		note := errors.Notice{
 			reporter:  errors.Reporter.checker
 			pos:       pos
-			file_path: c.file.path
+			file_path: file_path
 			message:   message
 			details:   details
 		}
@@ -122,20 +124,21 @@ fn (mut c Checker) warn_or_error(message string, pos token.Pos, warn bool) {
 		details = c.error_details.join('\n')
 		c.error_details = []
 	}
+	file_path := if pos.file_idx < 0 { c.file.path } else { c.table.filelist[pos.file_idx] }
 	if warn && !c.pref.skip_warnings {
 		c.nr_warnings++
 		if c.pref.message_limit >= 0 && c.nr_warnings >= c.pref.message_limit {
 			c.should_abort = true
 			return
 		}
 		// deduplicate warnings for the same line
-		kpos := '${c.file.path}:${pos.line_nr}:${message}'
+		kpos := '${file_path}:${pos.line_nr}:${message}'
 		if kpos !in c.warning_lines {
 			c.warning_lines[kpos] = true
 			wrn := errors.Warning{
 				reporter:  errors.Reporter.checker
 				pos:       pos
-				file_path: c.file.path
+				file_path: file_path
 				message:   message
 				details:   details
 			}
@@ -148,7 +151,7 @@ fn (mut c Checker) warn_or_error(message string, pos token.Pos, warn bool) {
 		if c.pref.fatal_errors {
 			util.show_compiler_message('error:', errors.CompilerMessage{
 				pos:       pos
-				file_path: c.file.path
+				file_path: file_path
 				message:   message
 				details:   details
 			})
@@ -160,13 +163,13 @@ fn (mut c Checker) warn_or_error(message string, pos token.Pos, warn bool) {
 			return
 		}
 		// deduplicate errors for the same line
-		kpos := '${c.file.path}:${pos.line_nr}:${message}'
+		kpos := '${file_path}:${pos.line_nr}:${message}'
 		if kpos !in c.error_lines {
 			c.error_lines[kpos] = true
 			err := errors.Error{
 				reporter:  errors.Reporter.checker
 				pos:       pos
-				file_path: c.file.path
+				file_path: file_path
 				message:   message
 				details:   details
 			}
diff --git a/vlib/v/checker/str.v b/vlib/v/checker/str.v
@@ -145,7 +145,7 @@ fn (mut c Checker) string_lit(mut node ast.StringLiteral) ast.Type {
 			`\\` {
 				mut start_pos := token.Pos{
 					...node.pos
-					col: node.pos.col + 1 + idx
+					col: u16(node.pos.col + 1 + idx)
 				}
 				start_idx := idx
 				idx++
diff --git a/vlib/v/gen/c/cgen.v b/vlib/v/gen/c/cgen.v
@@ -6609,7 +6609,8 @@ fn verror(s string) {
 
 @[noreturn]
 fn (g &Gen) error(s string, pos token.Pos) {
-	util.show_compiler_message('cgen error:', pos: pos, file_path: g.file.path, message: s)
+	file_path := if pos.file_idx < 0 { g.file.path } else { g.table.filelist[pos.file_idx] }
+	util.show_compiler_message('cgen error:', pos: pos, file_path: file_path, message: s)
 	exit(1)
 }
 
diff --git a/vlib/v/parser/fn.v b/vlib/v/parser/fn.v
@@ -61,7 +61,7 @@ fn (mut p Parser) call_expr(language ast.Language, mod string) ast.CallExpr {
 			ok_arg_pos := (args[params.len - 1] or { args[0] }).pos
 			pos := token.Pos{
 				...ok_arg_pos
-				col: ok_arg_pos.col + ok_arg_pos.len
+				col: u16(ok_arg_pos.col + ok_arg_pos.len)
 			}
 			p.unexpected_with_pos(pos.extend(p.tok.pos()), expecting: '`)`')
 		} else {
diff --git a/vlib/v/parser/messages.v b/vlib/v/parser/messages.v
@@ -68,20 +68,21 @@ fn (mut p Parser) note(s string) {
 fn (mut p Parser) error_with_pos(s string, pos token.Pos) ast.NodeError {
 	// print_backtrace()
 	mut kind := 'error:'
+	file_path := if pos.file_idx < 0 { p.file_path } else { p.table.filelist[pos.file_idx] }
 	if p.pref.fatal_errors {
-		util.show_compiler_message(kind, pos: pos, file_path: p.file_path, message: s)
+		util.show_compiler_message(kind, pos: pos, file_path: file_path, message: s)
 		exit(1)
 	}
 	if p.pref.output_mode == .stdout && !p.pref.check_only && !p.is_vls {
 		if p.pref.is_verbose {
 			print_backtrace()
 			kind = 'parser error:'
 		}
-		util.show_compiler_message(kind, pos: pos, file_path: p.file_path, message: s)
+		util.show_compiler_message(kind, pos: pos, file_path: file_path, message: s)
 		exit(1)
 	} else {
 		p.errors << errors.Error{
-			file_path: p.file_path
+			file_path: file_path
 			pos:       pos
 			reporter:  .parser
 			message:   s
@@ -145,15 +146,16 @@ fn (mut p Parser) warn_with_pos(s string, pos token.Pos) {
 	if p.pref.skip_warnings {
 		return
 	}
+	file_path := if pos.file_idx < 0 { p.file_path } else { p.table.filelist[pos.file_idx] }
 	if p.pref.output_mode == .stdout && !p.pref.check_only {
-		util.show_compiler_message('warning:', pos: pos, file_path: p.file_path, message: s)
+		util.show_compiler_message('warning:', pos: pos, file_path: file_path, message: s)
 	} else {
 		if p.pref.message_limit >= 0 && p.warnings.len >= p.pref.message_limit {
 			p.should_abort = true
 			return
 		}
 		p.warnings << errors.Warning{
-			file_path: p.file_path
+			file_path: file_path
 			pos:       pos
 			reporter:  .parser
 			message:   s
@@ -175,11 +177,12 @@ fn (mut p Parser) note_with_pos(s string, pos token.Pos) {
 		p.error_with_pos(s, pos)
 		return
 	}
+	file_path := if pos.file_idx < 0 { p.file_path } else { p.table.filelist[pos.file_idx] }
 	if p.pref.output_mode == .stdout && !p.pref.check_only {
-		util.show_compiler_message('notice:', pos: pos, file_path: p.file_path, message: s)
+		util.show_compiler_message('notice:', pos: pos, file_path: file_path, message: s)
 	} else {
 		p.notices << errors.Notice{
-			file_path: p.file_path
+			file_path: file_path
 			pos:       pos
 			reporter:  .parser
 			message:   s
diff --git a/vlib/v/parser/parser.v b/vlib/v/parser/parser.v
@@ -19,6 +19,7 @@ pub:
 mut:
 	file_base         string       // "hello.v"
 	file_path         string       // "/home/user/hello.v"
+	file_idx          i16          // file idx in the global table `filelist`
 	file_display_path string       // just "hello.v", when your current folder for the compilation is "/home/user/", otherwise the full path "/home/user/hello.v"
 	unique_prefix     string       // a hash of p.file_path, used for making anon fn generation unique
 	file_backend_mode ast.Language // .c for .c.v|.c.vv|.c.vsh files; .js for .js.v files, .amd64/.rv32/other arches for .amd64.v/.rv32.v/etc. files, .v otherwise.
@@ -255,8 +256,13 @@ pub fn parse_file(path string, mut table ast.Table, comments_mode scanner.Commen
 	$if trace_parse_file ? {
 		eprintln('> ${@MOD}.${@FN} comments_mode: ${comments_mode:-20} | path: ${path}')
 	}
+	mut file_idx := i16(table.filelist.index(path))
+	if file_idx == -1 {
+		file_idx = i16(table.filelist.len)
+		table.filelist << path
+	}
 	mut p := Parser{
-		scanner: scanner.new_scanner_file(path, comments_mode, pref_) or { panic(err) }
+		scanner: scanner.new_scanner_file(path, file_idx, comments_mode, pref_) or { panic(err) }
 		table:   table
 		pref:    pref_
 		// Only set vls mode if it's the file the user requested via `v -vls-mode file.v`
@@ -268,6 +274,7 @@ pub fn parse_file(path string, mut table ast.Table, comments_mode scanner.Commen
 		}
 		errors:   []errors.Error{}
 		warnings: []errors.Warning{}
+		file_idx: file_idx
 	}
 	p.set_path(path)
 	res := p.parse()
diff --git a/vlib/v/parser/tmpl.v b/vlib/v/parser/tmpl.v
@@ -103,7 +103,7 @@ pub:
 	calling_file string
 	line_nr      int
 	position     int
-	col          int
+	col          u16
 	message      string
 }
 
@@ -123,7 +123,7 @@ fn (err IncludeError) calling_file() string {
 	return err.calling_file
 }
 
-fn (err IncludeError) col() int {
+fn (err IncludeError) col() u16 {
 	return err.col
 }
 
@@ -140,7 +140,7 @@ fn (mut p Parser) process_includes(calling_file string, line_number int, line st
 			calling_file: calling_file
 			line_nr:      tline_number // line_number
 			position:     position + '@include '.len
-			col:          position + '@include '.len
+			col:          u16(position + '@include '.len)
 			message:      'path for @include must be quoted with \' or "'
 		}
 	}
diff --git a/vlib/v/scanner/scanner.v b/vlib/v/scanner/scanner.v
diff --git a/vlib/v/scanner/tests/shebang_outside_vsh_err.out b/vlib/v/scanner/tests/shebang_outside_vsh_err.out
diff --git a/vlib/v/token/pos.v b/vlib/v/token/pos.v
diff --git a/vlib/v/token/token.v b/vlib/v/token/token.v

Original file line number	Diff line number	Diff line change
`@@ -55,7 +55,7 @@ fn process_files(files []string) ! {`
`55`	`55`	`}`
`56`	`56`	`total_files++`
`57`	`57`	`sw.restart()`
`58`		`- s := scanner.new_scanner_file(f, comments_mode, pref_)!`
	`58`	`+ s := scanner.new_scanner_file(f, -1, comments_mode, pref_)!`
`59`	`59`	`f_us := sw.elapsed().microseconds()`
`60`	`60`	`total_us += f_us`
`61`	`61`	`total_bytes += s.text.len`
Original file line number	Diff line number	Diff line change
`@@ -2478,7 +2478,7 @@ pub fn (node Node) pos() token.Pos {`
`2478`	`2478`	`line_nr: -1`
`2479`	`2479`	`pos: -1`
`2480`	`2480`	`last_line: -1`
`2481`		`- col: -1`
	`2481`	`+ col: 0`
`2482`	`2482`	`}`
`2483`	`2483`	`}`
`2484`	`2484`	`}`
Original file line number	Diff line number	Diff line change
`@@ -102,6 +102,7 @@ pub mut:`
`102`	`102`	new_int bool // use 64bit/32bit platform dependent `int`
`103`	`103`	new_int_fmt_fix bool // vfmt will fix `int` to `i32`
`104`	`104`	`export_names map[string]string // @[export] names`
	`105`	`+ filelist []string // all files list`
`105`	`106`	`}`
`106`	`107`
`107`	`108`	`pub struct ComptTimeCondResult {`
Original file line number	Diff line number	Diff line change
`@@ -145,7 +145,7 @@ fn (mut c Checker) string_lit(mut node ast.StringLiteral) ast.Type {`
`145`	`145`	`\\` {
`146`	`146`	`mut start_pos := token.Pos{`
`147`	`147`	`...node.pos`
`148`		`- col: node.pos.col + 1 + idx`
	`148`	`+ col: u16(node.pos.col + 1 + idx)`
`149`	`149`	`}`
`150`	`150`	`start_idx := idx`
`151`	`151`	`idx++`
Original file line number	Diff line number	Diff line change
`@@ -6609,7 +6609,8 @@ fn verror(s string) {`
`6609`	`6609`
`6610`	`6610`	`@[noreturn]`
`6611`	`6611`	`fn (g &Gen) error(s string, pos token.Pos) {`
`6612`		`- util.show_compiler_message('cgen error:', pos: pos, file_path: g.file.path, message: s)`
	`6612`	`+ file_path := if pos.file_idx < 0 { g.file.path } else { g.table.filelist[pos.file_idx] }`
	`6613`	`+ util.show_compiler_message('cgen error:', pos: pos, file_path: file_path, message: s)`
`6613`	`6614`	`exit(1)`
`6614`	`6615`	`}`
`6615`	`6616`
Original file line number	Diff line number	Diff line change
`@@ -61,7 +61,7 @@ fn (mut p Parser) call_expr(language ast.Language, mod string) ast.CallExpr {`
`61`	`61`	`ok_arg_pos := (args[params.len - 1] or { args[0] }).pos`
`62`	`62`	`pos := token.Pos{`
`63`	`63`	`...ok_arg_pos`
`64`		`- col: ok_arg_pos.col + ok_arg_pos.len`
	`64`	`+ col: u16(ok_arg_pos.col + ok_arg_pos.len)`
`65`	`65`	`}`
`66`	`66`	p.unexpected_with_pos(pos.extend(p.tok.pos()), expecting: '`)`')
`67`	`67`	`} else {`