Skip to content

Commit d948bf1

Browse files
kbkpbotkbkpbot
authored
token,scanner,parser,checker,cgen: add file_idx to token.Pos (#25429)
1 parent 8287d5d commit d948bf1

11 files changed

Lines changed: 98 additions & 71 deletions

File tree

‎cmd/tools/measure/fmt_speed.v‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ fn process_files(files []string) ! {
8989

9090
fn new_parser(path string, comments_mode scanner.CommentsMode, table &ast.Table, pref_ &pref.Preferences) &parser.Parser {
9191
mut p := &parser.Parser{
92-
scanner: scanner.new_scanner_file(path, comments_mode, pref_) or { panic(err) }
92+
scanner: scanner.new_scanner_file(path, -1, comments_mode, pref_) or { panic(err) }
9393
table: table
9494
pref: pref_
9595
scope: &ast.Scope{

‎cmd/tools/measure/parser_speed.v‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ fn process_files(files []string) ! {
8787

8888
fn new_parser(path string, comments_mode scanner.CommentsMode, table &ast.Table, pref_ &pref.Preferences) &parser.Parser {
8989
mut p := &parser.Parser{
90-
scanner: scanner.new_scanner_file(path, comments_mode, pref_) or { panic(err) }
90+
scanner: scanner.new_scanner_file(path, -1, comments_mode, pref_) or { panic(err) }
9191
table: table
9292
pref: pref_
9393
scope: &ast.Scope{

‎cmd/tools/measure/scanner_speed.v‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ fn process_files(files []string) ! {
5555
}
5656
total_files++
5757
sw.restart()
58-
s := scanner.new_scanner_file(f, comments_mode, pref_)!
58+
s := scanner.new_scanner_file(f, -1, comments_mode, pref_)!
5959
f_us := sw.elapsed().microseconds()
6060
total_us += f_us
6161
total_bytes += s.text.len

‎vlib/v/ast/table.v‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,7 @@ pub mut:
103103
new_int bool // use 64bit/32bit platform dependent `int`
104104
new_int_fmt_fix bool // vfmt will fix `int` to `i32`
105105
export_names map[string]string // @[export] names
106+
filelist []string // all files list
106107
}
107108

108109
pub struct ComptTimeCondResult {

‎vlib/v/checker/errors.v‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,8 @@ fn (mut c Checker) add_error_detail(s string) {
1414
}
1515

1616
fn (mut c Checker) add_error_detail_with_pos(msg string, pos token.Pos) {
17-
c.add_error_detail(util.formatted_error('details:', msg, c.file.path, pos))
17+
file_path := if pos.file_idx < 0 { c.file.path } else { c.table.filelist[pos.file_idx] }
18+
c.add_error_detail(util.formatted_error('details:', msg, file_path, pos))
1819
}
1920

2021
fn (mut c Checker) add_instruction_for_option_type() {
@@ -90,13 +91,14 @@ fn (mut c Checker) note(message string, pos token.Pos) {
9091
c.error_details = []
9192
}
9293
// deduplicate notices for the same line
93-
kpos := '${c.file.path}:${pos.line_nr}:${message}'
94+
file_path := if pos.file_idx < 0 { c.file.path } else { c.table.filelist[pos.file_idx] }
95+
kpos := '${file_path}:${pos.line_nr}:${message}'
9496
if kpos !in c.notice_lines {
9597
c.notice_lines[kpos] = true
9698
note := errors.Notice{
9799
reporter: errors.Reporter.checker
98100
pos: pos
99-
file_path: c.file.path
101+
file_path: file_path
100102
message: message
101103
details: details
102104
}
@@ -122,20 +124,21 @@ fn (mut c Checker) warn_or_error(message string, pos token.Pos, warn bool) {
122124
details = c.error_details.join('\n')
123125
c.error_details = []
124126
}
127+
file_path := if pos.file_idx < 0 { c.file.path } else { c.table.filelist[pos.file_idx] }
125128
if warn && !c.pref.skip_warnings {
126129
c.nr_warnings++
127130
if c.pref.message_limit >= 0 && c.nr_warnings >= c.pref.message_limit {
128131
c.should_abort = true
129132
return
130133
}
131134
// deduplicate warnings for the same line
132-
kpos := '${c.file.path}:${pos.line_nr}:${message}'
135+
kpos := '${file_path}:${pos.line_nr}:${message}'
133136
if kpos !in c.warning_lines {
134137
c.warning_lines[kpos] = true
135138
wrn := errors.Warning{
136139
reporter: errors.Reporter.checker
137140
pos: pos
138-
file_path: c.file.path
141+
file_path: file_path
139142
message: message
140143
details: details
141144
}
@@ -148,7 +151,7 @@ fn (mut c Checker) warn_or_error(message string, pos token.Pos, warn bool) {
148151
if c.pref.fatal_errors {
149152
util.show_compiler_message('error:', errors.CompilerMessage{
150153
pos: pos
151-
file_path: c.file.path
154+
file_path: file_path
152155
message: message
153156
details: details
154157
})
@@ -160,13 +163,13 @@ fn (mut c Checker) warn_or_error(message string, pos token.Pos, warn bool) {
160163
return
161164
}
162165
// deduplicate errors for the same line
163-
kpos := '${c.file.path}:${pos.line_nr}:${message}'
166+
kpos := '${file_path}:${pos.line_nr}:${message}'
164167
if kpos !in c.error_lines {
165168
c.error_lines[kpos] = true
166169
err := errors.Error{
167170
reporter: errors.Reporter.checker
168171
pos: pos
169-
file_path: c.file.path
172+
file_path: file_path
170173
message: message
171174
details: details
172175
}

‎vlib/v/gen/c/cgen.v‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6590,7 +6590,8 @@ fn verror(s string) {
65906590

65916591
@[noreturn]
65926592
fn (g &Gen) error(s string, pos token.Pos) {
6593-
util.show_compiler_message('cgen error:', pos: pos, file_path: g.file.path, message: s)
6593+
file_path := if pos.file_idx < 0 { g.file.path } else { g.table.filelist[pos.file_idx] }
6594+
util.show_compiler_message('cgen error:', pos: pos, file_path: file_path, message: s)
65946595
exit(1)
65956596
}
65966597

‎vlib/v/parser/messages.v‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -68,20 +68,21 @@ fn (mut p Parser) note(s string) {
6868
fn (mut p Parser) error_with_pos(s string, pos token.Pos) ast.NodeError {
6969
// print_backtrace()
7070
mut kind := 'error:'
71+
file_path := if pos.file_idx < 0 { p.file_path } else { p.table.filelist[pos.file_idx] }
7172
if p.pref.fatal_errors {
72-
util.show_compiler_message(kind, pos: pos, file_path: p.file_path, message: s)
73+
util.show_compiler_message(kind, pos: pos, file_path: file_path, message: s)
7374
exit(1)
7475
}
7576
if p.pref.output_mode == .stdout && !p.pref.check_only && !p.is_vls {
7677
if p.pref.is_verbose {
7778
print_backtrace()
7879
kind = 'parser error:'
7980
}
80-
util.show_compiler_message(kind, pos: pos, file_path: p.file_path, message: s)
81+
util.show_compiler_message(kind, pos: pos, file_path: file_path, message: s)
8182
exit(1)
8283
} else {
8384
p.errors << errors.Error{
84-
file_path: p.file_path
85+
file_path: file_path
8586
pos: pos
8687
reporter: .parser
8788
message: s
@@ -145,15 +146,16 @@ fn (mut p Parser) warn_with_pos(s string, pos token.Pos) {
145146
if p.pref.skip_warnings {
146147
return
147148
}
149+
file_path := if pos.file_idx < 0 { p.file_path } else { p.table.filelist[pos.file_idx] }
148150
if p.pref.output_mode == .stdout && !p.pref.check_only {
149-
util.show_compiler_message('warning:', pos: pos, file_path: p.file_path, message: s)
151+
util.show_compiler_message('warning:', pos: pos, file_path: file_path, message: s)
150152
} else {
151153
if p.pref.message_limit >= 0 && p.warnings.len >= p.pref.message_limit {
152154
p.should_abort = true
153155
return
154156
}
155157
p.warnings << errors.Warning{
156-
file_path: p.file_path
158+
file_path: file_path
157159
pos: pos
158160
reporter: .parser
159161
message: s
@@ -175,11 +177,12 @@ fn (mut p Parser) note_with_pos(s string, pos token.Pos) {
175177
p.error_with_pos(s, pos)
176178
return
177179
}
180+
file_path := if pos.file_idx < 0 { p.file_path } else { p.table.filelist[pos.file_idx] }
178181
if p.pref.output_mode == .stdout && !p.pref.check_only {
179-
util.show_compiler_message('notice:', pos: pos, file_path: p.file_path, message: s)
182+
util.show_compiler_message('notice:', pos: pos, file_path: file_path, message: s)
180183
} else {
181184
p.notices << errors.Notice{
182-
file_path: p.file_path
185+
file_path: file_path
183186
pos: pos
184187
reporter: .parser
185188
message: s

‎vlib/v/parser/parser.v‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ pub:
1919
mut:
2020
file_base string // "hello.v"
2121
file_path string // "/home/user/hello.v"
22+
file_idx int // file idx in the global table `filelist`
2223
file_display_path string // just "hello.v", when your current folder for the compilation is "/home/user/", otherwise the full path "/home/user/hello.v"
2324
unique_prefix string // a hash of p.file_path, used for making anon fn generation unique
2425
file_backend_mode ast.Language // .c for .c.v|.c.vv|.c.vsh files; .js for .js.v files, .amd64/.rv32/other arches for .amd64.v/.rv32.v/etc. files, .v otherwise.
@@ -255,8 +256,13 @@ pub fn parse_file(path string, mut table ast.Table, comments_mode scanner.Commen
255256
$if trace_parse_file ? {
256257
eprintln('> ${@MOD}.${@FN} comments_mode: ${comments_mode:-20} | path: ${path}')
257258
}
259+
mut file_idx := table.filelist.index(path)
260+
if file_idx == -1 {
261+
file_idx = table.filelist.len
262+
table.filelist << path
263+
}
258264
mut p := Parser{
259-
scanner: scanner.new_scanner_file(path, comments_mode, pref_) or { panic(err) }
265+
scanner: scanner.new_scanner_file(path, file_idx, comments_mode, pref_) or { panic(err) }
260266
table: table
261267
pref: pref_
262268
// Only set vls mode if it's the file the user requested via `v -vls-mode file.v`
@@ -268,6 +274,7 @@ pub fn parse_file(path string, mut table ast.Table, comments_mode scanner.Commen
268274
}
269275
errors: []errors.Error{}
270276
warnings: []errors.Warning{}
277+
file_idx: file_idx
271278
}
272279
p.set_path(path)
273280
res := p.parse()

‎vlib/v/scanner/scanner.v‎

Lines changed: 52 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ pub struct Scanner {
2424
pub mut:
2525
file_path string // '/path/to/file.v'
2626
file_base string // 'file.v'
27+
file_idx int = -1 // file idx in the global table `filelist`
2728
text string // the whole text of the file
2829
pos int = -1 // current position in the file, first character is s.text[0]
2930
line_nr int // current line number
@@ -107,7 +108,7 @@ pub enum CommentsMode {
107108
}
108109

109110
// new scanner from file.
110-
pub fn new_scanner_file(file_path string, comments_mode CommentsMode, pref_ &pref.Preferences) !&Scanner {
111+
pub fn new_scanner_file(file_path string, file_idx int, comments_mode CommentsMode, pref_ &pref.Preferences) !&Scanner {
111112
if !os.is_file(file_path) {
112113
return error('${file_path} is not a .v file')
113114
}
@@ -123,6 +124,7 @@ pub fn new_scanner_file(file_path string, comments_mode CommentsMode, pref_ &pre
123124
comments_mode: comments_mode
124125
file_path: file_path
125126
file_base: os.base(file_path)
127+
file_idx: file_idx
126128
}
127129
s.scan_all_tokens_in_buffer()
128130
return s
@@ -186,26 +188,28 @@ fn (mut s Scanner) new_token(tok_kind token.Kind, lit string, len int) token.Tok
186188
max_column = 1
187189
}
188190
return token.Token{
189-
kind: tok_kind
190-
lit: lit
191-
line_nr: s.line_nr + line_offset
192-
col: max_column
193-
pos: s.pos - len + 1
194-
len: len
195-
tidx: cidx
191+
kind: tok_kind
192+
lit: lit
193+
line_nr: s.line_nr + line_offset
194+
col: max_column
195+
pos: s.pos - len + 1
196+
len: len
197+
tidx: cidx
198+
file_idx: s.file_idx
196199
}
197200
}
198201

199202
@[inline]
200203
fn (s &Scanner) new_eof_token() token.Token {
201204
return token.Token{
202-
kind: .eof
203-
lit: ''
204-
line_nr: s.line_nr + 1
205-
col: s.current_column()
206-
pos: s.pos
207-
len: 1
208-
tidx: s.tidx
205+
kind: .eof
206+
lit: ''
207+
line_nr: s.line_nr + 1
208+
col: s.current_column()
209+
pos: s.pos
210+
len: 1
211+
tidx: s.tidx
212+
file_idx: s.file_idx
209213
}
210214
}
211215

@@ -218,13 +222,14 @@ fn (mut s Scanner) new_multiline_token(tok_kind token.Kind, lit string, len int,
218222
max_column = 1
219223
}
220224
return token.Token{
221-
kind: tok_kind
222-
lit: lit
223-
line_nr: start_line + 1
224-
col: max_column
225-
pos: s.pos - len + 1
226-
len: len
227-
tidx: cidx
225+
kind: tok_kind
226+
lit: lit
227+
line_nr: start_line + 1
228+
col: max_column
229+
pos: s.pos - len + 1
230+
len: len
231+
tidx: cidx
232+
file_idx: s.file_idx
228233
}
229234
}
230235

@@ -945,10 +950,11 @@ pub fn (mut s Scanner) text_scan() token.Token {
945950
comment := s.text[start - 1..s.pos].trim_space()
946951
if s.line_nr != 1 {
947952
comment_pos := token.Pos{
948-
line_nr: s.line_nr - 1
949-
len: comment.len
950-
pos: start
951-
col: s.current_column() - comment.len
953+
line_nr: s.line_nr - 1
954+
len: comment.len
955+
pos: start
956+
col: s.current_column() - comment.len
957+
file_idx: s.file_idx
952958
}
953959
s.error_with_pos('a shebang is only valid at the top of the file',
954960
comment_pos)
@@ -1122,10 +1128,11 @@ pub fn (mut s Scanner) text_scan() token.Token {
11221128
mut comment := s.text[start..(s.pos - 1)]
11231129
if !comment.contains('\n') {
11241130
comment_pos := token.Pos{
1125-
line_nr: start_line
1126-
len: comment.len + 4
1127-
pos: start
1128-
col: s.current_column() - comment.len - 4
1131+
line_nr: start_line
1132+
len: comment.len + 4
1133+
pos: start
1134+
col: s.current_column() - comment.len - 4
1135+
file_idx: s.file_idx
11291136
}
11301137
s.error_with_pos('inline comment is deprecated, please use line comment',
11311138
comment_pos)
@@ -1193,9 +1200,10 @@ pub fn (mut s Scanner) ident_string() string {
11931200
s.is_nested_string = false
11941201
}
11951202
lspos := token.Pos{
1196-
line_nr: s.line_nr
1197-
pos: s.pos
1198-
col: s.pos - s.last_nl_pos - 1
1203+
line_nr: s.line_nr
1204+
pos: s.pos
1205+
col: s.pos - s.last_nl_pos - 1
1206+
file_idx: s.file_idx
11991207
}
12001208
q := s.text[s.pos]
12011209
is_quote := q in [single_quote, double_quote]
@@ -1511,9 +1519,10 @@ fn trim_slash_line_break(s string) string {
15111519
/// escaped utf8 runes in octal like `\342\230\205` => (★)
15121520
pub fn (mut s Scanner) ident_char() string {
15131521
lspos := token.Pos{
1514-
line_nr: s.line_nr
1515-
pos: s.pos
1516-
col: s.pos - s.last_nl_pos - 1
1522+
line_nr: s.line_nr
1523+
pos: s.pos
1524+
col: s.pos - s.last_nl_pos - 1
1525+
file_idx: s.file_idx
15171526
}
15181527

15191528
start := s.pos // the string position of the first backtick char
@@ -1669,9 +1678,10 @@ fn (mut s Scanner) inc_line_number() {
16691678

16701679
pub fn (mut s Scanner) current_pos() token.Pos {
16711680
return token.Pos{
1672-
line_nr: s.line_nr
1673-
pos: s.pos
1674-
col: s.current_column() - 1
1681+
line_nr: s.line_nr
1682+
pos: s.pos
1683+
col: s.current_column() - 1
1684+
file_idx: s.file_idx
16751685
}
16761686
}
16771687

@@ -1681,8 +1691,9 @@ pub fn (mut s Scanner) note(msg string) {
16811691
return
16821692
}
16831693
pos := token.Pos{
1684-
line_nr: s.line_nr
1685-
pos: s.pos
1694+
line_nr: s.line_nr
1695+
pos: s.pos
1696+
file_idx: s.file_idx
16861697
}
16871698
if s.pref.output_mode == .stdout && !s.pref.check_only {
16881699
util.show_compiler_message('notice:', pos: pos, file_path: s.file_path, message: msg)

0 commit comments

Comments
 (0)