vlib: revert x.json2 and net changes

Author: medvednikov

vlib/io/readerwriter.v

@@ -17,8 +17,7 @@ mut:
 // read reads up to `buf.len` bytes into `buf`. It returns
 // the number of bytes read or any error encountered.
 pub fn (mut r ReaderWriterImpl) read(mut buf []u8) !int {
-	mut local_buf := unsafe { buf[0..buf.len] }
-	return r.r.read(mut local_buf)
+	return r.r.read(mut buf)
 }
 
 // write writes `buf.len` bytes from `buf` to the underlying

vlib/net/http/backend.c.v

@@ -3,7 +3,7 @@
 // that can be found in the LICENSE file.
 module http
 
-import net.mbedtls
+import net.ssl
 import strings
 
 fn (req &Request) ssl_do(port int, method Method, host_name string, path string) !Response {
@@ -18,8 +18,13 @@ fn (req &Request) ssl_do(port int, method Method, host_name string, path string)
 }
 
 fn net_ssl_do(req &Request, port int, method Method, host_name string, path string) !Response {
-	mut ssl_conn := mbedtls.new_ssl_conn_with_values(req.verify, req.cert, req.cert_key,
-		req.validate, req.in_memory_verification)!
+	mut ssl_conn := ssl.new_ssl_conn(
+		verify:                 req.verify
+		cert:                   req.cert
+		cert_key:               req.cert_key
+		validate:               req.validate
+		in_memory_verification: req.in_memory_verification
+	)!
 	mut retries := 0
 	for {
 		ssl_conn.dial(host_name, port) or {
@@ -43,11 +48,11 @@ fn net_ssl_do(req &Request, port int, method Method, host_name string, path stri
 }
 
 fn read_from_ssl_connection_cb(con voidptr, buf &u8, bufsize int) !int {
-	mut ssl_conn := unsafe { &mbedtls.SSLConn(con) }
+	mut ssl_conn := unsafe { &ssl.SSLConn(con) }
 	return ssl_conn.socket_read_into_ptr(buf, bufsize)
 }
 
-fn (req &Request) do_request(req_headers string, mut ssl_conn mbedtls.SSLConn) !Response {
+fn (req &Request) do_request(req_headers string, mut ssl_conn ssl.SSLConn) !Response {
 	ssl_conn.write_string(req_headers) or { return err }
 	mut content := strings.new_builder(4096)
 	req.receive_all_data_from_cb_in_builder(mut content, voidptr(ssl_conn), read_from_ssl_connection_cb)!

vlib/net/http/download.v

@@ -12,7 +12,7 @@ pub fn download_file(url string, out_file_path string) ! {
 		println('http.download_file url=${url} out_file_path=${out_file_path}')
 	}
 	s := get(url) or { return err }
-	if s.status_code != 200 {
+	if s.status() != .ok {
 		return error_with_code(s.body, s.status_code)
 	}
 	$if debug_http ? {
@@ -26,7 +26,7 @@ pub fn download_file_with_cookies(url string, out_file_path string, cookies map[
 		println('http.download_file url=${url} out_file_path=${out_file_path}')
 	}
 	s := fetch(method: .get, url: url, cookies: cookies) or { return err }
-	if s.status_code != 200 {
+	if s.status() != .ok {
 		return error('received http code ${s.status_code}')
 	}
 	$if debug_http ? {

vlib/net/http/download_progress.v

@@ -1,7 +1,5 @@
 module http
 
-import os
-
 // Downloader is the interface that you have to implement, if you need to customise
 // how download_file_with_progress works, and what output it produces while a file
 // is downloaded.
@@ -47,20 +45,38 @@ pub mut:
 // fail, despite saving all the data in the file before that. The default is 65536 bytes.
 pub fn download_file_with_progress(url string, path string, params DownloaderParams) !Response {
 	mut d := unsafe { params.downloader }
-	mut req := Request{
-		method:   .get
-		url:      url
-		user_ptr: voidptr(d)
+	mut config := params.FetchConfig
+	config.url = url
+	config.user_ptr = voidptr(d)
+	config.on_progress_body = download_progres_cb
+	if config.stop_copying_limit == -1 {
+		// leave more than enough space for potential redirect headers
+		config.stop_copying_limit = 65536
 	}
+	mut req := prepare(config)!
 	d.on_start(mut req, path)!
 	response := req.do()!
-	if response.status_code != 200 {
-		return error_with_code(response.body, response.status_code)
-	}
-	if response.body.len > 0 {
-		d.on_chunk(&req, response.body.bytes(), 0, u64(response.body.len))!
+	$if windows && !no_vschannel ? {
+		// TODO: remove this, when windows supports streaming properly through vschannel
+		// For now though, just ensure that the complete body is "received" in one big chunk:
+		d.on_chunk(req, response.body.bytes(), 0, u64(response.body.len))!
 	}
-	os.write_file(path, response.body)!
-	d.on_finish(&req, &response)!
+	d.on_finish(req, response)!
 	return response
 }
+
+const zz = &Downloader(unsafe { nil })
+
+fn download_progres_cb(request &Request, chunk []u8, body_so_far u64, expected_size u64, status_code int) ! {
+	// TODO: remove this hack, when `unsafe { &Downloader( request.user_ptr ) }` works reliably,
+	// by just casting, without trying to promote the argument to the heap at all.
+	mut d := unsafe { zz }
+	pd := unsafe { &voidptr(&d) }
+	unsafe {
+		*pd = request.user_ptr
+	}
+	if status_code == 200 {
+		// ignore redirects, we are interested in the chunks of the final file:
+		d.on_chunk(request, chunk, body_so_far, expected_size)!
+	}
+}

vlib/net/http/download_terminal_downloader.v

@@ -4,16 +4,16 @@ import time
 
 // TerminalStreamingDownloader is the same as http.SilentStreamingDownloader, but produces a progress line on stdout.
 pub struct TerminalStreamingDownloader {
+	SilentStreamingDownloader
 mut:
-	silent        SilentStreamingDownloader
 	start_time    time.Time
 	past_time     time.Time
 	past_received u64
 }
 
 // on_start is called once at the start of the download.
 pub fn (mut d TerminalStreamingDownloader) on_start(mut request Request, path string) ! {
-	d.silent.on_start(mut request, path)!
+	d.SilentStreamingDownloader.on_start(mut request, path)!
 	d.start_time = time.now()
 	d.past_time = time.now()
 }
@@ -38,14 +38,14 @@ pub fn (mut d TerminalStreamingDownloader) on_chunk(request &Request, chunk []u8
 	estimated_s := estimated.seconds()
 	eta_s := f64_max(estimated_s - elapsed_s, 0.0)
 
-	d.silent.on_chunk(request, chunk, already_received, expected)!
-	print('\rDownloading to `${d.silent.path}` ${100.0 * ratio:6.2f}%, ${f64(already_received) / (1024 * 1024):7.3f}/${f64(expected) / (1024 * 1024):-7.3f}MB, ${speed:6.0f}KB/s, elapsed: ${elapsed_s:6.0f}s, eta: ${eta_s:6.0f}s')
+	d.SilentStreamingDownloader.on_chunk(request, chunk, already_received, expected)!
+	print('\rDownloading to `${d.path}` ${100.0 * ratio:6.2f}%, ${f64(already_received) / (1024 * 1024):7.3f}/${f64(expected) / (1024 * 1024):-7.3f}MB, ${speed:6.0f}KB/s, elapsed: ${elapsed_s:6.0f}s, eta: ${eta_s:6.0f}s')
 	flush_stdout()
 }
 
 // on_finish is called once at the end of the download.
 pub fn (mut d TerminalStreamingDownloader) on_finish(request &Request, response &Response) ! {
-	d.silent.on_finish(request, response)!
+	d.SilentStreamingDownloader.on_finish(request, response)!
 	println('')
 	flush_stdout()
 }

vlib/net/http/header.v

@@ -4,6 +4,7 @@
 module http
 
 import strings
+import arrays
 
 struct HeaderKV {
 	key   string
@@ -608,19 +609,14 @@ pub fn (h Header) custom_values(key string, flags HeaderQueryConfig) []string {
 // keys gets all header keys as strings
 pub fn (h Header) keys() []string {
 	mut res := []string{cap: h.cur_pos}
-	mut seen := map[string]bool{}
 	for i := 0; i < h.cur_pos; i++ {
 		if h.data[i].value == '' {
 			continue
 		}
-		key := h.data[i].key
-		if seen[key] {
-			continue
-		}
-		seen[key] = true
-		res << key
+		res << h.data[i].key
 	}
-	return res
+	// Make sure keys are lower case and unique
+	return arrays.uniq(res)
 }
 
 @[params]
@@ -698,7 +694,7 @@ pub fn (h Header) join(other Header) Header {
 		for v in other.custom_values(k, exact: true) {
 			combined.add_custom(k, v) or {
 				// panic because this should never fail
-				panic('unexpected header merge error')
+				panic('unexpected error: ' + err.str())
 			}
 		}
 	}

vlib/net/http/http_proxy.v

@@ -2,8 +2,8 @@ module http
 
 import encoding.base64
 import net
-import net.mbedtls
 import net.urllib
+import net.ssl
 import net.socks
 
 @[heap]
@@ -52,9 +52,9 @@ pub fn new_http_proxy(raw_url string) !&HttpProxy {
 		return error('Unknown port')
 	}
 
-	if url.user.username != '' || url.user.password != '' {
-		username = url.user.username
-		password = url.user.password
+	if u := url.user {
+		username = u.username
+		password = u.password
 	}
 
 	return &HttpProxy{
@@ -149,7 +149,7 @@ fn (pr &HttpProxy) dial(host string) !&net.TcpConn {
 	}
 }
 
-fn (pr &HttpProxy) ssl_dial(host string) !&mbedtls.SSLConn {
+fn (pr &HttpProxy) ssl_dial(host string) !&ssl.SSLConn {
 	if pr.scheme in ['http', 'https'] {
 		mut tcp := net.dial_tcp(pr.host)!
 		tcp.write(pr.build_proxy_headers(host).bytes())!
@@ -159,14 +159,17 @@ fn (pr &HttpProxy) ssl_dial(host string) !&mbedtls.SSLConn {
 			return error('ssl dial error: ${bf.bytestr()}')
 		}
 
-		mut ssl_conn := mbedtls.new_ssl_conn_with_values('', '', '', false, false)!
+		mut ssl_conn := ssl.new_ssl_conn(
+			verify:                 ''
+			cert:                   ''
+			cert_key:               ''
+			validate:               false
+			in_memory_verification: false
+		)!
 		ssl_conn.connect(mut tcp, host.all_before_last(':'))!
 		return ssl_conn
 	} else if pr.scheme == 'socks5' {
-		mut tcp := socks.socks5_dial(pr.host, host, pr.username, pr.password)!
-		mut ssl_conn := mbedtls.new_ssl_conn_with_values('', '', '', false, false)!
-		ssl_conn.connect(mut tcp, host.all_before_last(':'))!
-		return ssl_conn
+		return socks.socks5_ssl_dial(pr.host, host, pr.username, pr.password)!
 	} else {
 		return error('http_proxy ssl_dial: invalid proxy scheme')
 	}

vlib/net/http/request.v

@@ -174,7 +174,8 @@ pub fn (req &Request) do() !Response {
 		if !req.allow_redirect {
 			break
 		}
-		if resp.status_code !in [301, 302, 303, 307, 308] {
+		if resp.status() !in [.moved_permanently, .found, .see_other, .temporary_redirect,
+			.permanent_redirect] {
 			break
 		}
 		// follow any redirects
@@ -399,7 +400,7 @@ fn (req &Request) receive_all_data_from_cb_in_builder(mut content strings.Builde
 		if headers_end < 0 {
 			unsafe { header_buf.write_ptr(bp, len) }
 			if header_buf.len >= headers_body_boundary.len {
-				header_str := header_buf.spart(0, header_buf.len)
+				header_str := header_buf.bytestr()
 				hidx := header_str.index_(headers_body_boundary)
 				if hidx >= 0 {
 					headers_end = hidx + headers_body_boundary.len
@@ -505,10 +506,7 @@ pub fn parse_request(mut reader io.BufferedReader) !Request {
 pub fn parse_request_head(mut reader io.BufferedReader) !Request {
 	// request line
 	mut line := reader.read_line()!
-	request_line := parse_request_line(line)!
-	method := request_line.method
-	target := request_line.target
-	version := request_line.version
+	method, target, version := parse_request_line(line)!
 
 	// headers
 	mut header := new_header()
@@ -523,6 +521,7 @@ pub fn parse_request_head(mut reader io.BufferedReader) !Request {
 		}
 		if pos + 1 < line.len {
 			value := line[pos + 1..]
+			_, _ = key, value
 			// println('key,value=${key},${value}')
 			header.add_custom(key, value)!
 		}
@@ -552,10 +551,7 @@ pub fn parse_request_head_str(s string) !Request {
 		return error('malformed request: no request line found')
 	}
 	line0 := s[..pos0].trim_space()
-	request_line := parse_request_line(line0)!
-	method := request_line.method
-	target := request_line.target
-	version := request_line.version
+	method, target, version := parse_request_line(line0)!
 
 	// headers
 	mut header := new_header()
@@ -617,13 +613,7 @@ pub fn parse_request_str(s string) !Request {
 	return request
 }
 
-struct ParsedRequestLine {
-	method  Method
-	target  urllib.URL
-	version Version
-}
-
-fn parse_request_line(line string) !ParsedRequestLine {
+fn parse_request_line(line string) !(Method, urllib.URL, Version) {
 	// println('S=${s}')
 	words := line.split(' ')
 	// println('words=')
@@ -655,11 +645,7 @@ fn parse_request_line(line string) !ParsedRequestLine {
 	if version == .unknown {
 		return error('unsupported version')
 	}
-	return ParsedRequestLine{
-		method:  method
-		target:  target
-		version: version
-	}
+	return method, target, version
 }
 
 // Parse URL encoded key=value&key=value forms

vlib/net/http/server.v

@@ -73,12 +73,12 @@ pub fn (mut s Server) listen_and_serve() {
 	s.addr = l.str()
 	s.listener.set_accept_timeout(s.accept_timeout)
 
-	// Keep the request loop synchronous for now. It avoids the worker-channel path,
-	// while preserving the handler behavior for local server use.
-	mut worker := HandlerWorker{
-		id:                      0
-		handler:                 s.handler
-		max_keep_alive_requests: s.max_keep_alive_requests
+	// Create tcp connection channel
+	ch := chan &net.TcpConn{cap: s.pool_channel_slots}
+	// Create workers
+	mut ws := []thread{cap: s.worker_num}
+	for wid in 0 .. s.worker_num {
+		ws << new_handler_worker(wid, ch, s.handler, s.max_keep_alive_requests)
 	}
 
 	if s.show_startup_message {
@@ -102,7 +102,7 @@ pub fn (mut s Server) listen_and_serve() {
 		}
 		conn.set_read_timeout(s.read_timeout)
 		conn.set_write_timeout(s.write_timeout)
-		worker.handle_conn(mut conn)
+		ch <- conn
 	}
 	if s.state == .stopped {
 		s.close()
@@ -161,11 +161,29 @@ pub fn (mut s Server) wait_till_running(params WaitTillRunningParams) !int {
 
 struct HandlerWorker {
 	id                      int
+	ch                      chan &net.TcpConn
 	max_keep_alive_requests int
 pub mut:
 	handler Handler
 }
 
+fn new_handler_worker(wid int, ch chan &net.TcpConn, handler Handler, max_keep_alive_requests int) thread {
+	mut w := &HandlerWorker{
+		id:                      wid
+		ch:                      ch
+		handler:                 handler
+		max_keep_alive_requests: max_keep_alive_requests
+	}
+	return spawn w.process_requests()
+}
+
+fn (mut w HandlerWorker) process_requests() {
+	for {
+		mut conn := <-w.ch or { break }
+		w.handle_conn(mut conn)
+	}
+}
+
 fn (mut w HandlerWorker) handle_conn(mut conn net.TcpConn) {
 	defer {
 		conn.close() or { eprintln('close() failed: ${err}') }