Support emitting metrics after connection close

[lrowe]

Currently reset happens when the close syscall is called with the accepted connection fd. Any open fds are also closed. This has the huge advantage of not being blocked by any upstream connections left open in connection pools – so should remain the default – but means that it is impossible to emit metrics like request timings at request close since the vm will be reset before they can be sent.

I think the way around this is to add an option to defer reset until any fds opened after the vm is resumed are closed. This should be opt-in since it requires users carefully ensure fds are not left open (which would prevent reset and consumer resources.) With deno for instance it is possible to configure an http client without connection pooling:

strace deno run --v8-flags=--single-threaded --allow-net 'data:,const client = Deno.createHttpClient({ poolMaxIdlePerHost: 0 }); Deno.serve((req, info)=>{ info.completed.then(()=>fetch("http:127.0.0.1:8001/",{client})); return new Response("hello");})'

strace for a single curl to the deno server:

[{events=EPOLLIN, data={u32=1720821120, u64=111361632871808}}], 1024, -1) = 1
accept4(28, {sa_family=AF_INET, sin_port=htons(43270), sin_addr=inet_addr("127.0.0.1")}, [128 => 16], SOCK_CLOEXEC|SOCK_NONBLOCK) = 29
epoll_ctl(5, EPOLL_CTL_ADD, 29, {events=EPOLLIN|EPOLLOUT|EPOLLRDHUP|EPOLLET, data={u32=1721526272, u64=111361633576960}}) = 0
getpeername(29, {sa_family=AF_INET, sin_port=htons(43270), sin_addr=inet_addr("127.0.0.1")}, [128 => 16]) = 0
accept4(28, 0x7ffed2150330, [128], SOCK_CLOEXEC|SOCK_NONBLOCK) = -1 EAGAIN (Resource temporarily unavailable)
epoll_wait(3, [{events=EPOLLIN|EPOLLOUT, data={u32=1721526272, u64=111361633576960}}], 1024, -1) = 1
recvfrom(29, "GET / HTTP/1.1\r\nHost: localhost:"..., 512, 0, NULL, NULL) = 96

# outgoing socket fd created
socket(AF_INET, SOCK_STREAM|SOCK_CLOEXEC, IPPROTO_TCP) = 30
fcntl(30, F_GETFL)                      = 0x2 (flags O_RDWR)
fcntl(30, F_SETFL, O_RDWR|O_NONBLOCK)   = 0
connect(30, {sa_family=AF_INET, sin_port=htons(8001), sin_addr=inet_addr("127.0.0.1")}, 16) = -1 EINPROGRESS (Operation now in progress)
epoll_ctl(5, EPOLL_CTL_ADD, 30, {events=EPOLLIN|EPOLLOUT|EPOLLRDHUP|EPOLLET, data={u32=1722225408, u64=111361634276096}}) = 0
sendto(29, "HTTP/1.1 200 OK\r\ncontent-type: t"..., 162, MSG_NOSIGNAL, NULL, 0) = 162
shutdown(29, SHUT_WR)                   = 0
epoll_ctl(5, EPOLL_CTL_DEL, 29, NULL)   = 0

# currently the vm would be reset during the close of the incoming accept fd here
close(29)                               = 0
epoll_wait(3, [{events=EPOLLOUT, data={u32=1722225408, u64=111361634276096}}], 1024, -1) = 1
getsockopt(30, SOL_SOCKET, SO_ERROR, [0], [4]) = 0
setsockopt(30, SOL_TCP, TCP_NODELAY, [0], 4) = 0
getpeername(30, {sa_family=AF_INET, sin_port=htons(8001), sin_addr=inet_addr("127.0.0.1")}, [128 => 16]) = 0
getsockname(30, {sa_family=AF_INET, sin_port=htons(42964), sin_addr=inet_addr("127.0.0.1")}, [128 => 16]) = 0
sendto(30, "GET / HTTP/1.1\r\naccept: */*\r\nacc"..., 123, MSG_NOSIGNAL, NULL, 0) = 123
epoll_wait(3, [{events=EPOLLIN|EPOLLOUT, data={u32=1722225408, u64=111361634276096}}], 1024, -1) = 1
recvfrom(30, "HTTP/1.1 200 OK\r\ncontent-type: t"..., 8192, 0, NULL, NULL) = 143
shutdown(30, SHUT_WR)                   = 0
epoll_ctl(5, EPOLL_CTL_DEL, 30, NULL)   = 0

# but the outgoing socket fd is not closed until here
close(30)                               = 0
epoll_wait(3,