1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
|
// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package runtime
import (
"runtime/internal/atomic"
"unsafe"
)
const _DWORD_MAX = 0xffffffff
const _INVALID_HANDLE_VALUE = ^uintptr(0)
// net_op must be the same as beginning of internal/poll.operation.
// Keep these in sync.
type net_op struct {
// used by windows
o overlapped
// used by netpoll
pd *pollDesc
mode int32
errno int32
qty uint32
}
type overlappedEntry struct {
key uintptr
op *net_op // In reality it's *overlapped, but we cast it to *net_op anyway.
internal uintptr
qty uint32
}
var (
iocphandle uintptr = _INVALID_HANDLE_VALUE // completion port io handle
netpollWakeSig atomic.Uint32 // used to avoid duplicate calls of netpollBreak
)
func netpollinit() {
iocphandle = stdcall4(_CreateIoCompletionPort, _INVALID_HANDLE_VALUE, 0, 0, _DWORD_MAX)
if iocphandle == 0 {
println("runtime: CreateIoCompletionPort failed (errno=", getlasterror(), ")")
throw("runtime: netpollinit failed")
}
}
func netpollIsPollDescriptor(fd uintptr) bool {
return fd == iocphandle
}
func netpollopen(fd uintptr, pd *pollDesc) int32 {
if stdcall4(_CreateIoCompletionPort, fd, iocphandle, 0, 0) == 0 {
return int32(getlasterror())
}
return 0
}
func netpollclose(fd uintptr) int32 {
// nothing to do
return 0
}
func netpollarm(pd *pollDesc, mode int) {
throw("runtime: unused")
}
func netpollBreak() {
// Failing to cas indicates there is an in-flight wakeup, so we're done here.
if !netpollWakeSig.CompareAndSwap(0, 1) {
return
}
if stdcall4(_PostQueuedCompletionStatus, iocphandle, 0, 0, 0) == 0 {
println("runtime: netpoll: PostQueuedCompletionStatus failed (errno=", getlasterror(), ")")
throw("runtime: netpoll: PostQueuedCompletionStatus failed")
}
}
// netpoll checks for ready network connections.
// Returns list of goroutines that become runnable.
// delay < 0: blocks indefinitely
// delay == 0: does not block, just polls
// delay > 0: block for up to that many nanoseconds
func netpoll(delay int64) gList {
var entries [64]overlappedEntry
var wait, qty, flags, n, i uint32
var errno int32
var op *net_op
var toRun gList
mp := getg().m
if iocphandle == _INVALID_HANDLE_VALUE {
return gList{}
}
if delay < 0 {
wait = _INFINITE
} else if delay == 0 {
wait = 0
} else if delay < 1e6 {
wait = 1
} else if delay < 1e15 {
wait = uint32(delay / 1e6)
} else {
// An arbitrary cap on how long to wait for a timer.
// 1e9 ms == ~11.5 days.
wait = 1e9
}
n = uint32(len(entries) / int(gomaxprocs))
if n < 8 {
n = 8
}
if delay != 0 {
mp.blocked = true
}
if stdcall6(_GetQueuedCompletionStatusEx, iocphandle, uintptr(unsafe.Pointer(&entries[0])), uintptr(n), uintptr(unsafe.Pointer(&n)), uintptr(wait), 0) == 0 {
mp.blocked = false
errno = int32(getlasterror())
if errno == _WAIT_TIMEOUT {
return gList{}
}
println("runtime: GetQueuedCompletionStatusEx failed (errno=", errno, ")")
throw("runtime: netpoll failed")
}
mp.blocked = false
for i = 0; i < n; i++ {
op = entries[i].op
if op != nil {
errno = 0
qty = 0
if stdcall5(_WSAGetOverlappedResult, op.pd.fd, uintptr(unsafe.Pointer(op)), uintptr(unsafe.Pointer(&qty)), 0, uintptr(unsafe.Pointer(&flags))) == 0 {
errno = int32(getlasterror())
}
handlecompletion(&toRun, op, errno, qty)
} else {
netpollWakeSig.Store(0)
if delay == 0 {
// Forward the notification to the
// blocked poller.
netpollBreak()
}
}
}
return toRun
}
func handlecompletion(toRun *gList, op *net_op, errno int32, qty uint32) {
mode := op.mode
if mode != 'r' && mode != 'w' {
println("runtime: GetQueuedCompletionStatusEx returned invalid mode=", mode)
throw("runtime: netpoll failed")
}
op.errno = errno
op.qty = qty
netpollready(toRun, op.pd, mode)
}
|
