From bd07a46b6d85ab97e8bd7d83ea3e572aa1db4bf2 Mon Sep 17 00:00:00 2001 From: wangzhuowei Date: Mon, 18 Sep 2023 19:06:00 +0800 Subject: [PATCH 1/2] perf: use block syscall enter for epoll_wait --- poll_default_linux.go | 18 ++++++++++++--- sys_epoll_linux.go | 45 ++++++++++++++++++++++++++++---------- sys_epoll_linux_amd64.go | 30 +++++++++++++++++++++++++ sys_epoll_linux_arm64.go | 39 ++++----------------------------- sys_epoll_linux_loong64.go | 40 +++------------------------------ 5 files changed, 85 insertions(+), 87 deletions(-) create mode 100644 sys_epoll_linux_amd64.go diff --git a/poll_default_linux.go b/poll_default_linux.go index a0087ee0..fa6a5683 100644 --- a/poll_default_linux.go +++ b/poll_default_linux.go @@ -97,13 +97,25 @@ func (p *defaultPoll) Wait() (err error) { if n == p.size && p.size < 128*1024 { p.Reset(p.size<<1, caps) } - n, err = EpollWait(p.fd, p.events, msec) + // msec: 0(raw) => 1ms(sched,raw) => -1(block_syscall) + // poller's G will hold P at most 1ms + if msec > 0 { + n, err = EpollWaitRaw(p.fd, p.events, msec) + } else if msec == 0 { + n, err = EpollWaitRaw(p.fd, p.events, msec) + } else { // < 0 + n, err = EpollWaitBlock(p.fd, p.events, msec) + } if err != nil && err != syscall.EINTR { return err } if n <= 0 { - msec = -1 - runtime.Gosched() + if msec > 0 { + msec = -1 // no need to sched, because we will use block syscall + } else if msec == 0 { + msec = 1 + runtime.Gosched() + } continue } msec = 0 diff --git a/sys_epoll_linux.go b/sys_epoll_linux.go index 528c34ed..7bf18574 100644 --- a/sys_epoll_linux.go +++ b/sys_epoll_linux.go @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build !arm64 && !loong64 -// +build !arm64,!loong64 +//go:build linux +// +build linux package netpoll @@ -22,12 +22,11 @@ import ( "unsafe" ) -const EPOLLET = -syscall.EPOLLET +//go:linkname entersyscallblock runtime.entersyscallblock +func entersyscallblock() -type epollevent struct { - events uint32 - data [8]byte // unaligned uintptr -} +//go:linkname exitsyscall runtime.exitsyscall +func exitsyscall() // EpollCreate implements epoll_create1. func EpollCreate(flag int) (fd int, err error) { @@ -51,14 +50,36 @@ func EpollCtl(epfd int, op int, fd int, event *epollevent) (err error) { // EpollWait implements epoll_wait. func EpollWait(epfd int, events []epollevent, msec int) (n int, err error) { var r0 uintptr - var _p0 = unsafe.Pointer(&events[0]) - if msec == 0 { - r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_WAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), 0, 0, 0) - } else { - r0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_WAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0) + r0, _, err = syscall.Syscall6(SYS_EPOLL_WAIT, uintptr(epfd), uintptr(unsafe.Pointer(&events[0])), uintptr(len(events)), uintptr(msec), 0, 0) + if err == syscall.Errno(0) { + err = nil } + return int(r0), err +} + +func EpollWaitRaw(epfd int, events []epollevent, msec int) (n int, err error) { + var r0 uintptr + r0, _, err = syscall.RawSyscall6(SYS_EPOLL_WAIT, uintptr(epfd), uintptr(unsafe.Pointer(&events[0])), uintptr(len(events)), uintptr(msec), 0, 0) if err == syscall.Errno(0) { err = nil } return int(r0), err } + +func EpollWaitBlock(epfd int, events []epollevent, msec int) (n int, err error) { + r0, _, errno := BlockSyscall6(SYS_EPOLL_WAIT, uintptr(epfd), uintptr(unsafe.Pointer(&events[0])), uintptr(len(events)), uintptr(msec), 0, 0) + if errno == syscall.Errno(0) { + err = nil + } else { + err = errno + } + return int(r0), err +} + +//go:nosplit +func BlockSyscall6(trap, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err syscall.Errno) { + entersyscallblock() + r1, r2, err = syscall.RawSyscall6(trap, a1, a2, a3, a4, a5, a6) + exitsyscall() + return r1, r2, err +} diff --git a/sys_epoll_linux_amd64.go b/sys_epoll_linux_amd64.go new file mode 100644 index 00000000..f05e8bdc --- /dev/null +++ b/sys_epoll_linux_amd64.go @@ -0,0 +1,30 @@ +// Copyright 2022 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !arm64 && !loong64 +// +build !arm64,!loong64 + +package netpoll + +import ( + "syscall" +) + +const EPOLLET = -syscall.EPOLLET +const SYS_EPOLL_WAIT = syscall.SYS_EPOLL_WAIT + +type epollevent struct { + events uint32 + data [8]byte // unaligned uintptr +} diff --git a/sys_epoll_linux_arm64.go b/sys_epoll_linux_arm64.go index e8d6094d..6628e56f 100644 --- a/sys_epoll_linux_arm64.go +++ b/sys_epoll_linux_arm64.go @@ -12,51 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build arm64 +// +build arm64 + package netpoll import ( "syscall" - "unsafe" ) const EPOLLET = syscall.EPOLLET +const SYS_EPOLL_WAIT = syscall.SYS_EPOLL_PWAIT type epollevent struct { events uint32 _ int32 data [8]byte // unaligned uintptr } - -// EpollCreate implements epoll_create1. -func EpollCreate(flag int) (fd int, err error) { - var r0 uintptr - r0, _, err = syscall.RawSyscall(syscall.SYS_EPOLL_CREATE1, uintptr(flag), 0, 0) - if err == syscall.Errno(0) { - err = nil - } - return int(r0), err -} - -// EpollCtl implements epoll_ctl. -func EpollCtl(epfd int, op int, fd int, event *epollevent) (err error) { - _, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_CTL, uintptr(epfd), uintptr(op), uintptr(fd), uintptr(unsafe.Pointer(event)), 0, 0) - if err == syscall.Errno(0) { - err = nil - } - return err -} - -// EpollWait implements epoll_wait. -func EpollWait(epfd int, events []epollevent, msec int) (n int, err error) { - var r0 uintptr - var _p0 = unsafe.Pointer(&events[0]) - if msec == 0 { - r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), 0, 0, 0) - } else { - r0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0) - } - if err == syscall.Errno(0) { - err = nil - } - return int(r0), err -} diff --git a/sys_epoll_linux_loong64.go b/sys_epoll_linux_loong64.go index ecf36c13..6e761d55 100644 --- a/sys_epoll_linux_loong64.go +++ b/sys_epoll_linux_loong64.go @@ -12,54 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build linux && loong64 -// +build linux,loong64 +//go:build loong64 +// +build loong64 package netpoll import ( "syscall" - "unsafe" ) const EPOLLET = syscall.EPOLLET +const SYS_EPOLL_WAIT = syscall.SYS_EPOLL_PWAIT type epollevent struct { events uint32 _ int32 data [8]byte // unaligned uintptr } - -// EpollCreate implements epoll_create1. -func EpollCreate(flag int) (fd int, err error) { - var r0 uintptr - r0, _, err = syscall.RawSyscall(syscall.SYS_EPOLL_CREATE1, uintptr(flag), 0, 0) - if err == syscall.Errno(0) { - err = nil - } - return int(r0), err -} - -// EpollCtl implements epoll_ctl. -func EpollCtl(epfd int, op int, fd int, event *epollevent) (err error) { - _, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_CTL, uintptr(epfd), uintptr(op), uintptr(fd), uintptr(unsafe.Pointer(event)), 0, 0) - if err == syscall.Errno(0) { - err = nil - } - return err -} - -// EpollWait implements epoll_wait. -func EpollWait(epfd int, events []epollevent, msec int) (n int, err error) { - var r0 uintptr - var _p0 = unsafe.Pointer(&events[0]) - if msec == 0 { - r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), 0, 0, 0) - } else { - r0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0) - } - if err == syscall.Errno(0) { - err = nil - } - return int(r0), err -} From a47967327dd91faae52ddebc679fbc7efcdd5d4d Mon Sep 17 00:00:00 2001 From: wangzhuowei Date: Wed, 3 Jan 2024 11:37:55 +0800 Subject: [PATCH 2/2] fix: make block syscall in asm --- poll_default_linux.go | 4 +--- sys_epoll_linux.go | 25 ++++++++++++++---------- sys_epoll_linux_amd64.s | 41 ++++++++++++++++++++++++++++++++++++++++ sys_epoll_linux_arm64.s | 42 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 13 deletions(-) create mode 100644 sys_epoll_linux_amd64.s create mode 100644 sys_epoll_linux_arm64.s diff --git a/poll_default_linux.go b/poll_default_linux.go index fa6a5683..85e4b65f 100644 --- a/poll_default_linux.go +++ b/poll_default_linux.go @@ -99,9 +99,7 @@ func (p *defaultPoll) Wait() (err error) { } // msec: 0(raw) => 1ms(sched,raw) => -1(block_syscall) // poller's G will hold P at most 1ms - if msec > 0 { - n, err = EpollWaitRaw(p.fd, p.events, msec) - } else if msec == 0 { + if msec >= 0 { n, err = EpollWaitRaw(p.fd, p.events, msec) } else { // < 0 n, err = EpollWaitBlock(p.fd, p.events, msec) diff --git a/sys_epoll_linux.go b/sys_epoll_linux.go index 7bf18574..69410a46 100644 --- a/sys_epoll_linux.go +++ b/sys_epoll_linux.go @@ -28,6 +28,19 @@ func entersyscallblock() //go:linkname exitsyscall runtime.exitsyscall func exitsyscall() +//go:nosplit +func callEntersyscallblock() { + entersyscallblock() +} + +//go:nosplit +func callExitsyscall() { + exitsyscall() +} + +//go:noescape +func BlockSyscall6(num, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, errno uintptr) + // EpollCreate implements epoll_create1. func EpollCreate(flag int) (fd int, err error) { var r0 uintptr @@ -68,18 +81,10 @@ func EpollWaitRaw(epfd int, events []epollevent, msec int) (n int, err error) { func EpollWaitBlock(epfd int, events []epollevent, msec int) (n int, err error) { r0, _, errno := BlockSyscall6(SYS_EPOLL_WAIT, uintptr(epfd), uintptr(unsafe.Pointer(&events[0])), uintptr(len(events)), uintptr(msec), 0, 0) - if errno == syscall.Errno(0) { + if errno == 0 { err = nil } else { - err = errno + err = syscall.Errno(errno) } return int(r0), err } - -//go:nosplit -func BlockSyscall6(trap, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err syscall.Errno) { - entersyscallblock() - r1, r2, err = syscall.RawSyscall6(trap, a1, a2, a3, a4, a5, a6) - exitsyscall() - return r1, r2, err -} diff --git a/sys_epoll_linux_amd64.s b/sys_epoll_linux_amd64.s new file mode 100644 index 00000000..2d723a4e --- /dev/null +++ b/sys_epoll_linux_amd64.s @@ -0,0 +1,41 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "textflag.h" + +// func BlockSyscall6(num, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, errno uintptr) +TEXT ·BlockSyscall6(SB),NOSPLIT,$0-80 + CALL ·callEntersyscallblock(SB) + MOVQ num+0(FP), AX // syscall entry + MOVQ a1+8(FP), DI + MOVQ a2+16(FP), SI + MOVQ a3+24(FP), DX + MOVQ a4+32(FP), R10 + MOVQ a5+40(FP), R8 + MOVQ a6+48(FP), R9 + SYSCALL + CMPQ AX, $0xfffffffffffff001 + JLS ok + MOVQ $-1, r1+56(FP) + MOVQ $0, r2+64(FP) + NEGQ AX + MOVQ AX, errno+72(FP) + CALL ·callExitsyscall(SB) + RET +ok: + MOVQ AX, r1+56(FP) + MOVQ DX, r2+64(FP) + MOVQ $0, errno+72(FP) + CALL ·callExitsyscall(SB) + RET diff --git a/sys_epoll_linux_arm64.s b/sys_epoll_linux_arm64.s new file mode 100644 index 00000000..41930c78 --- /dev/null +++ b/sys_epoll_linux_arm64.s @@ -0,0 +1,42 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "textflag.h" + +// func BlockSyscall6(num, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, errno uintptr) +TEXT ·BlockSyscall6(SB),NOSPLIT,$0-80 + BL ·callEntersyscallblock(SB) + MOVD num+0(FP), R8 // syscall entry + MOVD a1+8(FP), R0 + MOVD a2+16(FP), R1 + MOVD a3+24(FP), R2 + MOVD a4+32(FP), R3 + MOVD a5+40(FP), R4 + MOVD a6+48(FP), R5 + SVC + CMN $4095, R0 + BCC ok + MOVD $-1, R4 + MOVD R4, r1+56(FP) + MOVD ZR, r2+64(FP) + NEG R0, R0 + MOVD R0, errno+72(FP) + BL ·callExitsyscall(SB) + RET +ok: + MOVD R0, r1+56(FP) + MOVD R1, r2+64(FP) + MOVD ZR, errno+72(FP) + BL ·callExitsyscall(SB) + RET