kdn | @K900 back to the topic of NetBird (or any golang program whatsoveer) building extremely slowly, I came up with this bpftrace syscalls trace for a netbird build of 10 minutes. They all are seem to be mostly related to runtime.netpoll, which might explain some issue while running in offline environment of Nix builds. I might be wrong though. A few of those are in ranges of 120/180 seconds suggesting some form of timeout
{
"event": "syscall_exit",
"timestamp": "2025-06-27T15:38:31.089",
"pid": 315688,
"syscall": "epoll_pwait",
"syscall_id": 281,
"duration_ms": 180008,
"ret": 1,
"stack": null
}
{
"event": "slow_syscall",
"timestamp": "2025-06-27T15:38:31.089",
"pid": 315688,
"syscall": "epoll_pwait",
"syscall_id": 281,
"duration_ms": 180008,
"stack": [
"40d9ee internal/runtime/syscall.Syscall6+14 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"43a7d2 runtime.netpoll+210 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"4476e5 runtime.findRunnable+2245 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"448cb1 runtime.schedule+177 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"449165 runtime.park_m+645 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"47b88e runtime.mcall+78 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"475b6e runtime.gopark+206 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"439dd7 runtime.netpollblock+247 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"474d85 internal/poll.runtime_pollWait+133 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"4f5d67 internal/poll.(*pollDesc).wait+39 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"4f705a internal/poll.(*FD).Read+634 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"50316f os.(*File).Read+79 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"5379b8 bytes.(*Buffer).ReadFrom+152 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"4ed351 io.copyBuffer+337 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"503aaf os.genericWriteTo+79 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"5039dc os.(*File).WriteTo+156 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"4ed29d io.copyBuffer+157 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"53d6b4 os/exec.(*Cmd).writerDescriptor.func1+52 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"53e2cc os/exec.(*Cmd).Start.func2+44 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"53e264 os/exec.(*Cmd).Start.gowrap1+36 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"47d741 runtime.goexit.abi0+1 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)"
]
}
{
"event": "syscall_exit",
"timestamp": "2025-06-27T15:40:32.103",
"pid": 315688,
"syscall": "epoll_pwait",
"syscall_id": 281,
"duration_ms": 120004,
"ret": 1,
"stack": null
}
{
"event": "slow_syscall",
"timestamp": "2025-06-27T15:40:32.103",
"pid": 315688,
"syscall": "epoll_pwait",
"syscall_id": 281,
"duration_ms": 120004,
"stack": [
"40d9ee internal/runtime/syscall.Syscall6+14 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"43a7d2 runtime.netpoll+210 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"4476e5 runtime.findRunnable+2245 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"448cb1 runtime.schedule+177 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"449165 runtime.park_m+645 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"47b88e runtime.mcall+78 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"0 0 ([unknown])"
]
}
{
"event": "syscall_exit",
"timestamp": "2025-06-27T15:46:11.110",
"pid": 315688,
"syscall": "epoll_pwait",
"syscall_id": 281,
"duration_ms": 339003,
"ret": 1,
"stack": null
}
{
"event": "slow_syscall",
"timestamp": "2025-06-27T15:46:11.110",
"pid": 315688,
"syscall": "epoll_pwait",
"syscall_id": 281,
"duration_ms": 339003,
"stack": [
"40d9ee internal/runtime/syscall.Syscall6+14 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"43a7d2 runtime.netpoll+210 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"4476e5 runtime.findRunnable+2245 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"448cb1 runtime.schedule+177 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"449165 runtime.park_m+645 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"47b88e runtime.mcall+78 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"475b6e runtime.gopark+206 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"429a49 runtime.(*scavengerState).park+73 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"429fd9 runtime.bgscavenge+89 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"420405 runtime.gcenable.gowrap2+37 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"47d741 runtime.goexit.abi0+1 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)"
]
}
{
"event": "syscall_exit",
"timestamp": "2025-06-27T15:46:11.111",
"pid": 315688,
"syscall": "waitid",
"syscall_id": 247,
"duration_ms": 639030,
"ret": 0,
"stack": null
}
{
"event": "slow_syscall",
"timestamp": "2025-06-27T15:46:11.111",
"pid": 315688,
"syscall": "waitid",
"syscall_id": 247,
"duration_ms": 639030,
"stack": [
"40d9ee internal/runtime/syscall.Syscall6+14 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"49bbca syscall.Syscall6+106 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"4f4859 internal/syscall/unix.Waitid+57 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"5079e9 os.(*Process).pidfdWait+521 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"502805 os.(*Process).wait+37 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"53ea45 os/exec.(*Cmd).Wait+69 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"53d72d os/exec.(*Cmd).Run+45 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"95e64a cmd/go/internal/work.(*Shell).runOut+1642 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"95de50 cmd/go/internal/work.(*Shell).run+112 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"94effb cmd/go/internal/work.gcToolchain.ld+4443 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"960a4f cmd/go/internal/work.(*gcToolchain).ld+79 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"9377b6 cmd/go/internal/work.(*Builder).link+918 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"919dcd cmd/go/internal/work.ActorFunc.Act+45 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"92b584 cmd/go/internal/work.(*Builder).Do.func3+2244 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"92aaee cmd/go/internal/work.(*Builder).Do.func4+174 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"47d741 runtime.goexit.abi0+1 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)"
]
}
| 14:28:36 |
kdn | * K900: back to the topic of NetBird (or any golang program whatsoveer) building extremely slowly, I came up with this bpftrace syscalls trace for a netbird build of 10 minutes. They all are seem to be mostly related to runtime.netpoll, which might explain some issue while running in offline environment of Nix builds. I might be wrong though. A few of those are in ranges of 120/180 seconds suggesting some form of timeout
{
"event": "syscall_exit",
"timestamp": "2025-06-27T15:38:31.089",
"pid": 315688,
"syscall": "epoll_pwait",
"syscall_id": 281,
"duration_ms": 180008,
"ret": 1,
"stack": null
}
{
"event": "slow_syscall",
"timestamp": "2025-06-27T15:38:31.089",
"pid": 315688,
"syscall": "epoll_pwait",
"syscall_id": 281,
"duration_ms": 180008,
"stack": [
"40d9ee internal/runtime/syscall.Syscall6+14 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"43a7d2 runtime.netpoll+210 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"4476e5 runtime.findRunnable+2245 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"448cb1 runtime.schedule+177 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"449165 runtime.park_m+645 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"47b88e runtime.mcall+78 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"475b6e runtime.gopark+206 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"439dd7 runtime.netpollblock+247 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"474d85 internal/poll.runtime_pollWait+133 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"4f5d67 internal/poll.(*pollDesc).wait+39 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"4f705a internal/poll.(*FD).Read+634 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"50316f os.(*File).Read+79 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"5379b8 bytes.(*Buffer).ReadFrom+152 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"4ed351 io.copyBuffer+337 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"503aaf os.genericWriteTo+79 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"5039dc os.(*File).WriteTo+156 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"4ed29d io.copyBuffer+157 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"53d6b4 os/exec.(*Cmd).writerDescriptor.func1+52 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"53e2cc os/exec.(*Cmd).Start.func2+44 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"53e264 os/exec.(*Cmd).Start.gowrap1+36 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"47d741 runtime.goexit.abi0+1 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)"
]
}
{
"event": "syscall_exit",
"timestamp": "2025-06-27T15:40:32.103",
"pid": 315688,
"syscall": "epoll_pwait",
"syscall_id": 281,
"duration_ms": 120004,
"ret": 1,
"stack": null
}
{
"event": "slow_syscall",
"timestamp": "2025-06-27T15:40:32.103",
"pid": 315688,
"syscall": "epoll_pwait",
"syscall_id": 281,
"duration_ms": 120004,
"stack": [
"40d9ee internal/runtime/syscall.Syscall6+14 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"43a7d2 runtime.netpoll+210 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"4476e5 runtime.findRunnable+2245 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"448cb1 runtime.schedule+177 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"449165 runtime.park_m+645 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"47b88e runtime.mcall+78 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"0 0 ([unknown])"
]
}
{
"event": "syscall_exit",
"timestamp": "2025-06-27T15:46:11.110",
"pid": 315688,
"syscall": "epoll_pwait",
"syscall_id": 281,
"duration_ms": 339003,
"ret": 1,
"stack": null
}
{
"event": "slow_syscall",
"timestamp": "2025-06-27T15:46:11.110",
"pid": 315688,
"syscall": "epoll_pwait",
"syscall_id": 281,
"duration_ms": 339003,
"stack": [
"40d9ee internal/runtime/syscall.Syscall6+14 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"43a7d2 runtime.netpoll+210 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"4476e5 runtime.findRunnable+2245 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"448cb1 runtime.schedule+177 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"449165 runtime.park_m+645 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"47b88e runtime.mcall+78 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"475b6e runtime.gopark+206 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"429a49 runtime.(*scavengerState).park+73 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"429fd9 runtime.bgscavenge+89 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"420405 runtime.gcenable.gowrap2+37 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"47d741 runtime.goexit.abi0+1 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)"
]
}
{
"event": "syscall_exit",
"timestamp": "2025-06-27T15:46:11.111",
"pid": 315688,
"syscall": "waitid",
"syscall_id": 247,
"duration_ms": 639030,
"ret": 0,
"stack": null
}
{
"event": "slow_syscall",
"timestamp": "2025-06-27T15:46:11.111",
"pid": 315688,
"syscall": "waitid",
"syscall_id": 247,
"duration_ms": 639030,
"stack": [
"40d9ee internal/runtime/syscall.Syscall6+14 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"49bbca syscall.Syscall6+106 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"4f4859 internal/syscall/unix.Waitid+57 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"5079e9 os.(*Process).pidfdWait+521 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"502805 os.(*Process).wait+37 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"53ea45 os/exec.(*Cmd).Wait+69 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"53d72d os/exec.(*Cmd).Run+45 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"95e64a cmd/go/internal/work.(*Shell).runOut+1642 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"95de50 cmd/go/internal/work.(*Shell).run+112 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"94effb cmd/go/internal/work.gcToolchain.ld+4443 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"960a4f cmd/go/internal/work.(*gcToolchain).ld+79 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"9377b6 cmd/go/internal/work.(*Builder).link+918 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"919dcd cmd/go/internal/work.ActorFunc.Act+45 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"92b584 cmd/go/internal/work.(*Builder).Do.func3+2244 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"92aaee cmd/go/internal/work.(*Builder).Do.func4+174 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)",
"47d741 runtime.goexit.abi0+1 (/nix/store/mdn9hnvf0qmxywbrpbywmxww2adxdn5v-go-1.24.4/share/go/bin/go)"
]
}
| 14:29:04 |