diff --git a/server/server.go b/server/server.go
index 7bfd923..539f0c5 100644
--- a/server/server.go
+++ b/server/server.go
@@ -1498,7 +1498,12 @@ func (server *Server) Start() (err error) {
 	// for the servers. Each network goroutine defers a call to
 	// netwg.Done(). In the Stop() we close all the connections
 	// and call netwg.Wait() to wait for the goroutines to end.
-	server.netwg.Add(3)
+	numWG := 2
+	if shouldListenWeb {
+		numWG++
+	}
+
+	server.netwg.Add(numWG)
 	go server.udpListenLoop()
 	go server.acceptLoop(server.tlsl)
 	if shouldListenWeb {