diff --git a/Makefile b/Makefile
index 721aab1..d6083da 100644
--- a/Makefile
+++ b/Makefile
@@ -19,7 +19,8 @@ GOFILES = \
 	message.go \
 	tlsserver.go \
 	server.go \
-	client.go
+	client.go \
+	channel.go
 
 .PHONY: grumble
 grumble: pkg
diff --git a/channel.go b/channel.go
new file mode 100644
index 0000000..034ba42
--- /dev/null
+++ b/channel.go
@@ -0,0 +1,48 @@
+package main
+
+// A Mumble channel
+type Channel struct {
+	Id          int
+	Name        string
+	Description string
+	Temporary   bool
+	Position    int
+
+	clients     map[uint32]*Client
+
+	parent      *Channel
+	children    map[int]*Channel
+}
+
+func NewChannel(id int, name string) (channel *Channel) {
+	channel = new(Channel)
+	channel.Id = id
+	channel.Name = name
+	channel.clients = make(map[uint32]*Client)
+	channel.children = make(map[int]*Channel)
+	return
+}
+
+// Add a child channel to a channel
+func (channel *Channel) AddChild(child *Channel) {
+	child.parent = channel
+	channel.children[child.Id] = child
+}
+
+// Remove a child channel from a parent
+func (channel *Channel) RemoveChild(child *Channel) {
+	child.parent = nil
+	channel.children[child.Id] = nil, false
+}
+
+// Add client
+func (channel *Channel) AddClient(client *Client) {
+	channel.clients[client.Session] = client
+	client.Channel = channel
+}
+
+// Remove client
+func (channel *Channel) RemoveClient(client *Client) {
+	channel.clients[client.Session] = nil, false
+	client.Channel = nil
+}
diff --git a/client.go b/client.go
index b4aa180..89da710 100644
--- a/client.go
+++ b/client.go
@@ -40,6 +40,7 @@ type Client struct {
 	Session  uint32
 	Username string
 	Tokens   []string
+	Channel  *Channel
 }
 
 // Something invalid happened on the wire.
@@ -48,11 +49,13 @@ func (client *Client) Panic(reason string) {
 }
 
 func (client *Client) Disconnect() {
-	client.disconnected = true
-	close(client.udprecv)
-	close(client.msgchan)
+	if !client.disconnected {
+		client.disconnected = true
+		close(client.udprecv)
+		close(client.msgchan)
 
-	client.server.RemoveClient(client)
+		client.server.RemoveClient(client)
+	}
 }
 
 // Read a protobuf message from a client
diff --git a/server.go b/server.go
index 5aef243..2bf8ea7 100644
--- a/server.go
+++ b/server.go
@@ -12,7 +12,6 @@ import (
 	"bufio"
 	"bytes"
 	"encoding/binary"
-	"container/list"
 	"sync"
 	"goprotobuf.googlecode.com/hg/proto"
 	"mumbleproto"
@@ -54,6 +53,7 @@ type Server struct {
 	session uint32
 	clients map[uint32]*Client
 
+	// Host, host/port -> client mapping
 	hmutex    sync.Mutex
 	hclients  map[string][]*Client
 	hpclients map[string]*Client
@@ -66,16 +66,6 @@ type Server struct {
 	root *Channel
 }
 
-// A Mumble channel
-type Channel struct {
-	Id          int
-	Name        string
-	Description string
-	Temporary   bool
-	Position    int
-	Channels    *list.List
-}
-
 // Allocate a new Murmur instance
 func NewServer(addr string, port int) (s *Server, err os.Error) {
 	s = new(Server)
@@ -96,10 +86,7 @@ func NewServer(addr string, port int) (s *Server, err os.Error) {
 	s.MaxBandwidth = 300000
 	s.MaxUsers = 10
 
-	s.root = &Channel{
-		Id:   0,
-		Name: "Root",
-	}
+	s.root = NewChannel(0, "Root")
 
 	go s.handler()
 	go s.multiplexer()
@@ -137,8 +124,6 @@ func (server *Server) NewClient(conn net.Conn) (err os.Error) {
 // internal representation.
 func (server *Server) RemoveClient(client *Client) {
 	server.hmutex.Lock()
-	defer server.hmutex.Unlock()
-
 	if client.udpaddr != nil {
 		host := client.udpaddr.IP.String()
 		oldclients := server.hclients[host]
@@ -151,7 +136,20 @@ func (server *Server) RemoveClient(client *Client) {
 		server.hclients[host] = newclients
 		server.hpclients[client.udpaddr.String()] = nil, false
 	}
+	server.hmutex.Unlock()
+
 	server.clients[client.Session] = nil, false
+
+	// Remove client from channel
+	channel := client.Channel
+	channel.RemoveClient(client)
+
+	err := server.broadcastProtoMessage(MessageUserRemove, &mumbleproto.UserRemove{
+		Session:   proto.Uint32(client.Session),
+	})
+	if err != nil {
+		// server panic
+	}
 }
 
 // This is the synchronous handler goroutine.
@@ -172,6 +170,15 @@ func (server *Server) handler() {
 		case vb := <-server.voicebroadcast:
 			log.Printf("VoiceBroadcast!")
 			if vb.target == 0 {
+				channel := vb.client.Channel
+				for _, client := range channel.clients {
+					if client != vb.client {
+						client.sendUdp(&Message{
+							buf:    vb.buf,
+							client: client,
+						})
+					}
+				}
 			}
 		}
 	}
@@ -255,6 +262,8 @@ func (server *Server) handleAuthenticate(client *Client, msg *Message) {
 	server.hmutex.Unlock()
 
 	// Broadcast the the user entered a channel
+	server.root.AddClient(client)
+	log.Printf("server.root = %p", server.root)
 	err = server.broadcastProtoMessage(MessageUserState, &mumbleproto.UserState{
 		Session:   proto.Uint32(client.Session),
 		Name:      proto.String(client.Username),