From e9801e6ca3c1d3dec1274451ff3ac3f8480cc7de Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Tue, 27 Oct 2015 18:47:27 -0700
Subject: [PATCH 001/176] FindFirstNewMessage can return -1

---
 socketserver/internal/server/backlog.go | 41 +++++++++++++------------
 1 file changed, 22 insertions(+), 19 deletions(-)

diff --git a/socketserver/internal/server/backlog.go b/socketserver/internal/server/backlog.go
index 66f09e93..c08565fb 100644
--- a/socketserver/internal/server/backlog.go
+++ b/socketserver/internal/server/backlog.go
@@ -124,7 +124,6 @@ func DumpCache() {
 
 	PersistentLSMLock.Lock()
 	PersistentLastMessages = make(map[Command]map[string]LastSavedMessage)
-	// TODO delete file?
 	PersistentLSMLock.Unlock()
 
 	CacheListsLock.Lock()
@@ -177,34 +176,38 @@ func SendTimedBacklogMessages(client *ClientInfo, disconnectTime time.Time) {
 
 	globIdx := FindFirstNewMessage(tgmarray(CachedGlobalMessages), disconnectTime)
 
-	for i := globIdx; i < len(CachedGlobalMessages); i++ {
-		item := CachedGlobalMessages[i]
-		msg := ClientMessage{MessageID: -1, Command: item.Command, origArguments: item.Data}
-		msg.parseOrigArguments()
-		client.MessageChannel <- msg
+	if globIdx != -1 {
+		for i := globIdx; i < len(CachedGlobalMessages); i++ {
+			item := CachedGlobalMessages[i]
+			msg := ClientMessage{MessageID: -1, Command: item.Command, origArguments: item.Data}
+			msg.parseOrigArguments()
+			client.MessageChannel <- msg
+		}
 	}
 
 	chanIdx := FindFirstNewMessage(tmmarray(CachedChannelMessages), disconnectTime)
 
-	for i := chanIdx; i < len(CachedChannelMessages); i++ {
-		item := CachedChannelMessages[i]
-		var send bool
-		for _, channel := range item.Channels {
-			for _, matchChannel := range client.CurrentChannels {
-				if channel == matchChannel {
-					send = true
+	if chanIdx != -1 {
+		for i := chanIdx; i < len(CachedChannelMessages); i++ {
+			item := CachedChannelMessages[i]
+			var send bool
+			for _, channel := range item.Channels {
+				for _, matchChannel := range client.CurrentChannels {
+					if channel == matchChannel {
+						send = true
+						break
+					}
+				}
+				if send {
 					break
 				}
 			}
 			if send {
-				break
+				msg := ClientMessage{MessageID: -1, Command: item.Command, origArguments: item.Data}
+				msg.parseOrigArguments()
+				client.MessageChannel <- msg
 			}
 		}
-		if send {
-			msg := ClientMessage{MessageID: -1, Command: item.Command, origArguments: item.Data}
-			msg.parseOrigArguments()
-			client.MessageChannel <- msg
-		}
 	}
 
 	CacheListsLock.RUnlock()

From 74bc15e0e90a67742f6a6deb4d8b338ba03d41f6 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Tue, 27 Oct 2015 21:21:06 -0700
Subject: [PATCH 002/176] lol commit messages

---
 socketserver/cmd/ffzsocketserver/console.go   | 47 +++++++++++++++++
 .../cmd/ffzsocketserver/socketserver.go       |  2 +
 socketserver/internal/server/backend.go       |  4 +-
 socketserver/internal/server/backlog.go       | 25 +++++++++
 socketserver/internal/server/backlog_test.go  |  4 ++
 socketserver/internal/server/commands.go      | 14 ++---
 socketserver/internal/server/handlecore.go    | 51 +++++++++++--------
 src/socket.js                                 |  1 -
 8 files changed, 117 insertions(+), 31 deletions(-)
 create mode 100644 socketserver/cmd/ffzsocketserver/console.go

diff --git a/socketserver/cmd/ffzsocketserver/console.go b/socketserver/cmd/ffzsocketserver/console.go
new file mode 100644
index 00000000..584b68bd
--- /dev/null
+++ b/socketserver/cmd/ffzsocketserver/console.go
@@ -0,0 +1,47 @@
+package main
+
+import (
+	"../../internal/server"
+	"fmt"
+	"github.com/abiosoft/ishell"
+	"runtime"
+)
+
+func commandLineConsole() {
+
+	shell := ishell.NewShell()
+
+	shell.Register("clientcount", func(args ...string) (string, error) {
+		server.GlobalSubscriptionInfo.RLock()
+		count := len(server.GlobalSubscriptionInfo.Members)
+		server.GlobalSubscriptionInfo.RUnlock()
+		return fmt.Sprintln(count, "clients connected"), nil
+	})
+
+	shell.Register("globalnotice", func(args ...string) (string, error) {
+		msg := server.ClientMessage{
+			MessageID: -1,
+			Command:   "message",
+			Arguments: args[0],
+		}
+		server.PublishToAll(msg)
+		return "Message sent.", nil
+	})
+
+	shell.Register("memstatsbysize", func(args ...string) (string, error) {
+		runtime.GC()
+
+		m := runtime.MemStats{}
+		runtime.ReadMemStats(&m)
+		for _, val := range m.BySize {
+			if val.Mallocs == 0 {
+				continue
+			}
+			shell.Println(val.Size, "bytes:", val.Mallocs, "allocs", val.Frees, "frees")
+		}
+		shell.Println(m.NumGC, "collections occurred")
+		return "", nil
+	})
+
+	shell.Start()
+}
diff --git a/socketserver/cmd/ffzsocketserver/socketserver.go b/socketserver/cmd/ffzsocketserver/socketserver.go
index 5de7a059..d0f7a6f6 100644
--- a/socketserver/cmd/ffzsocketserver/socketserver.go
+++ b/socketserver/cmd/ffzsocketserver/socketserver.go
@@ -46,6 +46,8 @@ func main() {
 
 	server.SetupServerAndHandle(conf, httpServer.TLSConfig, nil)
 
+	go commandLineConsole()
+
 	if conf.UseSSL {
 		err = httpServer.ListenAndServeTLS(conf.SSLCertificateFile, conf.SSLKeyFile)
 	} else {
diff --git a/socketserver/internal/server/backend.go b/socketserver/internal/server/backend.go
index 72d74b22..aae7ce79 100644
--- a/socketserver/internal/server/backend.go
+++ b/socketserver/internal/server/backend.go
@@ -182,9 +182,9 @@ func FetchBacklogData(chatSubs []string) ([]ClientMessage, error) {
 func GenerateKeys(outputFile, serverId, theirPublicStr string) {
 	var err error
 	output := ConfigFile{
-		ListenAddr: "0.0.0.0:8001",
+		ListenAddr:   "0.0.0.0:8001",
 		SocketOrigin: "localhost:8001",
-		BackendUrl: "http://localhost:8002/ffz",
+		BackendUrl:   "http://localhost:8002/ffz",
 		BannerHTML: `
 <!DOCTYPE html>
 <title>CatBag</title>
diff --git a/socketserver/internal/server/backlog.go b/socketserver/internal/server/backlog.go
index c08565fb..278944a0 100644
--- a/socketserver/internal/server/backlog.go
+++ b/socketserver/internal/server/backlog.go
@@ -214,6 +214,31 @@ func SendTimedBacklogMessages(client *ClientInfo, disconnectTime time.Time) {
 	client.Mutex.Unlock()
 }
 
+func TimedBacklogJanitor() {
+	for {
+		time.Sleep(1 * time.Hour)
+		CleanupTimedBacklogMessages()
+	}
+}
+
+func CleanupTimedBacklogMessages() {
+	CacheListsLock.Lock()
+	oneHourAgo := time.Now().Add(-24 * time.Hour)
+	globIdx := FindFirstNewMessage(tgmarray(CachedGlobalMessages), oneHourAgo)
+	if globIdx != -1 {
+		newGlobMsgs := make([]TimestampedGlobalMessage, len(CachedGlobalMessages)-globIdx)
+		copy(newGlobMsgs, CachedGlobalMessages[globIdx:])
+		CachedGlobalMessages = newGlobMsgs
+	}
+	chanIdx := FindFirstNewMessage(tmmarray(CachedChannelMessages), oneHourAgo)
+	if chanIdx != -1 {
+		newChanMsgs := make([]TimestampedMultichatMessage, len(CachedChannelMessages)-chanIdx)
+		copy(newChanMsgs, CachedChannelMessages[chanIdx:])
+		CachedChannelMessages = newChanMsgs
+	}
+	CacheListsLock.Unlock()
+}
+
 func InsertionSort(ary sort.Interface) {
 	for i := 1; i < ary.Len(); i++ {
 		for j := i; j > 0 && ary.Less(j, j-1); j-- {
diff --git a/socketserver/internal/server/backlog_test.go b/socketserver/internal/server/backlog_test.go
index 68757587..99ce4f5a 100644
--- a/socketserver/internal/server/backlog_test.go
+++ b/socketserver/internal/server/backlog_test.go
@@ -5,6 +5,10 @@ import (
 	"time"
 )
 
+func TestCleanupBacklogMessages(t *testing.T) {
+
+}
+
 func TestFindFirstNewMessageEmpty(t *testing.T) {
 	CachedGlobalMessages = []TimestampedGlobalMessage{}
 	i := FindFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
diff --git a/socketserver/internal/server/commands.go b/socketserver/internal/server/commands.go
index c947bf08..aba9df08 100644
--- a/socketserver/internal/server/commands.go
+++ b/socketserver/internal/server/commands.go
@@ -127,13 +127,13 @@ func HandleSub(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rms
 	AddToSliceS(&client.CurrentChannels, channel)
 	client.PendingSubscriptionsBacklog = append(client.PendingSubscriptionsBacklog, channel)
 
-	if client.MakePendingRequests == nil {
-		client.MakePendingRequests = time.AfterFunc(ChannelInfoDelay, GetSubscriptionBacklogFor(conn, client))
-	} else {
-		if !client.MakePendingRequests.Reset(ChannelInfoDelay) {
-			client.MakePendingRequests = time.AfterFunc(ChannelInfoDelay, GetSubscriptionBacklogFor(conn, client))
-		}
-	}
+	//	if client.MakePendingRequests == nil {
+	//		client.MakePendingRequests = time.AfterFunc(ChannelInfoDelay, GetSubscriptionBacklogFor(conn, client))
+	//	} else {
+	//		if !client.MakePendingRequests.Reset(ChannelInfoDelay) {
+	//			client.MakePendingRequests = time.AfterFunc(ChannelInfoDelay, GetSubscriptionBacklogFor(conn, client))
+	//		}
+	//	}
 
 	client.Mutex.Unlock()
 
diff --git a/socketserver/internal/server/handlecore.go b/socketserver/internal/server/handlecore.go
index f227db8b..b673571e 100644
--- a/socketserver/internal/server/handlecore.go
+++ b/socketserver/internal/server/handlecore.go
@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"golang.org/x/net/websocket"
+	"io"
 	"log"
 	"net/http"
 	"strconv"
@@ -74,43 +75,44 @@ var gconfig *ConfigFile
 // Create a websocket.Server with the options from the provided Config.
 func setupServer(config *ConfigFile, tlsConfig *tls.Config) *websocket.Server {
 	gconfig = config
-	sockConf, err := websocket.NewConfig("/", config.SocketOrigin)
-	if err != nil {
-		log.Fatal(err)
-	}
+	//	sockConf, err := websocket.NewConfig("/", config.SocketOrigin)
+	//	if err != nil {
+	//		log.Fatal(err)
+	//	}
 
 	SetupBackend(config)
 
-	if config.UseSSL {
-		cert, err := tls.LoadX509KeyPair(config.SSLCertificateFile, config.SSLKeyFile)
-		if err != nil {
-			log.Fatal(err)
-		}
-		tlsConfig.Certificates = []tls.Certificate{cert}
-		tlsConfig.ServerName = config.SocketOrigin
-		tlsConfig.BuildNameToCertificate()
-		sockConf.TlsConfig = tlsConfig
+	//	if config.UseSSL {
+	//		cert, err := tls.LoadX509KeyPair(config.SSLCertificateFile, config.SSLKeyFile)
+	//		if err != nil {
+	//			log.Fatal(err)
+	//		}
+	//		tlsConfig.Certificates = []tls.Certificate{cert}
+	//		tlsConfig.ServerName = config.SocketOrigin
+	//		tlsConfig.BuildNameToCertificate()
+	//		sockConf.TlsConfig = tlsConfig
+	//	}
 
-	}
-
-	sockServer := &websocket.Server{}
-	sockServer.Config = *sockConf
-	sockServer.Handler = HandleSocketConnection
+	//	sockServer := &websocket.Server{}
+	//	sockServer.Config = *sockConf
+	//	sockServer.Handler = HandleSocketConnection
 
 	go deadChannelReaper()
 
-	return sockServer
+	return nil
 }
 
 // Set up a websocket listener and register it on /.
 // (Uses http.DefaultServeMux .)
 func SetupServerAndHandle(config *ConfigFile, tlsConfig *tls.Config, serveMux *http.ServeMux) {
-	sockServer := setupServer(config, tlsConfig)
+	_ = setupServer(config, tlsConfig)
+	log.Print("hi")
 
 	if serveMux == nil {
 		serveMux = http.DefaultServeMux
 	}
-	serveMux.HandleFunc("/", ServeWebsocketOrCatbag(sockServer.ServeHTTP))
+	handler := websocket.Handler(HandleSocketConnection)
+	serveMux.HandleFunc("/", ServeWebsocketOrCatbag(handler.ServeHTTP))
 	serveMux.HandleFunc("/pub_msg", HBackendPublishRequest)
 	serveMux.HandleFunc("/dump_backlog", HBackendDumpBacklog)
 	serveMux.HandleFunc("/update_and_pub", HBackendUpdateAndPublish)
@@ -132,6 +134,8 @@ func ServeWebsocketOrCatbag(sockfunc func(http.ResponseWriter, *http.Request)) h
 func HandleSocketConnection(conn *websocket.Conn) {
 	// websocket.Conn is a ReadWriteCloser
 
+	fmt.Println("Got socket connection from", conn.Request().RemoteAddr)
+
 	var _closer sync.Once
 	closer := func() {
 		_closer.Do(func() {
@@ -156,6 +160,10 @@ func HandleSocketConnection(conn *websocket.Conn) {
 			}
 			clientChan <- msg
 		}
+
+		if err != io.EOF {
+			fmt.Println("Error while reading from client:", err)
+		}
 		errorChan <- err
 		close(errorChan)
 		close(clientChan)
@@ -198,6 +206,7 @@ RunLoop:
 	}
 
 	// Exit
+	fmt.Println("End socket connection from", conn.Request().RemoteAddr)
 
 	// Launch message draining goroutine - we aren't out of the pub/sub records
 	go func() {
diff --git a/src/socket.js b/src/socket.js
index 99847bfc..b9b9c210 100644
--- a/src/socket.js
+++ b/src/socket.js
@@ -37,7 +37,6 @@ FFZ.prototype.ws_iframe = function() {
 
 FFZ.prototype.ws_create = function() {
 	// Disable sockets for now.
-	return;
 
 	var f = this, ws;
 

From 9c4891db9f6fbf37c3392157405e86c97f31c945 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Tue, 27 Oct 2015 21:25:00 -0700
Subject: [PATCH 003/176] remove canonical path?

---
 socketserver/cmd/ffzsocketserver/socketserver.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/socketserver/cmd/ffzsocketserver/socketserver.go b/socketserver/cmd/ffzsocketserver/socketserver.go
index d0f7a6f6..59b33e3b 100644
--- a/socketserver/cmd/ffzsocketserver/socketserver.go
+++ b/socketserver/cmd/ffzsocketserver/socketserver.go
@@ -1,4 +1,4 @@
-package main // import "bitbucket.org/stendec/frankerfacez/socketserver/cmd/socketserver"
+package main
 
 import (
 	"../../internal/server"

From 9f1c369fdb07f706a06406760bc2da1a2dec6fd4 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 28 Oct 2015 15:19:22 -0700
Subject: [PATCH 004/176] switch to gorilla/websocket, post aggregate data

---
 .../cmd/ffzsocketserver/socketserver.go       |   2 +-
 socketserver/internal/server/backend.go       |  11 ++
 socketserver/internal/server/backlog.go       |   2 +-
 socketserver/internal/server/commands.go      |  83 ++++++---
 socketserver/internal/server/handlecore.go    | 161 ++++++++++--------
 .../internal/server/handlecore_test.go        |  10 +-
 .../internal/server/publisher_test.go         |  35 ++--
 7 files changed, 192 insertions(+), 112 deletions(-)

diff --git a/socketserver/cmd/ffzsocketserver/socketserver.go b/socketserver/cmd/ffzsocketserver/socketserver.go
index 59b33e3b..7382b736 100644
--- a/socketserver/cmd/ffzsocketserver/socketserver.go
+++ b/socketserver/cmd/ffzsocketserver/socketserver.go
@@ -44,7 +44,7 @@ func main() {
 		Addr: conf.ListenAddr,
 	}
 
-	server.SetupServerAndHandle(conf, httpServer.TLSConfig, nil)
+	server.SetupServerAndHandle(conf, nil)
 
 	go commandLineConsole()
 
diff --git a/socketserver/internal/server/backend.go b/socketserver/internal/server/backend.go
index aae7ce79..a7528a39 100644
--- a/socketserver/internal/server/backend.go
+++ b/socketserver/internal/server/backend.go
@@ -22,6 +22,7 @@ var backendUrl string
 var responseCache *cache.Cache
 
 var getBacklogUrl string
+var postStatisticsUrl string
 
 var backendSharedKey [32]byte
 var serverId int
@@ -37,6 +38,7 @@ func SetupBackend(config *ConfigFile) {
 	responseCache = cache.New(60*time.Second, 120*time.Second)
 
 	getBacklogUrl = fmt.Sprintf("%s/backlog", backendUrl)
+	postStatisticsUrl = fmt.Sprintf("%s/stats", backendUrl)
 
 	messageBufferPool.New = New4KByteBuffer
 
@@ -155,6 +157,15 @@ func RequestRemoteData(remoteCommand, data string, auth AuthInfo) (responseStr s
 	return
 }
 
+func SendAggregatedData(sealedForm url.Values) (error) {
+	resp, err := backendHttpClient.PostForm(postStatisticsUrl, sealedForm)
+	if err != nil {
+		return err
+	}
+
+	return resp.Body.Close()
+}
+
 func FetchBacklogData(chatSubs []string) ([]ClientMessage, error) {
 	formData := url.Values{
 		"subs": chatSubs,
diff --git a/socketserver/internal/server/backlog.go b/socketserver/internal/server/backlog.go
index 278944a0..9b4f981f 100644
--- a/socketserver/internal/server/backlog.go
+++ b/socketserver/internal/server/backlog.go
@@ -214,7 +214,7 @@ func SendTimedBacklogMessages(client *ClientInfo, disconnectTime time.Time) {
 	client.Mutex.Unlock()
 }
 
-func TimedBacklogJanitor() {
+func backlogJanitor() {
 	for {
 		time.Sleep(1 * time.Hour)
 		CleanupTimedBacklogMessages()
diff --git a/socketserver/internal/server/commands.go b/socketserver/internal/server/commands.go
index aba9df08..c0d3a9e3 100644
--- a/socketserver/internal/server/commands.go
+++ b/socketserver/internal/server/commands.go
@@ -1,13 +1,15 @@
 package server
 
 import (
+	"encoding/json"
 	"fmt"
+	"github.com/gorilla/websocket"
 	"github.com/satori/go.uuid"
-	"golang.org/x/net/websocket"
 	"log"
 	"strconv"
 	"sync"
 	"time"
+	"net/url"
 )
 
 var ResponseSuccess = ClientMessage{Command: SuccessCommand}
@@ -19,7 +21,7 @@ func HandleCommand(conn *websocket.Conn, client *ClientInfo, msg ClientMessage)
 	handler, ok := CommandHandlers[msg.Command]
 	if !ok {
 		log.Println("[!] Unknown command", msg.Command, "- sent by client", client.ClientID, "@", conn.RemoteAddr())
-		FFZCodec.Send(conn, ClientMessage{
+		SendMessage(conn, ClientMessage{
 			MessageID: msg.MessageID,
 			Command:   "error",
 			Arguments: fmt.Sprintf("Unknown command %s", msg.Command),
@@ -35,10 +37,10 @@ func HandleCommand(conn *websocket.Conn, client *ClientInfo, msg ClientMessage)
 			// The response will be delivered over client.MessageChannel / serverMessageChan
 		} else {
 			response.MessageID = msg.MessageID
-			FFZCodec.Send(conn, response)
+			SendMessage(conn, response)
 		}
 	} else {
-		FFZCodec.Send(conn, ClientMessage{
+		SendMessage(conn, ClientMessage{
 			MessageID: msg.MessageID,
 			Command:   "error",
 			Arguments: err.Error(),
@@ -196,27 +198,16 @@ func GetSubscriptionBacklog(conn *websocket.Conn, client *ClientInfo) {
 	}
 }
 
-type SurveySubmission struct {
-	User string
-	Json string
-}
-
-var SurveySubmissions []SurveySubmission
-var SurveySubmissionLock sync.Mutex
-
 func HandleSurvey(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
-	SurveySubmissionLock.Lock()
-	SurveySubmissions = append(SurveySubmissions, SurveySubmission{client.TwitchUsername, msg.origArguments})
-	SurveySubmissionLock.Unlock()
-
+	// Discard
 	return ResponseSuccess, nil
 }
 
 type FollowEvent struct {
-	User         string
-	Channel      string
-	NowFollowing bool
-	Timestamp    time.Time
+	User         string    `json:u`
+	Channel      string    `json:c`
+	NowFollowing bool      `json:f`
+	Timestamp    time.Time `json:t`
 }
 
 var FollowEvents []FollowEvent
@@ -270,14 +261,62 @@ func HandleEmoticonUses(conn *websocket.Conn, client *ClientInfo, msg ClientMess
 	return ResponseSuccess, nil
 }
 
+func sendAggregateData() {
+	for {
+		time.Sleep(15 * time.Minute)
+		DoSendAggregateData()
+	}
+}
+
+func DoSendAggregateData() {
+	FollowEventsLock.Lock()
+	follows := FollowEvents
+	FollowEvents = nil
+	FollowEventsLock.Unlock()
+	AggregateEmoteUsageLock.Lock()
+	emoteUsage := AggregateEmoteUsage
+	AggregateEmoteUsage = make(map[int]map[string]int)
+	AggregateEmoteUsageLock.Unlock()
+
+	reportForm := url.Values{}
+
+	followJson, err := json.Marshal(follows)
+	if err != nil {
+		log.Print(err)
+	} else {
+		reportForm.Set("follows", string(followJson))
+	}
+
+	emoteJson, err := json.Marshal(emoteUsage)
+	if err != nil {
+		log.Print(err)
+	} else {
+		reportForm.Set("emotes", string(emoteJson))
+	}
+
+	form, err := SealRequest(reportForm)
+	if err != nil {
+		log.Print(err)
+		return
+	}
+
+	err = SendAggregatedData(form)
+	if err != nil {
+		log.Print(err)
+		return
+	}
+
+	// done
+}
+
 func HandleRemoteCommand(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
 	go func(conn *websocket.Conn, msg ClientMessage, authInfo AuthInfo) {
 		resp, err := RequestRemoteDataCached(string(msg.Command), msg.origArguments, authInfo)
 
 		if err != nil {
-			FFZCodec.Send(conn, ClientMessage{MessageID: msg.MessageID, Command: ErrorCommand, Arguments: err.Error()})
+			SendMessage(conn, ClientMessage{MessageID: msg.MessageID, Command: ErrorCommand, Arguments: err.Error()})
 		} else {
-			FFZCodec.Send(conn, ClientMessage{MessageID: msg.MessageID, Command: SuccessCommand, origArguments: resp})
+			SendMessage(conn, ClientMessage{MessageID: msg.MessageID, Command: SuccessCommand, origArguments: resp})
 		}
 	}(conn, msg, client.AuthInfo)
 
diff --git a/socketserver/internal/server/handlecore.go b/socketserver/internal/server/handlecore.go
index b673571e..a285770b 100644
--- a/socketserver/internal/server/handlecore.go
+++ b/socketserver/internal/server/handlecore.go
@@ -1,17 +1,17 @@
 package server // import "bitbucket.org/stendec/frankerfacez/socketserver/internal/server"
 
 import (
-	"crypto/tls"
 	"encoding/json"
 	"errors"
 	"fmt"
-	"golang.org/x/net/websocket"
+	"github.com/gorilla/websocket"
 	"io"
 	"log"
 	"net/http"
 	"strconv"
 	"strings"
 	"sync"
+	"time"
 )
 
 const MAX_PACKET_SIZE = 1024
@@ -54,10 +54,12 @@ const HelloCommand Command = "hello"
 // It signals that the work has been handed off to a background goroutine.
 const AsyncResponseCommand Command = "_async"
 
-// A websocket.Codec that translates the protocol into ClientMessage objects.
-var FFZCodec websocket.Codec = websocket.Codec{
-	Marshal:   MarshalClientMessage,
-	Unmarshal: UnmarshalClientMessage,
+var SocketUpgrader = websocket.Upgrader{
+	ReadBufferSize:  1024,
+	WriteBufferSize: 1024,
+	CheckOrigin: func(r *http.Request) bool {
+		return r.Header.Get("Origin") == "http://www.twitch.tv"
+	},
 }
 
 // Errors that get returned to the client.
@@ -72,69 +74,54 @@ var ExpectedStringAndIntGotFloat = errors.New("Error: Second argument was a floa
 
 var gconfig *ConfigFile
 
-// Create a websocket.Server with the options from the provided Config.
-func setupServer(config *ConfigFile, tlsConfig *tls.Config) *websocket.Server {
-	gconfig = config
-	//	sockConf, err := websocket.NewConfig("/", config.SocketOrigin)
-	//	if err != nil {
-	//		log.Fatal(err)
-	//	}
-
-	SetupBackend(config)
-
-	//	if config.UseSSL {
-	//		cert, err := tls.LoadX509KeyPair(config.SSLCertificateFile, config.SSLKeyFile)
-	//		if err != nil {
-	//			log.Fatal(err)
-	//		}
-	//		tlsConfig.Certificates = []tls.Certificate{cert}
-	//		tlsConfig.ServerName = config.SocketOrigin
-	//		tlsConfig.BuildNameToCertificate()
-	//		sockConf.TlsConfig = tlsConfig
-	//	}
-
-	//	sockServer := &websocket.Server{}
-	//	sockServer.Config = *sockConf
-	//	sockServer.Handler = HandleSocketConnection
-
-	go deadChannelReaper()
-
-	return nil
-}
-
 // Set up a websocket listener and register it on /.
 // (Uses http.DefaultServeMux .)
-func SetupServerAndHandle(config *ConfigFile, tlsConfig *tls.Config, serveMux *http.ServeMux) {
-	_ = setupServer(config, tlsConfig)
-	log.Print("hi")
+func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
+	gconfig = config
+
+	SetupBackend(config)
 
 	if serveMux == nil {
 		serveMux = http.DefaultServeMux
 	}
-	handler := websocket.Handler(HandleSocketConnection)
-	serveMux.HandleFunc("/", ServeWebsocketOrCatbag(handler.ServeHTTP))
+
+	serveMux.HandleFunc("/", ServeWebsocketOrCatbag)
 	serveMux.HandleFunc("/pub_msg", HBackendPublishRequest)
 	serveMux.HandleFunc("/dump_backlog", HBackendDumpBacklog)
 	serveMux.HandleFunc("/update_and_pub", HBackendUpdateAndPublish)
+
+	go deadChannelReaper()
+	go backlogJanitor()
+	go sendAggregateData()
 }
 
-func ServeWebsocketOrCatbag(sockfunc func(http.ResponseWriter, *http.Request)) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		if r.Header.Get("Connection") == "Upgrade" {
-			sockfunc(w, r)
+func ServeWebsocketOrCatbag(w http.ResponseWriter, r *http.Request) {
+	fmt.Println("hi")
+	fmt.Println(r.Header)
+	if r.Header.Get("Connection") == "Upgrade" {
+		conn, err := SocketUpgrader.Upgrade(w, r, nil)
+		if err != nil {
+			fmt.Fprintf(w, "error: %v", err)
 			return
-		} else {
-			w.Write([]byte(gconfig.BannerHTML))
 		}
+		fmt.Println("upgraded!")
+		HandleSocketConnection(conn)
+
+		return
+	} else {
+		w.Write([]byte(gconfig.BannerHTML))
 	}
 }
 
+var CloseGotBinaryMessage = websocket.CloseError{Code: websocket.CloseUnsupportedData, Text: "got binary packet"}
+var CloseGotMessageId0 = websocket.CloseError{Code: websocket.ClosePolicyViolation, Text: "got messageid 0"}
+
 // Handle a new websocket connection from a FFZ client.
 // This runs in a goroutine started by net/http.
 func HandleSocketConnection(conn *websocket.Conn) {
 	// websocket.Conn is a ReadWriteCloser
 
-	fmt.Println("Got socket connection from", conn.Request().RemoteAddr)
+	log.Println("Got socket connection from", conn.RemoteAddr())
 
 	var _closer sync.Once
 	closer := func() {
@@ -150,19 +137,35 @@ func HandleSocketConnection(conn *websocket.Conn) {
 	_serverMessageChan := make(chan ClientMessage)
 	_errorChan := make(chan error)
 
+	var client ClientInfo
+	client.MessageChannel = _serverMessageChan
+
 	// Launch receiver goroutine
 	go func(errorChan chan<- error, clientChan chan<- ClientMessage) {
 		var msg ClientMessage
+		var messageType int
+		var packet []byte
 		var err error
-		for ; err == nil; err = FFZCodec.Receive(conn, &msg) {
+		for ; err == nil; messageType, packet, err = conn.ReadMessage() {
+			if messageType == websocket.BinaryMessage {
+				err = &CloseGotBinaryMessage
+				break
+			}
+			if messageType == websocket.CloseMessage {
+				err = io.EOF
+				break
+			}
+
+			UnmarshalClientMessage(packet, messageType, &msg)
 			if msg.MessageID == 0 {
 				continue
 			}
 			clientChan <- msg
 		}
 
-		if err != io.EOF {
-			fmt.Println("Error while reading from client:", err)
+		_, isClose := err.(*websocket.CloseError)
+		if err != io.EOF && !isClose {
+			log.Println("Error while reading from client:", err)
 		}
 		errorChan <- err
 		close(errorChan)
@@ -174,39 +177,42 @@ func HandleSocketConnection(conn *websocket.Conn) {
 	var clientChan <-chan ClientMessage = _clientChan
 	var serverMessageChan <-chan ClientMessage = _serverMessageChan
 
-	var client ClientInfo
-	client.MessageChannel = _serverMessageChan
-
 	// All set up, now enter the work loop
 
 RunLoop:
 	for {
 		select {
 		case err := <-errorChan:
-			FFZCodec.Send(conn, ClientMessage{
-				MessageID: -1,
-				Command:   "error",
-				Arguments: err.Error(),
-			}) // note - socket might be closed, but don't care
+			if err == io.EOF {
+				conn.Close() // no need to send a close frame :)
+				break RunLoop
+			} else if closeMsg, isClose := err.(*websocket.CloseError); isClose {
+				CloseConnection(conn, closeMsg)
+			} else {
+				CloseConnection(conn, &websocket.CloseError{
+					Code: websocket.CloseInternalServerErr,
+					Text: err.Error(),
+				})
+			}
+
 			break RunLoop
+
 		case msg := <-clientChan:
 			if client.Version == "" && msg.Command != HelloCommand {
-				FFZCodec.Send(conn, ClientMessage{
-					MessageID: msg.MessageID,
-					Command:   "error",
-					Arguments: "Error - the first message sent must be a 'hello'",
+				CloseConnection(conn, &websocket.CloseError{
+					Text: "Error - the first message sent must be a 'hello'",
+					Code: websocket.ClosePolicyViolation,
 				})
 				break RunLoop
 			}
 
 			HandleCommand(conn, &client, msg)
 		case smsg := <-serverMessageChan:
-			FFZCodec.Send(conn, smsg)
+			SendMessage(conn, smsg)
 		}
 	}
 
 	// Exit
-	fmt.Println("End socket connection from", conn.Request().RemoteAddr)
 
 	// Launch message draining goroutine - we aren't out of the pub/sub records
 	go func() {
@@ -220,6 +226,8 @@ RunLoop:
 	// And finished.
 	// Close the channel so the draining goroutine can finish, too.
 	close(_serverMessageChan)
+
+	log.Println("End socket connection from", conn.RemoteAddr())
 }
 
 func CallHandler(handler CommandHandler, conn *websocket.Conn, client *ClientInfo, cmsg ClientMessage) (rmsg ClientMessage, err error) {
@@ -236,8 +244,23 @@ func CallHandler(handler CommandHandler, conn *websocket.Conn, client *ClientInf
 	return handler(conn, client, cmsg)
 }
 
+func CloseConnection(conn *websocket.Conn, closeMsg *websocket.CloseError) {
+	fmt.Println("Terminating connection with", conn.RemoteAddr(), "-", closeMsg.Text)
+	conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(closeMsg.Code, closeMsg.Text), time.Now().Add(2*time.Minute))
+	conn.Close()
+}
+
+func SendMessage(conn *websocket.Conn, msg ClientMessage) {
+	messageType, packet, err := MarshalClientMessage(msg)
+	if err != nil {
+		panic(fmt.Sprintf("failed to marshal: %v %v", err, msg))
+	}
+	fmt.Println(string(packet))
+	conn.WriteMessage(messageType, packet)
+}
+
 // Unpack a message sent from the client into a ClientMessage.
-func UnmarshalClientMessage(data []byte, payloadType byte, v interface{}) (err error) {
+func UnmarshalClientMessage(data []byte, payloadType int, v interface{}) (err error) {
 	var spaceIdx int
 
 	out := v.(*ClientMessage)
@@ -282,7 +305,7 @@ func (cm *ClientMessage) parseOrigArguments() error {
 	return nil
 }
 
-func MarshalClientMessage(clientMessage interface{}) (data []byte, payloadType byte, err error) {
+func MarshalClientMessage(clientMessage interface{}) (payloadType int, data []byte, err error) {
 	var msg ClientMessage
 	var ok bool
 	msg, ok = clientMessage.(ClientMessage)
@@ -309,7 +332,7 @@ func MarshalClientMessage(clientMessage interface{}) (data []byte, payloadType b
 	if msg.Arguments != nil {
 		argBytes, err := json.Marshal(msg.Arguments)
 		if err != nil {
-			return nil, 0, err
+			return 0, nil, err
 		}
 
 		dataStr = fmt.Sprintf("%d %s %s", msg.MessageID, msg.Command, string(argBytes))
@@ -317,7 +340,7 @@ func MarshalClientMessage(clientMessage interface{}) (data []byte, payloadType b
 		dataStr = fmt.Sprintf("%d %s", msg.MessageID, msg.Command)
 	}
 
-	return []byte(dataStr), websocket.TextFrame, nil
+	return websocket.TextMessage, []byte(dataStr), nil
 }
 
 // Command handlers should use this to construct responses.
diff --git a/socketserver/internal/server/handlecore_test.go b/socketserver/internal/server/handlecore_test.go
index 161b5921..c6444c08 100644
--- a/socketserver/internal/server/handlecore_test.go
+++ b/socketserver/internal/server/handlecore_test.go
@@ -2,14 +2,14 @@ package server
 
 import (
 	"fmt"
-	"golang.org/x/net/websocket"
+	"github.com/gorilla/websocket"
 	"testing"
 )
 
 func ExampleUnmarshalClientMessage() {
 	sourceData := []byte("100 hello [\"ffz_3.5.30\",\"898b5bfa-b577-47bb-afb4-252c703b67d6\"]")
 	var cm ClientMessage
-	err := UnmarshalClientMessage(sourceData, websocket.TextFrame, &cm)
+	err := UnmarshalClientMessage(sourceData, websocket.TextMessage, &cm)
 	fmt.Println(err)
 	fmt.Println(cm.MessageID)
 	fmt.Println(cm.Command)
@@ -27,9 +27,9 @@ func ExampleMarshalClientMessage() {
 		Command:   "do_authorize",
 		Arguments: "1234567890",
 	}
-	data, payloadType, err := MarshalClientMessage(&cm)
+	payloadType, data, err := MarshalClientMessage(&cm)
 	fmt.Println(err)
-	fmt.Println(payloadType == websocket.TextFrame)
+	fmt.Println(payloadType == websocket.TextMessage)
 	fmt.Println(string(data))
 	// Output:
 	// <nil>
@@ -40,7 +40,7 @@ func ExampleMarshalClientMessage() {
 func TestArgumentsAsStringAndBool(t *testing.T) {
 	sourceData := []byte("1 foo [\"string\", false]")
 	var cm ClientMessage
-	err := UnmarshalClientMessage(sourceData, websocket.TextFrame, &cm)
+	err := UnmarshalClientMessage(sourceData, websocket.TextMessage, &cm)
 	if err != nil {
 		t.Fatal(err)
 	}
diff --git a/socketserver/internal/server/publisher_test.go b/socketserver/internal/server/publisher_test.go
index 2dc54ed6..7c8fbb89 100644
--- a/socketserver/internal/server/publisher_test.go
+++ b/socketserver/internal/server/publisher_test.go
@@ -3,8 +3,8 @@ package server
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/gorilla/websocket"
 	"github.com/satori/go.uuid"
-	"golang.org/x/net/websocket"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
@@ -27,7 +27,17 @@ const IgnoreReceivedArguments = 1 + 2i
 func TReceiveExpectedMessage(tb testing.TB, conn *websocket.Conn, messageId int, command Command, arguments interface{}) (ClientMessage, bool) {
 	var msg ClientMessage
 	var fail bool
-	err := FFZCodec.Receive(conn, &msg)
+	messageType, packet, err := conn.ReadMessage()
+	if err != nil {
+		tb.Error(err)
+		return msg, false
+	}
+	if messageType != websocket.TextMessage {
+		tb.Error("got non-text message", packet)
+		return msg, false
+	}
+
+	err = UnmarshalClientMessage(packet, messageType, &msg)
 	if err != nil {
 		tb.Error(err)
 		return msg, false
@@ -56,11 +66,8 @@ func TReceiveExpectedMessage(tb testing.TB, conn *websocket.Conn, messageId int,
 }
 
 func TSendMessage(tb testing.TB, conn *websocket.Conn, messageId int, command Command, arguments interface{}) bool {
-	err := FFZCodec.Send(conn, ClientMessage{MessageID: messageId, Command: command, Arguments: arguments})
-	if err != nil {
-		tb.Error(err)
-	}
-	return err == nil
+	SendMessage(conn, ClientMessage{MessageID: messageId, Command: command, Arguments: arguments})
+	return true
 }
 
 func TSealForSavePubMsg(tb testing.TB, cmd Command, channel string, arguments interface{}, deleteMode bool) (url.Values, error) {
@@ -157,7 +164,7 @@ func TSetup(testserver **httptest.Server, urls *TURLs) {
 
 	if testserver != nil {
 		serveMux := http.NewServeMux()
-		SetupServerAndHandle(conf, nil, serveMux)
+		SetupServerAndHandle(conf, serveMux)
 
 		tserv := httptest.NewUnstartedServer(serveMux)
 		*testserver = tserv
@@ -195,6 +202,7 @@ func TestSubscriptionAndPublish(t *testing.T) {
 	defer unsubscribeAllClients()
 
 	var conn *websocket.Conn
+	var resp *http.Response
 	var err error
 
 	// client 1: sub ch1, ch2
@@ -207,7 +215,7 @@ func TestSubscriptionAndPublish(t *testing.T) {
 	// msg 4: global
 
 	// Client 1
-	conn, err = websocket.Dial(urls.Websocket, "", urls.Origin)
+	conn, resp, err = websocket.DefaultDialer.Dial(urls.Websocket, http.Header{})
 	if err != nil {
 		t.Error(err)
 		return
@@ -236,7 +244,7 @@ func TestSubscriptionAndPublish(t *testing.T) {
 	}(conn)
 
 	// Client 2
-	conn, err = websocket.Dial(urls.Websocket, "", urls.Origin)
+	conn, resp, err = websocket.DefaultDialer.Dial(urls.Websocket, http.Header{})
 	if err != nil {
 		t.Error(err)
 		return
@@ -265,7 +273,7 @@ func TestSubscriptionAndPublish(t *testing.T) {
 	}(conn)
 
 	// Client 3
-	conn, err = websocket.Dial(urls.Websocket, "", urls.Origin)
+	conn, resp, err = websocket.DefaultDialer.Dial(urls.Websocket, http.Header{})
 	if err != nil {
 		t.Error(err)
 		return
@@ -291,7 +299,6 @@ func TestSubscriptionAndPublish(t *testing.T) {
 	readyWg.Wait()
 
 	var form url.Values
-	var resp *http.Response
 
 	// Publish message 1 - should go to clients 1, 2
 
@@ -338,7 +345,7 @@ func TestSubscriptionAndPublish(t *testing.T) {
 	}
 
 	// Start client 4
-	conn, err = websocket.Dial(urls.Websocket, "", urls.Origin)
+	conn, resp, err = websocket.DefaultDialer.Dial(urls.Websocket, http.Header{})
 	if err != nil {
 		t.Error(err)
 		return
@@ -401,7 +408,7 @@ func BenchmarkUserSubscriptionSinglePublish(b *testing.B) {
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		conn, err := websocket.Dial(urls.Websocket, "", urls.Origin)
+		conn, _, err := websocket.DefaultDialer.Dial(urls.Websocket, http.Header{})
 		if err != nil {
 			b.Error(err)
 			break

From 59ba6d52f6f2a9480c7c4a81e74189d1e5312859 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 28 Oct 2015 15:49:53 -0700
Subject: [PATCH 005/176] fix marshalling emote usage, log non-hello first
 messages

---
 socketserver/internal/server/commands.go   | 7 ++++++-
 socketserver/internal/server/handlecore.go | 1 +
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/socketserver/internal/server/commands.go b/socketserver/internal/server/commands.go
index c0d3a9e3..c0f2e8b7 100644
--- a/socketserver/internal/server/commands.go
+++ b/socketserver/internal/server/commands.go
@@ -287,7 +287,12 @@ func DoSendAggregateData() {
 		reportForm.Set("follows", string(followJson))
 	}
 
-	emoteJson, err := json.Marshal(emoteUsage)
+	strEmoteUsage := make(map[string]map[string]int)
+	for emoteId, usageByChannel := range emoteUsage {
+		strEmoteId := strconv.Itoa(emoteId)
+		strEmoteUsage[strEmoteId] = usageByChannel
+	}
+	emoteJson, err := json.Marshal(strEmoteUsage)
 	if err != nil {
 		log.Print(err)
 	} else {
diff --git a/socketserver/internal/server/handlecore.go b/socketserver/internal/server/handlecore.go
index a285770b..28b10b7d 100644
--- a/socketserver/internal/server/handlecore.go
+++ b/socketserver/internal/server/handlecore.go
@@ -199,6 +199,7 @@ RunLoop:
 
 		case msg := <-clientChan:
 			if client.Version == "" && msg.Command != HelloCommand {
+				log.Println("error - first message wasn't hello from", conn.RemoteAddr(), "-", msg)
 				CloseConnection(conn, &websocket.CloseError{
 					Text: "Error - the first message sent must be a 'hello'",
 					Code: websocket.ClosePolicyViolation,

From 43d596ad07c333c875c5f67b1673c054b3cf4250 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 28 Oct 2015 16:07:51 -0700
Subject: [PATCH 006/176] handle null clientid in ArgumetnsAsTwoStrings

---
 socketserver/internal/server/handlecore.go | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/socketserver/internal/server/handlecore.go b/socketserver/internal/server/handlecore.go
index 28b10b7d..9b480cee 100644
--- a/socketserver/internal/server/handlecore.go
+++ b/socketserver/internal/server/handlecore.go
@@ -96,8 +96,6 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 }
 
 func ServeWebsocketOrCatbag(w http.ResponseWriter, r *http.Request) {
-	fmt.Println("hi")
-	fmt.Println(r.Header)
 	if r.Header.Get("Connection") == "Upgrade" {
 		conn, err := SocketUpgrader.Upgrade(w, r, nil)
 		if err != nil {
@@ -397,6 +395,10 @@ func (cm *ClientMessage) ArgumentsAsTwoStrings() (string1, string2 string, err e
 			err = ExpectedTwoStrings
 			return
 		}
+		// clientID can be null
+		if ary[1] == nil {
+			return string1, "", nil
+		}
 		string2, ok = ary[1].(string)
 		if !ok {
 			err = ExpectedTwoStrings

From 014e7bc5e509af3ea944e6b06d2a2c17910a6e81 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 28 Oct 2015 17:24:21 -0700
Subject: [PATCH 007/176] Add publish command

---
 socketserver/cmd/ffzsocketserver/console.go | 25 +++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/socketserver/cmd/ffzsocketserver/console.go b/socketserver/cmd/ffzsocketserver/console.go
index 584b68bd..d202330f 100644
--- a/socketserver/cmd/ffzsocketserver/console.go
+++ b/socketserver/cmd/ffzsocketserver/console.go
@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"github.com/abiosoft/ishell"
 	"runtime"
+	"strings"
+	"github.com/gorilla/websocket"
 )
 
 func commandLineConsole() {
@@ -28,6 +30,28 @@ func commandLineConsole() {
 		return "Message sent.", nil
 	})
 
+	shell.Register("publish", func(args ...string) (string, error) {
+		if len(args) < 4 {
+			return "Usage: publish [room.sirstendec | _ALL] -1 reload_ff 23", nil
+		}
+
+		target := args[0]
+		line := strings.Join(args[1:], " ")
+		msg := server.ClientMessage{}
+		err := server.UnmarshalClientMessage([]byte(line), websocket.TextMessage, &msg)
+		if err != nil {
+			return "", err
+		}
+
+		var count int
+		if target == "_ALL" {
+			count = server.PublishToAll(msg)
+		} else {
+			count = server.PublishToChat(target, msg)
+		}
+		return fmt.Sprintf("Published to %d clients", count), nil
+	})
+
 	shell.Register("memstatsbysize", func(args ...string) (string, error) {
 		runtime.GC()
 
@@ -45,3 +69,4 @@ func commandLineConsole() {
 
 	shell.Start()
 }
+

From 6c422b8782d3cedb2a57da5dd9a73f503bd91ba2 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 28 Oct 2015 17:31:44 -0700
Subject: [PATCH 008/176] Write to logfile, cut some of logging

---
 socketserver/cmd/ffzsocketserver/socketserver.go |  6 ++++++
 socketserver/internal/server/handlecore.go       | 15 ++++++++-------
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/socketserver/cmd/ffzsocketserver/socketserver.go b/socketserver/cmd/ffzsocketserver/socketserver.go
index 7382b736..a804476a 100644
--- a/socketserver/cmd/ffzsocketserver/socketserver.go
+++ b/socketserver/cmd/ffzsocketserver/socketserver.go
@@ -44,6 +44,12 @@ func main() {
 		Addr: conf.ListenAddr,
 	}
 
+	logFile, err := os.OpenFile("output.log", os.O_WRONLY | os.O_APPEND | os.O_CREATE, 0644)
+	if err != nil {
+		log.Fatal("Could not create logfile: ", err)
+	}
+	log.SetOutput(logFile)
+
 	server.SetupServerAndHandle(conf, nil)
 
 	go commandLineConsole()
diff --git a/socketserver/internal/server/handlecore.go b/socketserver/internal/server/handlecore.go
index 9b480cee..6c2451ce 100644
--- a/socketserver/internal/server/handlecore.go
+++ b/socketserver/internal/server/handlecore.go
@@ -102,7 +102,6 @@ func ServeWebsocketOrCatbag(w http.ResponseWriter, r *http.Request) {
 			fmt.Fprintf(w, "error: %v", err)
 			return
 		}
-		fmt.Println("upgraded!")
 		HandleSocketConnection(conn)
 
 		return
@@ -113,6 +112,10 @@ func ServeWebsocketOrCatbag(w http.ResponseWriter, r *http.Request) {
 
 var CloseGotBinaryMessage = websocket.CloseError{Code: websocket.CloseUnsupportedData, Text: "got binary packet"}
 var CloseGotMessageId0 = websocket.CloseError{Code: websocket.ClosePolicyViolation, Text: "got messageid 0"}
+var CloseFirstMessageNotHello = websocket.CloseError{
+	Text: "Error - the first message sent must be a 'hello'",
+	Code: websocket.ClosePolicyViolation,
+}
 
 // Handle a new websocket connection from a FFZ client.
 // This runs in a goroutine started by net/http.
@@ -198,10 +201,7 @@ RunLoop:
 		case msg := <-clientChan:
 			if client.Version == "" && msg.Command != HelloCommand {
 				log.Println("error - first message wasn't hello from", conn.RemoteAddr(), "-", msg)
-				CloseConnection(conn, &websocket.CloseError{
-					Text: "Error - the first message sent must be a 'hello'",
-					Code: websocket.ClosePolicyViolation,
-				})
+				CloseConnection(conn, &CloseFirstMessageNotHello)
 				break RunLoop
 			}
 
@@ -244,7 +244,9 @@ func CallHandler(handler CommandHandler, conn *websocket.Conn, client *ClientInf
 }
 
 func CloseConnection(conn *websocket.Conn, closeMsg *websocket.CloseError) {
-	fmt.Println("Terminating connection with", conn.RemoteAddr(), "-", closeMsg.Text)
+	if closeMsg != &CloseFirstMessageNotHello {
+		log.Println("Terminating connection with", conn.RemoteAddr(), "-", closeMsg.Text)
+	}
 	conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(closeMsg.Code, closeMsg.Text), time.Now().Add(2*time.Minute))
 	conn.Close()
 }
@@ -254,7 +256,6 @@ func SendMessage(conn *websocket.Conn, msg ClientMessage) {
 	if err != nil {
 		panic(fmt.Sprintf("failed to marshal: %v %v", err, msg))
 	}
-	fmt.Println(string(packet))
 	conn.WriteMessage(messageType, packet)
 }
 

From cf49f38bf89e50981008eda3961f2cf2a31100e7 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 28 Oct 2015 18:12:20 -0700
Subject: [PATCH 009/176] Move BannerHTML to index.html

---
 socketserver/cmd/ffzsocketserver/console.go     |  3 +--
 socketserver/cmd/ffzsocketserver/index.html     | 13 +++++++++++++
 .../cmd/ffzsocketserver/socketserver.go         |  2 +-
 socketserver/internal/server/backend.go         | 17 +----------------
 socketserver/internal/server/commands.go        |  2 +-
 socketserver/internal/server/handlecore.go      | 11 ++++++++++-
 socketserver/internal/server/types.go           |  2 --
 7 files changed, 27 insertions(+), 23 deletions(-)
 create mode 100644 socketserver/cmd/ffzsocketserver/index.html

diff --git a/socketserver/cmd/ffzsocketserver/console.go b/socketserver/cmd/ffzsocketserver/console.go
index d202330f..a09399b7 100644
--- a/socketserver/cmd/ffzsocketserver/console.go
+++ b/socketserver/cmd/ffzsocketserver/console.go
@@ -4,9 +4,9 @@ import (
 	"../../internal/server"
 	"fmt"
 	"github.com/abiosoft/ishell"
+	"github.com/gorilla/websocket"
 	"runtime"
 	"strings"
-	"github.com/gorilla/websocket"
 )
 
 func commandLineConsole() {
@@ -69,4 +69,3 @@ func commandLineConsole() {
 
 	shell.Start()
 }
-
diff --git a/socketserver/cmd/ffzsocketserver/index.html b/socketserver/cmd/ffzsocketserver/index.html
new file mode 100644
index 00000000..e28ce2ee
--- /dev/null
+++ b/socketserver/cmd/ffzsocketserver/index.html
@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<title>CatBag</title>
+<link rel="stylesheet" href="//cdn.frankerfacez.com/script/catbag.css">
+<div id="container">
+<div id="zf0"></div><div id="zf1"></div><div id="zf2"></div>
+<div id="zf3"></div><div id="zf4"></div><div id="zf5"></div>
+<div id="zf6"></div><div id="zf7"></div><div id="zf8"></div>
+<div id="zf9"></div><div id="catbag"></div>
+<div id="bottom">
+	A <a href="http://www.frankerfacez.com/">FrankerFaceZ</a> Service
+	&mdash; CatBag by <a href="http://www.twitch.tv/wolsk">Wolsk</a>
+</div>
+</div>
diff --git a/socketserver/cmd/ffzsocketserver/socketserver.go b/socketserver/cmd/ffzsocketserver/socketserver.go
index a804476a..f8bf8c19 100644
--- a/socketserver/cmd/ffzsocketserver/socketserver.go
+++ b/socketserver/cmd/ffzsocketserver/socketserver.go
@@ -44,7 +44,7 @@ func main() {
 		Addr: conf.ListenAddr,
 	}
 
-	logFile, err := os.OpenFile("output.log", os.O_WRONLY | os.O_APPEND | os.O_CREATE, 0644)
+	logFile, err := os.OpenFile("output.log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
 	if err != nil {
 		log.Fatal("Could not create logfile: ", err)
 	}
diff --git a/socketserver/internal/server/backend.go b/socketserver/internal/server/backend.go
index a7528a39..08547026 100644
--- a/socketserver/internal/server/backend.go
+++ b/socketserver/internal/server/backend.go
@@ -157,7 +157,7 @@ func RequestRemoteData(remoteCommand, data string, auth AuthInfo) (responseStr s
 	return
 }
 
-func SendAggregatedData(sealedForm url.Values) (error) {
+func SendAggregatedData(sealedForm url.Values) error {
 	resp, err := backendHttpClient.PostForm(postStatisticsUrl, sealedForm)
 	if err != nil {
 		return err
@@ -196,21 +196,6 @@ func GenerateKeys(outputFile, serverId, theirPublicStr string) {
 		ListenAddr:   "0.0.0.0:8001",
 		SocketOrigin: "localhost:8001",
 		BackendUrl:   "http://localhost:8002/ffz",
-		BannerHTML: `
-<!DOCTYPE html>
-<title>CatBag</title>
-<link rel="stylesheet" href="//cdn.frankerfacez.com/script/catbag.css">
-<div id="container">
-<div id="zf0"></div><div id="zf1"></div><div id="zf2"></div>
-<div id="zf3"></div><div id="zf4"></div><div id="zf5"></div>
-<div id="zf6"></div><div id="zf7"></div><div id="zf8"></div>
-<div id="zf9"></div><div id="catbag"></div>
-<div id="bottom">
-	A <a href="http://www.frankerfacez.com/">FrankerFaceZ</a> Service
-	&mdash; CatBag by <a href="http://www.twitch.tv/wolsk">Wolsk</a>
-</div>
-</div>
-`,
 	}
 
 	output.ServerId, err = strconv.Atoi(serverId)
diff --git a/socketserver/internal/server/commands.go b/socketserver/internal/server/commands.go
index c0f2e8b7..0a2cf538 100644
--- a/socketserver/internal/server/commands.go
+++ b/socketserver/internal/server/commands.go
@@ -6,10 +6,10 @@ import (
 	"github.com/gorilla/websocket"
 	"github.com/satori/go.uuid"
 	"log"
+	"net/url"
 	"strconv"
 	"sync"
 	"time"
-	"net/url"
 )
 
 var ResponseSuccess = ClientMessage{Command: SuccessCommand}
diff --git a/socketserver/internal/server/handlecore.go b/socketserver/internal/server/handlecore.go
index 6c2451ce..ff3b786c 100644
--- a/socketserver/internal/server/handlecore.go
+++ b/socketserver/internal/server/handlecore.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"github.com/gorilla/websocket"
 	"io"
+	"io/ioutil"
 	"log"
 	"net/http"
 	"strconv"
@@ -74,6 +75,8 @@ var ExpectedStringAndIntGotFloat = errors.New("Error: Second argument was a floa
 
 var gconfig *ConfigFile
 
+var BannerHTML []byte
+
 // Set up a websocket listener and register it on /.
 // (Uses http.DefaultServeMux .)
 func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
@@ -85,6 +88,12 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 		serveMux = http.DefaultServeMux
 	}
 
+	bannerBytes, err := ioutil.ReadFile("index.html")
+	if err != nil {
+		log.Fatal("Could not open index.html", err)
+	}
+	BannerHTML = bannerBytes
+
 	serveMux.HandleFunc("/", ServeWebsocketOrCatbag)
 	serveMux.HandleFunc("/pub_msg", HBackendPublishRequest)
 	serveMux.HandleFunc("/dump_backlog", HBackendDumpBacklog)
@@ -106,7 +115,7 @@ func ServeWebsocketOrCatbag(w http.ResponseWriter, r *http.Request) {
 
 		return
 	} else {
-		w.Write([]byte(gconfig.BannerHTML))
+		w.Write(BannerHTML)
 	}
 }
 
diff --git a/socketserver/internal/server/types.go b/socketserver/internal/server/types.go
index cc9ba947..76123277 100644
--- a/socketserver/internal/server/types.go
+++ b/socketserver/internal/server/types.go
@@ -17,8 +17,6 @@ type ConfigFile struct {
 	SocketOrigin string
 	// URL to the backend server
 	BackendUrl string
-	// Memes go here
-	BannerHTML string
 
 	// SSL/TLS
 	UseSSL             bool

From ab6eb2d53006a6adef2975278bdd13fd8d230edf Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 28 Oct 2015 22:59:27 -0700
Subject: [PATCH 010/176] Send pings to the client

---
 socketserver/internal/server/handlecore.go | 24 +++++++++++++++++++++-
 socketserver/internal/server/types.go      |  3 +++
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/socketserver/internal/server/handlecore.go b/socketserver/internal/server/handlecore.go
index ff3b786c..b76762e4 100644
--- a/socketserver/internal/server/handlecore.go
+++ b/socketserver/internal/server/handlecore.go
@@ -121,6 +121,7 @@ func ServeWebsocketOrCatbag(w http.ResponseWriter, r *http.Request) {
 
 var CloseGotBinaryMessage = websocket.CloseError{Code: websocket.CloseUnsupportedData, Text: "got binary packet"}
 var CloseGotMessageId0 = websocket.CloseError{Code: websocket.ClosePolicyViolation, Text: "got messageid 0"}
+var CloseTimedOut = websocket.CloseError{Code: websocket.CloseNoStatusReceived, Text: "no ping replies for 5 minutes"}
 var CloseFirstMessageNotHello = websocket.CloseError{
 	Text: "Error - the first message sent must be a 'hello'",
 	Code: websocket.ClosePolicyViolation,
@@ -183,6 +184,12 @@ func HandleSocketConnection(conn *websocket.Conn) {
 		// exit
 	}(_errorChan, _clientChan)
 
+	conn.SetPongHandler(func(pongBody string) error {
+		fmt.Println("got pong")
+		client.pingCount = 0
+		return nil
+	})
+
 	var errorChan <-chan error = _errorChan
 	var clientChan <-chan ClientMessage = _clientChan
 	var serverMessageChan <-chan ClientMessage = _serverMessageChan
@@ -215,8 +222,18 @@ RunLoop:
 			}
 
 			HandleCommand(conn, &client, msg)
+
 		case smsg := <-serverMessageChan:
 			SendMessage(conn, smsg)
+
+		case <- time.After(1 * time.Minute):
+			client.pingCount++
+			if client.pingCount == 5 {
+				CloseConnection(conn, &CloseTimedOut)
+				break RunLoop
+			} else {
+				conn.WriteControl(websocket.PingMessage, []byte(strconv.FormatInt(time.Now().Unix(), 10)), getDeadline())
+			}
 		}
 	}
 
@@ -238,6 +255,10 @@ RunLoop:
 	log.Println("End socket connection from", conn.RemoteAddr())
 }
 
+func getDeadline() time.Time {
+	return time.Now().Add(1 * time.Minute)
+}
+
 func CallHandler(handler CommandHandler, conn *websocket.Conn, client *ClientInfo, cmsg ClientMessage) (rmsg ClientMessage, err error) {
 	defer func() {
 		if r := recover(); r != nil {
@@ -256,7 +277,7 @@ func CloseConnection(conn *websocket.Conn, closeMsg *websocket.CloseError) {
 	if closeMsg != &CloseFirstMessageNotHello {
 		log.Println("Terminating connection with", conn.RemoteAddr(), "-", closeMsg.Text)
 	}
-	conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(closeMsg.Code, closeMsg.Text), time.Now().Add(2*time.Minute))
+	conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(closeMsg.Code, closeMsg.Text), getDeadline())
 	conn.Close()
 }
 
@@ -265,6 +286,7 @@ func SendMessage(conn *websocket.Conn, msg ClientMessage) {
 	if err != nil {
 		panic(fmt.Sprintf("failed to marshal: %v %v", err, msg))
 	}
+	conn.SetWriteDeadline(getDeadline())
 	conn.WriteMessage(messageType, packet)
 }
 
diff --git a/socketserver/internal/server/types.go b/socketserver/internal/server/types.go
index 76123277..d3d1224e 100644
--- a/socketserver/internal/server/types.go
+++ b/socketserver/internal/server/types.go
@@ -87,6 +87,9 @@ type ClientInfo struct {
 	// Server-initiated messages should be sent here
 	// Never nil.
 	MessageChannel chan<- ClientMessage
+
+	// The number of pings sent without a response
+	pingCount int
 }
 
 type tgmarray []TimestampedGlobalMessage

From 0786f6aa07c4c6b5d67ce230ba977833f6eda272 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 28 Oct 2015 23:00:22 -0700
Subject: [PATCH 011/176] oops

---
 socketserver/internal/server/handlecore.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/socketserver/internal/server/handlecore.go b/socketserver/internal/server/handlecore.go
index b76762e4..1a61b0ef 100644
--- a/socketserver/internal/server/handlecore.go
+++ b/socketserver/internal/server/handlecore.go
@@ -185,7 +185,6 @@ func HandleSocketConnection(conn *websocket.Conn) {
 	}(_errorChan, _clientChan)
 
 	conn.SetPongHandler(func(pongBody string) error {
-		fmt.Println("got pong")
 		client.pingCount = 0
 		return nil
 	})

From 1fdef8b846f466ba0d0e06d428f4c91d93d23fd6 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 28 Oct 2015 23:27:04 -0700
Subject: [PATCH 012/176] report emote data more often

---
 socketserver/internal/server/commands.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/socketserver/internal/server/commands.go b/socketserver/internal/server/commands.go
index 0a2cf538..b9efa0bc 100644
--- a/socketserver/internal/server/commands.go
+++ b/socketserver/internal/server/commands.go
@@ -263,7 +263,7 @@ func HandleEmoticonUses(conn *websocket.Conn, client *ClientInfo, msg ClientMess
 
 func sendAggregateData() {
 	for {
-		time.Sleep(15 * time.Minute)
+		time.Sleep(5 * time.Minute)
 		DoSendAggregateData()
 	}
 }

From 601b5501a76c7cd909ea298f06be88b27380cdfc Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Thu, 29 Oct 2015 00:17:13 -0700
Subject: [PATCH 013/176] Fix multithreaded call to SendMessage()

---
 socketserver/internal/server/commands.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/socketserver/internal/server/commands.go b/socketserver/internal/server/commands.go
index b9efa0bc..ee819650 100644
--- a/socketserver/internal/server/commands.go
+++ b/socketserver/internal/server/commands.go
@@ -319,9 +319,9 @@ func HandleRemoteCommand(conn *websocket.Conn, client *ClientInfo, msg ClientMes
 		resp, err := RequestRemoteDataCached(string(msg.Command), msg.origArguments, authInfo)
 
 		if err != nil {
-			SendMessage(conn, ClientMessage{MessageID: msg.MessageID, Command: ErrorCommand, Arguments: err.Error()})
+			client.MessageChannel <- ClientMessage{MessageID: msg.MessageID, Command: ErrorCommand, Arguments: err.Error()}
 		} else {
-			SendMessage(conn, ClientMessage{MessageID: msg.MessageID, Command: SuccessCommand, origArguments: resp})
+			client.MessageChannel <- ClientMessage{MessageID: msg.MessageID, Command: SuccessCommand, origArguments: resp}
 		}
 	}(conn, msg, client.AuthInfo)
 

From 33bf762a007941209902738bbb41283ca8a952cd Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Thu, 29 Oct 2015 01:23:58 -0700
Subject: [PATCH 014/176] Fix panic sending on closed client.MessageChannel

---
 socketserver/internal/server/commands.go   | 48 ++++++++++++----------
 socketserver/internal/server/handlecore.go |  4 ++
 socketserver/internal/server/types.go      |  5 ++-
 3 files changed, 35 insertions(+), 22 deletions(-)

diff --git a/socketserver/internal/server/commands.go b/socketserver/internal/server/commands.go
index ee819650..db91c66a 100644
--- a/socketserver/internal/server/commands.go
+++ b/socketserver/internal/server/commands.go
@@ -85,22 +85,20 @@ func HandleReady(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (r
 	client.MakePendingRequests = nil
 	client.Mutex.Unlock()
 
-	if disconnectAt == 0 {
-		// backlog only
-		go func() {
-			client.MessageChannel <- ClientMessage{MessageID: msg.MessageID, Command: SuccessCommand}
-			SendBacklogForNewClient(client)
-		}()
-		return ClientMessage{Command: AsyncResponseCommand}, nil
-	} else {
-		// backlog and timed
-		go func() {
-			client.MessageChannel <- ClientMessage{MessageID: msg.MessageID, Command: SuccessCommand}
-			SendBacklogForNewClient(client)
+	go func() {
+		client.MsgChannelKeepalive.RLock()
+		if client.MessageChannel == nil {
+			return
+		}
+
+		client.MessageChannel <- ClientMessage{MessageID: msg.MessageID, Command: SuccessCommand}
+		SendBacklogForNewClient(client)
+		if disconnectAt != 0 {
 			SendTimedBacklogMessages(client, time.Unix(disconnectAt, 0))
-		}()
-		return ClientMessage{Command: AsyncResponseCommand}, nil
-	}
+		}
+		client.MsgChannelKeepalive.RUnlock()
+	}()
+	return ClientMessage{Command: AsyncResponseCommand}, nil
 }
 
 func HandleSetUser(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
@@ -193,9 +191,13 @@ func GetSubscriptionBacklog(conn *websocket.Conn, client *ClientInfo) {
 	}
 
 	// Deliver to client
-	for _, msg := range messages {
-		client.MessageChannel <- msg
+	client.MsgChannelKeepalive.RLock()
+	if client.MessageChannel != nil {
+		for _, msg := range messages {
+			client.MessageChannel <- msg
+		}
 	}
+	client.MsgChannelKeepalive.RUnlock()
 }
 
 func HandleSurvey(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
@@ -318,11 +320,15 @@ func HandleRemoteCommand(conn *websocket.Conn, client *ClientInfo, msg ClientMes
 	go func(conn *websocket.Conn, msg ClientMessage, authInfo AuthInfo) {
 		resp, err := RequestRemoteDataCached(string(msg.Command), msg.origArguments, authInfo)
 
-		if err != nil {
-			client.MessageChannel <- ClientMessage{MessageID: msg.MessageID, Command: ErrorCommand, Arguments: err.Error()}
-		} else {
-			client.MessageChannel <- ClientMessage{MessageID: msg.MessageID, Command: SuccessCommand, origArguments: resp}
+		client.MsgChannelKeepalive.RLock()
+		if client.MessageChannel != nil {
+			if err != nil {
+				client.MessageChannel <- ClientMessage{MessageID: msg.MessageID, Command: ErrorCommand, Arguments: err.Error()}
+			} else {
+				client.MessageChannel <- ClientMessage{MessageID: msg.MessageID, Command: SuccessCommand, origArguments: resp}
+			}
 		}
+		client.MsgChannelKeepalive.RUnlock()
 	}(conn, msg, client.AuthInfo)
 
 	return ClientMessage{Command: AsyncResponseCommand}, nil
diff --git a/socketserver/internal/server/handlecore.go b/socketserver/internal/server/handlecore.go
index 1a61b0ef..c2d05ce8 100644
--- a/socketserver/internal/server/handlecore.go
+++ b/socketserver/internal/server/handlecore.go
@@ -247,6 +247,10 @@ RunLoop:
 	// Stop getting messages...
 	UnsubscribeAll(&client)
 
+	client.MsgChannelKeepalive.Lock()
+	client.MessageChannel = nil
+	client.MsgChannelKeepalive.Unlock()
+
 	// And finished.
 	// Close the channel so the draining goroutine can finish, too.
 	close(_serverMessageChan)
diff --git a/socketserver/internal/server/types.go b/socketserver/internal/server/types.go
index d3d1224e..0fd79a39 100644
--- a/socketserver/internal/server/types.go
+++ b/socketserver/internal/server/types.go
@@ -85,9 +85,12 @@ type ClientInfo struct {
 	MakePendingRequests *time.Timer
 
 	// Server-initiated messages should be sent here
-	// Never nil.
+	// This field will be nil before it is closed.
 	MessageChannel chan<- ClientMessage
 
+	// Take a read-lock on this before checking whether MessageChannel is nil.
+	MsgChannelKeepalive sync.RWMutex
+
 	// The number of pings sent without a response
 	pingCount int
 }

From 37833043bd946bd9741a9c05f3c40d422af75b9e Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Thu, 29 Oct 2015 01:30:25 -0700
Subject: [PATCH 015/176] Make sure that the reader thread finishes when main
 thread

calls CloseConnection
---
 socketserver/internal/server/handlecore.go | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/socketserver/internal/server/handlecore.go b/socketserver/internal/server/handlecore.go
index c2d05ce8..f22bf842 100644
--- a/socketserver/internal/server/handlecore.go
+++ b/socketserver/internal/server/handlecore.go
@@ -147,12 +147,13 @@ func HandleSocketConnection(conn *websocket.Conn) {
 	_clientChan := make(chan ClientMessage)
 	_serverMessageChan := make(chan ClientMessage)
 	_errorChan := make(chan error)
+	stoppedChan := make(chan struct{})
 
 	var client ClientInfo
 	client.MessageChannel = _serverMessageChan
 
 	// Launch receiver goroutine
-	go func(errorChan chan<- error, clientChan chan<- ClientMessage) {
+	go func(errorChan chan<- error, clientChan chan<- ClientMessage, stoppedChan <-chan struct{}) {
 		var msg ClientMessage
 		var messageType int
 		var packet []byte
@@ -171,7 +172,13 @@ func HandleSocketConnection(conn *websocket.Conn) {
 			if msg.MessageID == 0 {
 				continue
 			}
-			clientChan <- msg
+			select {
+			case clientChan <- msg:
+			case <-stoppedChan:
+				close(errorChan)
+				close(clientChan)
+				return
+			}
 		}
 
 		_, isClose := err.(*websocket.CloseError)
@@ -182,7 +189,7 @@ func HandleSocketConnection(conn *websocket.Conn) {
 		close(errorChan)
 		close(clientChan)
 		// exit
-	}(_errorChan, _clientChan)
+	}(_errorChan, _clientChan, stoppedChan)
 
 	conn.SetPongHandler(func(pongBody string) error {
 		client.pingCount = 0
@@ -244,6 +251,8 @@ RunLoop:
 		}
 	}()
 
+	close(stoppedChan)
+
 	// Stop getting messages...
 	UnsubscribeAll(&client)
 

From a14038b215581c5c6cf207304b1a1f0b7017d64c Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Thu, 29 Oct 2015 10:29:16 -0700
Subject: [PATCH 016/176] missing list != nil check

---
 socketserver/internal/server/publisher.go | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/socketserver/internal/server/publisher.go b/socketserver/internal/server/publisher.go
index d9658ac7..5c1c71dd 100644
--- a/socketserver/internal/server/publisher.go
+++ b/socketserver/internal/server/publisher.go
@@ -144,9 +144,11 @@ func UnsubscribeAll(client *ClientInfo) {
 func UnsubscribeSingleChat(client *ClientInfo, channelName string) {
 	ChatSubscriptionLock.RLock()
 	list := ChatSubscriptionInfo[channelName]
-	list.Lock()
-	RemoveFromSliceC(&list.Members, client.MessageChannel)
-	list.Unlock()
+	if list != nil {
+		list.Lock()
+		RemoveFromSliceC(&list.Members, client.MessageChannel)
+		list.Unlock()
+	}
 	ChatSubscriptionLock.RUnlock()
 }
 

From e11c6e64794777bb93c2ae346353a6b1111ab85c Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Thu, 29 Oct 2015 15:01:42 -0700
Subject: [PATCH 017/176] val can be nil

---
 socketserver/internal/server/publisher.go | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/socketserver/internal/server/publisher.go b/socketserver/internal/server/publisher.go
index 5c1c71dd..9febcf9d 100644
--- a/socketserver/internal/server/publisher.go
+++ b/socketserver/internal/server/publisher.go
@@ -161,8 +161,10 @@ func deadChannelReaper() {
 		time.Sleep(ReapingDelay)
 		ChatSubscriptionLock.Lock()
 		for key, val := range ChatSubscriptionInfo {
-			if len(val.Members) == 0 {
-				ChatSubscriptionInfo[key] = nil
+			if val != nil {
+				if len(val.Members) == 0 {
+					ChatSubscriptionInfo[key] = nil
+				}
 			}
 		}
 		ChatSubscriptionLock.Unlock()

From b2624ec77fcbf38b75692b2d7bb3d1eb91f9f3c9 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Thu, 29 Oct 2015 15:02:28 -0700
Subject: [PATCH 018/176] Should be deleting the key too

---
 socketserver/internal/server/publisher.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/socketserver/internal/server/publisher.go b/socketserver/internal/server/publisher.go
index 9febcf9d..db1f74ac 100644
--- a/socketserver/internal/server/publisher.go
+++ b/socketserver/internal/server/publisher.go
@@ -163,7 +163,7 @@ func deadChannelReaper() {
 		for key, val := range ChatSubscriptionInfo {
 			if val != nil {
 				if len(val.Members) == 0 {
-					ChatSubscriptionInfo[key] = nil
+					delete(ChatSubscriptionInfo, key)
 				}
 			}
 		}

From 18619eddd3088fa4709ea7085e790d5d70c1a190 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sat, 31 Oct 2015 23:16:58 -0700
Subject: [PATCH 019/176] Align memstats, error on non-200 responses

---
 socketserver/cmd/ffzsocketserver/console.go |  2 +-
 socketserver/internal/server/backend.go     | 17 ++++++++++++++++-
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/socketserver/cmd/ffzsocketserver/console.go b/socketserver/cmd/ffzsocketserver/console.go
index a09399b7..a6638d39 100644
--- a/socketserver/cmd/ffzsocketserver/console.go
+++ b/socketserver/cmd/ffzsocketserver/console.go
@@ -61,7 +61,7 @@ func commandLineConsole() {
 			if val.Mallocs == 0 {
 				continue
 			}
-			shell.Println(val.Size, "bytes:", val.Mallocs, "allocs", val.Frees, "frees")
+			shell.Print(fmt.Sprintf("%5d: %6d outstanding (%d total)\n", val.Size, val.Mallocs - val.Frees, val.Mallocs))
 		}
 		shell.Println(m.NumGC, "collections occurred")
 		return "", nil
diff --git a/socketserver/internal/server/backend.go b/socketserver/internal/server/backend.go
index 08547026..10b2bb73 100644
--- a/socketserver/internal/server/backend.go
+++ b/socketserver/internal/server/backend.go
@@ -136,9 +136,12 @@ func RequestRemoteData(remoteCommand, data string, auth AuthInfo) (responseStr s
 	if err != nil {
 		return "", err
 	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return "", httpError(resp.StatusCode)
+	}
 
 	respBytes, err := ioutil.ReadAll(resp.Body)
-	resp.Body.Close()
 	if err != nil {
 		return "", err
 	}
@@ -162,6 +165,10 @@ func SendAggregatedData(sealedForm url.Values) error {
 	if err != nil {
 		return err
 	}
+	if resp.StatusCode != 200 {
+		resp.Body.Close()
+		return httpError(resp.StatusCode)
+	}
 
 	return resp.Body.Close()
 }
@@ -180,6 +187,10 @@ func FetchBacklogData(chatSubs []string) ([]ClientMessage, error) {
 	if err != nil {
 		return nil, err
 	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return nil, httpError(resp.StatusCode)
+	}
 	dec := json.NewDecoder(resp.Body)
 	var messages []ClientMessage
 	err = dec.Decode(messages)
@@ -190,6 +201,10 @@ func FetchBacklogData(chatSubs []string) ([]ClientMessage, error) {
 	return messages, nil
 }
 
+func httpError(statusCode int) error {
+	return fmt.Errorf("backend http error: %d", statusCode)
+}
+
 func GenerateKeys(outputFile, serverId, theirPublicStr string) {
 	var err error
 	output := ConfigFile{

From d6f5b28ef571d9da0a79634545e009bee6253d33 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sat, 31 Oct 2015 23:49:34 -0700
Subject: [PATCH 020/176] this is a pretty screwed-up api if you ask me

---
 socketserver/internal/server/commands.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/socketserver/internal/server/commands.go b/socketserver/internal/server/commands.go
index db91c66a..0bb44efa 100644
--- a/socketserver/internal/server/commands.go
+++ b/socketserver/internal/server/commands.go
@@ -325,7 +325,9 @@ func HandleRemoteCommand(conn *websocket.Conn, client *ClientInfo, msg ClientMes
 			if err != nil {
 				client.MessageChannel <- ClientMessage{MessageID: msg.MessageID, Command: ErrorCommand, Arguments: err.Error()}
 			} else {
-				client.MessageChannel <- ClientMessage{MessageID: msg.MessageID, Command: SuccessCommand, origArguments: resp}
+				cm := ClientMessage{MessageID: msg.MessageID, Command: SuccessCommand, origArguments: resp}
+				cm.parseOrigArguments()
+				client.MessageChannel <- cm
 			}
 		}
 		client.MsgChannelKeepalive.RUnlock()

From 40e26b55350cc9b9ee657cc0a8356e805889a01d Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 1 Nov 2015 00:26:46 -0700
Subject: [PATCH 021/176] Add bunching get_link, without enabling it

---
 socketserver/internal/server/commands.go   | 120 ++++++++++++++++++++-
 socketserver/internal/server/handlecore.go |  10 +-
 socketserver/internal/server/utils.go      |  14 +++
 3 files changed, 137 insertions(+), 7 deletions(-)

diff --git a/socketserver/internal/server/commands.go b/socketserver/internal/server/commands.go
index 0bb44efa..7d029b82 100644
--- a/socketserver/internal/server/commands.go
+++ b/socketserver/internal/server/commands.go
@@ -316,6 +316,122 @@ func DoSendAggregateData() {
 	// done
 }
 
+type BunchedRequest struct {
+	Command Command
+	Param string
+}
+func BunchedRequestFromCM(msg *ClientMessage) BunchedRequest {
+	return BunchedRequest{Command: msg.Command, Param: msg.origArguments}
+}
+type BunchedResponse struct {
+	Response string
+	Timestamp time.Time
+}
+type BunchSubscriber struct {
+	Client *ClientInfo
+	MessageID int
+}
+
+type BunchSubscriberList struct {
+	sync.Mutex
+	Members []BunchSubscriber
+}
+
+var PendingBunchedRequests map[BunchedRequest]BunchSubscriberList = make(map[BunchedRequest]BunchSubscriberList)
+var PendingBunchLock sync.RWMutex
+var CompletedBunchedRequests map[BunchedRequest]BunchedResponse
+var CompletedBunchLock sync.RWMutex
+
+func HandleBunchedRemotecommand(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
+	br := BunchedRequestFromCM(&msg)
+
+	CompletedBunchLock.RLock()
+	resp, ok := CompletedBunchedRequests[br]
+	if ok && !resp.Timestamp.After(time.Now().Add(5 * time.Minute)) {
+		CompletedBunchLock.RUnlock()
+		return SuccessMessageFromString(resp.Response), nil
+	} else if ok {
+		CompletedBunchLock.RUnlock()
+
+		// Entry expired, let's remove it...
+		CompletedBunchLock.Lock()
+		// recheck condition
+		resp, ok = CompletedBunchedRequests[br]
+		if ok && resp.Timestamp.After(time.Now().Add(5 * time.Minute)) {
+			delete(CompletedBunchedRequests, br)
+		}
+		CompletedBunchLock.Unlock()
+	} else {
+		CompletedBunchLock.RUnlock()
+	}
+
+	// !!! unlocked on reply
+	client.MsgChannelKeepalive.RLock()
+
+	PendingBunchLock.RLock()
+	list, ok := PendingBunchedRequests[br]
+	var needToStart bool
+	if ok {
+		list.Lock()
+		AddToSliceB(&list.Members, client, msg.MessageID)
+		list.Unlock()
+		PendingBunchLock.RUnlock()
+
+		return ClientMessage{Command: AsyncResponseCommand}, nil
+ 	} else {
+		PendingBunchLock.RUnlock()
+		PendingBunchLock.Lock()
+		// RECHECK because someone else might have added it
+		list, ok = PendingBunchedRequests[br]
+		if ok {
+			list.Lock()
+			AddToSliceB(&list.Members, client, msg.MessageID)
+			list.Unlock()
+			PendingBunchLock.Unlock()
+			return ClientMessage{Command: AsyncResponseCommand}, nil
+		} else {
+			PendingBunchedRequests[br] = BunchSubscriberList{Members: []BunchSubscriber{{Client: client, MessageID: msg.MessageID}}}
+			needToStart = true
+			PendingBunchLock.Unlock()
+		}
+	}
+
+	if needToStart {
+		go func(request BunchedRequest) {
+			resp, err := RequestRemoteDataCached(string(request.Command), request.Param, AuthInfo{})
+
+			PendingBunchLock.Lock() // Prevent new signups
+			var msg ClientMessage
+			if err != nil {
+				CompletedBunchLock.Lock() // mutex on map
+				CompletedBunchedRequests[request] = BunchedResponse{Response: resp, Timestamp: time.Now()}
+				CompletedBunchLock.Unlock()
+
+				msg = SuccessMessageFromString(resp)
+			} else {
+				msg.Command = ErrorCommand
+				msg.Arguments = err.Error()
+			}
+
+			bsl := PendingBunchedRequests[request]
+			bsl.Lock()
+			for _, member := range bsl.Members {
+				msg.MessageID = member.MessageID
+				member.Client.MessageChannel <- msg
+				member.Client.MsgChannelKeepalive.RUnlock()
+			}
+			bsl.Unlock()
+
+			delete(PendingBunchedRequests, request)
+			PendingBunchLock.Unlock()
+		}(br)
+
+		return ClientMessage{Command: AsyncResponseCommand}, nil
+	} else {
+		panic("logic error")
+	}
+}
+
 func HandleRemoteCommand(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
 	go func(conn *websocket.Conn, msg ClientMessage, authInfo AuthInfo) {
 		resp, err := RequestRemoteDataCached(string(msg.Command), msg.origArguments, authInfo)
@@ -325,9 +441,7 @@ func HandleRemoteCommand(conn *websocket.Conn, client *ClientInfo, msg ClientMes
 			if err != nil {
 				client.MessageChannel <- ClientMessage{MessageID: msg.MessageID, Command: ErrorCommand, Arguments: err.Error()}
 			} else {
-				cm := ClientMessage{MessageID: msg.MessageID, Command: SuccessCommand, origArguments: resp}
-				cm.parseOrigArguments()
-				client.MessageChannel <- cm
+				client.MessageChannel <- SuccessMessageFromString(resp)
 			}
 		}
 		client.MsgChannelKeepalive.RUnlock()
diff --git a/socketserver/internal/server/handlecore.go b/socketserver/internal/server/handlecore.go
index f22bf842..be28f511 100644
--- a/socketserver/internal/server/handlecore.go
+++ b/socketserver/internal/server/handlecore.go
@@ -387,12 +387,14 @@ func MarshalClientMessage(clientMessage interface{}) (payloadType int, data []by
 }
 
 // Command handlers should use this to construct responses.
-func NewClientMessage(arguments interface{}) ClientMessage {
-	return ClientMessage{
-		MessageID: 0, // filled by the select loop
+func SuccessMessageFromString(arguments string) ClientMessage {
+	cm := ClientMessage{
+		MessageID: -1, // filled by the select loop
 		Command:   SuccessCommand,
-		Arguments: arguments,
+		origArguments: arguments,
 	}
+	cm.parseOrigArguments()
+	return cm
 }
 
 // Convenience method: Parse the arguments of the ClientMessage as a single string.
diff --git a/socketserver/internal/server/utils.go b/socketserver/internal/server/utils.go
index 8dbff0f4..48f8749f 100644
--- a/socketserver/internal/server/utils.go
+++ b/socketserver/internal/server/utils.go
@@ -159,3 +159,17 @@ func RemoveFromSliceC(ary *[]chan<- ClientMessage, val chan<- ClientMessage) boo
 	*ary = slice
 	return true
 }
+
+func AddToSliceB(ary *[]BunchSubscriber, client *ClientInfo, mid int) bool {
+	newSub := BunchSubscriber{Client: client, MessageID: mid}
+	slice := *ary
+	for _, v := range slice {
+		if v == newSub {
+			return false
+		}
+	}
+
+	slice = append(slice, newSub)
+	*ary = slice
+	return true
+}

From 41e9a8f1f84bd33000b444478bae787c667b8ee5 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 1 Nov 2015 00:30:49 -0700
Subject: [PATCH 022/176] Change SuccessCommand to 'ok'

---
 socketserver/internal/server/handlecore.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/socketserver/internal/server/handlecore.go b/socketserver/internal/server/handlecore.go
index be28f511..9fd110f1 100644
--- a/socketserver/internal/server/handlecore.go
+++ b/socketserver/internal/server/handlecore.go
@@ -43,7 +43,7 @@ var CommandHandlers = map[Command]CommandHandler{
 }
 
 // Sent by the server in ClientMessage.Command to indicate success.
-const SuccessCommand Command = "True"
+const SuccessCommand Command = "ok"
 
 // Sent by the server in ClientMessage.Command to indicate failure.
 const ErrorCommand Command = "error"

From 7c89ed98e353fb2022c678e3b0e7a70bbbcf6703 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 1 Nov 2015 12:44:41 -0800
Subject: [PATCH 023/176] These are replies, so they need the mid

---
 socketserver/internal/server/commands.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/socketserver/internal/server/commands.go b/socketserver/internal/server/commands.go
index 7d029b82..f1f0cf9f 100644
--- a/socketserver/internal/server/commands.go
+++ b/socketserver/internal/server/commands.go
@@ -441,7 +441,9 @@ func HandleRemoteCommand(conn *websocket.Conn, client *ClientInfo, msg ClientMes
 			if err != nil {
 				client.MessageChannel <- ClientMessage{MessageID: msg.MessageID, Command: ErrorCommand, Arguments: err.Error()}
 			} else {
-				client.MessageChannel <- SuccessMessageFromString(resp)
+				msg := SuccessMessageFromString(resp)
+				msg.MessageID = msg.MessageID
+				client.MessageChannel <- msg
 			}
 		}
 		client.MsgChannelKeepalive.RUnlock()

From f9323413aa25dfe1ce2dfb08103b5243f8d96779 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 1 Nov 2015 13:17:35 -0800
Subject: [PATCH 024/176] Fix bunching commands and enable

---
 socketserver/cmd/ffzsocketserver/console.go |  9 +++++-
 socketserver/internal/server/commands.go    | 36 +++++++++++++++------
 socketserver/internal/server/handlecore.go  | 11 ++++---
 socketserver/internal/server/types.go       |  3 ++
 4 files changed, 43 insertions(+), 16 deletions(-)

diff --git a/socketserver/cmd/ffzsocketserver/console.go b/socketserver/cmd/ffzsocketserver/console.go
index a6638d39..a82904bd 100644
--- a/socketserver/cmd/ffzsocketserver/console.go
+++ b/socketserver/cmd/ffzsocketserver/console.go
@@ -61,11 +61,18 @@ func commandLineConsole() {
 			if val.Mallocs == 0 {
 				continue
 			}
-			shell.Print(fmt.Sprintf("%5d: %6d outstanding (%d total)\n", val.Size, val.Mallocs - val.Frees, val.Mallocs))
+			shell.Print(fmt.Sprintf("%5d: %6d outstanding (%d total)\n", val.Size, val.Mallocs-val.Frees, val.Mallocs))
 		}
 		shell.Println(m.NumGC, "collections occurred")
 		return "", nil
 	})
 
+	shell.Register("panic", func(args ...string) (string, error) {
+		go func() {
+			panic("requested panic")
+		}()
+		return "", nil
+	})
+
 	shell.Start()
 }
diff --git a/socketserver/internal/server/commands.go b/socketserver/internal/server/commands.go
index f1f0cf9f..8d157722 100644
--- a/socketserver/internal/server/commands.go
+++ b/socketserver/internal/server/commands.go
@@ -318,17 +318,19 @@ func DoSendAggregateData() {
 
 type BunchedRequest struct {
 	Command Command
-	Param string
+	Param   string
 }
+
 func BunchedRequestFromCM(msg *ClientMessage) BunchedRequest {
 	return BunchedRequest{Command: msg.Command, Param: msg.origArguments}
 }
+
 type BunchedResponse struct {
-	Response string
+	Response  string
 	Timestamp time.Time
 }
 type BunchSubscriber struct {
-	Client *ClientInfo
+	Client    *ClientInfo
 	MessageID int
 }
 
@@ -337,17 +339,31 @@ type BunchSubscriberList struct {
 	Members []BunchSubscriber
 }
 
-var PendingBunchedRequests map[BunchedRequest]BunchSubscriberList = make(map[BunchedRequest]BunchSubscriberList)
+var PendingBunchedRequests map[BunchedRequest]*BunchSubscriberList = make(map[BunchedRequest]*BunchSubscriberList)
 var PendingBunchLock sync.RWMutex
-var CompletedBunchedRequests map[BunchedRequest]BunchedResponse
+var CompletedBunchedRequests map[BunchedRequest]BunchedResponse = make(map[BunchedRequest]BunchedResponse)
 var CompletedBunchLock sync.RWMutex
 
+func bunchingJanitor() {
+	for {
+		time.Sleep(5 * time.Minute)
+		keepIfAfter := time.Now().Add(-5 * time.Minute)
+		CompletedBunchLock.Lock()
+		for req, resp := range CompletedBunchedRequests {
+			if !resp.Timestamp.After(keepIfAfter) {
+				delete(CompletedBunchedRequests, req)
+			}
+		}
+		CompletedBunchLock.Unlock()
+	}
+}
+
 func HandleBunchedRemotecommand(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
 	br := BunchedRequestFromCM(&msg)
 
 	CompletedBunchLock.RLock()
 	resp, ok := CompletedBunchedRequests[br]
-	if ok && !resp.Timestamp.After(time.Now().Add(5 * time.Minute)) {
+	if ok && resp.Timestamp.After(time.Now().Add(-5*time.Minute)) {
 		CompletedBunchLock.RUnlock()
 		return SuccessMessageFromString(resp.Response), nil
 	} else if ok {
@@ -357,7 +373,7 @@ func HandleBunchedRemotecommand(conn *websocket.Conn, client *ClientInfo, msg Cl
 		CompletedBunchLock.Lock()
 		// recheck condition
 		resp, ok = CompletedBunchedRequests[br]
-		if ok && resp.Timestamp.After(time.Now().Add(5 * time.Minute)) {
+		if ok && !resp.Timestamp.After(time.Now().Add(-5*time.Minute)) {
 			delete(CompletedBunchedRequests, br)
 		}
 		CompletedBunchLock.Unlock()
@@ -378,7 +394,7 @@ func HandleBunchedRemotecommand(conn *websocket.Conn, client *ClientInfo, msg Cl
 		PendingBunchLock.RUnlock()
 
 		return ClientMessage{Command: AsyncResponseCommand}, nil
- 	} else {
+	} else {
 		PendingBunchLock.RUnlock()
 		PendingBunchLock.Lock()
 		// RECHECK because someone else might have added it
@@ -390,7 +406,7 @@ func HandleBunchedRemotecommand(conn *websocket.Conn, client *ClientInfo, msg Cl
 			PendingBunchLock.Unlock()
 			return ClientMessage{Command: AsyncResponseCommand}, nil
 		} else {
-			PendingBunchedRequests[br] = BunchSubscriberList{Members: []BunchSubscriber{{Client: client, MessageID: msg.MessageID}}}
+			PendingBunchedRequests[br] = &BunchSubscriberList{Members: []BunchSubscriber{{Client: client, MessageID: msg.MessageID}}}
 			needToStart = true
 			PendingBunchLock.Unlock()
 		}
@@ -402,7 +418,7 @@ func HandleBunchedRemotecommand(conn *websocket.Conn, client *ClientInfo, msg Cl
 
 			PendingBunchLock.Lock() // Prevent new signups
 			var msg ClientMessage
-			if err != nil {
+			if err == nil {
 				CompletedBunchLock.Lock() // mutex on map
 				CompletedBunchedRequests[request] = BunchedResponse{Response: resp, Timestamp: time.Now()}
 				CompletedBunchLock.Unlock()
diff --git a/socketserver/internal/server/handlecore.go b/socketserver/internal/server/handlecore.go
index 9fd110f1..06ae7a93 100644
--- a/socketserver/internal/server/handlecore.go
+++ b/socketserver/internal/server/handlecore.go
@@ -36,8 +36,8 @@ var CommandHandlers = map[Command]CommandHandler{
 	"survey":        HandleSurvey,
 
 	"twitch_emote":          HandleRemoteCommand,
-	"get_link":              HandleRemoteCommand,
-	"get_display_name":      HandleRemoteCommand,
+	"get_link":              HandleBunchedRemotecommand,
+	"get_display_name":      HandleBunchedRemotecommand,
 	"update_follow_buttons": HandleRemoteCommand,
 	"chat_history":          HandleRemoteCommand,
 }
@@ -151,6 +151,7 @@ func HandleSocketConnection(conn *websocket.Conn) {
 
 	var client ClientInfo
 	client.MessageChannel = _serverMessageChan
+	client.RemoteAddr = conn.RemoteAddr()
 
 	// Launch receiver goroutine
 	go func(errorChan chan<- error, clientChan chan<- ClientMessage, stoppedChan <-chan struct{}) {
@@ -232,7 +233,7 @@ RunLoop:
 		case smsg := <-serverMessageChan:
 			SendMessage(conn, smsg)
 
-		case <- time.After(1 * time.Minute):
+		case <-time.After(1 * time.Minute):
 			client.pingCount++
 			if client.pingCount == 5 {
 				CloseConnection(conn, &CloseTimedOut)
@@ -389,8 +390,8 @@ func MarshalClientMessage(clientMessage interface{}) (payloadType int, data []by
 // Command handlers should use this to construct responses.
 func SuccessMessageFromString(arguments string) ClientMessage {
 	cm := ClientMessage{
-		MessageID: -1, // filled by the select loop
-		Command:   SuccessCommand,
+		MessageID:     -1, // filled by the select loop
+		Command:       SuccessCommand,
 		origArguments: arguments,
 	}
 	cm.parseOrigArguments()
diff --git a/socketserver/internal/server/types.go b/socketserver/internal/server/types.go
index 0fd79a39..1420401d 100644
--- a/socketserver/internal/server/types.go
+++ b/socketserver/internal/server/types.go
@@ -3,6 +3,7 @@ package server
 import (
 	"encoding/json"
 	"github.com/satori/go.uuid"
+	"net"
 	"sync"
 	"time"
 )
@@ -68,6 +69,8 @@ type ClientInfo struct {
 	// TODO(riking) - does this need to be protected cross-thread?
 	AuthInfo
 
+	RemoteAddr net.Addr
+
 	// Username validation nonce.
 	ValidationNonce string
 

From 5f0e1ffc4ee988a3efd0b407d950db7471be20b9 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 1 Nov 2015 14:47:50 -0800
Subject: [PATCH 025/176] revert change for a clean merge

---
 src/socket.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/socket.js b/src/socket.js
index b9b9c210..99847bfc 100644
--- a/src/socket.js
+++ b/src/socket.js
@@ -37,6 +37,7 @@ FFZ.prototype.ws_iframe = function() {
 
 FFZ.prototype.ws_create = function() {
 	// Disable sockets for now.
+	return;
 
 	var f = this, ws;
 

From 46887cdb5d2dec68ecce19bcae2a3e1f9e554bae Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 1 Nov 2015 18:23:22 -0800
Subject: [PATCH 026/176] Cap emote usage reports, error on negative

---
 socketserver/internal/server/commands.go | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/socketserver/internal/server/commands.go b/socketserver/internal/server/commands.go
index 8d157722..4fbb8688 100644
--- a/socketserver/internal/server/commands.go
+++ b/socketserver/internal/server/commands.go
@@ -2,6 +2,7 @@ package server
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"github.com/gorilla/websocket"
 	"github.com/satori/go.uuid"
@@ -231,12 +232,28 @@ func HandleTrackFollow(conn *websocket.Conn, client *ClientInfo, msg ClientMessa
 
 var AggregateEmoteUsage map[int]map[string]int = make(map[int]map[string]int)
 var AggregateEmoteUsageLock sync.Mutex
+var ErrorNegativeEmoteUsage = errors.New("Emote usage count cannot be negative")
 
 func HandleEmoticonUses(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
 	// arguments is [1]map[EmoteId]map[RoomName]float64
 
 	mapRoot := msg.Arguments.([]interface{})[0].(map[string]interface{})
 
+	for strEmote, val1 := range mapRoot {
+		_, err = strconv.Atoi(strEmote)
+		if err != nil {
+			return
+		}
+		mapInner := val1.(map[string]interface{})
+		for _, val2 := range mapInner {
+			var count int = int(val2.(float64))
+			if count <= 0 {
+				err = ErrorNegativeEmoteUsage
+				return
+			}
+		}
+	}
+
 	AggregateEmoteUsageLock.Lock()
 	defer AggregateEmoteUsageLock.Unlock()
 
@@ -256,6 +273,9 @@ func HandleEmoticonUses(conn *websocket.Conn, client *ClientInfo, msg ClientMess
 		mapInner := val1.(map[string]interface{})
 		for roomName, val2 := range mapInner {
 			var count int = int(val2.(float64))
+			if count > 200 {
+				count = 200
+			}
 			destMapInner[roomName] += count
 		}
 	}

From 013e49e2c581bb9b289390356a5d0c91f29ce2f1 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Mon, 2 Nov 2015 22:54:53 -0800
Subject: [PATCH 027/176] Change MsgChannelKeepalive to a sync.WaitGroup

---
 socketserver/internal/server/commands.go   | 35 +++++++++-------------
 socketserver/internal/server/handlecore.go |  6 ++--
 socketserver/internal/server/types.go      |  4 +--
 3 files changed, 19 insertions(+), 26 deletions(-)

diff --git a/socketserver/internal/server/commands.go b/socketserver/internal/server/commands.go
index 4fbb8688..2b13eb09 100644
--- a/socketserver/internal/server/commands.go
+++ b/socketserver/internal/server/commands.go
@@ -86,18 +86,14 @@ func HandleReady(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (r
 	client.MakePendingRequests = nil
 	client.Mutex.Unlock()
 
+	client.MsgChannelKeepalive.Add(1)
 	go func() {
-		client.MsgChannelKeepalive.RLock()
-		if client.MessageChannel == nil {
-			return
-		}
-
 		client.MessageChannel <- ClientMessage{MessageID: msg.MessageID, Command: SuccessCommand}
 		SendBacklogForNewClient(client)
 		if disconnectAt != 0 {
 			SendTimedBacklogMessages(client, time.Unix(disconnectAt, 0))
 		}
-		client.MsgChannelKeepalive.RUnlock()
+		client.MsgChannelKeepalive.Done()
 	}()
 	return ClientMessage{Command: AsyncResponseCommand}, nil
 }
@@ -192,13 +188,13 @@ func GetSubscriptionBacklog(conn *websocket.Conn, client *ClientInfo) {
 	}
 
 	// Deliver to client
-	client.MsgChannelKeepalive.RLock()
+	client.MsgChannelKeepalive.Add(1)
 	if client.MessageChannel != nil {
 		for _, msg := range messages {
 			client.MessageChannel <- msg
 		}
 	}
-	client.MsgChannelKeepalive.RUnlock()
+	client.MsgChannelKeepalive.Done()
 }
 
 func HandleSurvey(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
@@ -401,8 +397,7 @@ func HandleBunchedRemotecommand(conn *websocket.Conn, client *ClientInfo, msg Cl
 		CompletedBunchLock.RUnlock()
 	}
 
-	// !!! unlocked on reply
-	client.MsgChannelKeepalive.RLock()
+	client.MsgChannelKeepalive.Add(1)
 
 	PendingBunchLock.RLock()
 	list, ok := PendingBunchedRequests[br]
@@ -454,7 +449,7 @@ func HandleBunchedRemotecommand(conn *websocket.Conn, client *ClientInfo, msg Cl
 			for _, member := range bsl.Members {
 				msg.MessageID = member.MessageID
 				member.Client.MessageChannel <- msg
-				member.Client.MsgChannelKeepalive.RUnlock()
+				member.Client.MsgChannelKeepalive.Done()
 			}
 			bsl.Unlock()
 
@@ -469,20 +464,18 @@ func HandleBunchedRemotecommand(conn *websocket.Conn, client *ClientInfo, msg Cl
 }
 
 func HandleRemoteCommand(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
+	client.MsgChannelKeepalive.Add(1)
 	go func(conn *websocket.Conn, msg ClientMessage, authInfo AuthInfo) {
 		resp, err := RequestRemoteDataCached(string(msg.Command), msg.origArguments, authInfo)
 
-		client.MsgChannelKeepalive.RLock()
-		if client.MessageChannel != nil {
-			if err != nil {
-				client.MessageChannel <- ClientMessage{MessageID: msg.MessageID, Command: ErrorCommand, Arguments: err.Error()}
-			} else {
-				msg := SuccessMessageFromString(resp)
-				msg.MessageID = msg.MessageID
-				client.MessageChannel <- msg
-			}
+		if err != nil {
+			client.MessageChannel <- ClientMessage{MessageID: msg.MessageID, Command: ErrorCommand, Arguments: err.Error()}
+		} else {
+			msg := SuccessMessageFromString(resp)
+			msg.MessageID = msg.MessageID
+			client.MessageChannel <- msg
 		}
-		client.MsgChannelKeepalive.RUnlock()
+		client.MsgChannelKeepalive.Done()
 	}(conn, msg, client.AuthInfo)
 
 	return ClientMessage{Command: AsyncResponseCommand}, nil
diff --git a/socketserver/internal/server/handlecore.go b/socketserver/internal/server/handlecore.go
index 06ae7a93..35215577 100644
--- a/socketserver/internal/server/handlecore.go
+++ b/socketserver/internal/server/handlecore.go
@@ -257,11 +257,11 @@ RunLoop:
 	// Stop getting messages...
 	UnsubscribeAll(&client)
 
-	client.MsgChannelKeepalive.Lock()
+	// Wait for pending jobs to finish...
+	client.MsgChannelKeepalive.Wait()
 	client.MessageChannel = nil
-	client.MsgChannelKeepalive.Unlock()
 
-	// And finished.
+	// And done.
 	// Close the channel so the draining goroutine can finish, too.
 	close(_serverMessageChan)
 
diff --git a/socketserver/internal/server/types.go b/socketserver/internal/server/types.go
index 1420401d..97dae35a 100644
--- a/socketserver/internal/server/types.go
+++ b/socketserver/internal/server/types.go
@@ -91,8 +91,8 @@ type ClientInfo struct {
 	// This field will be nil before it is closed.
 	MessageChannel chan<- ClientMessage
 
-	// Take a read-lock on this before checking whether MessageChannel is nil.
-	MsgChannelKeepalive sync.RWMutex
+	// Take out an Add() on this during a command if you need to use the MessageChannel later.
+	MsgChannelKeepalive sync.WaitGroup
 
 	// The number of pings sent without a response
 	pingCount int

From db486e4ebaa446347ea1d13f3a06e5aef0138b02 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Mon, 2 Nov 2015 22:59:38 -0800
Subject: [PATCH 028/176] Refactor bunched remote command

---
 socketserver/internal/server/commands.go   | 96 +++++++++-------------
 socketserver/internal/server/handlecore.go |  4 +-
 2 files changed, 40 insertions(+), 60 deletions(-)

diff --git a/socketserver/internal/server/commands.go b/socketserver/internal/server/commands.go
index 2b13eb09..54557192 100644
--- a/socketserver/internal/server/commands.go
+++ b/socketserver/internal/server/commands.go
@@ -356,7 +356,7 @@ type BunchSubscriberList struct {
 }
 
 var PendingBunchedRequests map[BunchedRequest]*BunchSubscriberList = make(map[BunchedRequest]*BunchSubscriberList)
-var PendingBunchLock sync.RWMutex
+var PendingBunchLock sync.Mutex
 var CompletedBunchedRequests map[BunchedRequest]BunchedResponse = make(map[BunchedRequest]BunchedResponse)
 var CompletedBunchLock sync.RWMutex
 
@@ -374,16 +374,16 @@ func bunchingJanitor() {
 	}
 }
 
-func HandleBunchedRemotecommand(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
+func HandleBunchedRemoteCommand(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
 	br := BunchedRequestFromCM(&msg)
 
 	CompletedBunchLock.RLock()
 	resp, ok := CompletedBunchedRequests[br]
+	CompletedBunchLock.RUnlock()
+
 	if ok && resp.Timestamp.After(time.Now().Add(-5*time.Minute)) {
-		CompletedBunchLock.RUnlock()
 		return SuccessMessageFromString(resp.Response), nil
 	} else if ok {
-		CompletedBunchLock.RUnlock()
 
 		// Entry expired, let's remove it...
 		CompletedBunchLock.Lock()
@@ -393,74 +393,54 @@ func HandleBunchedRemotecommand(conn *websocket.Conn, client *ClientInfo, msg Cl
 			delete(CompletedBunchedRequests, br)
 		}
 		CompletedBunchLock.Unlock()
-	} else {
-		CompletedBunchLock.RUnlock()
 	}
 
-	client.MsgChannelKeepalive.Add(1)
-
-	PendingBunchLock.RLock()
+	PendingBunchLock.Lock()
 	list, ok := PendingBunchedRequests[br]
-	var needToStart bool
 	if ok {
 		list.Lock()
 		AddToSliceB(&list.Members, client, msg.MessageID)
 		list.Unlock()
-		PendingBunchLock.RUnlock()
+		PendingBunchLock.Unlock()
+		client.MsgChannelKeepalive.Add(1)
 
 		return ClientMessage{Command: AsyncResponseCommand}, nil
-	} else {
-		PendingBunchLock.RUnlock()
-		PendingBunchLock.Lock()
-		// RECHECK because someone else might have added it
-		list, ok = PendingBunchedRequests[br]
-		if ok {
-			list.Lock()
-			AddToSliceB(&list.Members, client, msg.MessageID)
-			list.Unlock()
-			PendingBunchLock.Unlock()
-			return ClientMessage{Command: AsyncResponseCommand}, nil
+	}
+
+	PendingBunchedRequests[br] = &BunchSubscriberList{Members: []BunchSubscriber{{Client: client, MessageID: msg.MessageID}}}
+	PendingBunchLock.Unlock()
+	client.MsgChannelKeepalive.Add(1)
+
+	go func(request BunchedRequest) {
+		resp, err := RequestRemoteDataCached(string(request.Command), request.Param, AuthInfo{})
+
+		PendingBunchLock.Lock() // Prevent new signups
+		var msg ClientMessage
+		if err == nil {
+			CompletedBunchLock.Lock() // mutex on map
+			CompletedBunchedRequests[request] = BunchedResponse{Response: resp, Timestamp: time.Now()}
+			CompletedBunchLock.Unlock()
+
+			msg = SuccessMessageFromString(resp)
 		} else {
-			PendingBunchedRequests[br] = &BunchSubscriberList{Members: []BunchSubscriber{{Client: client, MessageID: msg.MessageID}}}
-			needToStart = true
-			PendingBunchLock.Unlock()
+			msg.Command = ErrorCommand
+			msg.Arguments = err.Error()
 		}
-	}
 
-	if needToStart {
-		go func(request BunchedRequest) {
-			resp, err := RequestRemoteDataCached(string(request.Command), request.Param, AuthInfo{})
+		bsl := PendingBunchedRequests[request]
+		bsl.Lock()
+		for _, member := range bsl.Members {
+			msg.MessageID = member.MessageID
+			member.Client.MessageChannel <- msg
+			member.Client.MsgChannelKeepalive.Done()
+		}
+		bsl.Unlock()
 
-			PendingBunchLock.Lock() // Prevent new signups
-			var msg ClientMessage
-			if err == nil {
-				CompletedBunchLock.Lock() // mutex on map
-				CompletedBunchedRequests[request] = BunchedResponse{Response: resp, Timestamp: time.Now()}
-				CompletedBunchLock.Unlock()
+		delete(PendingBunchedRequests, request)
+		PendingBunchLock.Unlock()
+	}(br)
 
-				msg = SuccessMessageFromString(resp)
-			} else {
-				msg.Command = ErrorCommand
-				msg.Arguments = err.Error()
-			}
-
-			bsl := PendingBunchedRequests[request]
-			bsl.Lock()
-			for _, member := range bsl.Members {
-				msg.MessageID = member.MessageID
-				member.Client.MessageChannel <- msg
-				member.Client.MsgChannelKeepalive.Done()
-			}
-			bsl.Unlock()
-
-			delete(PendingBunchedRequests, request)
-			PendingBunchLock.Unlock()
-		}(br)
-
-		return ClientMessage{Command: AsyncResponseCommand}, nil
-	} else {
-		panic("logic error")
-	}
+	return ClientMessage{Command: AsyncResponseCommand}, nil
 }
 
 func HandleRemoteCommand(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
diff --git a/socketserver/internal/server/handlecore.go b/socketserver/internal/server/handlecore.go
index 35215577..935fbafd 100644
--- a/socketserver/internal/server/handlecore.go
+++ b/socketserver/internal/server/handlecore.go
@@ -36,8 +36,8 @@ var CommandHandlers = map[Command]CommandHandler{
 	"survey":        HandleSurvey,
 
 	"twitch_emote":          HandleRemoteCommand,
-	"get_link":              HandleBunchedRemotecommand,
-	"get_display_name":      HandleBunchedRemotecommand,
+	"get_link":              HandleBunchedRemoteCommand,
+	"get_display_name":      HandleBunchedRemoteCommand,
 	"update_follow_buttons": HandleRemoteCommand,
 	"chat_history":          HandleRemoteCommand,
 }

From 7b095542fcc6677c995b400186336fd69ca1cd60 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Tue, 3 Nov 2015 16:44:42 -0800
Subject: [PATCH 029/176] Attempt to fix locking issue

---
 socketserver/internal/server/commands.go   | 5 ++++-
 socketserver/internal/server/handlecore.go | 6 +++++-
 socketserver/internal/server/types.go      | 2 ++
 3 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/socketserver/internal/server/commands.go b/socketserver/internal/server/commands.go
index 54557192..57d1e496 100644
--- a/socketserver/internal/server/commands.go
+++ b/socketserver/internal/server/commands.go
@@ -431,7 +431,10 @@ func HandleBunchedRemoteCommand(conn *websocket.Conn, client *ClientInfo, msg Cl
 		bsl.Lock()
 		for _, member := range bsl.Members {
 			msg.MessageID = member.MessageID
-			member.Client.MessageChannel <- msg
+			select {
+			case member.Client.MessageChannel <- msg:
+			case <-member.Client.MsgChannelIsDone:
+			}
 			member.Client.MsgChannelKeepalive.Done()
 		}
 		bsl.Unlock()
diff --git a/socketserver/internal/server/handlecore.go b/socketserver/internal/server/handlecore.go
index 935fbafd..f6778d55 100644
--- a/socketserver/internal/server/handlecore.go
+++ b/socketserver/internal/server/handlecore.go
@@ -152,6 +152,7 @@ func HandleSocketConnection(conn *websocket.Conn) {
 	var client ClientInfo
 	client.MessageChannel = _serverMessageChan
 	client.RemoteAddr = conn.RemoteAddr()
+	client.MsgChannelIsDone = stoppedChan
 
 	// Launch receiver goroutine
 	go func(errorChan chan<- error, clientChan chan<- ClientMessage, stoppedChan <-chan struct{}) {
@@ -186,7 +187,10 @@ func HandleSocketConnection(conn *websocket.Conn) {
 		if err != io.EOF && !isClose {
 			log.Println("Error while reading from client:", err)
 		}
-		errorChan <- err
+		select {
+		case errorChan <- err:
+		case <-stoppedChan:
+		}
 		close(errorChan)
 		close(clientChan)
 		// exit
diff --git a/socketserver/internal/server/types.go b/socketserver/internal/server/types.go
index 97dae35a..2f6013b3 100644
--- a/socketserver/internal/server/types.go
+++ b/socketserver/internal/server/types.go
@@ -91,6 +91,8 @@ type ClientInfo struct {
 	// This field will be nil before it is closed.
 	MessageChannel chan<- ClientMessage
 
+	MsgChannelIsDone <-chan struct{}
+
 	// Take out an Add() on this during a command if you need to use the MessageChannel later.
 	MsgChannelKeepalive sync.WaitGroup
 

From 64cc80c79a783ac0a7e40b907b02b82974527235 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Tue, 3 Nov 2015 17:57:35 -0800
Subject: [PATCH 030/176] maybe the helper function is wrong

---
 socketserver/internal/server/commands.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/socketserver/internal/server/commands.go b/socketserver/internal/server/commands.go
index 57d1e496..018c8c97 100644
--- a/socketserver/internal/server/commands.go
+++ b/socketserver/internal/server/commands.go
@@ -454,8 +454,8 @@ func HandleRemoteCommand(conn *websocket.Conn, client *ClientInfo, msg ClientMes
 		if err != nil {
 			client.MessageChannel <- ClientMessage{MessageID: msg.MessageID, Command: ErrorCommand, Arguments: err.Error()}
 		} else {
-			msg := SuccessMessageFromString(resp)
-			msg.MessageID = msg.MessageID
+			msg := ClientMessage{MessageID: msg.MessageID, Command: SuccessCommand, origArguments: resp}
+			msg.parseOrigArguments()
 			client.MessageChannel <- msg
 		}
 		client.MsgChannelKeepalive.Done()

From d09e4e592c6f0bf7f667acad10763cdc64984f23 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 4 Nov 2015 12:09:24 -0800
Subject: [PATCH 031/176] Remove response cache for bunched commands, simplify
 locks

---
 socketserver/internal/server/commands.go | 56 ++++--------------------
 1 file changed, 8 insertions(+), 48 deletions(-)

diff --git a/socketserver/internal/server/commands.go b/socketserver/internal/server/commands.go
index 018c8c97..cd735d35 100644
--- a/socketserver/internal/server/commands.go
+++ b/socketserver/internal/server/commands.go
@@ -357,77 +357,41 @@ type BunchSubscriberList struct {
 
 var PendingBunchedRequests map[BunchedRequest]*BunchSubscriberList = make(map[BunchedRequest]*BunchSubscriberList)
 var PendingBunchLock sync.Mutex
-var CompletedBunchedRequests map[BunchedRequest]BunchedResponse = make(map[BunchedRequest]BunchedResponse)
-var CompletedBunchLock sync.RWMutex
-
-func bunchingJanitor() {
-	for {
-		time.Sleep(5 * time.Minute)
-		keepIfAfter := time.Now().Add(-5 * time.Minute)
-		CompletedBunchLock.Lock()
-		for req, resp := range CompletedBunchedRequests {
-			if !resp.Timestamp.After(keepIfAfter) {
-				delete(CompletedBunchedRequests, req)
-			}
-		}
-		CompletedBunchLock.Unlock()
-	}
-}
 
 func HandleBunchedRemoteCommand(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
 	br := BunchedRequestFromCM(&msg)
 
-	CompletedBunchLock.RLock()
-	resp, ok := CompletedBunchedRequests[br]
-	CompletedBunchLock.RUnlock()
-
-	if ok && resp.Timestamp.After(time.Now().Add(-5*time.Minute)) {
-		return SuccessMessageFromString(resp.Response), nil
-	} else if ok {
-
-		// Entry expired, let's remove it...
-		CompletedBunchLock.Lock()
-		// recheck condition
-		resp, ok = CompletedBunchedRequests[br]
-		if ok && !resp.Timestamp.After(time.Now().Add(-5*time.Minute)) {
-			delete(CompletedBunchedRequests, br)
-		}
-		CompletedBunchLock.Unlock()
-	}
-
 	PendingBunchLock.Lock()
+	defer PendingBunchLock.Unlock()
 	list, ok := PendingBunchedRequests[br]
 	if ok {
 		list.Lock()
 		AddToSliceB(&list.Members, client, msg.MessageID)
 		list.Unlock()
-		PendingBunchLock.Unlock()
-		client.MsgChannelKeepalive.Add(1)
 
 		return ClientMessage{Command: AsyncResponseCommand}, nil
 	}
 
 	PendingBunchedRequests[br] = &BunchSubscriberList{Members: []BunchSubscriber{{Client: client, MessageID: msg.MessageID}}}
-	PendingBunchLock.Unlock()
-	client.MsgChannelKeepalive.Add(1)
 
 	go func(request BunchedRequest) {
 		resp, err := RequestRemoteDataCached(string(request.Command), request.Param, AuthInfo{})
 
-		PendingBunchLock.Lock() // Prevent new signups
 		var msg ClientMessage
 		if err == nil {
-			CompletedBunchLock.Lock() // mutex on map
-			CompletedBunchedRequests[request] = BunchedResponse{Response: resp, Timestamp: time.Now()}
-			CompletedBunchLock.Unlock()
-
-			msg = SuccessMessageFromString(resp)
+			msg.Command = SuccessCommand
+			msg.origArguments = resp
+			msg.parseOrigArguments()
 		} else {
 			msg.Command = ErrorCommand
 			msg.Arguments = err.Error()
 		}
 
+		PendingBunchLock.Lock()
 		bsl := PendingBunchedRequests[request]
+		delete(PendingBunchedRequests, request)
+		PendingBunchLock.Unlock()
+
 		bsl.Lock()
 		for _, member := range bsl.Members {
 			msg.MessageID = member.MessageID
@@ -435,12 +399,8 @@ func HandleBunchedRemoteCommand(conn *websocket.Conn, client *ClientInfo, msg Cl
 			case member.Client.MessageChannel <- msg:
 			case <-member.Client.MsgChannelIsDone:
 			}
-			member.Client.MsgChannelKeepalive.Done()
 		}
 		bsl.Unlock()
-
-		delete(PendingBunchedRequests, request)
-		PendingBunchLock.Unlock()
 	}(br)
 
 	return ClientMessage{Command: AsyncResponseCommand}, nil

From 3ad095acf4573ab91b23252ea2749f77b069cb34 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 4 Nov 2015 12:25:42 -0800
Subject: [PATCH 032/176] Forward application/json errors to client

---
 socketserver/internal/server/backend.go | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/socketserver/internal/server/backend.go b/socketserver/internal/server/backend.go
index 10b2bb73..68be3b7e 100644
--- a/socketserver/internal/server/backend.go
+++ b/socketserver/internal/server/backend.go
@@ -105,6 +105,12 @@ func HBackendPublishRequest(w http.ResponseWriter, r *http.Request) {
 	fmt.Fprint(w, count)
 }
 
+type BackendForwardedError string
+
+func (bfe BackendForwardedError) Error() string {
+	return string(bfe)
+}
+
 func RequestRemoteDataCached(remoteCommand, data string, auth AuthInfo) (string, error) {
 	cached, ok := responseCache.Get(getCacheKey(remoteCommand, data))
 	if ok {
@@ -137,9 +143,6 @@ func RequestRemoteData(remoteCommand, data string, auth AuthInfo) (responseStr s
 		return "", err
 	}
 	defer resp.Body.Close()
-	if resp.StatusCode != 200 {
-		return "", httpError(resp.StatusCode)
-	}
 
 	respBytes, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
@@ -148,6 +151,14 @@ func RequestRemoteData(remoteCommand, data string, auth AuthInfo) (responseStr s
 
 	responseStr = string(respBytes)
 
+	if resp.StatusCode != 200 {
+		if resp.Header.Get("Content-Type") == "application/json" {
+			return "", BackendForwardedError(responseStr)
+		} else {
+			return "", httpError(resp.StatusCode)
+		}
+	}
+
 	if resp.Header.Get("FFZ-Cache") != "" {
 		durSecs, err := strconv.ParseInt(resp.Header.Get("FFZ-Cache"), 10, 64)
 		if err != nil {

From 525b19eccb3f211eb9d36836a8209c7193d88cc0 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 4 Nov 2015 15:11:49 -0800
Subject: [PATCH 033/176] Add reporting of added/removed subcriptions

---
 socketserver/internal/server/backend.go   | 63 +++++++++++++++++++++--
 socketserver/internal/server/publisher.go | 26 ++++++++--
 socketserver/internal/server/types.go     |  6 +--
 3 files changed, 84 insertions(+), 11 deletions(-)

diff --git a/socketserver/internal/server/backend.go b/socketserver/internal/server/backend.go
index 68be3b7e..3e98e742 100644
--- a/socketserver/internal/server/backend.go
+++ b/socketserver/internal/server/backend.go
@@ -15,6 +15,7 @@ import (
 	"strings"
 	"sync"
 	"time"
+	"github.com/gorilla/websocket"
 )
 
 var backendHttpClient http.Client
@@ -23,6 +24,7 @@ var responseCache *cache.Cache
 
 var getBacklogUrl string
 var postStatisticsUrl string
+var addTopicUrl string
 
 var backendSharedKey [32]byte
 var serverId int
@@ -35,10 +37,11 @@ func SetupBackend(config *ConfigFile) {
 	if responseCache != nil {
 		responseCache.Flush()
 	}
-	responseCache = cache.New(60*time.Second, 120*time.Second)
+	responseCache = cache.New(60 * time.Second, 120 * time.Second)
 
 	getBacklogUrl = fmt.Sprintf("%s/backlog", backendUrl)
 	postStatisticsUrl = fmt.Sprintf("%s/stats", backendUrl)
+	addTopicUrl = fmt.Sprintf("%s/topics", backendUrl)
 
 	messageBufferPool.New = New4KByteBuffer
 
@@ -203,15 +206,69 @@ func FetchBacklogData(chatSubs []string) ([]ClientMessage, error) {
 		return nil, httpError(resp.StatusCode)
 	}
 	dec := json.NewDecoder(resp.Body)
-	var messages []ClientMessage
-	err = dec.Decode(messages)
+	var messageStrings []string
+	err = dec.Decode(messageStrings)
 	if err != nil {
 		return nil, err
 	}
 
+	var messages = make([]ClientMessage, len(messageStrings))
+	for i, str := range messageStrings {
+		UnmarshalClientMessage([]byte(str), websocket.TextMessage, &messages[i])
+	}
+
 	return messages, nil
 }
 
+type NotOkError struct {
+	Response string
+	Code int
+}
+func (noe NotOkError) Error() string {
+	return fmt.Sprintf("backend returned %d: %s", noe.Code, noe.Response)
+}
+
+func SendNewTopicNotice(topic string) error {
+	return sendTopicNotice(topic, true)
+}
+
+func SendCleanupTopicsNotice(topics []string) error {
+	return sendTopicNotice(strings.Join(topics, ","), false)
+}
+
+func sendTopicNotice(topic string, added bool) error {
+	formData := url.Values{}
+	formData.Set("channels", topic)
+	if added {
+		formData.Set("added", "t")
+	} else {
+		formData.Set("added", "f")
+	}
+
+	sealedForm, err := SealRequest(formData)
+	if err != nil {
+		return err
+	}
+
+	resp, err := backendHttpClient.PostForm(addTopicUrl, sealedForm)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	respBytes, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	respStr := string(respBytes)
+	if respStr != "ok" {
+		return NotOkError{Code: resp.StatusCode, Response: respStr}
+	}
+
+	return nil
+}
+
 func httpError(statusCode int) error {
 	return fmt.Errorf("backend http error: %d", statusCode)
 }
diff --git a/socketserver/internal/server/publisher.go b/socketserver/internal/server/publisher.go
index db1f74ac..54a5b3d1 100644
--- a/socketserver/internal/server/publisher.go
+++ b/socketserver/internal/server/publisher.go
@@ -6,6 +6,7 @@ package server
 import (
 	"sync"
 	"time"
+	"log"
 )
 
 type SubscriberList struct {
@@ -82,6 +83,14 @@ func _subscribeWhileRlocked(channelName string, value chan<- ClientMessage) {
 		list.Members = []chan<- ClientMessage{value} // Create it populated, to avoid reaper
 		ChatSubscriptionInfo[channelName] = list
 		ChatSubscriptionLock.Unlock()
+
+		go func(topic string) {
+			err := SendNewTopicNotice(topic)
+			if err != nil {
+				log.Println("error reporting new sub:", err)
+			}
+		}(channelName)
+
 		ChatSubscriptionLock.RLock()
 	} else {
 		list.Lock()
@@ -152,21 +161,28 @@ func UnsubscribeSingleChat(client *ClientInfo, channelName string) {
 	ChatSubscriptionLock.RUnlock()
 }
 
-const ReapingDelay = 120 * time.Minute
+const ReapingDelay = 20 * time.Minute
 
 // Checks ChatSubscriptionInfo for entries with no subscribers every ReapingDelay.
 // Started from SetupServer().
 func deadChannelReaper() {
 	for {
 		time.Sleep(ReapingDelay)
+		var cleanedUp = make([]string, 0, 6)
 		ChatSubscriptionLock.Lock()
 		for key, val := range ChatSubscriptionInfo {
-			if val != nil {
-				if len(val.Members) == 0 {
-					delete(ChatSubscriptionInfo, key)
-				}
+			if val == nil || len(val.Members) == 0 {
+				delete(ChatSubscriptionInfo, key)
+				cleanedUp = append(cleanedUp, key)
 			}
 		}
 		ChatSubscriptionLock.Unlock()
+
+		if len(cleanedUp) != 0 {
+			err := SendCleanupTopicsNotice(cleanedUp)
+			if err != nil {
+				log.Println("error reporting cleaned subs:", err)
+			}
+		}
 	}
 }
diff --git a/socketserver/internal/server/types.go b/socketserver/internal/server/types.go
index 2f6013b3..48d3534d 100644
--- a/socketserver/internal/server/types.go
+++ b/socketserver/internal/server/types.go
@@ -34,13 +34,13 @@ type ClientMessage struct {
 	// Message ID. Increments by 1 for each message sent from the client.
 	// When replying to a command, the message ID must be echoed.
 	// When sending a server-initiated message, this is -1.
-	MessageID int
+	MessageID int `json:m`
 	// The command that the client wants from the server.
 	// When sent from the server, the literal string 'True' indicates success.
 	// Before sending, a blank Command will be converted into SuccessCommand.
-	Command Command
+	Command Command `json:c`
 	// Result of json.Unmarshal on the third field send from the client
-	Arguments interface{}
+	Arguments interface{} `json:a`
 
 	origArguments string
 }

From 0dffc494e4399c812331f484f5d51657894d09e6 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Thu, 5 Nov 2015 23:24:35 -0800
Subject: [PATCH 034/176] Fix go vet problems

---
 socketserver/internal/server/backlog.go  | 2 +-
 socketserver/internal/server/commands.go | 8 ++++----
 socketserver/internal/server/types.go    | 6 +++---
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/socketserver/internal/server/backlog.go b/socketserver/internal/server/backlog.go
index 9b4f981f..6ced8141 100644
--- a/socketserver/internal/server/backlog.go
+++ b/socketserver/internal/server/backlog.go
@@ -366,7 +366,7 @@ func HBackendUpdateAndPublish(w http.ResponseWriter, r *http.Request) {
 	cacheinfo, ok := ServerInitiatedCommands[cmd]
 	if !ok {
 		w.WriteHeader(422)
-		fmt.Fprintf(w, "Caching semantics unknown for command '%s'. Post to /addcachedcommand first.")
+		fmt.Fprintf(w, "Caching semantics unknown for command '%s'. Post to /addcachedcommand first.", cmd)
 		return
 	}
 
diff --git a/socketserver/internal/server/commands.go b/socketserver/internal/server/commands.go
index cd735d35..9594362e 100644
--- a/socketserver/internal/server/commands.go
+++ b/socketserver/internal/server/commands.go
@@ -203,10 +203,10 @@ func HandleSurvey(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (
 }
 
 type FollowEvent struct {
-	User         string    `json:u`
-	Channel      string    `json:c`
-	NowFollowing bool      `json:f`
-	Timestamp    time.Time `json:t`
+	User         string    `json:"u"`
+	Channel      string    `json:"c"`
+	NowFollowing bool      `json:"f"`
+	Timestamp    time.Time `json:"t"`
 }
 
 var FollowEvents []FollowEvent
diff --git a/socketserver/internal/server/types.go b/socketserver/internal/server/types.go
index 48d3534d..7147bb1c 100644
--- a/socketserver/internal/server/types.go
+++ b/socketserver/internal/server/types.go
@@ -34,13 +34,13 @@ type ClientMessage struct {
 	// Message ID. Increments by 1 for each message sent from the client.
 	// When replying to a command, the message ID must be echoed.
 	// When sending a server-initiated message, this is -1.
-	MessageID int `json:m`
+	MessageID int `json:"m"`
 	// The command that the client wants from the server.
 	// When sent from the server, the literal string 'True' indicates success.
 	// Before sending, a blank Command will be converted into SuccessCommand.
-	Command Command `json:c`
+	Command Command `json:"c"`
 	// Result of json.Unmarshal on the third field send from the client
-	Arguments interface{} `json:a`
+	Arguments interface{} `json:"a"`
 
 	origArguments string
 }

From 95a8f710f8db5650157fd8271627da900d6c6955 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 8 Nov 2015 16:44:16 -0800
Subject: [PATCH 035/176] Implement username validation

---
 socketserver/cmd/ffzsocketserver/console.go |  23 +++
 socketserver/internal/server/backend.go     |  14 +-
 socketserver/internal/server/commands.go    |  46 ++++--
 socketserver/internal/server/handlecore.go  |  11 +-
 socketserver/internal/server/irc.go         | 163 ++++++++++++++++++++
 socketserver/internal/server/publisher.go   |   4 +-
 socketserver/internal/server/types.go       |   2 +
 7 files changed, 242 insertions(+), 21 deletions(-)
 create mode 100644 socketserver/internal/server/irc.go

diff --git a/socketserver/cmd/ffzsocketserver/console.go b/socketserver/cmd/ffzsocketserver/console.go
index a82904bd..e6ceeefb 100644
--- a/socketserver/cmd/ffzsocketserver/console.go
+++ b/socketserver/cmd/ffzsocketserver/console.go
@@ -13,6 +13,11 @@ func commandLineConsole() {
 
 	shell := ishell.NewShell()
 
+	shell.Register("help", func(args ...string) (string, error) {
+		shell.PrintCommands()
+		return "", nil
+	})
+
 	shell.Register("clientcount", func(args ...string) (string, error) {
 		server.GlobalSubscriptionInfo.RLock()
 		count := len(server.GlobalSubscriptionInfo.Members)
@@ -67,6 +72,24 @@ func commandLineConsole() {
 		return "", nil
 	})
 
+	shell.Register("authorizeeveryone", func(args ...string) (string, error) {
+		if len(args) == 0 {
+			if server.Configuation.SendAuthToNewClients {
+				return "All clients are recieving auth challenges upon claiming a name.", nil
+			} else {
+				return "All clients are not recieving auth challenges upon claiming a name.", nil
+			}
+		} else if args[0] == "on" {
+			server.Configuation.SendAuthToNewClients = true
+			return "All new clients will recieve auth challenges upon claiming a name.", nil
+		} else if args[0] == "off" {
+			server.Configuation.SendAuthToNewClients = false
+			return "All new clients will not recieve auth challenges upon claiming a name.", nil
+		} else {
+			return "Usage: authorizeeveryone [ on | off ]", nil
+		}
+	})
+
 	shell.Register("panic", func(args ...string) (string, error) {
 		go func() {
 			panic("requested panic")
diff --git a/socketserver/internal/server/backend.go b/socketserver/internal/server/backend.go
index 3e98e742..40ebfcf9 100644
--- a/socketserver/internal/server/backend.go
+++ b/socketserver/internal/server/backend.go
@@ -4,7 +4,9 @@ import (
 	"crypto/rand"
 	"encoding/base64"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"github.com/gorilla/websocket"
 	"github.com/pmylund/go-cache"
 	"golang.org/x/crypto/nacl/box"
 	"io/ioutil"
@@ -15,7 +17,6 @@ import (
 	"strings"
 	"sync"
 	"time"
-	"github.com/gorilla/websocket"
 )
 
 var backendHttpClient http.Client
@@ -37,7 +38,7 @@ func SetupBackend(config *ConfigFile) {
 	if responseCache != nil {
 		responseCache.Flush()
 	}
-	responseCache = cache.New(60 * time.Second, 120 * time.Second)
+	responseCache = cache.New(60*time.Second, 120*time.Second)
 
 	getBacklogUrl = fmt.Sprintf("%s/backlog", backendUrl)
 	postStatisticsUrl = fmt.Sprintf("%s/stats", backendUrl)
@@ -114,6 +115,8 @@ func (bfe BackendForwardedError) Error() string {
 	return string(bfe)
 }
 
+var AuthorizationNeededError = errors.New("Must authenticate Twitch username to use this command")
+
 func RequestRemoteDataCached(remoteCommand, data string, auth AuthInfo) (string, error) {
 	cached, ok := responseCache.Get(getCacheKey(remoteCommand, data))
 	if ok {
@@ -154,7 +157,9 @@ func RequestRemoteData(remoteCommand, data string, auth AuthInfo) (responseStr s
 
 	responseStr = string(respBytes)
 
-	if resp.StatusCode != 200 {
+	if resp.StatusCode == 401 {
+		return "", AuthorizationNeededError
+	} else if resp.StatusCode != 200 {
 		if resp.Header.Get("Content-Type") == "application/json" {
 			return "", BackendForwardedError(responseStr)
 		} else {
@@ -222,8 +227,9 @@ func FetchBacklogData(chatSubs []string) ([]ClientMessage, error) {
 
 type NotOkError struct {
 	Response string
-	Code int
+	Code     int
 }
+
 func (noe NotOkError) Error() string {
 	return fmt.Sprintf("backend returned %d: %s", noe.Code, noe.Response)
 }
diff --git a/socketserver/internal/server/commands.go b/socketserver/internal/server/commands.go
index 9594362e..02f51dae 100644
--- a/socketserver/internal/server/commands.go
+++ b/socketserver/internal/server/commands.go
@@ -109,6 +109,14 @@ func HandleSetUser(conn *websocket.Conn, client *ClientInfo, msg ClientMessage)
 	client.UsernameValidated = false
 	client.Mutex.Unlock()
 
+	if Configuation.SendAuthToNewClients {
+		client.MsgChannelKeepalive.Add(1)
+		go client.StartAuthorization(func(_ *ClientInfo, _ bool) {
+			client.MsgChannelKeepalive.Done()
+		})
+
+	}
+
 	return ResponseSuccess, nil
 }
 
@@ -408,18 +416,32 @@ func HandleBunchedRemoteCommand(conn *websocket.Conn, client *ClientInfo, msg Cl
 
 func HandleRemoteCommand(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
 	client.MsgChannelKeepalive.Add(1)
-	go func(conn *websocket.Conn, msg ClientMessage, authInfo AuthInfo) {
-		resp, err := RequestRemoteDataCached(string(msg.Command), msg.origArguments, authInfo)
-
-		if err != nil {
-			client.MessageChannel <- ClientMessage{MessageID: msg.MessageID, Command: ErrorCommand, Arguments: err.Error()}
-		} else {
-			msg := ClientMessage{MessageID: msg.MessageID, Command: SuccessCommand, origArguments: resp}
-			msg.parseOrigArguments()
-			client.MessageChannel <- msg
-		}
-		client.MsgChannelKeepalive.Done()
-	}(conn, msg, client.AuthInfo)
+	go doRemoteCommand(conn, msg, client)
 
 	return ClientMessage{Command: AsyncResponseCommand}, nil
 }
+
+const AuthorizationFailedErrorString = "Failed to verify your Twitch username."
+
+func doRemoteCommand(conn *websocket.Conn, msg ClientMessage, client *ClientInfo) {
+	resp, err := RequestRemoteDataCached(string(msg.Command), msg.origArguments, client.AuthInfo)
+
+	if err == AuthorizationNeededError {
+		client.StartAuthorization(func(_ *ClientInfo, success bool) {
+			if success {
+				doRemoteCommand(conn, msg, client)
+			} else {
+				client.MessageChannel <- ClientMessage{MessageID: msg.MessageID, Command: ErrorCommand, Arguments: AuthorizationFailedErrorString}
+				client.MsgChannelKeepalive.Done()
+			}
+		})
+		return // without keepalive.Done()
+	} else if err != nil {
+		client.MessageChannel <- ClientMessage{MessageID: msg.MessageID, Command: ErrorCommand, Arguments: err.Error()}
+	} else {
+		msg := ClientMessage{MessageID: msg.MessageID, Command: SuccessCommand, origArguments: resp}
+		msg.parseOrigArguments()
+		client.MessageChannel <- msg
+	}
+	client.MsgChannelKeepalive.Done()
+}
diff --git a/socketserver/internal/server/handlecore.go b/socketserver/internal/server/handlecore.go
index f6778d55..21b4c6d8 100644
--- a/socketserver/internal/server/handlecore.go
+++ b/socketserver/internal/server/handlecore.go
@@ -51,6 +51,8 @@ const ErrorCommand Command = "error"
 // This must be the first command sent by the client once the connection is established.
 const HelloCommand Command = "hello"
 
+const AuthorizeCommand Command = "do_authorize"
+
 // A handler returning a ClientMessage with this Command will prevent replying to the client.
 // It signals that the work has been handed off to a background goroutine.
 const AsyncResponseCommand Command = "_async"
@@ -73,14 +75,14 @@ var ExpectedStringAndInt = errors.New("Error: Expected array of string, int as a
 var ExpectedStringAndBool = errors.New("Error: Expected array of string, bool as arguments.")
 var ExpectedStringAndIntGotFloat = errors.New("Error: Second argument was a float, expected an integer.")
 
-var gconfig *ConfigFile
+var Configuation *ConfigFile
 
 var BannerHTML []byte
 
 // Set up a websocket listener and register it on /.
 // (Uses http.DefaultServeMux .)
 func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
-	gconfig = config
+	Configuation = config
 
 	SetupBackend(config)
 
@@ -99,9 +101,12 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 	serveMux.HandleFunc("/dump_backlog", HBackendDumpBacklog)
 	serveMux.HandleFunc("/update_and_pub", HBackendUpdateAndPublish)
 
-	go deadChannelReaper()
+	go pubsubJanitor()
 	go backlogJanitor()
+	go authorizationJanitor()
 	go sendAggregateData()
+
+	go ircConnection()
 }
 
 func ServeWebsocketOrCatbag(w http.ResponseWriter, r *http.Request) {
diff --git a/socketserver/internal/server/irc.go b/socketserver/internal/server/irc.go
new file mode 100644
index 00000000..a9a95edf
--- /dev/null
+++ b/socketserver/internal/server/irc.go
@@ -0,0 +1,163 @@
+package server
+
+import (
+	"bytes"
+	"crypto/rand"
+	"encoding/base64"
+	"fmt"
+	irc "github.com/fluffle/goirc/client"
+	"log"
+	"strings"
+	"sync"
+	"time"
+)
+
+type AuthCallback func(client *ClientInfo, successful bool)
+
+type PendingAuthorization struct {
+	Client    *ClientInfo
+	Challenge string
+	Callback  AuthCallback
+	EnteredAt time.Time
+}
+
+var PendingAuths []PendingAuthorization
+var PendingAuthLock sync.Mutex
+
+func AddPendingAuthorization(client *ClientInfo, challenge string, callback AuthCallback) {
+	PendingAuthLock.Lock()
+	defer PendingAuthLock.Unlock()
+
+	PendingAuths = append(PendingAuths, PendingAuthorization{
+		Client:    client,
+		Challenge: challenge,
+		Callback:  callback,
+		EnteredAt: time.Now(),
+	})
+}
+
+func authorizationJanitor() {
+	for {
+		time.Sleep(5 * time.Minute)
+
+		func() {
+			cullTime := time.Now().Add(-30 * time.Minute)
+
+			PendingAuthLock.Lock()
+			defer PendingAuthLock.Unlock()
+
+			newPendingAuths := make([]PendingAuthorization, 0, len(PendingAuths))
+
+			for _, v := range PendingAuths {
+				if !cullTime.After(v.EnteredAt) {
+					newPendingAuths = append(newPendingAuths, v)
+				}
+			}
+
+			PendingAuths = newPendingAuths
+		}()
+	}
+}
+
+func (client *ClientInfo) StartAuthorization(callback AuthCallback) {
+	fmt.Println(DEBUG, "startig auth for user", client.TwitchUsername, client.RemoteAddr)
+	var nonce [32]byte
+	_, err := rand.Read(nonce[:])
+	if err != nil {
+		go func(client *ClientInfo, callback AuthCallback) {
+			callback(client, false)
+		}(client, callback)
+		return
+	}
+	buf := bytes.NewBuffer(nil)
+	enc := base64.NewEncoder(base64.RawURLEncoding, buf)
+	enc.Write(nonce[:])
+	enc.Close()
+	challenge := buf.String()
+
+	fmt.Println(DEBUG, "adding to auth array")
+	AddPendingAuthorization(client, challenge, callback)
+
+	fmt.Println(DEBUG, "sending auth message")
+	client.MessageChannel <- ClientMessage{MessageID: -1, Command: AuthorizeCommand, Arguments: challenge}
+}
+
+const AuthChannelName = "frankerfacezauthorizer"
+const AuthChannel = "#" + AuthChannelName
+const AuthCommand = "AUTH"
+
+const DEBUG = "DEBUG"
+
+func ircConnection() {
+
+	c := irc.SimpleClient("justinfan123")
+
+	c.HandleFunc(irc.CONNECTED, func(conn *irc.Conn, line *irc.Line) {
+		conn.Join(AuthChannel)
+	})
+
+	c.HandleFunc(irc.PRIVMSG, func(conn *irc.Conn, line *irc.Line) {
+		channel := line.Args[0]
+		msg := line.Args[1]
+		if channel != AuthChannel || !strings.HasPrefix(msg, AuthCommand) || !line.Public() {
+			fmt.Println(DEBUG, "discarded msg", line.Raw)
+			return
+		}
+
+		msgArray := strings.Split(msg, " ")
+		if len(msgArray) != 2 {
+			fmt.Println(DEBUG, "discarded msg - not 2 strings", line.Raw)
+			return
+		}
+
+		submittedUser := line.Nick
+		submittedChallenge := msgArray[1]
+
+		var auth PendingAuthorization
+		var idx int = -1
+
+		PendingAuthLock.Lock()
+		for i, v := range PendingAuths {
+			if v.Client.TwitchUsername == submittedUser && v.Challenge == submittedChallenge {
+				auth = v
+				idx = i
+				break
+			}
+		}
+		if idx != -1 {
+			PendingAuths = append(PendingAuths[:idx], PendingAuths[idx+1:]...)
+		}
+		PendingAuthLock.Unlock()
+
+		if idx == -1 {
+			fmt.Println(DEBUG, "discarded msg - challenge not found", line.Raw)
+			return
+		}
+
+		// auth is valid, and removed from pending list
+
+		fmt.Println(DEBUG, "authorization success for user", auth.Client.TwitchUsername)
+		var usernameChanged bool
+		auth.Client.Mutex.Lock()
+		if auth.Client.TwitchUsername == submittedUser { // recheck condition
+			auth.Client.UsernameValidated = true
+		} else {
+			usernameChanged = true
+		}
+		auth.Client.Mutex.Unlock()
+
+		if auth.Callback != nil {
+			if !usernameChanged {
+				auth.Callback(auth.Client, true)
+			} else {
+				auth.Callback(auth.Client, false)
+			}
+		}
+	})
+
+	err := c.ConnectTo("irc.twitch.tv")
+	if err != nil {
+		log.Fatalln("Cannot connect to IRC:", err)
+	}
+
+}
diff --git a/socketserver/internal/server/publisher.go b/socketserver/internal/server/publisher.go
index 54a5b3d1..92b14f68 100644
--- a/socketserver/internal/server/publisher.go
+++ b/socketserver/internal/server/publisher.go
@@ -4,9 +4,9 @@ package server
 // If I screwed up the locking, I won't know until it's too late.
 
 import (
+	"log"
 	"sync"
 	"time"
-	"log"
 )
 
 type SubscriberList struct {
@@ -165,7 +165,7 @@ const ReapingDelay = 20 * time.Minute
 
 // Checks ChatSubscriptionInfo for entries with no subscribers every ReapingDelay.
 // Started from SetupServer().
-func deadChannelReaper() {
+func pubsubJanitor() {
 	for {
 		time.Sleep(ReapingDelay)
 		var cleanedUp = make([]string, 0, 6)
diff --git a/socketserver/internal/server/types.go b/socketserver/internal/server/types.go
index 7147bb1c..0c889e55 100644
--- a/socketserver/internal/server/types.go
+++ b/socketserver/internal/server/types.go
@@ -28,6 +28,8 @@ type ConfigFile struct {
 	OurPrivateKey    []byte
 	OurPublicKey     []byte
 	BackendPublicKey []byte
+
+	SendAuthToNewClients bool
 }
 
 type ClientMessage struct {

From a4bac68d029c7e2d5343f364d05111a997506a77 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 8 Nov 2015 21:10:24 -0800
Subject: [PATCH 036/176] Post to backend when server start

---
 socketserver/cmd/ffzsocketserver/socketserver.go |  3 ++-
 socketserver/internal/server/backend.go          |  2 ++
 socketserver/internal/server/handlecore.go       | 16 +++++++++++++++-
 3 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/socketserver/cmd/ffzsocketserver/socketserver.go b/socketserver/cmd/ffzsocketserver/socketserver.go
index f8bf8c19..a7e1b9f3 100644
--- a/socketserver/cmd/ffzsocketserver/socketserver.go
+++ b/socketserver/cmd/ffzsocketserver/socketserver.go
@@ -48,12 +48,13 @@ func main() {
 	if err != nil {
 		log.Fatal("Could not create logfile: ", err)
 	}
-	log.SetOutput(logFile)
 
 	server.SetupServerAndHandle(conf, nil)
 
 	go commandLineConsole()
 
+	log.SetOutput(logFile)
+
 	if conf.UseSSL {
 		err = httpServer.ListenAndServeTLS(conf.SSLCertificateFile, conf.SSLKeyFile)
 	} else {
diff --git a/socketserver/internal/server/backend.go b/socketserver/internal/server/backend.go
index 40ebfcf9..6af4d068 100644
--- a/socketserver/internal/server/backend.go
+++ b/socketserver/internal/server/backend.go
@@ -26,6 +26,7 @@ var responseCache *cache.Cache
 var getBacklogUrl string
 var postStatisticsUrl string
 var addTopicUrl string
+var announceStartupUrl string
 
 var backendSharedKey [32]byte
 var serverId int
@@ -43,6 +44,7 @@ func SetupBackend(config *ConfigFile) {
 	getBacklogUrl = fmt.Sprintf("%s/backlog", backendUrl)
 	postStatisticsUrl = fmt.Sprintf("%s/stats", backendUrl)
 	addTopicUrl = fmt.Sprintf("%s/topics", backendUrl)
+	announceStartupUrl = fmt.Sprintf("%s/startup", backendUrl)
 
 	messageBufferPool.New = New4KByteBuffer
 
diff --git a/socketserver/internal/server/handlecore.go b/socketserver/internal/server/handlecore.go
index 21b4c6d8..526111bb 100644
--- a/socketserver/internal/server/handlecore.go
+++ b/socketserver/internal/server/handlecore.go
@@ -9,6 +9,7 @@ import (
 	"io/ioutil"
 	"log"
 	"net/http"
+	"net/url"
 	"strconv"
 	"strings"
 	"sync"
@@ -92,7 +93,7 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 
 	bannerBytes, err := ioutil.ReadFile("index.html")
 	if err != nil {
-		log.Fatal("Could not open index.html", err)
+		log.Fatalln("Could not open index.html:", err)
 	}
 	BannerHTML = bannerBytes
 
@@ -101,6 +102,19 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 	serveMux.HandleFunc("/dump_backlog", HBackendDumpBacklog)
 	serveMux.HandleFunc("/update_and_pub", HBackendUpdateAndPublish)
 
+	announceForm, err := SealRequest(url.Values{
+		"startup": []string{"1"},
+	})
+	if err != nil {
+		log.Fatalln("Unable to seal requests:", err)
+	}
+	resp, err := backendHttpClient.PostForm(announceStartupUrl, announceForm)
+	if err != nil {
+		log.Println(err)
+	} else {
+		resp.Body.Close()
+	}
+
 	go pubsubJanitor()
 	go backlogJanitor()
 	go authorizationJanitor()

From 736deacf4ce8748b247d6c493769540bda747527 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 8 Nov 2015 21:20:32 -0800
Subject: [PATCH 037/176] Default to remote command

---
 socketserver/internal/server/commands.go | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/socketserver/internal/server/commands.go b/socketserver/internal/server/commands.go
index 02f51dae..1a60151f 100644
--- a/socketserver/internal/server/commands.go
+++ b/socketserver/internal/server/commands.go
@@ -21,13 +21,7 @@ const ChannelInfoDelay = 2 * time.Second
 func HandleCommand(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) {
 	handler, ok := CommandHandlers[msg.Command]
 	if !ok {
-		log.Println("[!] Unknown command", msg.Command, "- sent by client", client.ClientID, "@", conn.RemoteAddr())
-		SendMessage(conn, ClientMessage{
-			MessageID: msg.MessageID,
-			Command:   "error",
-			Arguments: fmt.Sprintf("Unknown command %s", msg.Command),
-		})
-		return
+		handler = HandleRemoteCommand
 	}
 
 	response, err := CallHandler(handler, conn, client, msg)

From 4f3473e8d934c689e1077a5c1825c5c3e2ff8bd6 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 8 Nov 2015 21:27:44 -0800
Subject: [PATCH 038/176] imported and not used: "fmt"

---
 socketserver/internal/server/commands.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/socketserver/internal/server/commands.go b/socketserver/internal/server/commands.go
index 1a60151f..6b9ea72a 100644
--- a/socketserver/internal/server/commands.go
+++ b/socketserver/internal/server/commands.go
@@ -3,7 +3,6 @@ package server
 import (
 	"encoding/json"
 	"errors"
-	"fmt"
 	"github.com/gorilla/websocket"
 	"github.com/satori/go.uuid"
 	"log"

From a4ecc790b9f0902b83ac7f3b0e17061ae8cd0466 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 8 Nov 2015 22:01:32 -0800
Subject: [PATCH 039/176] Handle multichat in single-publish

---
 socketserver/internal/server/backend.go  | 8 ++++----
 socketserver/internal/server/commands.go | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/socketserver/internal/server/backend.go b/socketserver/internal/server/backend.go
index 6af4d068..42adb5ff 100644
--- a/socketserver/internal/server/backend.go
+++ b/socketserver/internal/server/backend.go
@@ -99,7 +99,7 @@ func HBackendPublishRequest(w http.ResponseWriter, r *http.Request) {
 	case MsgTargetTypeChat:
 		count = PublishToChat(channel, cm)
 	case MsgTargetTypeMultichat:
-		// TODO
+		count = PublishToMultiple(strings.Split(channel, ","), cm)
 	case MsgTargetTypeGlobal:
 		count = PublishToAll(cm)
 	case MsgTargetTypeInvalid:
@@ -119,15 +119,15 @@ func (bfe BackendForwardedError) Error() string {
 
 var AuthorizationNeededError = errors.New("Must authenticate Twitch username to use this command")
 
-func RequestRemoteDataCached(remoteCommand, data string, auth AuthInfo) (string, error) {
+func SendRemoteCommandCached(remoteCommand, data string, auth AuthInfo) (string, error) {
 	cached, ok := responseCache.Get(getCacheKey(remoteCommand, data))
 	if ok {
 		return cached.(string), nil
 	}
-	return RequestRemoteData(remoteCommand, data, auth)
+	return SendRemoteCommand(remoteCommand, data, auth)
 }
 
-func RequestRemoteData(remoteCommand, data string, auth AuthInfo) (responseStr string, err error) {
+func SendRemoteCommand(remoteCommand, data string, auth AuthInfo) (responseStr string, err error) {
 	destUrl := fmt.Sprintf("%s/cmd/%s", backendUrl, remoteCommand)
 	var authKey string
 	if auth.UsernameValidated {
diff --git a/socketserver/internal/server/commands.go b/socketserver/internal/server/commands.go
index 6b9ea72a..00adc88a 100644
--- a/socketserver/internal/server/commands.go
+++ b/socketserver/internal/server/commands.go
@@ -376,7 +376,7 @@ func HandleBunchedRemoteCommand(conn *websocket.Conn, client *ClientInfo, msg Cl
 	PendingBunchedRequests[br] = &BunchSubscriberList{Members: []BunchSubscriber{{Client: client, MessageID: msg.MessageID}}}
 
 	go func(request BunchedRequest) {
-		resp, err := RequestRemoteDataCached(string(request.Command), request.Param, AuthInfo{})
+		resp, err := SendRemoteCommandCached(string(request.Command), request.Param, AuthInfo{})
 
 		var msg ClientMessage
 		if err == nil {
@@ -417,7 +417,7 @@ func HandleRemoteCommand(conn *websocket.Conn, client *ClientInfo, msg ClientMes
 const AuthorizationFailedErrorString = "Failed to verify your Twitch username."
 
 func doRemoteCommand(conn *websocket.Conn, msg ClientMessage, client *ClientInfo) {
-	resp, err := RequestRemoteDataCached(string(msg.Command), msg.origArguments, client.AuthInfo)
+	resp, err := SendRemoteCommandCached(string(msg.Command), msg.origArguments, client.AuthInfo)
 
 	if err == AuthorizationNeededError {
 		client.StartAuthorization(func(_ *ClientInfo, success bool) {

From 29eb07f58ce2e64e577a6c4f8a02f8b603e5223c Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 8 Nov 2015 22:34:06 -0800
Subject: [PATCH 040/176] Rename & Reorganize pub/sub

---
 socketserver/internal/server/backend.go       |   2 +-
 socketserver/internal/server/backlog.go       | 392 -------------
 socketserver/internal/server/backlog_test.go  |  80 ---
 socketserver/internal/server/commands.go      |  28 +-
 socketserver/internal/server/handlecore.go    |  72 +--
 socketserver/internal/server/publisher.go     | 522 ++++++++++++------
 .../internal/server/publisher_test.go         | 488 ++--------------
 socketserver/internal/server/subscriptions.go | 188 +++++++
 .../internal/server/subscriptions_test.go     | 448 +++++++++++++++
 9 files changed, 1110 insertions(+), 1110 deletions(-)
 delete mode 100644 socketserver/internal/server/backlog.go
 delete mode 100644 socketserver/internal/server/backlog_test.go
 create mode 100644 socketserver/internal/server/subscriptions.go
 create mode 100644 socketserver/internal/server/subscriptions_test.go

diff --git a/socketserver/internal/server/backend.go b/socketserver/internal/server/backend.go
index 42adb5ff..4557f534 100644
--- a/socketserver/internal/server/backend.go
+++ b/socketserver/internal/server/backend.go
@@ -97,7 +97,7 @@ func HBackendPublishRequest(w http.ResponseWriter, r *http.Request) {
 	case MsgTargetTypeSingle:
 		// TODO
 	case MsgTargetTypeChat:
-		count = PublishToChat(channel, cm)
+		count = PublishToChannel(channel, cm)
 	case MsgTargetTypeMultichat:
 		count = PublishToMultiple(strings.Split(channel, ","), cm)
 	case MsgTargetTypeGlobal:
diff --git a/socketserver/internal/server/backlog.go b/socketserver/internal/server/backlog.go
deleted file mode 100644
index 6ced8141..00000000
--- a/socketserver/internal/server/backlog.go
+++ /dev/null
@@ -1,392 +0,0 @@
-package server
-
-import (
-	"errors"
-	"fmt"
-	"net/http"
-	"sort"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-)
-
-type PushCommandCacheInfo struct {
-	Caching BacklogCacheType
-	Target  MessageTargetType
-}
-
-// this value is just docs right now
-var ServerInitiatedCommands = map[Command]PushCommandCacheInfo{
-	/// Global updates & notices
-	"update_news": {CacheTypeTimestamps, MsgTargetTypeGlobal}, // timecache:global
-	"message":     {CacheTypeTimestamps, MsgTargetTypeGlobal}, // timecache:global
-	"reload_ff":   {CacheTypeTimestamps, MsgTargetTypeGlobal}, // timecache:global
-
-	/// Emote updates
-	"reload_badges": {CacheTypeTimestamps, MsgTargetTypeGlobal},    // timecache:global
-	"set_badge":     {CacheTypeTimestamps, MsgTargetTypeMultichat}, // timecache:multichat
-	"reload_set":    {},                                            // timecache:multichat
-	"load_set":      {},                                            // TODO what are the semantics of this?
-
-	/// User auth
-	"do_authorize": {CacheTypeNever, MsgTargetTypeSingle}, // nocache:single
-
-	/// Channel data
-	// follow_sets: extra emote sets included in the chat
-	// follow_buttons: extra follow buttons below the stream
-	"follow_sets":    {CacheTypePersistent, MsgTargetTypeChat}, // mustcache:chat
-	"follow_buttons": {CacheTypePersistent, MsgTargetTypeChat}, // mustcache:watching
-	"srl_race":       {CacheTypeLastOnly, MsgTargetTypeChat},   // cachelast:watching
-
-	/// Chatter/viewer counts
-	"chatters": {CacheTypeLastOnly, MsgTargetTypeChat}, // cachelast:watching
-	"viewers":  {CacheTypeLastOnly, MsgTargetTypeChat}, // cachelast:watching
-}
-
-type BacklogCacheType int
-
-const (
-	// This is not a cache type.
-	CacheTypeInvalid BacklogCacheType = iota
-	// This message cannot be cached.
-	CacheTypeNever
-	// Save the last 24 hours of this message.
-	// If a client indicates that it has reconnected, replay the messages sent after the disconnect.
-	// Do not replay if the client indicates that this is a firstload.
-	CacheTypeTimestamps
-	// Save only the last copy of this message, and always send it when the backlog is requested.
-	CacheTypeLastOnly
-	// Save this backlog data to disk with its timestamp.
-	// Send it when the backlog is requested, or after a reconnect if it was updated.
-	CacheTypePersistent
-)
-
-type MessageTargetType int
-
-const (
-	// This is not a message target.
-	MsgTargetTypeInvalid MessageTargetType = iota
-	// This message is targeted to a single TODO(user or connection)
-	MsgTargetTypeSingle
-	// This message is targeted to all users in a chat
-	MsgTargetTypeChat
-	// This message is targeted to all users in multiple chats
-	MsgTargetTypeMultichat
-	// This message is sent to all FFZ users.
-	MsgTargetTypeGlobal
-)
-
-// note: see types.go for methods on these
-
-// Returned by BacklogCacheType.UnmarshalJSON()
-var ErrorUnrecognizedCacheType = errors.New("Invalid value for cachetype")
-
-// Returned by MessageTargetType.UnmarshalJSON()
-var ErrorUnrecognizedTargetType = errors.New("Invalid value for message target")
-
-type TimestampedGlobalMessage struct {
-	Timestamp time.Time
-	Command   Command
-	Data      string
-}
-
-type TimestampedMultichatMessage struct {
-	Timestamp time.Time
-	Channels  []string
-	Command   Command
-	Data      string
-}
-
-type LastSavedMessage struct {
-	Timestamp time.Time
-	Data      string
-}
-
-// map is command -> channel -> data
-
-// CacheTypeLastOnly. Cleaned up by reaper goroutine every ~hour.
-var CachedLastMessages map[Command]map[string]LastSavedMessage
-var CachedLSMLock sync.RWMutex
-
-// CacheTypePersistent. Never cleaned.
-var PersistentLastMessages map[Command]map[string]LastSavedMessage
-var PersistentLSMLock sync.RWMutex
-
-var CachedGlobalMessages []TimestampedGlobalMessage
-var CachedChannelMessages []TimestampedMultichatMessage
-var CacheListsLock sync.RWMutex
-
-func DumpCache() {
-	CachedLSMLock.Lock()
-	CachedLastMessages = make(map[Command]map[string]LastSavedMessage)
-	CachedLSMLock.Unlock()
-
-	PersistentLSMLock.Lock()
-	PersistentLastMessages = make(map[Command]map[string]LastSavedMessage)
-	PersistentLSMLock.Unlock()
-
-	CacheListsLock.Lock()
-	CachedGlobalMessages = make(tgmarray, 0)
-	CachedChannelMessages = make(tmmarray, 0)
-	CacheListsLock.Unlock()
-}
-
-func SendBacklogForNewClient(client *ClientInfo) {
-	client.Mutex.Lock() // reading CurrentChannels
-	PersistentLSMLock.RLock()
-	for _, cmd := range GetCommandsOfType(PushCommandCacheInfo{CacheTypePersistent, MsgTargetTypeChat}) {
-		chanMap := CachedLastMessages[cmd]
-		if chanMap == nil {
-			continue
-		}
-		for _, channel := range client.CurrentChannels {
-			msg, ok := chanMap[channel]
-			if ok {
-				msg := ClientMessage{MessageID: -1, Command: cmd, origArguments: msg.Data}
-				msg.parseOrigArguments()
-				client.MessageChannel <- msg
-			}
-		}
-	}
-	PersistentLSMLock.RUnlock()
-
-	CachedLSMLock.RLock()
-	for _, cmd := range GetCommandsOfType(PushCommandCacheInfo{CacheTypeLastOnly, MsgTargetTypeChat}) {
-		chanMap := CachedLastMessages[cmd]
-		if chanMap == nil {
-			continue
-		}
-		for _, channel := range client.CurrentChannels {
-			msg, ok := chanMap[channel]
-			if ok {
-				msg := ClientMessage{MessageID: -1, Command: cmd, origArguments: msg.Data}
-				msg.parseOrigArguments()
-				client.MessageChannel <- msg
-			}
-		}
-	}
-	CachedLSMLock.RUnlock()
-	client.Mutex.Unlock()
-}
-
-func SendTimedBacklogMessages(client *ClientInfo, disconnectTime time.Time) {
-	client.Mutex.Lock() // reading CurrentChannels
-	CacheListsLock.RLock()
-
-	globIdx := FindFirstNewMessage(tgmarray(CachedGlobalMessages), disconnectTime)
-
-	if globIdx != -1 {
-		for i := globIdx; i < len(CachedGlobalMessages); i++ {
-			item := CachedGlobalMessages[i]
-			msg := ClientMessage{MessageID: -1, Command: item.Command, origArguments: item.Data}
-			msg.parseOrigArguments()
-			client.MessageChannel <- msg
-		}
-	}
-
-	chanIdx := FindFirstNewMessage(tmmarray(CachedChannelMessages), disconnectTime)
-
-	if chanIdx != -1 {
-		for i := chanIdx; i < len(CachedChannelMessages); i++ {
-			item := CachedChannelMessages[i]
-			var send bool
-			for _, channel := range item.Channels {
-				for _, matchChannel := range client.CurrentChannels {
-					if channel == matchChannel {
-						send = true
-						break
-					}
-				}
-				if send {
-					break
-				}
-			}
-			if send {
-				msg := ClientMessage{MessageID: -1, Command: item.Command, origArguments: item.Data}
-				msg.parseOrigArguments()
-				client.MessageChannel <- msg
-			}
-		}
-	}
-
-	CacheListsLock.RUnlock()
-	client.Mutex.Unlock()
-}
-
-func backlogJanitor() {
-	for {
-		time.Sleep(1 * time.Hour)
-		CleanupTimedBacklogMessages()
-	}
-}
-
-func CleanupTimedBacklogMessages() {
-	CacheListsLock.Lock()
-	oneHourAgo := time.Now().Add(-24 * time.Hour)
-	globIdx := FindFirstNewMessage(tgmarray(CachedGlobalMessages), oneHourAgo)
-	if globIdx != -1 {
-		newGlobMsgs := make([]TimestampedGlobalMessage, len(CachedGlobalMessages)-globIdx)
-		copy(newGlobMsgs, CachedGlobalMessages[globIdx:])
-		CachedGlobalMessages = newGlobMsgs
-	}
-	chanIdx := FindFirstNewMessage(tmmarray(CachedChannelMessages), oneHourAgo)
-	if chanIdx != -1 {
-		newChanMsgs := make([]TimestampedMultichatMessage, len(CachedChannelMessages)-chanIdx)
-		copy(newChanMsgs, CachedChannelMessages[chanIdx:])
-		CachedChannelMessages = newChanMsgs
-	}
-	CacheListsLock.Unlock()
-}
-
-func InsertionSort(ary sort.Interface) {
-	for i := 1; i < ary.Len(); i++ {
-		for j := i; j > 0 && ary.Less(j, j-1); j-- {
-			ary.Swap(j, j-1)
-		}
-	}
-}
-
-type TimestampArray interface {
-	Len() int
-	GetTime(int) time.Time
-}
-
-func FindFirstNewMessage(ary TimestampArray, disconnectTime time.Time) (idx int) {
-	// TODO needs tests
-	len := ary.Len()
-	i := len
-
-	// Walk backwards until we find GetTime() before disconnectTime
-	step := 1
-	for i > 0 {
-		i -= step
-		if i < 0 {
-			i = 0
-		}
-		if !ary.GetTime(i).After(disconnectTime) {
-			break
-		}
-		step = int(float64(step)*1.5) + 1
-	}
-
-	// Walk forwards until we find GetTime() after disconnectTime
-	for i < len && !ary.GetTime(i).After(disconnectTime) {
-		i++
-	}
-
-	if i == len {
-		return -1
-	}
-	return i
-}
-
-func SaveLastMessage(which map[Command]map[string]LastSavedMessage, locker sync.Locker, cmd Command, channel string, timestamp time.Time, data string, deleting bool) {
-	locker.Lock()
-	defer locker.Unlock()
-
-	chanMap, ok := CachedLastMessages[cmd]
-	if !ok {
-		if deleting {
-			return
-		}
-		chanMap = make(map[string]LastSavedMessage)
-		CachedLastMessages[cmd] = chanMap
-	}
-
-	if deleting {
-		delete(chanMap, channel)
-	} else {
-		chanMap[channel] = LastSavedMessage{timestamp, data}
-	}
-}
-
-func SaveGlobalMessage(cmd Command, timestamp time.Time, data string) {
-	CacheListsLock.Lock()
-	CachedGlobalMessages = append(CachedGlobalMessages, TimestampedGlobalMessage{timestamp, cmd, data})
-	InsertionSort(tgmarray(CachedGlobalMessages))
-	CacheListsLock.Unlock()
-}
-
-func SaveMultichanMessage(cmd Command, channels string, timestamp time.Time, data string) {
-	CacheListsLock.Lock()
-	CachedChannelMessages = append(CachedChannelMessages, TimestampedMultichatMessage{timestamp, strings.Split(channels, ","), cmd, data})
-	InsertionSort(tmmarray(CachedChannelMessages))
-	CacheListsLock.Unlock()
-}
-
-func GetCommandsOfType(match PushCommandCacheInfo) []Command {
-	var ret []Command
-	for cmd, info := range ServerInitiatedCommands {
-		if info == match {
-			ret = append(ret, cmd)
-		}
-	}
-	return ret
-}
-
-func HBackendDumpBacklog(w http.ResponseWriter, r *http.Request) {
-	r.ParseForm()
-	formData, err := UnsealRequest(r.Form)
-	if err != nil {
-		w.WriteHeader(403)
-		fmt.Fprintf(w, "Error: %v", err)
-		return
-	}
-
-	confirm := formData.Get("confirm")
-	if confirm == "1" {
-		DumpCache()
-	}
-}
-
-// Publish a message to clients, and update the in-server cache for the message.
-// notes:
-// `scope` is implicit in the command
-func HBackendUpdateAndPublish(w http.ResponseWriter, r *http.Request) {
-	r.ParseForm()
-	formData, err := UnsealRequest(r.Form)
-	if err != nil {
-		w.WriteHeader(403)
-		fmt.Fprintf(w, "Error: %v", err)
-		return
-	}
-
-	cmd := Command(formData.Get("cmd"))
-	json := formData.Get("args")
-	channel := formData.Get("channel")
-	deleteMode := formData.Get("delete") != ""
-	timeStr := formData.Get("time")
-	timestamp, err := time.Parse(time.UnixDate, timeStr)
-	if err != nil {
-		w.WriteHeader(422)
-		fmt.Fprintf(w, "error parsing time: %v", err)
-	}
-
-	cacheinfo, ok := ServerInitiatedCommands[cmd]
-	if !ok {
-		w.WriteHeader(422)
-		fmt.Fprintf(w, "Caching semantics unknown for command '%s'. Post to /addcachedcommand first.", cmd)
-		return
-	}
-
-	var count int
-	msg := ClientMessage{MessageID: -1, Command: cmd, origArguments: json}
-	msg.parseOrigArguments()
-
-	if cacheinfo.Caching == CacheTypeLastOnly && cacheinfo.Target == MsgTargetTypeChat {
-		SaveLastMessage(CachedLastMessages, &CachedLSMLock, cmd, channel, timestamp, json, deleteMode)
-		count = PublishToChat(channel, msg)
-	} else if cacheinfo.Caching == CacheTypePersistent && cacheinfo.Target == MsgTargetTypeChat {
-		SaveLastMessage(PersistentLastMessages, &PersistentLSMLock, cmd, channel, timestamp, json, deleteMode)
-		count = PublishToChat(channel, msg)
-	} else if cacheinfo.Caching == CacheTypeTimestamps && cacheinfo.Target == MsgTargetTypeMultichat {
-		SaveMultichanMessage(cmd, channel, timestamp, json)
-		count = PublishToMultiple(strings.Split(channel, ","), msg)
-	} else if cacheinfo.Caching == CacheTypeTimestamps && cacheinfo.Target == MsgTargetTypeGlobal {
-		SaveGlobalMessage(cmd, timestamp, json)
-		count = PublishToAll(msg)
-	}
-
-	w.Write([]byte(strconv.Itoa(count)))
-}
diff --git a/socketserver/internal/server/backlog_test.go b/socketserver/internal/server/backlog_test.go
deleted file mode 100644
index 99ce4f5a..00000000
--- a/socketserver/internal/server/backlog_test.go
+++ /dev/null
@@ -1,80 +0,0 @@
-package server
-
-import (
-	"testing"
-	"time"
-)
-
-func TestCleanupBacklogMessages(t *testing.T) {
-
-}
-
-func TestFindFirstNewMessageEmpty(t *testing.T) {
-	CachedGlobalMessages = []TimestampedGlobalMessage{}
-	i := FindFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
-	if i != -1 {
-		t.Errorf("Expected -1, got %d", i)
-	}
-}
-func TestFindFirstNewMessageOneBefore(t *testing.T) {
-	CachedGlobalMessages = []TimestampedGlobalMessage{
-		{Timestamp: time.Unix(8, 0)},
-	}
-	i := FindFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
-	if i != -1 {
-		t.Errorf("Expected -1, got %d", i)
-	}
-}
-func TestFindFirstNewMessageSeveralBefore(t *testing.T) {
-	CachedGlobalMessages = []TimestampedGlobalMessage{
-		{Timestamp: time.Unix(1, 0)},
-		{Timestamp: time.Unix(2, 0)},
-		{Timestamp: time.Unix(3, 0)},
-		{Timestamp: time.Unix(4, 0)},
-		{Timestamp: time.Unix(5, 0)},
-	}
-	i := FindFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
-	if i != -1 {
-		t.Errorf("Expected -1, got %d", i)
-	}
-}
-func TestFindFirstNewMessageInMiddle(t *testing.T) {
-	CachedGlobalMessages = []TimestampedGlobalMessage{
-		{Timestamp: time.Unix(1, 0)},
-		{Timestamp: time.Unix(2, 0)},
-		{Timestamp: time.Unix(3, 0)},
-		{Timestamp: time.Unix(4, 0)},
-		{Timestamp: time.Unix(5, 0)},
-		{Timestamp: time.Unix(11, 0)},
-		{Timestamp: time.Unix(12, 0)},
-		{Timestamp: time.Unix(13, 0)},
-		{Timestamp: time.Unix(14, 0)},
-		{Timestamp: time.Unix(15, 0)},
-	}
-	i := FindFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
-	if i != 5 {
-		t.Errorf("Expected 5, got %d", i)
-	}
-}
-func TestFindFirstNewMessageOneAfter(t *testing.T) {
-	CachedGlobalMessages = []TimestampedGlobalMessage{
-		{Timestamp: time.Unix(15, 0)},
-	}
-	i := FindFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
-	if i != 0 {
-		t.Errorf("Expected 0, got %d", i)
-	}
-}
-func TestFindFirstNewMessageSeveralAfter(t *testing.T) {
-	CachedGlobalMessages = []TimestampedGlobalMessage{
-		{Timestamp: time.Unix(11, 0)},
-		{Timestamp: time.Unix(12, 0)},
-		{Timestamp: time.Unix(13, 0)},
-		{Timestamp: time.Unix(14, 0)},
-		{Timestamp: time.Unix(15, 0)},
-	}
-	i := FindFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
-	if i != 0 {
-		t.Errorf("Expected 0, got %d", i)
-	}
-}
diff --git a/socketserver/internal/server/commands.go b/socketserver/internal/server/commands.go
index 00adc88a..9f8fed86 100644
--- a/socketserver/internal/server/commands.go
+++ b/socketserver/internal/server/commands.go
@@ -12,8 +12,30 @@ import (
 	"time"
 )
 
-var ResponseSuccess = ClientMessage{Command: SuccessCommand}
-var ResponseFailure = ClientMessage{Command: "False"}
+// A command is how the client refers to a function on the server. It's just a string.
+type Command string
+
+// A function that is called to respond to a Command.
+type CommandHandler func(*websocket.Conn, *ClientInfo, ClientMessage) (ClientMessage, error)
+
+var CommandHandlers = map[Command]CommandHandler{
+	HelloCommand: HandleHello,
+	"setuser":    HandleSetUser,
+	"ready":      HandleReady,
+
+	"sub":   HandleSub,
+	"unsub": HandleUnsub,
+
+	"track_follow":  HandleTrackFollow,
+	"emoticon_uses": HandleEmoticonUses,
+	"survey":        HandleSurvey,
+
+	"twitch_emote":          HandleRemoteCommand,
+	"get_link":              HandleBunchedRemoteCommand,
+	"get_display_name":      HandleBunchedRemoteCommand,
+	"update_follow_buttons": HandleRemoteCommand,
+	"chat_history":          HandleRemoteCommand,
+}
 
 const ChannelInfoDelay = 2 * time.Second
 
@@ -135,7 +157,7 @@ func HandleSub(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rms
 
 	client.Mutex.Unlock()
 
-	SubscribeChat(client, channel)
+	SubscribeChannel(client, channel)
 
 	return ResponseSuccess, nil
 }
diff --git a/socketserver/internal/server/handlecore.go b/socketserver/internal/server/handlecore.go
index 526111bb..d0d0c694 100644
--- a/socketserver/internal/server/handlecore.go
+++ b/socketserver/internal/server/handlecore.go
@@ -18,31 +18,6 @@ import (
 
 const MAX_PACKET_SIZE = 1024
 
-// A command is how the client refers to a function on the server. It's just a string.
-type Command string
-
-// A function that is called to respond to a Command.
-type CommandHandler func(*websocket.Conn, *ClientInfo, ClientMessage) (ClientMessage, error)
-
-var CommandHandlers = map[Command]CommandHandler{
-	HelloCommand: HandleHello,
-	"setuser":    HandleSetUser,
-	"ready":      HandleReady,
-
-	"sub":   HandleSub,
-	"unsub": HandleUnsub,
-
-	"track_follow":  HandleTrackFollow,
-	"emoticon_uses": HandleEmoticonUses,
-	"survey":        HandleSurvey,
-
-	"twitch_emote":          HandleRemoteCommand,
-	"get_link":              HandleBunchedRemoteCommand,
-	"get_display_name":      HandleBunchedRemoteCommand,
-	"update_follow_buttons": HandleRemoteCommand,
-	"chat_history":          HandleRemoteCommand,
-}
-
 // Sent by the server in ClientMessage.Command to indicate success.
 const SuccessCommand Command = "ok"
 
@@ -58,28 +33,11 @@ const AuthorizeCommand Command = "do_authorize"
 // It signals that the work has been handed off to a background goroutine.
 const AsyncResponseCommand Command = "_async"
 
-var SocketUpgrader = websocket.Upgrader{
-	ReadBufferSize:  1024,
-	WriteBufferSize: 1024,
-	CheckOrigin: func(r *http.Request) bool {
-		return r.Header.Get("Origin") == "http://www.twitch.tv"
-	},
-}
-
-// Errors that get returned to the client.
-var ProtocolError error = errors.New("FFZ Socket protocol error.")
-var ProtocolErrorNegativeID error = errors.New("FFZ Socket protocol error: negative or zero message ID.")
-var ExpectedSingleString = errors.New("Error: Expected single string as arguments.")
-var ExpectedSingleInt = errors.New("Error: Expected single integer as arguments.")
-var ExpectedTwoStrings = errors.New("Error: Expected array of string, string as arguments.")
-var ExpectedStringAndInt = errors.New("Error: Expected array of string, int as arguments.")
-var ExpectedStringAndBool = errors.New("Error: Expected array of string, bool as arguments.")
-var ExpectedStringAndIntGotFloat = errors.New("Error: Second argument was a float, expected an integer.")
+var ResponseSuccess = ClientMessage{Command: SuccessCommand}
+var ResponseFailure = ClientMessage{Command: "False"}
 
 var Configuation *ConfigFile
 
-var BannerHTML []byte
-
 // Set up a websocket listener and register it on /.
 // (Uses http.DefaultServeMux .)
 func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
@@ -98,9 +56,9 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 	BannerHTML = bannerBytes
 
 	serveMux.HandleFunc("/", ServeWebsocketOrCatbag)
-	serveMux.HandleFunc("/pub_msg", HBackendPublishRequest)
-	serveMux.HandleFunc("/dump_backlog", HBackendDumpBacklog)
-	serveMux.HandleFunc("/update_and_pub", HBackendUpdateAndPublish)
+	serveMux.HandleFunc("/drop_backlog", HBackendDropBacklog)
+	serveMux.HandleFunc("/uncached_pub", HBackendPublishRequest)
+	serveMux.HandleFunc("/cached_pub", HBackendUpdateAndPublish)
 
 	announceForm, err := SealRequest(url.Values{
 		"startup": []string{"1"},
@@ -123,6 +81,16 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 	go ircConnection()
 }
 
+var SocketUpgrader = websocket.Upgrader{
+	ReadBufferSize:  1024,
+	WriteBufferSize: 1024,
+	CheckOrigin: func(r *http.Request) bool {
+		return r.Header.Get("Origin") == "http://www.twitch.tv"
+	},
+}
+
+var BannerHTML []byte
+
 func ServeWebsocketOrCatbag(w http.ResponseWriter, r *http.Request) {
 	if r.Header.Get("Connection") == "Upgrade" {
 		conn, err := SocketUpgrader.Upgrade(w, r, nil)
@@ -138,6 +106,16 @@ func ServeWebsocketOrCatbag(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// Errors that get returned to the client.
+var ProtocolError error = errors.New("FFZ Socket protocol error.")
+var ProtocolErrorNegativeID error = errors.New("FFZ Socket protocol error: negative or zero message ID.")
+var ExpectedSingleString = errors.New("Error: Expected single string as arguments.")
+var ExpectedSingleInt = errors.New("Error: Expected single integer as arguments.")
+var ExpectedTwoStrings = errors.New("Error: Expected array of string, string as arguments.")
+var ExpectedStringAndInt = errors.New("Error: Expected array of string, int as arguments.")
+var ExpectedStringAndBool = errors.New("Error: Expected array of string, bool as arguments.")
+var ExpectedStringAndIntGotFloat = errors.New("Error: Second argument was a float, expected an integer.")
+
 var CloseGotBinaryMessage = websocket.CloseError{Code: websocket.CloseUnsupportedData, Text: "got binary packet"}
 var CloseGotMessageId0 = websocket.CloseError{Code: websocket.ClosePolicyViolation, Text: "got messageid 0"}
 var CloseTimedOut = websocket.CloseError{Code: websocket.CloseNoStatusReceived, Text: "no ping replies for 5 minutes"}
diff --git a/socketserver/internal/server/publisher.go b/socketserver/internal/server/publisher.go
index 92b14f68..a90efb1c 100644
--- a/socketserver/internal/server/publisher.go
+++ b/socketserver/internal/server/publisher.go
@@ -1,188 +1,392 @@
 package server
 
-// This is the scariest code I've written yet for the server.
-// If I screwed up the locking, I won't know until it's too late.
-
 import (
-	"log"
+	"errors"
+	"fmt"
+	"net/http"
+	"sort"
+	"strconv"
+	"strings"
 	"sync"
 	"time"
 )
 
-type SubscriberList struct {
-	sync.RWMutex
-	Members []chan<- ClientMessage
+type PushCommandCacheInfo struct {
+	Caching BacklogCacheType
+	Target  MessageTargetType
 }
 
-var ChatSubscriptionInfo map[string]*SubscriberList = make(map[string]*SubscriberList)
-var ChatSubscriptionLock sync.RWMutex
-var GlobalSubscriptionInfo SubscriberList
+// this value is just docs right now
+var ServerInitiatedCommands = map[Command]PushCommandCacheInfo{
+	/// Global updates & notices
+	"update_news": {CacheTypeTimestamps, MsgTargetTypeGlobal}, // timecache:global
+	"message":     {CacheTypeTimestamps, MsgTargetTypeGlobal}, // timecache:global
+	"reload_ff":   {CacheTypeTimestamps, MsgTargetTypeGlobal}, // timecache:global
 
-func PublishToChat(channel string, msg ClientMessage) (count int) {
-	ChatSubscriptionLock.RLock()
-	list := ChatSubscriptionInfo[channel]
-	if list != nil {
-		list.RLock()
-		for _, msgChan := range list.Members {
-			msgChan <- msg
-			count++
+	/// Emote updates
+	"reload_badges": {CacheTypeTimestamps, MsgTargetTypeGlobal},    // timecache:global
+	"set_badge":     {CacheTypeTimestamps, MsgTargetTypeMultichat}, // timecache:multichat
+	"reload_set":    {},                                            // timecache:multichat
+	"load_set":      {},                                            // TODO what are the semantics of this?
+
+	/// User auth
+	"do_authorize": {CacheTypeNever, MsgTargetTypeSingle}, // nocache:single
+
+	/// Channel data
+	// follow_sets: extra emote sets included in the chat
+	// follow_buttons: extra follow buttons below the stream
+	"follow_sets":    {CacheTypePersistent, MsgTargetTypeChat}, // mustcache:chat
+	"follow_buttons": {CacheTypePersistent, MsgTargetTypeChat}, // mustcache:watching
+	"srl_race":       {CacheTypeLastOnly, MsgTargetTypeChat},   // cachelast:watching
+
+	/// Chatter/viewer counts
+	"chatters": {CacheTypeLastOnly, MsgTargetTypeChat}, // cachelast:watching
+	"viewers":  {CacheTypeLastOnly, MsgTargetTypeChat}, // cachelast:watching
+}
+
+type BacklogCacheType int
+
+const (
+	// This is not a cache type.
+	CacheTypeInvalid BacklogCacheType = iota
+	// This message cannot be cached.
+	CacheTypeNever
+	// Save the last 24 hours of this message.
+	// If a client indicates that it has reconnected, replay the messages sent after the disconnect.
+	// Do not replay if the client indicates that this is a firstload.
+	CacheTypeTimestamps
+	// Save only the last copy of this message, and always send it when the backlog is requested.
+	CacheTypeLastOnly
+	// Save this backlog data to disk with its timestamp.
+	// Send it when the backlog is requested, or after a reconnect if it was updated.
+	CacheTypePersistent
+)
+
+type MessageTargetType int
+
+const (
+	// This is not a message target.
+	MsgTargetTypeInvalid MessageTargetType = iota
+	// This message is targeted to a single TODO(user or connection)
+	MsgTargetTypeSingle
+	// This message is targeted to all users in a chat
+	MsgTargetTypeChat
+	// This message is targeted to all users in multiple chats
+	MsgTargetTypeMultichat
+	// This message is sent to all FFZ users.
+	MsgTargetTypeGlobal
+)
+
+// note: see types.go for methods on these
+
+// Returned by BacklogCacheType.UnmarshalJSON()
+var ErrorUnrecognizedCacheType = errors.New("Invalid value for cachetype")
+
+// Returned by MessageTargetType.UnmarshalJSON()
+var ErrorUnrecognizedTargetType = errors.New("Invalid value for message target")
+
+type TimestampedGlobalMessage struct {
+	Timestamp time.Time
+	Command   Command
+	Data      string
+}
+
+type TimestampedMultichatMessage struct {
+	Timestamp time.Time
+	Channels  []string
+	Command   Command
+	Data      string
+}
+
+type LastSavedMessage struct {
+	Timestamp time.Time
+	Data      string
+}
+
+// map is command -> channel -> data
+
+// CacheTypeLastOnly. Cleaned up by reaper goroutine every ~hour.
+var CachedLastMessages map[Command]map[string]LastSavedMessage
+var CachedLSMLock sync.RWMutex
+
+// CacheTypePersistent. Never cleaned.
+var PersistentLastMessages map[Command]map[string]LastSavedMessage
+var PersistentLSMLock sync.RWMutex
+
+var CachedGlobalMessages []TimestampedGlobalMessage
+var CachedChannelMessages []TimestampedMultichatMessage
+var CacheListsLock sync.RWMutex
+
+func DumpCache() {
+	CachedLSMLock.Lock()
+	CachedLastMessages = make(map[Command]map[string]LastSavedMessage)
+	CachedLSMLock.Unlock()
+
+	PersistentLSMLock.Lock()
+	PersistentLastMessages = make(map[Command]map[string]LastSavedMessage)
+	PersistentLSMLock.Unlock()
+
+	CacheListsLock.Lock()
+	CachedGlobalMessages = make(tgmarray, 0)
+	CachedChannelMessages = make(tmmarray, 0)
+	CacheListsLock.Unlock()
+}
+
+func SendBacklogForNewClient(client *ClientInfo) {
+	client.Mutex.Lock() // reading CurrentChannels
+	PersistentLSMLock.RLock()
+	for _, cmd := range GetCommandsOfType(PushCommandCacheInfo{CacheTypePersistent, MsgTargetTypeChat}) {
+		chanMap := CachedLastMessages[cmd]
+		if chanMap == nil {
+			continue
 		}
-		list.RUnlock()
-	}
-	ChatSubscriptionLock.RUnlock()
-	return
-}
-
-func PublishToMultiple(channels []string, msg ClientMessage) (count int) {
-	found := make(map[chan<- ClientMessage]struct{})
-
-	ChatSubscriptionLock.RLock()
-
-	for _, channel := range channels {
-		list := ChatSubscriptionInfo[channel]
-		if list != nil {
-			list.RLock()
-			for _, msgChan := range list.Members {
-				found[msgChan] = struct{}{}
+		for _, channel := range client.CurrentChannels {
+			msg, ok := chanMap[channel]
+			if ok {
+				msg := ClientMessage{MessageID: -1, Command: cmd, origArguments: msg.Data}
+				msg.parseOrigArguments()
+				client.MessageChannel <- msg
 			}
-			list.RUnlock()
 		}
 	}
+	PersistentLSMLock.RUnlock()
 
-	ChatSubscriptionLock.RUnlock()
-
-	for msgChan, _ := range found {
-		msgChan <- msg
-		count++
-	}
-	return
-}
-
-func PublishToAll(msg ClientMessage) (count int) {
-	GlobalSubscriptionInfo.RLock()
-	for _, msgChan := range GlobalSubscriptionInfo.Members {
-		msgChan <- msg
-		count++
-	}
-	GlobalSubscriptionInfo.RUnlock()
-	return
-}
-
-// Add a channel to the subscriptions while holding a read-lock to the map.
-// Locks:
-//   - ALREADY HOLDING a read-lock to the 'which' top-level map via the rlocker object
-//   - possible write lock to the 'which' top-level map via the wlocker object
-//   - write lock to SubscriptionInfo (if not creating new)
-func _subscribeWhileRlocked(channelName string, value chan<- ClientMessage) {
-	list := ChatSubscriptionInfo[channelName]
-	if list == nil {
-		// Not found, so create it
-		ChatSubscriptionLock.RUnlock()
-		ChatSubscriptionLock.Lock()
-		list = &SubscriberList{}
-		list.Members = []chan<- ClientMessage{value} // Create it populated, to avoid reaper
-		ChatSubscriptionInfo[channelName] = list
-		ChatSubscriptionLock.Unlock()
-
-		go func(topic string) {
-			err := SendNewTopicNotice(topic)
-			if err != nil {
-				log.Println("error reporting new sub:", err)
+	CachedLSMLock.RLock()
+	for _, cmd := range GetCommandsOfType(PushCommandCacheInfo{CacheTypeLastOnly, MsgTargetTypeChat}) {
+		chanMap := CachedLastMessages[cmd]
+		if chanMap == nil {
+			continue
+		}
+		for _, channel := range client.CurrentChannels {
+			msg, ok := chanMap[channel]
+			if ok {
+				msg := ClientMessage{MessageID: -1, Command: cmd, origArguments: msg.Data}
+				msg.parseOrigArguments()
+				client.MessageChannel <- msg
 			}
-		}(channelName)
-
-		ChatSubscriptionLock.RLock()
-	} else {
-		list.Lock()
-		AddToSliceC(&list.Members, value)
-		list.Unlock()
-	}
-}
-
-func SubscribeGlobal(client *ClientInfo) {
-	GlobalSubscriptionInfo.Lock()
-	AddToSliceC(&GlobalSubscriptionInfo.Members, client.MessageChannel)
-	GlobalSubscriptionInfo.Unlock()
-}
-
-func SubscribeChat(client *ClientInfo, channelName string) {
-	ChatSubscriptionLock.RLock()
-	_subscribeWhileRlocked(channelName, client.MessageChannel)
-	ChatSubscriptionLock.RUnlock()
-}
-
-func unsubscribeAllClients() {
-	GlobalSubscriptionInfo.Lock()
-	GlobalSubscriptionInfo.Members = nil
-	GlobalSubscriptionInfo.Unlock()
-	ChatSubscriptionLock.Lock()
-	ChatSubscriptionInfo = make(map[string]*SubscriberList)
-	ChatSubscriptionLock.Unlock()
-}
-
-// Unsubscribe the client from all channels, AND clear the CurrentChannels / WatchingChannels fields.
-// Locks:
-//   - read lock to top-level maps
-//   - write lock to SubscriptionInfos
-//   - write lock to ClientInfo
-func UnsubscribeAll(client *ClientInfo) {
-	client.Mutex.Lock()
-	client.PendingSubscriptionsBacklog = nil
-	client.PendingSubscriptionsBacklog = nil
-	client.Mutex.Unlock()
-
-	GlobalSubscriptionInfo.Lock()
-	RemoveFromSliceC(&GlobalSubscriptionInfo.Members, client.MessageChannel)
-	GlobalSubscriptionInfo.Unlock()
-
-	ChatSubscriptionLock.RLock()
-	client.Mutex.Lock()
-	for _, v := range client.CurrentChannels {
-		list := ChatSubscriptionInfo[v]
-		if list != nil {
-			list.Lock()
-			RemoveFromSliceC(&list.Members, client.MessageChannel)
-			list.Unlock()
 		}
 	}
-	client.CurrentChannels = nil
+	CachedLSMLock.RUnlock()
 	client.Mutex.Unlock()
-	ChatSubscriptionLock.RUnlock()
 }
 
-func UnsubscribeSingleChat(client *ClientInfo, channelName string) {
-	ChatSubscriptionLock.RLock()
-	list := ChatSubscriptionInfo[channelName]
-	if list != nil {
-		list.Lock()
-		RemoveFromSliceC(&list.Members, client.MessageChannel)
-		list.Unlock()
+func SendTimedBacklogMessages(client *ClientInfo, disconnectTime time.Time) {
+	client.Mutex.Lock() // reading CurrentChannels
+	CacheListsLock.RLock()
+
+	globIdx := FindFirstNewMessage(tgmarray(CachedGlobalMessages), disconnectTime)
+
+	if globIdx != -1 {
+		for i := globIdx; i < len(CachedGlobalMessages); i++ {
+			item := CachedGlobalMessages[i]
+			msg := ClientMessage{MessageID: -1, Command: item.Command, origArguments: item.Data}
+			msg.parseOrigArguments()
+			client.MessageChannel <- msg
+		}
 	}
-	ChatSubscriptionLock.RUnlock()
+
+	chanIdx := FindFirstNewMessage(tmmarray(CachedChannelMessages), disconnectTime)
+
+	if chanIdx != -1 {
+		for i := chanIdx; i < len(CachedChannelMessages); i++ {
+			item := CachedChannelMessages[i]
+			var send bool
+			for _, channel := range item.Channels {
+				for _, matchChannel := range client.CurrentChannels {
+					if channel == matchChannel {
+						send = true
+						break
+					}
+				}
+				if send {
+					break
+				}
+			}
+			if send {
+				msg := ClientMessage{MessageID: -1, Command: item.Command, origArguments: item.Data}
+				msg.parseOrigArguments()
+				client.MessageChannel <- msg
+			}
+		}
+	}
+
+	CacheListsLock.RUnlock()
+	client.Mutex.Unlock()
 }
 
-const ReapingDelay = 20 * time.Minute
-
-// Checks ChatSubscriptionInfo for entries with no subscribers every ReapingDelay.
-// Started from SetupServer().
-func pubsubJanitor() {
+func backlogJanitor() {
 	for {
-		time.Sleep(ReapingDelay)
-		var cleanedUp = make([]string, 0, 6)
-		ChatSubscriptionLock.Lock()
-		for key, val := range ChatSubscriptionInfo {
-			if val == nil || len(val.Members) == 0 {
-				delete(ChatSubscriptionInfo, key)
-				cleanedUp = append(cleanedUp, key)
-			}
-		}
-		ChatSubscriptionLock.Unlock()
+		time.Sleep(1 * time.Hour)
+		CleanupTimedBacklogMessages()
+	}
+}
 
-		if len(cleanedUp) != 0 {
-			err := SendCleanupTopicsNotice(cleanedUp)
-			if err != nil {
-				log.Println("error reporting cleaned subs:", err)
-			}
+func CleanupTimedBacklogMessages() {
+	CacheListsLock.Lock()
+	oneHourAgo := time.Now().Add(-24 * time.Hour)
+	globIdx := FindFirstNewMessage(tgmarray(CachedGlobalMessages), oneHourAgo)
+	if globIdx != -1 {
+		newGlobMsgs := make([]TimestampedGlobalMessage, len(CachedGlobalMessages)-globIdx)
+		copy(newGlobMsgs, CachedGlobalMessages[globIdx:])
+		CachedGlobalMessages = newGlobMsgs
+	}
+	chanIdx := FindFirstNewMessage(tmmarray(CachedChannelMessages), oneHourAgo)
+	if chanIdx != -1 {
+		newChanMsgs := make([]TimestampedMultichatMessage, len(CachedChannelMessages)-chanIdx)
+		copy(newChanMsgs, CachedChannelMessages[chanIdx:])
+		CachedChannelMessages = newChanMsgs
+	}
+	CacheListsLock.Unlock()
+}
+
+func InsertionSort(ary sort.Interface) {
+	for i := 1; i < ary.Len(); i++ {
+		for j := i; j > 0 && ary.Less(j, j-1); j-- {
+			ary.Swap(j, j-1)
 		}
 	}
 }
+
+type TimestampArray interface {
+	Len() int
+	GetTime(int) time.Time
+}
+
+func FindFirstNewMessage(ary TimestampArray, disconnectTime time.Time) (idx int) {
+	// TODO needs tests
+	len := ary.Len()
+	i := len
+
+	// Walk backwards until we find GetTime() before disconnectTime
+	step := 1
+	for i > 0 {
+		i -= step
+		if i < 0 {
+			i = 0
+		}
+		if !ary.GetTime(i).After(disconnectTime) {
+			break
+		}
+		step = int(float64(step)*1.5) + 1
+	}
+
+	// Walk forwards until we find GetTime() after disconnectTime
+	for i < len && !ary.GetTime(i).After(disconnectTime) {
+		i++
+	}
+
+	if i == len {
+		return -1
+	}
+	return i
+}
+
+func SaveLastMessage(which map[Command]map[string]LastSavedMessage, locker sync.Locker, cmd Command, channel string, timestamp time.Time, data string, deleting bool) {
+	locker.Lock()
+	defer locker.Unlock()
+
+	chanMap, ok := CachedLastMessages[cmd]
+	if !ok {
+		if deleting {
+			return
+		}
+		chanMap = make(map[string]LastSavedMessage)
+		CachedLastMessages[cmd] = chanMap
+	}
+
+	if deleting {
+		delete(chanMap, channel)
+	} else {
+		chanMap[channel] = LastSavedMessage{timestamp, data}
+	}
+}
+
+func SaveGlobalMessage(cmd Command, timestamp time.Time, data string) {
+	CacheListsLock.Lock()
+	CachedGlobalMessages = append(CachedGlobalMessages, TimestampedGlobalMessage{timestamp, cmd, data})
+	InsertionSort(tgmarray(CachedGlobalMessages))
+	CacheListsLock.Unlock()
+}
+
+func SaveMultichanMessage(cmd Command, channels string, timestamp time.Time, data string) {
+	CacheListsLock.Lock()
+	CachedChannelMessages = append(CachedChannelMessages, TimestampedMultichatMessage{timestamp, strings.Split(channels, ","), cmd, data})
+	InsertionSort(tmmarray(CachedChannelMessages))
+	CacheListsLock.Unlock()
+}
+
+func GetCommandsOfType(match PushCommandCacheInfo) []Command {
+	var ret []Command
+	for cmd, info := range ServerInitiatedCommands {
+		if info == match {
+			ret = append(ret, cmd)
+		}
+	}
+	return ret
+}
+
+func HBackendDropBacklog(w http.ResponseWriter, r *http.Request) {
+	r.ParseForm()
+	formData, err := UnsealRequest(r.Form)
+	if err != nil {
+		w.WriteHeader(403)
+		fmt.Fprintf(w, "Error: %v", err)
+		return
+	}
+
+	confirm := formData.Get("confirm")
+	if confirm == "1" {
+		DumpCache()
+	}
+}
+
+// Publish a message to clients, and update the in-server cache for the message.
+// notes:
+// `scope` is implicit in the command
+func HBackendUpdateAndPublish(w http.ResponseWriter, r *http.Request) {
+	r.ParseForm()
+	formData, err := UnsealRequest(r.Form)
+	if err != nil {
+		w.WriteHeader(403)
+		fmt.Fprintf(w, "Error: %v", err)
+		return
+	}
+
+	cmd := Command(formData.Get("cmd"))
+	json := formData.Get("args")
+	channel := formData.Get("channel")
+	deleteMode := formData.Get("delete") != ""
+	timeStr := formData.Get("time")
+	timestamp, err := time.Parse(time.UnixDate, timeStr)
+	if err != nil {
+		w.WriteHeader(422)
+		fmt.Fprintf(w, "error parsing time: %v", err)
+	}
+
+	cacheinfo, ok := ServerInitiatedCommands[cmd]
+	if !ok {
+		w.WriteHeader(422)
+		fmt.Fprintf(w, "Caching semantics unknown for command '%s'. Post to /addcachedcommand first.", cmd)
+		return
+	}
+
+	var count int
+	msg := ClientMessage{MessageID: -1, Command: cmd, origArguments: json}
+	msg.parseOrigArguments()
+
+	if cacheinfo.Caching == CacheTypeLastOnly && cacheinfo.Target == MsgTargetTypeChat {
+		SaveLastMessage(CachedLastMessages, &CachedLSMLock, cmd, channel, timestamp, json, deleteMode)
+		count = PublishToChannel(channel, msg)
+	} else if cacheinfo.Caching == CacheTypePersistent && cacheinfo.Target == MsgTargetTypeChat {
+		SaveLastMessage(PersistentLastMessages, &PersistentLSMLock, cmd, channel, timestamp, json, deleteMode)
+		count = PublishToChannel(channel, msg)
+	} else if cacheinfo.Caching == CacheTypeTimestamps && cacheinfo.Target == MsgTargetTypeMultichat {
+		SaveMultichanMessage(cmd, channel, timestamp, json)
+		count = PublishToMultiple(strings.Split(channel, ","), msg)
+	} else if cacheinfo.Caching == CacheTypeTimestamps && cacheinfo.Target == MsgTargetTypeGlobal {
+		SaveGlobalMessage(cmd, timestamp, json)
+		count = PublishToAll(msg)
+	}
+
+	w.Write([]byte(strconv.Itoa(count)))
+}
diff --git a/socketserver/internal/server/publisher_test.go b/socketserver/internal/server/publisher_test.go
index 7c8fbb89..99ce4f5a 100644
--- a/socketserver/internal/server/publisher_test.go
+++ b/socketserver/internal/server/publisher_test.go
@@ -1,448 +1,80 @@
 package server
 
 import (
-	"encoding/json"
-	"fmt"
-	"github.com/gorilla/websocket"
-	"github.com/satori/go.uuid"
-	"io/ioutil"
-	"net/http"
-	"net/http/httptest"
-	"net/url"
-	"os"
-	"strconv"
-	"sync"
-	"syscall"
 	"testing"
 	"time"
 )
 
-func TCountOpenFDs() uint64 {
-	ary, _ := ioutil.ReadDir(fmt.Sprintf("/proc/%d/fd", os.Getpid()))
-	return uint64(len(ary))
+func TestCleanupBacklogMessages(t *testing.T) {
+
 }
 
-const IgnoreReceivedArguments = 1 + 2i
-
-func TReceiveExpectedMessage(tb testing.TB, conn *websocket.Conn, messageId int, command Command, arguments interface{}) (ClientMessage, bool) {
-	var msg ClientMessage
-	var fail bool
-	messageType, packet, err := conn.ReadMessage()
-	if err != nil {
-		tb.Error(err)
-		return msg, false
-	}
-	if messageType != websocket.TextMessage {
-		tb.Error("got non-text message", packet)
-		return msg, false
-	}
-
-	err = UnmarshalClientMessage(packet, messageType, &msg)
-	if err != nil {
-		tb.Error(err)
-		return msg, false
-	}
-	if msg.MessageID != messageId {
-		tb.Error("Message ID was wrong. Expected", messageId, ", got", msg.MessageID, ":", msg)
-		fail = true
-	}
-	if msg.Command != command {
-		tb.Error("Command was wrong. Expected", command, ", got", msg.Command, ":", msg)
-		fail = true
-	}
-	if arguments != IgnoreReceivedArguments {
-		if arguments == nil {
-			if msg.origArguments != "" {
-				tb.Error("Arguments are wrong. Expected", arguments, ", got", msg.Arguments, ":", msg)
-			}
-		} else {
-			argBytes, _ := json.Marshal(arguments)
-			if msg.origArguments != string(argBytes) {
-				tb.Error("Arguments are wrong. Expected", arguments, ", got", msg.Arguments, ":", msg)
-			}
-		}
-	}
-	return msg, !fail
-}
-
-func TSendMessage(tb testing.TB, conn *websocket.Conn, messageId int, command Command, arguments interface{}) bool {
-	SendMessage(conn, ClientMessage{MessageID: messageId, Command: command, Arguments: arguments})
-	return true
-}
-
-func TSealForSavePubMsg(tb testing.TB, cmd Command, channel string, arguments interface{}, deleteMode bool) (url.Values, error) {
-	form := url.Values{}
-	form.Set("cmd", string(cmd))
-	argsBytes, err := json.Marshal(arguments)
-	if err != nil {
-		tb.Error(err)
-		return nil, err
-	}
-	form.Set("args", string(argsBytes))
-	form.Set("channel", channel)
-	if deleteMode {
-		form.Set("delete", "1")
-	}
-	form.Set("time", time.Now().Format(time.UnixDate))
-
-	sealed, err := SealRequest(form)
-	if err != nil {
-		tb.Error(err)
-		return nil, err
-	}
-	return sealed, nil
-}
-
-func TCheckResponse(tb testing.TB, resp *http.Response, expected string) bool {
-	var failed bool
-	respBytes, err := ioutil.ReadAll(resp.Body)
-	resp.Body.Close()
-	respStr := string(respBytes)
-
-	if err != nil {
-		tb.Error(err)
-		failed = true
-	}
-
-	if resp.StatusCode != 200 {
-		tb.Error("Publish failed: ", resp.StatusCode, respStr)
-		failed = true
-	}
-
-	if respStr != expected {
-		tb.Errorf("Got wrong response from server. Expected: '%s' Got: '%s'", expected, respStr)
-		failed = true
-	}
-	return !failed
-}
-
-type TURLs struct {
-	Websocket  string
-	Origin     string
-	PubMsg     string
-	SavePubMsg string // update_and_pub
-}
-
-func TGetUrls(testserver *httptest.Server) TURLs {
-	addr := testserver.Listener.Addr().String()
-	return TURLs{
-		Websocket:  fmt.Sprintf("ws://%s/", addr),
-		Origin:     fmt.Sprintf("http://%s", addr),
-		PubMsg:     fmt.Sprintf("http://%s/pub_msg", addr),
-		SavePubMsg: fmt.Sprintf("http://%s/update_and_pub", addr),
+func TestFindFirstNewMessageEmpty(t *testing.T) {
+	CachedGlobalMessages = []TimestampedGlobalMessage{}
+	i := FindFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
+	if i != -1 {
+		t.Errorf("Expected -1, got %d", i)
 	}
 }
-
-func TSetup(testserver **httptest.Server, urls *TURLs) {
-	DumpCache()
-
-	conf := &ConfigFile{
-		ServerId:     20,
-		UseSSL:       false,
-		SocketOrigin: "localhost:2002",
-		BannerHTML: `
-<!DOCTYPE html>
-<title>CatBag</title>
-<link rel="stylesheet" href="//cdn.frankerfacez.com/script/catbag.css">
-<div id="container">
-<div id="zf0"></div><div id="zf1"></div><div id="zf2"></div>
-<div id="zf3"></div><div id="zf4"></div><div id="zf5"></div>
-<div id="zf6"></div><div id="zf7"></div><div id="zf8"></div>
-<div id="zf9"></div><div id="catbag"></div>
-<div id="bottom">
-	A <a href="http://www.frankerfacez.com/">FrankerFaceZ</a> Service
-	&mdash; CatBag by <a href="http://www.twitch.tv/wolsk">Wolsk</a>
-</div>
-</div>
-`,
-		OurPublicKey:     []byte{176, 149, 72, 209, 35, 42, 110, 220, 22, 236, 212, 129, 213, 199, 1, 227, 185, 167, 150, 159, 117, 202, 164, 100, 9, 107, 45, 141, 122, 221, 155, 73},
-		OurPrivateKey:    []byte{247, 133, 147, 194, 70, 240, 211, 216, 223, 16, 241, 253, 120, 14, 198, 74, 237, 180, 89, 33, 146, 146, 140, 58, 88, 160, 2, 246, 112, 35, 239, 87},
-		BackendPublicKey: []byte{19, 163, 37, 157, 50, 139, 193, 85, 229, 47, 166, 21, 153, 231, 31, 133, 41, 158, 8, 53, 73, 0, 113, 91, 13, 181, 131, 248, 176, 18, 1, 107},
+func TestFindFirstNewMessageOneBefore(t *testing.T) {
+	CachedGlobalMessages = []TimestampedGlobalMessage{
+		{Timestamp: time.Unix(8, 0)},
 	}
-	gconfig = conf
-	SetupBackend(conf)
-
-	if testserver != nil {
-		serveMux := http.NewServeMux()
-		SetupServerAndHandle(conf, serveMux)
-
-		tserv := httptest.NewUnstartedServer(serveMux)
-		*testserver = tserv
-		tserv.Start()
-		if urls != nil {
-			*urls = TGetUrls(tserv)
-		}
+	i := FindFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
+	if i != -1 {
+		t.Errorf("Expected -1, got %d", i)
 	}
 }
-
-func TestSubscriptionAndPublish(t *testing.T) {
-	var doneWg sync.WaitGroup
-	var readyWg sync.WaitGroup
-
-	const TestChannelName1 = "room.testchannel"
-	const TestChannelName2 = "room.chan2"
-	const TestChannelName3 = "room.chan3"
-	const TestChannelNameUnused = "room.empty"
-	const TestCommandChan = "testdata_single"
-	const TestCommandMulti = "testdata_multi"
-	const TestCommandGlobal = "testdata_global"
-	const TestData1 = "123456789"
-	const TestData2 = 42
-	const TestData3 = false
-	var TestData4 = []interface{}{"str1", "str2", "str3"}
-
-	ServerInitiatedCommands[TestCommandChan] = PushCommandCacheInfo{CacheTypeLastOnly, MsgTargetTypeChat}
-	ServerInitiatedCommands[TestCommandMulti] = PushCommandCacheInfo{CacheTypeTimestamps, MsgTargetTypeMultichat}
-	ServerInitiatedCommands[TestCommandGlobal] = PushCommandCacheInfo{CacheTypeTimestamps, MsgTargetTypeGlobal}
-
-	var server *httptest.Server
-	var urls TURLs
-	TSetup(&server, &urls)
-	defer server.CloseClientConnections()
-	defer unsubscribeAllClients()
-
-	var conn *websocket.Conn
-	var resp *http.Response
-	var err error
-
-	// client 1: sub ch1, ch2
-	// client 2: sub ch1, ch3
-	// client 3: sub none
-	// client 4: delayed sub ch1
-	// msg 1: ch1
-	// msg 2: ch2, ch3
-	// msg 3: chEmpty
-	// msg 4: global
-
-	// Client 1
-	conn, resp, err = websocket.DefaultDialer.Dial(urls.Websocket, http.Header{})
-	if err != nil {
-		t.Error(err)
-		return
+func TestFindFirstNewMessageSeveralBefore(t *testing.T) {
+	CachedGlobalMessages = []TimestampedGlobalMessage{
+		{Timestamp: time.Unix(1, 0)},
+		{Timestamp: time.Unix(2, 0)},
+		{Timestamp: time.Unix(3, 0)},
+		{Timestamp: time.Unix(4, 0)},
+		{Timestamp: time.Unix(5, 0)},
 	}
-
-	doneWg.Add(1)
-	readyWg.Add(1)
-	go func(conn *websocket.Conn) {
-		TSendMessage(t, conn, 1, HelloCommand, []interface{}{"ffz_0.0-test", uuid.NewV4().String()})
-		TReceiveExpectedMessage(t, conn, 1, SuccessCommand, IgnoreReceivedArguments)
-		TSendMessage(t, conn, 2, "sub", TestChannelName1)
-		TReceiveExpectedMessage(t, conn, 2, SuccessCommand, nil)
-		TSendMessage(t, conn, 3, "sub", TestChannelName2) // 2
-		TReceiveExpectedMessage(t, conn, 3, SuccessCommand, nil)
-		TSendMessage(t, conn, 4, "ready", 0)
-		TReceiveExpectedMessage(t, conn, 4, SuccessCommand, nil)
-
-		readyWg.Done()
-
-		TReceiveExpectedMessage(t, conn, -1, TestCommandChan, TestData1)
-		TReceiveExpectedMessage(t, conn, -1, TestCommandMulti, TestData2)
-		TReceiveExpectedMessage(t, conn, -1, TestCommandGlobal, TestData4)
-
-		conn.Close()
-		doneWg.Done()
-	}(conn)
-
-	// Client 2
-	conn, resp, err = websocket.DefaultDialer.Dial(urls.Websocket, http.Header{})
-	if err != nil {
-		t.Error(err)
-		return
+	i := FindFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
+	if i != -1 {
+		t.Errorf("Expected -1, got %d", i)
 	}
-
-	doneWg.Add(1)
-	readyWg.Add(1)
-	go func(conn *websocket.Conn) {
-		TSendMessage(t, conn, 1, HelloCommand, []interface{}{"ffz_0.0-test", uuid.NewV4().String()})
-		TReceiveExpectedMessage(t, conn, 1, SuccessCommand, IgnoreReceivedArguments)
-		TSendMessage(t, conn, 2, "sub", TestChannelName1)
-		TReceiveExpectedMessage(t, conn, 2, SuccessCommand, nil)
-		TSendMessage(t, conn, 3, "sub", TestChannelName3) // 3
-		TReceiveExpectedMessage(t, conn, 3, SuccessCommand, nil)
-		TSendMessage(t, conn, 4, "ready", 0)
-		TReceiveExpectedMessage(t, conn, 4, SuccessCommand, nil)
-
-		readyWg.Done()
-
-		TReceiveExpectedMessage(t, conn, -1, TestCommandChan, TestData1)
-		TReceiveExpectedMessage(t, conn, -1, TestCommandMulti, TestData2)
-		TReceiveExpectedMessage(t, conn, -1, TestCommandGlobal, TestData4)
-
-		conn.Close()
-		doneWg.Done()
-	}(conn)
-
-	// Client 3
-	conn, resp, err = websocket.DefaultDialer.Dial(urls.Websocket, http.Header{})
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
-	doneWg.Add(1)
-	readyWg.Add(1)
-	go func(conn *websocket.Conn) {
-		TSendMessage(t, conn, 1, HelloCommand, []interface{}{"ffz_0.0-test", uuid.NewV4().String()})
-		TReceiveExpectedMessage(t, conn, 1, SuccessCommand, IgnoreReceivedArguments)
-		TSendMessage(t, conn, 2, "ready", 0)
-		TReceiveExpectedMessage(t, conn, 2, SuccessCommand, nil)
-
-		readyWg.Done()
-
-		TReceiveExpectedMessage(t, conn, -1, TestCommandGlobal, TestData4)
-
-		conn.Close()
-		doneWg.Done()
-	}(conn)
-
-	// Wait for clients 1-3
-	readyWg.Wait()
-
-	var form url.Values
-
-	// Publish message 1 - should go to clients 1, 2
-
-	form, err = TSealForSavePubMsg(t, TestCommandChan, TestChannelName1, TestData1, false)
-	if err != nil {
-		t.FailNow()
-	}
-	resp, err = http.PostForm(urls.SavePubMsg, form)
-	if !TCheckResponse(t, resp, strconv.Itoa(2)) {
-		t.FailNow()
-	}
-
-	// Publish message 2 - should go to clients 1, 2
-
-	form, err = TSealForSavePubMsg(t, TestCommandMulti, TestChannelName2+","+TestChannelName3, TestData2, false)
-	if err != nil {
-		t.FailNow()
-	}
-	resp, err = http.PostForm(urls.SavePubMsg, form)
-	if !TCheckResponse(t, resp, strconv.Itoa(2)) {
-		t.FailNow()
-	}
-
-	// Publish message 3 - should go to no clients
-
-	form, err = TSealForSavePubMsg(t, TestCommandChan, TestChannelNameUnused, TestData3, false)
-	if err != nil {
-		t.FailNow()
-	}
-	resp, err = http.PostForm(urls.SavePubMsg, form)
-	if !TCheckResponse(t, resp, strconv.Itoa(0)) {
-		t.FailNow()
-	}
-
-	// Publish message 4 - should go to clients 1, 2, 3
-
-	form, err = TSealForSavePubMsg(t, TestCommandGlobal, "", TestData4, false)
-	if err != nil {
-		t.FailNow()
-	}
-	resp, err = http.PostForm(urls.SavePubMsg, form)
-	if !TCheckResponse(t, resp, strconv.Itoa(3)) {
-		t.FailNow()
-	}
-
-	// Start client 4
-	conn, resp, err = websocket.DefaultDialer.Dial(urls.Websocket, http.Header{})
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
-	doneWg.Add(1)
-	readyWg.Add(1)
-	go func(conn *websocket.Conn) {
-		TSendMessage(t, conn, 1, HelloCommand, []interface{}{"ffz_0.0-test", uuid.NewV4().String()})
-		TReceiveExpectedMessage(t, conn, 1, SuccessCommand, IgnoreReceivedArguments)
-		TSendMessage(t, conn, 2, "sub", TestChannelName1)
-		TReceiveExpectedMessage(t, conn, 2, SuccessCommand, nil)
-		TSendMessage(t, conn, 3, "ready", 0)
-		TReceiveExpectedMessage(t, conn, 3, SuccessCommand, nil)
-
-		// backlog message
-		TReceiveExpectedMessage(t, conn, -1, TestCommandChan, TestData1)
-
-		readyWg.Done()
-
-		conn.Close()
-		doneWg.Done()
-	}(conn)
-
-	readyWg.Wait()
-
-	doneWg.Wait()
-	server.Close()
 }
-
-func BenchmarkUserSubscriptionSinglePublish(b *testing.B) {
-	var doneWg sync.WaitGroup
-	var readyWg sync.WaitGroup
-
-	const TestChannelName = "room.testchannel"
-	const TestCommand = "testdata"
-	const TestData = "123456789"
-
-	message := ClientMessage{MessageID: -1, Command: "testdata", Arguments: TestData}
-
-	fmt.Println()
-	fmt.Println(b.N)
-
-	var limit syscall.Rlimit
-	syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit)
-
-	limit.Cur = TCountOpenFDs() + uint64(b.N)*2 + 100
-
-	if limit.Cur > limit.Max {
-		b.Skip("Open file limit too low")
-		return
+func TestFindFirstNewMessageInMiddle(t *testing.T) {
+	CachedGlobalMessages = []TimestampedGlobalMessage{
+		{Timestamp: time.Unix(1, 0)},
+		{Timestamp: time.Unix(2, 0)},
+		{Timestamp: time.Unix(3, 0)},
+		{Timestamp: time.Unix(4, 0)},
+		{Timestamp: time.Unix(5, 0)},
+		{Timestamp: time.Unix(11, 0)},
+		{Timestamp: time.Unix(12, 0)},
+		{Timestamp: time.Unix(13, 0)},
+		{Timestamp: time.Unix(14, 0)},
+		{Timestamp: time.Unix(15, 0)},
 	}
-
-	syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit)
-
-	var server *httptest.Server
-	var urls TURLs
-	TSetup(&server, &urls)
-	defer unsubscribeAllClients()
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		conn, _, err := websocket.DefaultDialer.Dial(urls.Websocket, http.Header{})
-		if err != nil {
-			b.Error(err)
-			break
-		}
-		doneWg.Add(1)
-		readyWg.Add(1)
-		go func(i int, conn *websocket.Conn) {
-			TSendMessage(b, conn, 1, HelloCommand, []interface{}{"ffz_0.0-test", uuid.NewV4().String()})
-			TSendMessage(b, conn, 2, "sub", TestChannelName)
-
-			TReceiveExpectedMessage(b, conn, 1, SuccessCommand, IgnoreReceivedArguments)
-			TReceiveExpectedMessage(b, conn, 2, SuccessCommand, nil)
-
-			readyWg.Done()
-
-			TReceiveExpectedMessage(b, conn, -1, TestCommand, TestData)
-
-			conn.Close()
-			doneWg.Done()
-		}(i, conn)
+	i := FindFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
+	if i != 5 {
+		t.Errorf("Expected 5, got %d", i)
+	}
+}
+func TestFindFirstNewMessageOneAfter(t *testing.T) {
+	CachedGlobalMessages = []TimestampedGlobalMessage{
+		{Timestamp: time.Unix(15, 0)},
+	}
+	i := FindFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
+	if i != 0 {
+		t.Errorf("Expected 0, got %d", i)
+	}
+}
+func TestFindFirstNewMessageSeveralAfter(t *testing.T) {
+	CachedGlobalMessages = []TimestampedGlobalMessage{
+		{Timestamp: time.Unix(11, 0)},
+		{Timestamp: time.Unix(12, 0)},
+		{Timestamp: time.Unix(13, 0)},
+		{Timestamp: time.Unix(14, 0)},
+		{Timestamp: time.Unix(15, 0)},
+	}
+	i := FindFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
+	if i != 0 {
+		t.Errorf("Expected 0, got %d", i)
 	}
-
-	readyWg.Wait()
-
-	fmt.Println("publishing...")
-	if PublishToChat(TestChannelName, message) != b.N {
-		b.Error("not enough sent")
-		server.CloseClientConnections()
-		panic("halting test instead of waiting")
-	}
-	doneWg.Wait()
-	fmt.Println("...done.")
-
-	b.StopTimer()
-	server.Close()
-	server.CloseClientConnections()
 }
diff --git a/socketserver/internal/server/subscriptions.go b/socketserver/internal/server/subscriptions.go
new file mode 100644
index 00000000..338224b8
--- /dev/null
+++ b/socketserver/internal/server/subscriptions.go
@@ -0,0 +1,188 @@
+package server
+
+// This is the scariest code I've written yet for the server.
+// If I screwed up the locking, I won't know until it's too late.
+
+import (
+	"log"
+	"sync"
+	"time"
+)
+
+type SubscriberList struct {
+	sync.RWMutex
+	Members []chan<- ClientMessage
+}
+
+var ChatSubscriptionInfo map[string]*SubscriberList = make(map[string]*SubscriberList)
+var ChatSubscriptionLock sync.RWMutex
+var GlobalSubscriptionInfo SubscriberList
+
+func SubscribeGlobal(client *ClientInfo) {
+	GlobalSubscriptionInfo.Lock()
+	AddToSliceC(&GlobalSubscriptionInfo.Members, client.MessageChannel)
+	GlobalSubscriptionInfo.Unlock()
+}
+
+func SubscribeChannel(client *ClientInfo, channelName string) {
+	ChatSubscriptionLock.RLock()
+	_subscribeWhileRlocked(channelName, client.MessageChannel)
+	ChatSubscriptionLock.RUnlock()
+}
+
+func PublishToChannel(channel string, msg ClientMessage) (count int) {
+	ChatSubscriptionLock.RLock()
+	list := ChatSubscriptionInfo[channel]
+	if list != nil {
+		list.RLock()
+		for _, msgChan := range list.Members {
+			msgChan <- msg
+			count++
+		}
+		list.RUnlock()
+	}
+	ChatSubscriptionLock.RUnlock()
+	return
+}
+
+func PublishToMultiple(channels []string, msg ClientMessage) (count int) {
+	found := make(map[chan<- ClientMessage]struct{})
+
+	ChatSubscriptionLock.RLock()
+
+	for _, channel := range channels {
+		list := ChatSubscriptionInfo[channel]
+		if list != nil {
+			list.RLock()
+			for _, msgChan := range list.Members {
+				found[msgChan] = struct{}{}
+			}
+			list.RUnlock()
+		}
+	}
+
+	ChatSubscriptionLock.RUnlock()
+
+	for msgChan, _ := range found {
+		msgChan <- msg
+		count++
+	}
+	return
+}
+
+func PublishToAll(msg ClientMessage) (count int) {
+	GlobalSubscriptionInfo.RLock()
+	for _, msgChan := range GlobalSubscriptionInfo.Members {
+		msgChan <- msg
+		count++
+	}
+	GlobalSubscriptionInfo.RUnlock()
+	return
+}
+
+func UnsubscribeSingleChat(client *ClientInfo, channelName string) {
+	ChatSubscriptionLock.RLock()
+	list := ChatSubscriptionInfo[channelName]
+	if list != nil {
+		list.Lock()
+		RemoveFromSliceC(&list.Members, client.MessageChannel)
+		list.Unlock()
+	}
+	ChatSubscriptionLock.RUnlock()
+}
+
+// Unsubscribe the client from all channels, AND clear the CurrentChannels / WatchingChannels fields.
+// Locks:
+//   - read lock to top-level maps
+//   - write lock to SubscriptionInfos
+//   - write lock to ClientInfo
+func UnsubscribeAll(client *ClientInfo) {
+	client.Mutex.Lock()
+	client.PendingSubscriptionsBacklog = nil
+	client.PendingSubscriptionsBacklog = nil
+	client.Mutex.Unlock()
+
+	GlobalSubscriptionInfo.Lock()
+	RemoveFromSliceC(&GlobalSubscriptionInfo.Members, client.MessageChannel)
+	GlobalSubscriptionInfo.Unlock()
+
+	ChatSubscriptionLock.RLock()
+	client.Mutex.Lock()
+	for _, v := range client.CurrentChannels {
+		list := ChatSubscriptionInfo[v]
+		if list != nil {
+			list.Lock()
+			RemoveFromSliceC(&list.Members, client.MessageChannel)
+			list.Unlock()
+		}
+	}
+	client.CurrentChannels = nil
+	client.Mutex.Unlock()
+	ChatSubscriptionLock.RUnlock()
+}
+
+func unsubscribeAllClients() {
+	GlobalSubscriptionInfo.Lock()
+	GlobalSubscriptionInfo.Members = nil
+	GlobalSubscriptionInfo.Unlock()
+	ChatSubscriptionLock.Lock()
+	ChatSubscriptionInfo = make(map[string]*SubscriberList)
+	ChatSubscriptionLock.Unlock()
+}
+
+const ReapingDelay = 20 * time.Minute
+
+// Checks ChatSubscriptionInfo for entries with no subscribers every ReapingDelay.
+// Started from SetupServer().
+func pubsubJanitor() {
+	for {
+		time.Sleep(ReapingDelay)
+		var cleanedUp = make([]string, 0, 6)
+		ChatSubscriptionLock.Lock()
+		for key, val := range ChatSubscriptionInfo {
+			if val == nil || len(val.Members) == 0 {
+				delete(ChatSubscriptionInfo, key)
+				cleanedUp = append(cleanedUp, key)
+			}
+		}
+		ChatSubscriptionLock.Unlock()
+
+		if len(cleanedUp) != 0 {
+			err := SendCleanupTopicsNotice(cleanedUp)
+			if err != nil {
+				log.Println("error reporting cleaned subs:", err)
+			}
+		}
+	}
+}
+
+// Add a channel to the subscriptions while holding a read-lock to the map.
+// Locks:
+//   - ALREADY HOLDING a read-lock to the 'which' top-level map via the rlocker object
+//   - possible write lock to the 'which' top-level map via the wlocker object
+//   - write lock to SubscriptionInfo (if not creating new)
+func _subscribeWhileRlocked(channelName string, value chan<- ClientMessage) {
+	list := ChatSubscriptionInfo[channelName]
+	if list == nil {
+		// Not found, so create it
+		ChatSubscriptionLock.RUnlock()
+		ChatSubscriptionLock.Lock()
+		list = &SubscriberList{}
+		list.Members = []chan<- ClientMessage{value} // Create it populated, to avoid reaper
+		ChatSubscriptionInfo[channelName] = list
+		ChatSubscriptionLock.Unlock()
+
+		go func(topic string) {
+			err := SendNewTopicNotice(topic)
+			if err != nil {
+				log.Println("error reporting new sub:", err)
+			}
+		}(channelName)
+
+		ChatSubscriptionLock.RLock()
+	} else {
+		list.Lock()
+		AddToSliceC(&list.Members, value)
+		list.Unlock()
+	}
+}
\ No newline at end of file
diff --git a/socketserver/internal/server/subscriptions_test.go b/socketserver/internal/server/subscriptions_test.go
new file mode 100644
index 00000000..cd0364bc
--- /dev/null
+++ b/socketserver/internal/server/subscriptions_test.go
@@ -0,0 +1,448 @@
+package server
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/gorilla/websocket"
+	"github.com/satori/go.uuid"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"strconv"
+	"sync"
+	"syscall"
+	"testing"
+	"time"
+)
+
+func TCountOpenFDs() uint64 {
+	ary, _ := ioutil.ReadDir(fmt.Sprintf("/proc/%d/fd", os.Getpid()))
+	return uint64(len(ary))
+}
+
+const IgnoreReceivedArguments = 1 + 2i
+
+func TReceiveExpectedMessage(tb testing.TB, conn *websocket.Conn, messageId int, command Command, arguments interface{}) (ClientMessage, bool) {
+	var msg ClientMessage
+	var fail bool
+	messageType, packet, err := conn.ReadMessage()
+	if err != nil {
+		tb.Error(err)
+		return msg, false
+	}
+	if messageType != websocket.TextMessage {
+		tb.Error("got non-text message", packet)
+		return msg, false
+	}
+
+	err = UnmarshalClientMessage(packet, messageType, &msg)
+	if err != nil {
+		tb.Error(err)
+		return msg, false
+	}
+	if msg.MessageID != messageId {
+		tb.Error("Message ID was wrong. Expected", messageId, ", got", msg.MessageID, ":", msg)
+		fail = true
+	}
+	if msg.Command != command {
+		tb.Error("Command was wrong. Expected", command, ", got", msg.Command, ":", msg)
+		fail = true
+	}
+	if arguments != IgnoreReceivedArguments {
+		if arguments == nil {
+			if msg.origArguments != "" {
+				tb.Error("Arguments are wrong. Expected", arguments, ", got", msg.Arguments, ":", msg)
+			}
+		} else {
+			argBytes, _ := json.Marshal(arguments)
+			if msg.origArguments != string(argBytes) {
+				tb.Error("Arguments are wrong. Expected", arguments, ", got", msg.Arguments, ":", msg)
+			}
+		}
+	}
+	return msg, !fail
+}
+
+func TSendMessage(tb testing.TB, conn *websocket.Conn, messageId int, command Command, arguments interface{}) bool {
+	SendMessage(conn, ClientMessage{MessageID: messageId, Command: command, Arguments: arguments})
+	return true
+}
+
+func TSealForSavePubMsg(tb testing.TB, cmd Command, channel string, arguments interface{}, deleteMode bool) (url.Values, error) {
+	form := url.Values{}
+	form.Set("cmd", string(cmd))
+	argsBytes, err := json.Marshal(arguments)
+	if err != nil {
+		tb.Error(err)
+		return nil, err
+	}
+	form.Set("args", string(argsBytes))
+	form.Set("channel", channel)
+	if deleteMode {
+		form.Set("delete", "1")
+	}
+	form.Set("time", time.Now().Format(time.UnixDate))
+
+	sealed, err := SealRequest(form)
+	if err != nil {
+		tb.Error(err)
+		return nil, err
+	}
+	return sealed, nil
+}
+
+func TCheckResponse(tb testing.TB, resp *http.Response, expected string) bool {
+	var failed bool
+	respBytes, err := ioutil.ReadAll(resp.Body)
+	resp.Body.Close()
+	respStr := string(respBytes)
+
+	if err != nil {
+		tb.Error(err)
+		failed = true
+	}
+
+	if resp.StatusCode != 200 {
+		tb.Error("Publish failed: ", resp.StatusCode, respStr)
+		failed = true
+	}
+
+	if respStr != expected {
+		tb.Errorf("Got wrong response from server. Expected: '%s' Got: '%s'", expected, respStr)
+		failed = true
+	}
+	return !failed
+}
+
+type TURLs struct {
+	Websocket  string
+	Origin     string
+	PubMsg     string
+	SavePubMsg string // update_and_pub
+}
+
+func TGetUrls(testserver *httptest.Server) TURLs {
+	addr := testserver.Listener.Addr().String()
+	return TURLs{
+		Websocket:  fmt.Sprintf("ws://%s/", addr),
+		Origin:     fmt.Sprintf("http://%s", addr),
+		PubMsg:     fmt.Sprintf("http://%s/pub_msg", addr),
+		SavePubMsg: fmt.Sprintf("http://%s/update_and_pub", addr),
+	}
+}
+
+func TSetup(testserver **httptest.Server, urls *TURLs) {
+	DumpCache()
+
+	conf := &ConfigFile{
+		ServerId:     20,
+		UseSSL:       false,
+		SocketOrigin: "localhost:2002",
+		BannerHTML: `
+<!DOCTYPE html>
+<title>CatBag</title>
+<link rel="stylesheet" href="//cdn.frankerfacez.com/script/catbag.css">
+<div id="container">
+<div id="zf0"></div><div id="zf1"></div><div id="zf2"></div>
+<div id="zf3"></div><div id="zf4"></div><div id="zf5"></div>
+<div id="zf6"></div><div id="zf7"></div><div id="zf8"></div>
+<div id="zf9"></div><div id="catbag"></div>
+<div id="bottom">
+	A <a href="http://www.frankerfacez.com/">FrankerFaceZ</a> Service
+	&mdash; CatBag by <a href="http://www.twitch.tv/wolsk">Wolsk</a>
+</div>
+</div>
+`,
+		OurPublicKey:     []byte{176, 149, 72, 209, 35, 42, 110, 220, 22, 236, 212, 129, 213, 199, 1, 227, 185, 167, 150, 159, 117, 202, 164, 100, 9, 107, 45, 141, 122, 221, 155, 73},
+		OurPrivateKey:    []byte{247, 133, 147, 194, 70, 240, 211, 216, 223, 16, 241, 253, 120, 14, 198, 74, 237, 180, 89, 33, 146, 146, 140, 58, 88, 160, 2, 246, 112, 35, 239, 87},
+		BackendPublicKey: []byte{19, 163, 37, 157, 50, 139, 193, 85, 229, 47, 166, 21, 153, 231, 31, 133, 41, 158, 8, 53, 73, 0, 113, 91, 13, 181, 131, 248, 176, 18, 1, 107},
+	}
+	gconfig = conf
+	SetupBackend(conf)
+
+	if testserver != nil {
+		serveMux := http.NewServeMux()
+		SetupServerAndHandle(conf, serveMux)
+
+		tserv := httptest.NewUnstartedServer(serveMux)
+		*testserver = tserv
+		tserv.Start()
+		if urls != nil {
+			*urls = TGetUrls(tserv)
+		}
+	}
+}
+
+func TestSubscriptionAndPublish(t *testing.T) {
+	var doneWg sync.WaitGroup
+	var readyWg sync.WaitGroup
+
+	const TestChannelName1 = "room.testchannel"
+	const TestChannelName2 = "room.chan2"
+	const TestChannelName3 = "room.chan3"
+	const TestChannelNameUnused = "room.empty"
+	const TestCommandChan = "testdata_single"
+	const TestCommandMulti = "testdata_multi"
+	const TestCommandGlobal = "testdata_global"
+	const TestData1 = "123456789"
+	const TestData2 = 42
+	const TestData3 = false
+	var TestData4 = []interface{}{"str1", "str2", "str3"}
+
+	ServerInitiatedCommands[TestCommandChan] = PushCommandCacheInfo{CacheTypeLastOnly, MsgTargetTypeChat}
+	ServerInitiatedCommands[TestCommandMulti] = PushCommandCacheInfo{CacheTypeTimestamps, MsgTargetTypeMultichat}
+	ServerInitiatedCommands[TestCommandGlobal] = PushCommandCacheInfo{CacheTypeTimestamps, MsgTargetTypeGlobal}
+
+	var server *httptest.Server
+	var urls TURLs
+	TSetup(&server, &urls)
+	defer server.CloseClientConnections()
+	defer unsubscribeAllClients()
+
+	var conn *websocket.Conn
+	var resp *http.Response
+	var err error
+
+	// client 1: sub ch1, ch2
+	// client 2: sub ch1, ch3
+	// client 3: sub none
+	// client 4: delayed sub ch1
+	// msg 1: ch1
+	// msg 2: ch2, ch3
+	// msg 3: chEmpty
+	// msg 4: global
+
+	// Client 1
+	conn, resp, err = websocket.DefaultDialer.Dial(urls.Websocket, http.Header{})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	doneWg.Add(1)
+	readyWg.Add(1)
+	go func(conn *websocket.Conn) {
+		TSendMessage(t, conn, 1, HelloCommand, []interface{}{"ffz_0.0-test", uuid.NewV4().String()})
+		TReceiveExpectedMessage(t, conn, 1, SuccessCommand, IgnoreReceivedArguments)
+		TSendMessage(t, conn, 2, "sub", TestChannelName1)
+		TReceiveExpectedMessage(t, conn, 2, SuccessCommand, nil)
+		TSendMessage(t, conn, 3, "sub", TestChannelName2) // 2
+		TReceiveExpectedMessage(t, conn, 3, SuccessCommand, nil)
+		TSendMessage(t, conn, 4, "ready", 0)
+		TReceiveExpectedMessage(t, conn, 4, SuccessCommand, nil)
+
+		readyWg.Done()
+
+		TReceiveExpectedMessage(t, conn, -1, TestCommandChan, TestData1)
+		TReceiveExpectedMessage(t, conn, -1, TestCommandMulti, TestData2)
+		TReceiveExpectedMessage(t, conn, -1, TestCommandGlobal, TestData4)
+
+		conn.Close()
+		doneWg.Done()
+	}(conn)
+
+	// Client 2
+	conn, resp, err = websocket.DefaultDialer.Dial(urls.Websocket, http.Header{})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	doneWg.Add(1)
+	readyWg.Add(1)
+	go func(conn *websocket.Conn) {
+		TSendMessage(t, conn, 1, HelloCommand, []interface{}{"ffz_0.0-test", uuid.NewV4().String()})
+		TReceiveExpectedMessage(t, conn, 1, SuccessCommand, IgnoreReceivedArguments)
+		TSendMessage(t, conn, 2, "sub", TestChannelName1)
+		TReceiveExpectedMessage(t, conn, 2, SuccessCommand, nil)
+		TSendMessage(t, conn, 3, "sub", TestChannelName3) // 3
+		TReceiveExpectedMessage(t, conn, 3, SuccessCommand, nil)
+		TSendMessage(t, conn, 4, "ready", 0)
+		TReceiveExpectedMessage(t, conn, 4, SuccessCommand, nil)
+
+		readyWg.Done()
+
+		TReceiveExpectedMessage(t, conn, -1, TestCommandChan, TestData1)
+		TReceiveExpectedMessage(t, conn, -1, TestCommandMulti, TestData2)
+		TReceiveExpectedMessage(t, conn, -1, TestCommandGlobal, TestData4)
+
+		conn.Close()
+		doneWg.Done()
+	}(conn)
+
+	// Client 3
+	conn, resp, err = websocket.DefaultDialer.Dial(urls.Websocket, http.Header{})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	doneWg.Add(1)
+	readyWg.Add(1)
+	go func(conn *websocket.Conn) {
+		TSendMessage(t, conn, 1, HelloCommand, []interface{}{"ffz_0.0-test", uuid.NewV4().String()})
+		TReceiveExpectedMessage(t, conn, 1, SuccessCommand, IgnoreReceivedArguments)
+		TSendMessage(t, conn, 2, "ready", 0)
+		TReceiveExpectedMessage(t, conn, 2, SuccessCommand, nil)
+
+		readyWg.Done()
+
+		TReceiveExpectedMessage(t, conn, -1, TestCommandGlobal, TestData4)
+
+		conn.Close()
+		doneWg.Done()
+	}(conn)
+
+	// Wait for clients 1-3
+	readyWg.Wait()
+
+	var form url.Values
+
+	// Publish message 1 - should go to clients 1, 2
+
+	form, err = TSealForSavePubMsg(t, TestCommandChan, TestChannelName1, TestData1, false)
+	if err != nil {
+		t.FailNow()
+	}
+	resp, err = http.PostForm(urls.SavePubMsg, form)
+	if !TCheckResponse(t, resp, strconv.Itoa(2)) {
+		t.FailNow()
+	}
+
+	// Publish message 2 - should go to clients 1, 2
+
+	form, err = TSealForSavePubMsg(t, TestCommandMulti, TestChannelName2+","+TestChannelName3, TestData2, false)
+	if err != nil {
+		t.FailNow()
+	}
+	resp, err = http.PostForm(urls.SavePubMsg, form)
+	if !TCheckResponse(t, resp, strconv.Itoa(2)) {
+		t.FailNow()
+	}
+
+	// Publish message 3 - should go to no clients
+
+	form, err = TSealForSavePubMsg(t, TestCommandChan, TestChannelNameUnused, TestData3, false)
+	if err != nil {
+		t.FailNow()
+	}
+	resp, err = http.PostForm(urls.SavePubMsg, form)
+	if !TCheckResponse(t, resp, strconv.Itoa(0)) {
+		t.FailNow()
+	}
+
+	// Publish message 4 - should go to clients 1, 2, 3
+
+	form, err = TSealForSavePubMsg(t, TestCommandGlobal, "", TestData4, false)
+	if err != nil {
+		t.FailNow()
+	}
+	resp, err = http.PostForm(urls.SavePubMsg, form)
+	if !TCheckResponse(t, resp, strconv.Itoa(3)) {
+		t.FailNow()
+	}
+
+	// Start client 4
+	conn, resp, err = websocket.DefaultDialer.Dial(urls.Websocket, http.Header{})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	doneWg.Add(1)
+	readyWg.Add(1)
+	go func(conn *websocket.Conn) {
+		TSendMessage(t, conn, 1, HelloCommand, []interface{}{"ffz_0.0-test", uuid.NewV4().String()})
+		TReceiveExpectedMessage(t, conn, 1, SuccessCommand, IgnoreReceivedArguments)
+		TSendMessage(t, conn, 2, "sub", TestChannelName1)
+		TReceiveExpectedMessage(t, conn, 2, SuccessCommand, nil)
+		TSendMessage(t, conn, 3, "ready", 0)
+		TReceiveExpectedMessage(t, conn, 3, SuccessCommand, nil)
+
+		// backlog message
+		TReceiveExpectedMessage(t, conn, -1, TestCommandChan, TestData1)
+
+		readyWg.Done()
+
+		conn.Close()
+		doneWg.Done()
+	}(conn)
+
+	readyWg.Wait()
+
+	doneWg.Wait()
+	server.Close()
+}
+
+func BenchmarkUserSubscriptionSinglePublish(b *testing.B) {
+	var doneWg sync.WaitGroup
+	var readyWg sync.WaitGroup
+
+	const TestChannelName = "room.testchannel"
+	const TestCommand = "testdata"
+	const TestData = "123456789"
+
+	message := ClientMessage{MessageID: -1, Command: "testdata", Arguments: TestData}
+
+	fmt.Println()
+	fmt.Println(b.N)
+
+	var limit syscall.Rlimit
+	syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit)
+
+	limit.Cur = TCountOpenFDs() + uint64(b.N)*2 + 100
+
+	if limit.Cur > limit.Max {
+		b.Skip("Open file limit too low")
+		return
+	}
+
+	syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit)
+
+	var server *httptest.Server
+	var urls TURLs
+	TSetup(&server, &urls)
+	defer unsubscribeAllClients()
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		conn, _, err := websocket.DefaultDialer.Dial(urls.Websocket, http.Header{})
+		if err != nil {
+			b.Error(err)
+			break
+		}
+		doneWg.Add(1)
+		readyWg.Add(1)
+		go func(i int, conn *websocket.Conn) {
+			TSendMessage(b, conn, 1, HelloCommand, []interface{}{"ffz_0.0-test", uuid.NewV4().String()})
+			TSendMessage(b, conn, 2, "sub", TestChannelName)
+
+			TReceiveExpectedMessage(b, conn, 1, SuccessCommand, IgnoreReceivedArguments)
+			TReceiveExpectedMessage(b, conn, 2, SuccessCommand, nil)
+
+			readyWg.Done()
+
+			TReceiveExpectedMessage(b, conn, -1, TestCommand, TestData)
+
+			conn.Close()
+			doneWg.Done()
+		}(i, conn)
+	}
+
+	readyWg.Wait()
+
+	fmt.Println("publishing...")
+	if PublishToChannel(TestChannelName, message) != b.N {
+		b.Error("not enough sent")
+		server.CloseClientConnections()
+		panic("halting test instead of waiting")
+	}
+	doneWg.Wait()
+	fmt.Println("...done.")
+
+	b.StopTimer()
+	server.Close()
+	server.CloseClientConnections()
+}

From 377afb7a6ba2c3fa7023690e88e18f94ebe8809b Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Mon, 9 Nov 2015 14:44:33 -0800
Subject: [PATCH 041/176] Speed up subscription janitor to 1min

---
 socketserver/cmd/ffzsocketserver/console.go   |  2 +-
 socketserver/internal/server/commands.go      |  1 +
 socketserver/internal/server/subscriptions.go | 18 +++++++++++-------
 3 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/socketserver/cmd/ffzsocketserver/console.go b/socketserver/cmd/ffzsocketserver/console.go
index e6ceeefb..5aacd702 100644
--- a/socketserver/cmd/ffzsocketserver/console.go
+++ b/socketserver/cmd/ffzsocketserver/console.go
@@ -52,7 +52,7 @@ func commandLineConsole() {
 		if target == "_ALL" {
 			count = server.PublishToAll(msg)
 		} else {
-			count = server.PublishToChat(target, msg)
+			count = server.PublishToChannel(target, msg)
 		}
 		return fmt.Sprintf("Published to %d clients", count), nil
 	})
diff --git a/socketserver/internal/server/commands.go b/socketserver/internal/server/commands.go
index 9f8fed86..24f60284 100644
--- a/socketserver/internal/server/commands.go
+++ b/socketserver/internal/server/commands.go
@@ -77,6 +77,7 @@ func HandleHello(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (r
 	}
 
 	SubscribeGlobal(client)
+	SubscribeDefaults(client)
 
 	return ClientMessage{
 		Arguments: client.ClientID.String(),
diff --git a/socketserver/internal/server/subscriptions.go b/socketserver/internal/server/subscriptions.go
index 338224b8..2c2cfaa8 100644
--- a/socketserver/internal/server/subscriptions.go
+++ b/socketserver/internal/server/subscriptions.go
@@ -18,18 +18,22 @@ var ChatSubscriptionInfo map[string]*SubscriberList = make(map[string]*Subscribe
 var ChatSubscriptionLock sync.RWMutex
 var GlobalSubscriptionInfo SubscriberList
 
-func SubscribeGlobal(client *ClientInfo) {
-	GlobalSubscriptionInfo.Lock()
-	AddToSliceC(&GlobalSubscriptionInfo.Members, client.MessageChannel)
-	GlobalSubscriptionInfo.Unlock()
-}
-
 func SubscribeChannel(client *ClientInfo, channelName string) {
 	ChatSubscriptionLock.RLock()
 	_subscribeWhileRlocked(channelName, client.MessageChannel)
 	ChatSubscriptionLock.RUnlock()
 }
 
+func SubscribeDefaults(client *ClientInfo) {
+
+}
+
+func SubscribeGlobal(client *ClientInfo) {
+	GlobalSubscriptionInfo.Lock()
+	AddToSliceC(&GlobalSubscriptionInfo.Members, client.MessageChannel)
+	GlobalSubscriptionInfo.Unlock()
+}
+
 func PublishToChannel(channel string, msg ClientMessage) (count int) {
 	ChatSubscriptionLock.RLock()
 	list := ChatSubscriptionInfo[channel]
@@ -130,7 +134,7 @@ func unsubscribeAllClients() {
 	ChatSubscriptionLock.Unlock()
 }
 
-const ReapingDelay = 20 * time.Minute
+const ReapingDelay = 1 * time.Minute
 
 // Checks ChatSubscriptionInfo for entries with no subscribers every ReapingDelay.
 // Started from SetupServer().

From d518759fa012bcec4979ef9c0e770d4497e6c3c4 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 15 Nov 2015 15:52:37 -0800
Subject: [PATCH 042/176] Re-add bunched request caching with a lazy janitor

And this time, only one lock is held at a time during the response.
---
 socketserver/internal/server/commands.go   | 80 +++++++++++++++++++++-
 socketserver/internal/server/handlecore.go |  5 +-
 2 files changed, 81 insertions(+), 4 deletions(-)

diff --git a/socketserver/internal/server/commands.go b/socketserver/internal/server/commands.go
index 24f60284..68b29d5c 100644
--- a/socketserver/internal/server/commands.go
+++ b/socketserver/internal/server/commands.go
@@ -379,12 +379,82 @@ type BunchSubscriberList struct {
 	Members []BunchSubscriber
 }
 
+type CacheStatus byte
+const (
+	CacheStatusNotFound = iota
+	CacheStatusFound
+	CacheStatusExpired
+)
+
 var PendingBunchedRequests map[BunchedRequest]*BunchSubscriberList = make(map[BunchedRequest]*BunchSubscriberList)
 var PendingBunchLock sync.Mutex
+var CachedBunchedRequests map[BunchedRequest]BunchedResponse = make(map[BunchedRequest]BunchedResponse)
+var CachedBunchLock sync.RWMutex
+var BunchCacheCleanupSignal *sync.Cond = sync.NewCond(&CachedBunchLock)
+var BunchCacheLastCleanup time.Time
+
+func bunchCacheJanitor() {
+	go func() {
+		for {
+			time.Sleep(30*time.Minute)
+			BunchCacheCleanupSignal.Signal()
+		}
+	}()
+
+	CachedBunchLock.Lock()
+	for {
+		// Unlocks CachedBunchLock, waits for signal, re-locks
+		BunchCacheCleanupSignal.Wait()
+
+		if BunchCacheLastCleanup.After(time.Now().Add(-1*time.Second)) {
+			// skip if it's been less than 1 second
+			continue
+		}
+
+		// CachedBunchLock is held here
+		keepIfAfter := time.Now().Add(-5*time.Minute)
+		for req, resp := range CachedBunchedRequests {
+			if !resp.Timestamp.After(keepIfAfter) {
+				delete(CachedBunchedRequests, req)
+			}
+		}
+		BunchCacheLastCleanup = time.Now()
+		// Loop and Wait(), which re-locks
+	}
+}
 
 func HandleBunchedRemoteCommand(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
 	br := BunchedRequestFromCM(&msg)
 
+	cacheStatus := func() byte {
+		CachedBunchLock.RLock()
+		defer CachedBunchLock.RUnlock()
+		bresp, ok := CachedBunchedRequests[br]
+		if ok && bresp.Timestamp.After(time.Now().Add(-5*time.Minute)) {
+			client.MsgChannelKeepalive.Add(1)
+			go func() {
+				var rmsg ClientMessage
+				rmsg.Command = SuccessCommand
+				rmsg.MessageID = msg.MessageID
+				rmsg.origArguments = bresp.Response
+				rmsg.parseOrigArguments()
+				client.MessageChannel <- rmsg
+				client.MsgChannelKeepalive.Done()
+			}()
+			return CacheStatusFound
+		} else if ok {
+			return CacheStatusExpired
+		}
+		return CacheStatusNotFound
+	}()
+
+	if cacheStatus == CacheStatusFound {
+		return ClientMessage{Command: AsyncResponseCommand}, nil
+	} else if cacheStatus == CacheStatusExpired {
+		// Wake up the lazy janitor
+		BunchCacheCleanupSignal.Signal()
+	}
+
 	PendingBunchLock.Lock()
 	defer PendingBunchLock.Unlock()
 	list, ok := PendingBunchedRequests[br]
@@ -399,18 +469,24 @@ func HandleBunchedRemoteCommand(conn *websocket.Conn, client *ClientInfo, msg Cl
 	PendingBunchedRequests[br] = &BunchSubscriberList{Members: []BunchSubscriber{{Client: client, MessageID: msg.MessageID}}}
 
 	go func(request BunchedRequest) {
-		resp, err := SendRemoteCommandCached(string(request.Command), request.Param, AuthInfo{})
+		respStr, err := SendRemoteCommandCached(string(request.Command), request.Param, AuthInfo{})
 
 		var msg ClientMessage
 		if err == nil {
 			msg.Command = SuccessCommand
-			msg.origArguments = resp
+			msg.origArguments = respStr
 			msg.parseOrigArguments()
 		} else {
 			msg.Command = ErrorCommand
 			msg.Arguments = err.Error()
 		}
 
+		if err == nil {
+			CachedBunchLock.Lock()
+			CachedBunchedRequests[request] = BunchedResponse{Response: respStr, Timestamp: time.Now()}
+			CachedBunchLock.Unlock()
+		}
+
 		PendingBunchLock.Lock()
 		bsl := PendingBunchedRequests[request]
 		delete(PendingBunchedRequests, request)
diff --git a/socketserver/internal/server/handlecore.go b/socketserver/internal/server/handlecore.go
index d0d0c694..6ecb1d28 100644
--- a/socketserver/internal/server/handlecore.go
+++ b/socketserver/internal/server/handlecore.go
@@ -73,9 +73,10 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 		resp.Body.Close()
 	}
 
-	go pubsubJanitor()
-	go backlogJanitor()
 	go authorizationJanitor()
+	go backlogJanitor()
+	go bunchCacheJanitor()
+	go pubsubJanitor()
 	go sendAggregateData()
 
 	go ircConnection()

From aa6f090fccd49cf0fcf8046ccd6071c6caf9569b Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 15 Nov 2015 15:55:20 -0800
Subject: [PATCH 043/176] Rename a few variables

---
 socketserver/internal/server/commands.go | 26 ++++++++++++------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/socketserver/internal/server/commands.go b/socketserver/internal/server/commands.go
index 68b29d5c..79a8b6d7 100644
--- a/socketserver/internal/server/commands.go
+++ b/socketserver/internal/server/commands.go
@@ -365,7 +365,7 @@ func BunchedRequestFromCM(msg *ClientMessage) BunchedRequest {
 	return BunchedRequest{Command: msg.Command, Param: msg.origArguments}
 }
 
-type BunchedResponse struct {
+type CachedBunchedResponse struct {
 	Response  string
 	Timestamp time.Time
 }
@@ -388,9 +388,9 @@ const (
 
 var PendingBunchedRequests map[BunchedRequest]*BunchSubscriberList = make(map[BunchedRequest]*BunchSubscriberList)
 var PendingBunchLock sync.Mutex
-var CachedBunchedRequests map[BunchedRequest]BunchedResponse = make(map[BunchedRequest]BunchedResponse)
-var CachedBunchLock sync.RWMutex
-var BunchCacheCleanupSignal *sync.Cond = sync.NewCond(&CachedBunchLock)
+var BunchCache map[BunchedRequest]CachedBunchedResponse = make(map[BunchedRequest]CachedBunchedResponse)
+var BunchCacheLock sync.RWMutex
+var BunchCacheCleanupSignal *sync.Cond = sync.NewCond(&BunchCacheLock)
 var BunchCacheLastCleanup time.Time
 
 func bunchCacheJanitor() {
@@ -401,7 +401,7 @@ func bunchCacheJanitor() {
 		}
 	}()
 
-	CachedBunchLock.Lock()
+	BunchCacheLock.Lock()
 	for {
 		// Unlocks CachedBunchLock, waits for signal, re-locks
 		BunchCacheCleanupSignal.Wait()
@@ -413,9 +413,9 @@ func bunchCacheJanitor() {
 
 		// CachedBunchLock is held here
 		keepIfAfter := time.Now().Add(-5*time.Minute)
-		for req, resp := range CachedBunchedRequests {
+		for req, resp := range BunchCache {
 			if !resp.Timestamp.After(keepIfAfter) {
-				delete(CachedBunchedRequests, req)
+				delete(BunchCache, req)
 			}
 		}
 		BunchCacheLastCleanup = time.Now()
@@ -427,9 +427,9 @@ func HandleBunchedRemoteCommand(conn *websocket.Conn, client *ClientInfo, msg Cl
 	br := BunchedRequestFromCM(&msg)
 
 	cacheStatus := func() byte {
-		CachedBunchLock.RLock()
-		defer CachedBunchLock.RUnlock()
-		bresp, ok := CachedBunchedRequests[br]
+		BunchCacheLock.RLock()
+		defer BunchCacheLock.RUnlock()
+		bresp, ok := BunchCache[br]
 		if ok && bresp.Timestamp.After(time.Now().Add(-5*time.Minute)) {
 			client.MsgChannelKeepalive.Add(1)
 			go func() {
@@ -482,9 +482,9 @@ func HandleBunchedRemoteCommand(conn *websocket.Conn, client *ClientInfo, msg Cl
 		}
 
 		if err == nil {
-			CachedBunchLock.Lock()
-			CachedBunchedRequests[request] = BunchedResponse{Response: respStr, Timestamp: time.Now()}
-			CachedBunchLock.Unlock()
+			BunchCacheLock.Lock()
+			BunchCache[request] = CachedBunchedResponse{Response: respStr, Timestamp: time.Now()}
+			BunchCacheLock.Unlock()
 		}
 
 		PendingBunchLock.Lock()

From 66cc124e372af280620cb71446b7acf89b901ebe Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 15 Nov 2015 18:43:34 -0800
Subject: [PATCH 044/176] golint part 1

---
 socketserver/cmd/ffzsocketserver/console.go   | 12 ++-
 .../cmd/ffzsocketserver/socketserver.go       | 10 +--
 socketserver/internal/server/backend.go       | 83 +++++++++++--------
 socketserver/internal/server/commands.go      | 59 +++++++------
 socketserver/internal/server/handlecore.go    | 50 ++++++-----
 socketserver/internal/server/subscriptions.go |  2 +-
 .../internal/server/subscriptions_test.go     | 14 ++--
 socketserver/internal/server/types.go         | 16 ++--
 socketserver/internal/server/utils.go         |  2 +-
 9 files changed, 136 insertions(+), 112 deletions(-)

diff --git a/socketserver/cmd/ffzsocketserver/console.go b/socketserver/cmd/ffzsocketserver/console.go
index 5aacd702..88cc4b44 100644
--- a/socketserver/cmd/ffzsocketserver/console.go
+++ b/socketserver/cmd/ffzsocketserver/console.go
@@ -74,20 +74,18 @@ func commandLineConsole() {
 
 	shell.Register("authorizeeveryone", func(args ...string) (string, error) {
 		if len(args) == 0 {
-			if server.Configuation.SendAuthToNewClients {
+			if server.Configuration.SendAuthToNewClients {
 				return "All clients are recieving auth challenges upon claiming a name.", nil
-			} else {
-				return "All clients are not recieving auth challenges upon claiming a name.", nil
 			}
+			return "All clients are not recieving auth challenges upon claiming a name.", nil
 		} else if args[0] == "on" {
-			server.Configuation.SendAuthToNewClients = true
+			server.Configuration.SendAuthToNewClients = true
 			return "All new clients will recieve auth challenges upon claiming a name.", nil
 		} else if args[0] == "off" {
-			server.Configuation.SendAuthToNewClients = false
+			server.Configuration.SendAuthToNewClients = false
 			return "All new clients will not recieve auth challenges upon claiming a name.", nil
-		} else {
-			return "Usage: authorizeeveryone [ on | off ]", nil
 		}
+		return "Usage: authorizeeveryone [ on | off ]", nil
 	})
 
 	shell.Register("panic", func(args ...string) (string, error) {
diff --git a/socketserver/cmd/ffzsocketserver/socketserver.go b/socketserver/cmd/ffzsocketserver/socketserver.go
index a7e1b9f3..c280218f 100644
--- a/socketserver/cmd/ffzsocketserver/socketserver.go
+++ b/socketserver/cmd/ffzsocketserver/socketserver.go
@@ -11,14 +11,14 @@ import (
 	"os"
 )
 
-var configFilename *string = flag.String("config", "config.json", "Configuration file, including the keypairs for the NaCl crypto library, for communicating with the backend.")
-var generateKeys *bool = flag.Bool("genkeys", false, "Generate NaCl keys instead of serving requests.\nArguments: [int serverId] [base64 backendPublic]\nThe backend public key can either be specified in base64 on the command line, or put in the json file later.")
+var configFilename = flag.String("config", "config.json", "Configuration file, including the keypairs for the NaCl crypto library, for communicating with the backend.")
+var flagGenerateKeys = flag.Bool("genkeys", false, "Generate NaCl keys instead of serving requests.\nArguments: [int serverId] [base64 backendPublic]\nThe backend public key can either be specified in base64 on the command line, or put in the json file later.")
 
 func main() {
 	flag.Parse()
 
-	if *generateKeys {
-		GenerateKeys(*configFilename)
+	if *flagGenerateKeys {
+		generateKeys(*configFilename)
 		return
 	}
 
@@ -66,7 +66,7 @@ func main() {
 	}
 }
 
-func GenerateKeys(outputFile string) {
+func generateKeys(outputFile string) {
 	if flag.NArg() < 1 {
 		fmt.Println("Specify a numeric server ID after -genkeys")
 		os.Exit(2)
diff --git a/socketserver/internal/server/backend.go b/socketserver/internal/server/backend.go
index 4557f534..d4cc5bf2 100644
--- a/socketserver/internal/server/backend.go
+++ b/socketserver/internal/server/backend.go
@@ -19,39 +19,39 @@ import (
 	"time"
 )
 
-var backendHttpClient http.Client
-var backendUrl string
+var backendHTTPClient http.Client
+var backendURL string
 var responseCache *cache.Cache
 
-var getBacklogUrl string
-var postStatisticsUrl string
-var addTopicUrl string
-var announceStartupUrl string
+var getBacklogURL string
+var postStatisticsURL string
+var addTopicURL string
+var announceStartupURL string
 
 var backendSharedKey [32]byte
-var serverId int
+var serverID int
 
 var messageBufferPool sync.Pool
 
-func SetupBackend(config *ConfigFile) {
-	backendHttpClient.Timeout = 60 * time.Second
-	backendUrl = config.BackendUrl
+func setupBackend(config *ConfigFile) {
+	backendHTTPClient.Timeout = 60 * time.Second
+	backendURL = config.BackendURL
 	if responseCache != nil {
 		responseCache.Flush()
 	}
 	responseCache = cache.New(60*time.Second, 120*time.Second)
 
-	getBacklogUrl = fmt.Sprintf("%s/backlog", backendUrl)
-	postStatisticsUrl = fmt.Sprintf("%s/stats", backendUrl)
-	addTopicUrl = fmt.Sprintf("%s/topics", backendUrl)
-	announceStartupUrl = fmt.Sprintf("%s/startup", backendUrl)
+	getBacklogURL = fmt.Sprintf("%s/backlog", backendURL)
+	postStatisticsURL = fmt.Sprintf("%s/stats", backendURL)
+	addTopicURL = fmt.Sprintf("%s/topics", backendURL)
+	announceStartupURL = fmt.Sprintf("%s/startup", backendURL)
 
 	messageBufferPool.New = New4KByteBuffer
 
 	var theirPublic, ourPrivate [32]byte
 	copy(theirPublic[:], config.BackendPublicKey)
 	copy(ourPrivate[:], config.OurPrivateKey)
-	serverId = config.ServerId
+	serverID = config.ServerID
 
 	box.Precompute(&backendSharedKey, &theirPublic, &ourPrivate)
 }
@@ -60,8 +60,10 @@ func getCacheKey(remoteCommand, data string) string {
 	return fmt.Sprintf("%s/%s", remoteCommand, data)
 }
 
-// Publish a message to clients with no caching.
-// The scope must be specified because no attempt is made to recognize the command.
+// HBackendPublishRequest handles the /uncached_pub route.
+// The backend can POST here to publish a message to clients with no caching.
+// The POST arguments are `cmd`, `args`, `channel`, and `scope`.
+// The `scope` argument is required because no attempt is made to infer the scope from the command, unlike /cached_pub.
 func HBackendPublishRequest(w http.ResponseWriter, r *http.Request) {
 	r.ParseForm()
 	formData, err := UnsealRequest(r.Form)
@@ -95,7 +97,7 @@ func HBackendPublishRequest(w http.ResponseWriter, r *http.Request) {
 
 	switch target {
 	case MsgTargetTypeSingle:
-		// TODO
+	// TODO
 	case MsgTargetTypeChat:
 		count = PublishToChannel(channel, cm)
 	case MsgTargetTypeMultichat:
@@ -111,14 +113,18 @@ func HBackendPublishRequest(w http.ResponseWriter, r *http.Request) {
 	fmt.Fprint(w, count)
 }
 
-type BackendForwardedError string
+// ErrForwardedFromBackend is an error returned by the backend server.
+type ErrForwardedFromBackend string
 
-func (bfe BackendForwardedError) Error() string {
+func (bfe ErrForwardedFromBackend) Error() string {
 	return string(bfe)
 }
 
-var AuthorizationNeededError = errors.New("Must authenticate Twitch username to use this command")
+// ErrAuthorizationNeeded is emitted when the backend replies with HTTP 401.
+// Indicates that an attempt to validate `ClientInfo.TwitchUsername` should be attempted.
+var ErrAuthorizationNeeded = errors.New("Must authenticate Twitch username to use this command")
 
+// SendRemoteCommandCached performs a RPC call on the backend, but caches responses.
 func SendRemoteCommandCached(remoteCommand, data string, auth AuthInfo) (string, error) {
 	cached, ok := responseCache.Get(getCacheKey(remoteCommand, data))
 	if ok {
@@ -127,8 +133,12 @@ func SendRemoteCommandCached(remoteCommand, data string, auth AuthInfo) (string,
 	return SendRemoteCommand(remoteCommand, data, auth)
 }
 
+// SendRemoteCommand performs a RPC call on the backend by POSTing to `/cmd/$remoteCommand`.
+// The form data is as follows: `clientData` is the JSON in the `data` parameter
+// (should be retrieved from ClientMessage.Arguments), and either `username` or
+// `usernameClaimed` depending on whether AuthInfo.UsernameValidates is true is AuthInfo.TwitchUsername.
 func SendRemoteCommand(remoteCommand, data string, auth AuthInfo) (responseStr string, err error) {
-	destUrl := fmt.Sprintf("%s/cmd/%s", backendUrl, remoteCommand)
+	destURL := fmt.Sprintf("%s/cmd/%s", backendURL, remoteCommand)
 	var authKey string
 	if auth.UsernameValidated {
 		authKey = "usernameClaimed"
@@ -146,7 +156,7 @@ func SendRemoteCommand(remoteCommand, data string, auth AuthInfo) (responseStr s
 		return "", err
 	}
 
-	resp, err := backendHttpClient.PostForm(destUrl, sealedForm)
+	resp, err := backendHTTPClient.PostForm(destURL, sealedForm)
 	if err != nil {
 		return "", err
 	}
@@ -160,13 +170,12 @@ func SendRemoteCommand(remoteCommand, data string, auth AuthInfo) (responseStr s
 	responseStr = string(respBytes)
 
 	if resp.StatusCode == 401 {
-		return "", AuthorizationNeededError
+		return "", ErrAuthorizationNeeded
 	} else if resp.StatusCode != 200 {
 		if resp.Header.Get("Content-Type") == "application/json" {
-			return "", BackendForwardedError(responseStr)
-		} else {
-			return "", httpError(resp.StatusCode)
+			return "", ErrForwardedFromBackend(responseStr)
 		}
+		return "", httpError(resp.StatusCode)
 	}
 
 	if resp.Header.Get("FFZ-Cache") != "" {
@@ -182,7 +191,7 @@ func SendRemoteCommand(remoteCommand, data string, auth AuthInfo) (responseStr s
 }
 
 func SendAggregatedData(sealedForm url.Values) error {
-	resp, err := backendHttpClient.PostForm(postStatisticsUrl, sealedForm)
+	resp, err := backendHTTPClient.PostForm(postStatisticsURL, sealedForm)
 	if err != nil {
 		return err
 	}
@@ -204,7 +213,7 @@ func FetchBacklogData(chatSubs []string) ([]ClientMessage, error) {
 		return nil, err
 	}
 
-	resp, err := backendHttpClient.PostForm(getBacklogUrl, sealedForm)
+	resp, err := backendHTTPClient.PostForm(getBacklogURL, sealedForm)
 	if err != nil {
 		return nil, err
 	}
@@ -227,12 +236,14 @@ func FetchBacklogData(chatSubs []string) ([]ClientMessage, error) {
 	return messages, nil
 }
 
-type NotOkError struct {
+// ErrBackendNotOK indicates that the backend replied with something other than the string "ok".
+type ErrBackendNotOK struct {
 	Response string
 	Code     int
 }
 
-func (noe NotOkError) Error() string {
+// Implements the error interface.
+func (noe ErrBackendNotOK) Error() string {
 	return fmt.Sprintf("backend returned %d: %s", noe.Code, noe.Response)
 }
 
@@ -258,7 +269,7 @@ func sendTopicNotice(topic string, added bool) error {
 		return err
 	}
 
-	resp, err := backendHttpClient.PostForm(addTopicUrl, sealedForm)
+	resp, err := backendHTTPClient.PostForm(addTopicURL, sealedForm)
 	if err != nil {
 		return err
 	}
@@ -271,7 +282,7 @@ func sendTopicNotice(topic string, added bool) error {
 
 	respStr := string(respBytes)
 	if respStr != "ok" {
-		return NotOkError{Code: resp.StatusCode, Response: respStr}
+		return ErrBackendNotOK{Code: resp.StatusCode, Response: respStr}
 	}
 
 	return nil
@@ -281,15 +292,15 @@ func httpError(statusCode int) error {
 	return fmt.Errorf("backend http error: %d", statusCode)
 }
 
-func GenerateKeys(outputFile, serverId, theirPublicStr string) {
+func GenerateKeys(outputFile, serverID, theirPublicStr string) {
 	var err error
 	output := ConfigFile{
 		ListenAddr:   "0.0.0.0:8001",
 		SocketOrigin: "localhost:8001",
-		BackendUrl:   "http://localhost:8002/ffz",
+		BackendURL:   "http://localhost:8002/ffz",
 	}
 
-	output.ServerId, err = strconv.Atoi(serverId)
+	output.ServerID, err = strconv.Atoi(serverID)
 	if err != nil {
 		log.Fatal(err)
 	}
diff --git a/socketserver/internal/server/commands.go b/socketserver/internal/server/commands.go
index 79a8b6d7..2f26b72e 100644
--- a/socketserver/internal/server/commands.go
+++ b/socketserver/internal/server/commands.go
@@ -12,13 +12,14 @@ import (
 	"time"
 )
 
-// A command is how the client refers to a function on the server. It's just a string.
+// Command is a string indicating which RPC is requested.
+// The Commands sent from Client -> Server and Server -> Client are disjoint sets.
 type Command string
 
-// A function that is called to respond to a Command.
+// CommandHandler is a RPC handler assosciated with a Command.
 type CommandHandler func(*websocket.Conn, *ClientInfo, ClientMessage) (ClientMessage, error)
 
-var CommandHandlers = map[Command]CommandHandler{
+var commandHandlers = map[Command]CommandHandler{
 	HelloCommand: HandleHello,
 	"setuser":    HandleSetUser,
 	"ready":      HandleReady,
@@ -40,7 +41,7 @@ var CommandHandlers = map[Command]CommandHandler{
 const ChannelInfoDelay = 2 * time.Second
 
 func HandleCommand(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) {
-	handler, ok := CommandHandlers[msg.Command]
+	handler, ok := commandHandlers[msg.Command]
 	if !ok {
 		handler = HandleRemoteCommand
 	}
@@ -65,13 +66,13 @@ func HandleCommand(conn *websocket.Conn, client *ClientInfo, msg ClientMessage)
 }
 
 func HandleHello(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
-	version, clientId, err := msg.ArgumentsAsTwoStrings()
+	version, clientID, err := msg.ArgumentsAsTwoStrings()
 	if err != nil {
 		return
 	}
 
 	client.Version = version
-	client.ClientID = uuid.FromStringOrNil(clientId)
+	client.ClientID = uuid.FromStringOrNil(clientID)
 	if client.ClientID == uuid.Nil {
 		client.ClientID = uuid.NewV4()
 	}
@@ -125,7 +126,7 @@ func HandleSetUser(conn *websocket.Conn, client *ClientInfo, msg ClientMessage)
 	client.UsernameValidated = false
 	client.Mutex.Unlock()
 
-	if Configuation.SendAuthToNewClients {
+	if Configuration.SendAuthToNewClients {
 		client.MsgChannelKeepalive.Add(1)
 		go client.StartAuthorization(func(_ *ClientInfo, _ bool) {
 			client.MsgChannelKeepalive.Done()
@@ -200,7 +201,7 @@ func GetSubscriptionBacklog(conn *websocket.Conn, client *ClientInfo) {
 		return
 	}
 
-	if backendUrl == "" {
+	if backendURL == "" {
 		return // for testing runs
 	}
 	messages, err := FetchBacklogData(subs)
@@ -250,12 +251,17 @@ func HandleTrackFollow(conn *websocket.Conn, client *ClientInfo, msg ClientMessa
 	return ResponseSuccess, nil
 }
 
+// AggregateEmoteUsage is a map from emoteID to a map from chatroom name to usage count.
 var AggregateEmoteUsage map[int]map[string]int = make(map[int]map[string]int)
+
+// AggregateEmoteUsageLock is the lock for AggregateEmoteUsage.
 var AggregateEmoteUsageLock sync.Mutex
-var ErrorNegativeEmoteUsage = errors.New("Emote usage count cannot be negative")
+
+// ErrNegativeEmoteUsage is emitted when the submitted emote usage is negative.
+var ErrNegativeEmoteUsage = errors.New("Emote usage count cannot be negative")
 
 func HandleEmoticonUses(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
-	// arguments is [1]map[EmoteId]map[RoomName]float64
+	// arguments is [1]map[emoteID]map[ChatroomName]float64
 
 	mapRoot := msg.Arguments.([]interface{})[0].(map[string]interface{})
 
@@ -268,7 +274,7 @@ func HandleEmoticonUses(conn *websocket.Conn, client *ClientInfo, msg ClientMess
 		for _, val2 := range mapInner {
 			var count int = int(val2.(float64))
 			if count <= 0 {
-				err = ErrorNegativeEmoteUsage
+				err = ErrNegativeEmoteUsage
 				return
 			}
 		}
@@ -278,16 +284,16 @@ func HandleEmoticonUses(conn *websocket.Conn, client *ClientInfo, msg ClientMess
 	defer AggregateEmoteUsageLock.Unlock()
 
 	for strEmote, val1 := range mapRoot {
-		var emoteId int
-		emoteId, err = strconv.Atoi(strEmote)
+		var emoteID int
+		emoteID, err = strconv.Atoi(strEmote)
 		if err != nil {
 			return
 		}
 
-		destMapInner, ok := AggregateEmoteUsage[emoteId]
+		destMapInner, ok := AggregateEmoteUsage[emoteID]
 		if !ok {
 			destMapInner = make(map[string]int)
-			AggregateEmoteUsage[emoteId] = destMapInner
+			AggregateEmoteUsage[emoteID] = destMapInner
 		}
 
 		mapInner := val1.(map[string]interface{})
@@ -322,23 +328,23 @@ func DoSendAggregateData() {
 
 	reportForm := url.Values{}
 
-	followJson, err := json.Marshal(follows)
+	followJSON, err := json.Marshal(follows)
 	if err != nil {
 		log.Print(err)
 	} else {
-		reportForm.Set("follows", string(followJson))
+		reportForm.Set("follows", string(followJSON))
 	}
 
 	strEmoteUsage := make(map[string]map[string]int)
-	for emoteId, usageByChannel := range emoteUsage {
-		strEmoteId := strconv.Itoa(emoteId)
-		strEmoteUsage[strEmoteId] = usageByChannel
+	for emoteID, usageByChannel := range emoteUsage {
+		strEmoteID := strconv.Itoa(emoteID)
+		strEmoteUsage[strEmoteID] = usageByChannel
 	}
-	emoteJson, err := json.Marshal(strEmoteUsage)
+	emoteJSON, err := json.Marshal(strEmoteUsage)
 	if err != nil {
 		log.Print(err)
 	} else {
-		reportForm.Set("emotes", string(emoteJson))
+		reportForm.Set("emotes", string(emoteJSON))
 	}
 
 	form, err := SealRequest(reportForm)
@@ -380,6 +386,7 @@ type BunchSubscriberList struct {
 }
 
 type CacheStatus byte
+
 const (
 	CacheStatusNotFound = iota
 	CacheStatusFound
@@ -396,7 +403,7 @@ var BunchCacheLastCleanup time.Time
 func bunchCacheJanitor() {
 	go func() {
 		for {
-			time.Sleep(30*time.Minute)
+			time.Sleep(30 * time.Minute)
 			BunchCacheCleanupSignal.Signal()
 		}
 	}()
@@ -406,13 +413,13 @@ func bunchCacheJanitor() {
 		// Unlocks CachedBunchLock, waits for signal, re-locks
 		BunchCacheCleanupSignal.Wait()
 
-		if BunchCacheLastCleanup.After(time.Now().Add(-1*time.Second)) {
+		if BunchCacheLastCleanup.After(time.Now().Add(-1 * time.Second)) {
 			// skip if it's been less than 1 second
 			continue
 		}
 
 		// CachedBunchLock is held here
-		keepIfAfter := time.Now().Add(-5*time.Minute)
+		keepIfAfter := time.Now().Add(-5 * time.Minute)
 		for req, resp := range BunchCache {
 			if !resp.Timestamp.After(keepIfAfter) {
 				delete(BunchCache, req)
@@ -518,7 +525,7 @@ const AuthorizationFailedErrorString = "Failed to verify your Twitch username."
 func doRemoteCommand(conn *websocket.Conn, msg ClientMessage, client *ClientInfo) {
 	resp, err := SendRemoteCommandCached(string(msg.Command), msg.origArguments, client.AuthInfo)
 
-	if err == AuthorizationNeededError {
+	if err == ErrAuthorizationNeeded {
 		client.StartAuthorization(func(_ *ClientInfo, success bool) {
 			if success {
 				doRemoteCommand(conn, msg, client)
diff --git a/socketserver/internal/server/handlecore.go b/socketserver/internal/server/handlecore.go
index 6ecb1d28..e38c8d03 100644
--- a/socketserver/internal/server/handlecore.go
+++ b/socketserver/internal/server/handlecore.go
@@ -16,34 +16,38 @@ import (
 	"time"
 )
 
-const MAX_PACKET_SIZE = 1024
-
-// Sent by the server in ClientMessage.Command to indicate success.
+// SuccessCommand is a Reply Command to indicate success in reply to a C2S Command.
 const SuccessCommand Command = "ok"
 
-// Sent by the server in ClientMessage.Command to indicate failure.
+// ErrorCommand is a Reply Command to indicate that a C2S Command failed.
 const ErrorCommand Command = "error"
 
-// This must be the first command sent by the client once the connection is established.
+// HelloCommand is a C2S Command.
+// HelloCommand must be the Command of the first ClientMessage sent during a connection.
+// Sending any other command will result in a CloseFirstMessageNotHello.
 const HelloCommand Command = "hello"
 
+// AuthorizeCommand is a S2C Command sent as part of Twitch username validation.
 const AuthorizeCommand Command = "do_authorize"
 
-// A handler returning a ClientMessage with this Command will prevent replying to the client.
-// It signals that the work has been handed off to a background goroutine.
+// AsyncResponseCommand is a pseudo-Reply Command.
+// It indicates that the Reply Command to the client's C2S Command will be delivered
+// on a goroutine over the ClientInfo.MessageChannel and should not be delivered immediately.
 const AsyncResponseCommand Command = "_async"
 
+// ResponseSuccess is a Reply ClientMessage with the MessageID not yet filled out.
 var ResponseSuccess = ClientMessage{Command: SuccessCommand}
-var ResponseFailure = ClientMessage{Command: "False"}
 
-var Configuation *ConfigFile
+// Configuration is the active ConfigFile.
+var Configuration *ConfigFile
 
-// Set up a websocket listener and register it on /.
-// (Uses http.DefaultServeMux .)
+// SetupServerAndHandle starts all background goroutines and registers HTTP listeners on the given ServeMux.
+// Essentially, this function completely preps the server for a http.ListenAndServe call.
+// (Uses http.DefaultServeMux if `serveMux` is nil.)
 func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
-	Configuation = config
+	Configuration = config
 
-	SetupBackend(config)
+	setupBackend(config)
 
 	if serveMux == nil {
 		serveMux = http.DefaultServeMux
@@ -66,7 +70,7 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 	if err != nil {
 		log.Fatalln("Unable to seal requests:", err)
 	}
-	resp, err := backendHttpClient.PostForm(announceStartupUrl, announceForm)
+	resp, err := backendHTTPClient.PostForm(announceStartupURL, announceForm)
 	if err != nil {
 		log.Println(err)
 	} else {
@@ -82,6 +86,7 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 	go ircConnection()
 }
 
+// SocketUpgrader is the websocket.Upgrader currently in use.
 var SocketUpgrader = websocket.Upgrader{
 	ReadBufferSize:  1024,
 	WriteBufferSize: 1024,
@@ -90,6 +95,8 @@ var SocketUpgrader = websocket.Upgrader{
 	},
 }
 
+// BannerHTML is the content served to web browsers viewing the socket server website.
+// Memes go here.
 var BannerHTML []byte
 
 func ServeWebsocketOrCatbag(w http.ResponseWriter, r *http.Request) {
@@ -118,7 +125,6 @@ var ExpectedStringAndBool = errors.New("Error: Expected array of string, bool as
 var ExpectedStringAndIntGotFloat = errors.New("Error: Second argument was a float, expected an integer.")
 
 var CloseGotBinaryMessage = websocket.CloseError{Code: websocket.CloseUnsupportedData, Text: "got binary packet"}
-var CloseGotMessageId0 = websocket.CloseError{Code: websocket.ClosePolicyViolation, Text: "got messageid 0"}
 var CloseTimedOut = websocket.CloseError{Code: websocket.CloseNoStatusReceived, Text: "no ping replies for 5 minutes"}
 var CloseFirstMessageNotHello = websocket.CloseError{
 	Text: "Error - the first message sent must be a 'hello'",
@@ -296,6 +302,8 @@ func CloseConnection(conn *websocket.Conn, closeMsg *websocket.CloseError) {
 	conn.Close()
 }
 
+// SendMessage sends a ClientMessage over the websocket connection with a timeout.
+// If marshalling the ClientMessage fails, this function will panic.
 func SendMessage(conn *websocket.Conn, msg ClientMessage) {
 	messageType, packet, err := MarshalClientMessage(msg)
 	if err != nil {
@@ -305,7 +313,7 @@ func SendMessage(conn *websocket.Conn, msg ClientMessage) {
 	conn.WriteMessage(messageType, packet)
 }
 
-// Unpack a message sent from the client into a ClientMessage.
+// UnmarshalClientMessage unpacks websocket TextMessage into a ClientMessage provided in the `v` parameter.
 func UnmarshalClientMessage(data []byte, payloadType int, v interface{}) (err error) {
 	var spaceIdx int
 
@@ -317,12 +325,12 @@ func UnmarshalClientMessage(data []byte, payloadType int, v interface{}) (err er
 	if spaceIdx == -1 {
 		return ProtocolError
 	}
-	messageId, err := strconv.Atoi(dataStr[:spaceIdx])
-	if messageId < -1 || messageId == 0 {
+	messageID, err := strconv.Atoi(dataStr[:spaceIdx])
+	if messageID < -1 || messageID == 0 {
 		return ProtocolErrorNegativeID
 	}
 
-	out.MessageID = messageId
+	out.MessageID = messageID
 	dataStr = dataStr[spaceIdx+1:]
 
 	spaceIdx = strings.IndexRune(dataStr, ' ')
@@ -334,8 +342,8 @@ func UnmarshalClientMessage(data []byte, payloadType int, v interface{}) (err er
 		out.Command = Command(dataStr[:spaceIdx])
 	}
 	dataStr = dataStr[spaceIdx+1:]
-	argumentsJson := dataStr
-	out.origArguments = argumentsJson
+	argumentsJSON := dataStr
+	out.origArguments = argumentsJSON
 	err = out.parseOrigArguments()
 	if err != nil {
 		return
diff --git a/socketserver/internal/server/subscriptions.go b/socketserver/internal/server/subscriptions.go
index 2c2cfaa8..35b6020b 100644
--- a/socketserver/internal/server/subscriptions.go
+++ b/socketserver/internal/server/subscriptions.go
@@ -189,4 +189,4 @@ func _subscribeWhileRlocked(channelName string, value chan<- ClientMessage) {
 		AddToSliceC(&list.Members, value)
 		list.Unlock()
 	}
-}
\ No newline at end of file
+}
diff --git a/socketserver/internal/server/subscriptions_test.go b/socketserver/internal/server/subscriptions_test.go
index cd0364bc..9e87f504 100644
--- a/socketserver/internal/server/subscriptions_test.go
+++ b/socketserver/internal/server/subscriptions_test.go
@@ -24,7 +24,7 @@ func TCountOpenFDs() uint64 {
 
 const IgnoreReceivedArguments = 1 + 2i
 
-func TReceiveExpectedMessage(tb testing.TB, conn *websocket.Conn, messageId int, command Command, arguments interface{}) (ClientMessage, bool) {
+func TReceiveExpectedMessage(tb testing.TB, conn *websocket.Conn, messageID int, command Command, arguments interface{}) (ClientMessage, bool) {
 	var msg ClientMessage
 	var fail bool
 	messageType, packet, err := conn.ReadMessage()
@@ -42,8 +42,8 @@ func TReceiveExpectedMessage(tb testing.TB, conn *websocket.Conn, messageId int,
 		tb.Error(err)
 		return msg, false
 	}
-	if msg.MessageID != messageId {
-		tb.Error("Message ID was wrong. Expected", messageId, ", got", msg.MessageID, ":", msg)
+	if msg.MessageID != messageID {
+		tb.Error("Message ID was wrong. Expected", messageID, ", got", msg.MessageID, ":", msg)
 		fail = true
 	}
 	if msg.Command != command {
@@ -65,8 +65,8 @@ func TReceiveExpectedMessage(tb testing.TB, conn *websocket.Conn, messageId int,
 	return msg, !fail
 }
 
-func TSendMessage(tb testing.TB, conn *websocket.Conn, messageId int, command Command, arguments interface{}) bool {
-	SendMessage(conn, ClientMessage{MessageID: messageId, Command: command, Arguments: arguments})
+func TSendMessage(tb testing.TB, conn *websocket.Conn, messageID int, command Command, arguments interface{}) bool {
+	SendMessage(conn, ClientMessage{MessageID: messageID, Command: command, Arguments: arguments})
 	return true
 }
 
@@ -137,7 +137,7 @@ func TSetup(testserver **httptest.Server, urls *TURLs) {
 	DumpCache()
 
 	conf := &ConfigFile{
-		ServerId:     20,
+		ServerID:     20,
 		UseSSL:       false,
 		SocketOrigin: "localhost:2002",
 		BannerHTML: `
@@ -160,7 +160,7 @@ func TSetup(testserver **httptest.Server, urls *TURLs) {
 		BackendPublicKey: []byte{19, 163, 37, 157, 50, 139, 193, 85, 229, 47, 166, 21, 153, 231, 31, 133, 41, 158, 8, 53, 73, 0, 113, 91, 13, 181, 131, 248, 176, 18, 1, 107},
 	}
 	gconfig = conf
-	SetupBackend(conf)
+	setupBackend(conf)
 
 	if testserver != nil {
 		serveMux := http.NewServeMux()
diff --git a/socketserver/internal/server/types.go b/socketserver/internal/server/types.go
index 0c889e55..1088b0fd 100644
--- a/socketserver/internal/server/types.go
+++ b/socketserver/internal/server/types.go
@@ -12,12 +12,12 @@ const CryptoBoxKeyLength = 32
 
 type ConfigFile struct {
 	// Numeric server id known to the backend
-	ServerId   int
+	ServerID   int
 	ListenAddr string
 	// Hostname of the socket server
 	SocketOrigin string
 	// URL to the backend server
-	BackendUrl string
+	BackendURL string
 
 	// SSL/TLS
 	UseSSL             bool
@@ -168,19 +168,19 @@ func (bct BacklogCacheType) MarshalJSON() ([]byte, error) {
 }
 
 // Implements json.Unmarshaler
-func (pbct *BacklogCacheType) UnmarshalJSON(data []byte) error {
+func (bct *BacklogCacheType) UnmarshalJSON(data []byte) error {
 	var str string
 	err := json.Unmarshal(data, &str)
 	if err != nil {
 		return err
 	}
 	if str == "" {
-		*pbct = CacheTypeInvalid
+		*bct = CacheTypeInvalid
 		return nil
 	}
 	val := BacklogCacheTypeByName(str)
 	if val != CacheTypeInvalid {
-		*pbct = val
+		*bct = val
 		return nil
 	}
 	return ErrorUnrecognizedCacheType
@@ -224,19 +224,19 @@ func (mtt MessageTargetType) MarshalJSON() ([]byte, error) {
 }
 
 // Implements json.Unmarshaler
-func (pmtt *MessageTargetType) UnmarshalJSON(data []byte) error {
+func (mtt *MessageTargetType) UnmarshalJSON(data []byte) error {
 	var str string
 	err := json.Unmarshal(data, &str)
 	if err != nil {
 		return err
 	}
 	if str == "" {
-		*pmtt = MsgTargetTypeInvalid
+		*mtt = MsgTargetTypeInvalid
 		return nil
 	}
 	mtt := MessageTargetTypeByName(str)
 	if mtt != MsgTargetTypeInvalid {
-		*pmtt = mtt
+		*mtt = mtt
 		return nil
 	}
 	return ErrorUnrecognizedTargetType
diff --git a/socketserver/internal/server/utils.go b/socketserver/internal/server/utils.go
index 48f8749f..11c9c77b 100644
--- a/socketserver/internal/server/utils.go
+++ b/socketserver/internal/server/utils.go
@@ -54,7 +54,7 @@ func SealRequest(form url.Values) (url.Values, error) {
 	retval := url.Values{
 		"nonce": []string{nonceString},
 		"msg":   []string{cipherString},
-		"id":    []string{strconv.Itoa(serverId)},
+		"id":    []string{strconv.Itoa(serverID)},
 	}
 
 	return retval, nil

From cc3a160c296bc5e177fcc1c63546ac0a52e2efba Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Mon, 16 Nov 2015 12:50:00 -0800
Subject: [PATCH 045/176] more golint & cleanup

---
 socketserver/cmd/ffzsocketserver/console.go   |   2 +-
 .../cmd/ffzsocketserver/socketserver.go       |   4 +-
 socketserver/internal/server/backend.go       |  12 +
 socketserver/internal/server/commands.go      | 277 ++++++++----------
 socketserver/internal/server/handlecore.go    | 128 ++++----
 socketserver/internal/server/publisher.go     |  38 ++-
 .../internal/server/publisher_test.go         |  12 +-
 .../internal/server/subscriptions_test.go     |   2 +-
 socketserver/internal/server/types.go         |  17 +-
 socketserver/internal/server/utils.go         |   4 +-
 10 files changed, 251 insertions(+), 245 deletions(-)

diff --git a/socketserver/cmd/ffzsocketserver/console.go b/socketserver/cmd/ffzsocketserver/console.go
index 88cc4b44..7da8e6fa 100644
--- a/socketserver/cmd/ffzsocketserver/console.go
+++ b/socketserver/cmd/ffzsocketserver/console.go
@@ -1,7 +1,7 @@
 package main
 
 import (
-	"../../internal/server"
+	"bitbucket.org/stendec/frankerfacez/socketserver/internal/server"
 	"fmt"
 	"github.com/abiosoft/ishell"
 	"github.com/gorilla/websocket"
diff --git a/socketserver/cmd/ffzsocketserver/socketserver.go b/socketserver/cmd/ffzsocketserver/socketserver.go
index c280218f..67f982c5 100644
--- a/socketserver/cmd/ffzsocketserver/socketserver.go
+++ b/socketserver/cmd/ffzsocketserver/socketserver.go
@@ -1,7 +1,7 @@
-package main
+package main // import "bitbucket.org/stendec/frankerfacez/socketserver/cmd/ffzsocketserver"
 
 import (
-	"../../internal/server"
+	"bitbucket.org/stendec/frankerfacez/socketserver/internal/server"
 	"encoding/json"
 	"flag"
 	"fmt"
diff --git a/socketserver/internal/server/backend.go b/socketserver/internal/server/backend.go
index d4cc5bf2..c1241752 100644
--- a/socketserver/internal/server/backend.go
+++ b/socketserver/internal/server/backend.go
@@ -190,6 +190,7 @@ func SendRemoteCommand(remoteCommand, data string, auth AuthInfo) (responseStr s
 	return
 }
 
+// SendAggregatedData sends aggregated emote usage and following data to the backend server.
 func SendAggregatedData(sealedForm url.Values) error {
 	resp, err := backendHTTPClient.PostForm(postStatisticsURL, sealedForm)
 	if err != nil {
@@ -203,6 +204,8 @@ func SendAggregatedData(sealedForm url.Values) error {
 	return resp.Body.Close()
 }
 
+// FetchBacklogData makes a request to the backend for backlog data on a set of pub/sub topics.
+// TODO scrap this, replaced by /cached_pub
 func FetchBacklogData(chatSubs []string) ([]ClientMessage, error) {
 	formData := url.Values{
 		"subs": chatSubs,
@@ -247,10 +250,18 @@ func (noe ErrBackendNotOK) Error() string {
 	return fmt.Sprintf("backend returned %d: %s", noe.Code, noe.Response)
 }
 
+// SendNewTopicNotice notifies the backend that a client has performed the first subscription to a pub/sub topic.
+// POST data:
+// channels=room.trihex
+// added=t
 func SendNewTopicNotice(topic string) error {
 	return sendTopicNotice(topic, true)
 }
 
+// SendCleanupTopicsNotice notifies the backend that pub/sub topics have no subscribers anymore.
+// POST data:
+// channels=room.sirstendec,room.bobross,feature.foo
+// added=f
 func SendCleanupTopicsNotice(topics []string) error {
 	return sendTopicNotice(strings.Join(topics, ","), false)
 }
@@ -292,6 +303,7 @@ func httpError(statusCode int) error {
 	return fmt.Errorf("backend http error: %d", statusCode)
 }
 
+// GenerateKeys generates a new NaCl keypair for the server and writes out the default configuration file.
 func GenerateKeys(outputFile, serverID, theirPublicStr string) {
 	var err error
 	output := ConfigFile{
diff --git a/socketserver/internal/server/commands.go b/socketserver/internal/server/commands.go
index 2f26b72e..8552f768 100644
--- a/socketserver/internal/server/commands.go
+++ b/socketserver/internal/server/commands.go
@@ -3,6 +3,7 @@ package server
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"github.com/gorilla/websocket"
 	"github.com/satori/go.uuid"
 	"log"
@@ -20,33 +21,34 @@ type Command string
 type CommandHandler func(*websocket.Conn, *ClientInfo, ClientMessage) (ClientMessage, error)
 
 var commandHandlers = map[Command]CommandHandler{
-	HelloCommand: HandleHello,
-	"setuser":    HandleSetUser,
-	"ready":      HandleReady,
+	HelloCommand: C2SHello,
+	"setuser":    C2SSetUser,
+	"ready":      C2SReady,
 
-	"sub":   HandleSub,
-	"unsub": HandleUnsub,
+	"sub":   C2SSubscribe,
+	"unsub": C2SUnsubscribe,
 
-	"track_follow":  HandleTrackFollow,
-	"emoticon_uses": HandleEmoticonUses,
-	"survey":        HandleSurvey,
+	"track_follow":  C2STrackFollow,
+	"emoticon_uses": C2SEmoticonUses,
+	"survey":        C2SSurvey,
 
-	"twitch_emote":          HandleRemoteCommand,
-	"get_link":              HandleBunchedRemoteCommand,
-	"get_display_name":      HandleBunchedRemoteCommand,
-	"update_follow_buttons": HandleRemoteCommand,
-	"chat_history":          HandleRemoteCommand,
+	"twitch_emote":          C2SHandleRemoteCommand,
+	"get_link":              C2SHandleBunchedCommand,
+	"get_display_name":      C2SHandleBunchedCommand,
+	"update_follow_buttons": C2SHandleRemoteCommand,
+	"chat_history":          C2SHandleRemoteCommand,
 }
 
-const ChannelInfoDelay = 2 * time.Second
-
-func HandleCommand(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) {
+// DispatchC2SCommand handles a C2S Command in the provided ClientMessage.
+// It calls the correct CommandHandler function, catching panics.
+// It sends either the returned Reply ClientMessage, setting the correct messageID, or sends an ErrorCommand
+func DispatchC2SCommand(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) {
 	handler, ok := commandHandlers[msg.Command]
 	if !ok {
-		handler = HandleRemoteCommand
+		handler = C2SHandleRemoteCommand
 	}
 
-	response, err := CallHandler(handler, conn, client, msg)
+	response, err := callHandler(handler, conn, client, msg)
 
 	if err == nil {
 		if response.Command == AsyncResponseCommand {
@@ -59,13 +61,29 @@ func HandleCommand(conn *websocket.Conn, client *ClientInfo, msg ClientMessage)
 	} else {
 		SendMessage(conn, ClientMessage{
 			MessageID: msg.MessageID,
-			Command:   "error",
+			Command:   ErrorCommand,
 			Arguments: err.Error(),
 		})
 	}
 }
 
-func HandleHello(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
+func callHandler(handler CommandHandler, conn *websocket.Conn, client *ClientInfo, cmsg ClientMessage) (rmsg ClientMessage, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			var ok bool
+			fmt.Print("[!] Error executing command", cmsg.Command, "--", r)
+			err, ok = r.(error)
+			if !ok {
+				err = fmt.Errorf("command handler: %v", r)
+			}
+		}
+	}()
+	return handler(conn, client, cmsg)
+}
+
+// C2SHello implements the `hello` C2S Command.
+// It calls SubscribeGlobal() and SubscribeDefaults() with the client, and fills out ClientInfo.Version and ClientInfo.ClientID.
+func C2SHello(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
 	version, clientID, err := msg.ArgumentsAsTwoStrings()
 	if err != nil {
 		return
@@ -85,7 +103,7 @@ func HandleHello(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (r
 	}, nil
 }
 
-func HandleReady(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
+func C2SReady(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
 	disconnectAt, err := msg.ArgumentsAsInt()
 	if err != nil {
 		return
@@ -115,7 +133,7 @@ func HandleReady(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (r
 	return ClientMessage{Command: AsyncResponseCommand}, nil
 }
 
-func HandleSetUser(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
+func C2SSetUser(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
 	username, err := msg.ArgumentsAsString()
 	if err != nil {
 		return
@@ -137,7 +155,7 @@ func HandleSetUser(conn *websocket.Conn, client *ClientInfo, msg ClientMessage)
 	return ResponseSuccess, nil
 }
 
-func HandleSub(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
+func C2SSubscribe(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
 	channel, err := msg.ArgumentsAsString()
 
 	if err != nil {
@@ -145,18 +163,10 @@ func HandleSub(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rms
 	}
 
 	client.Mutex.Lock()
-
 	AddToSliceS(&client.CurrentChannels, channel)
-	client.PendingSubscriptionsBacklog = append(client.PendingSubscriptionsBacklog, channel)
-
-	//	if client.MakePendingRequests == nil {
-	//		client.MakePendingRequests = time.AfterFunc(ChannelInfoDelay, GetSubscriptionBacklogFor(conn, client))
-	//	} else {
-	//		if !client.MakePendingRequests.Reset(ChannelInfoDelay) {
-	//			client.MakePendingRequests = time.AfterFunc(ChannelInfoDelay, GetSubscriptionBacklogFor(conn, client))
-	//		}
-	//	}
-
+	if usePendingSubscrptionsBacklog {
+		client.PendingSubscriptionsBacklog = append(client.PendingSubscriptionsBacklog, channel)
+	}
 	client.Mutex.Unlock()
 
 	SubscribeChannel(client, channel)
@@ -164,7 +174,9 @@ func HandleSub(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rms
 	return ResponseSuccess, nil
 }
 
-func HandleUnsub(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
+// C2SUnsubscribe implements the `unsub` C2S Command.
+// It removes the channel from ClientInfo.CurrentChannels and calls UnsubscribeSingleChat.
+func C2SUnsubscribe(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
 	channel, err := msg.ArgumentsAsString()
 
 	if err != nil {
@@ -180,91 +192,57 @@ func HandleUnsub(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (r
 	return ResponseSuccess, nil
 }
 
-func GetSubscriptionBacklogFor(conn *websocket.Conn, client *ClientInfo) func() {
-	return func() {
-		GetSubscriptionBacklog(conn, client)
-	}
-}
-
-// On goroutine
-func GetSubscriptionBacklog(conn *websocket.Conn, client *ClientInfo) {
-	var subs []string
-
-	// Lock, grab the data, and reset it
-	client.Mutex.Lock()
-	subs = client.PendingSubscriptionsBacklog
-	client.PendingSubscriptionsBacklog = nil
-	client.MakePendingRequests = nil
-	client.Mutex.Unlock()
-
-	if len(subs) == 0 {
-		return
-	}
-
-	if backendURL == "" {
-		return // for testing runs
-	}
-	messages, err := FetchBacklogData(subs)
-
-	if err != nil {
-		// Oh well.
-		log.Print("error in GetSubscriptionBacklog:", err)
-		return
-	}
-
-	// Deliver to client
-	client.MsgChannelKeepalive.Add(1)
-	if client.MessageChannel != nil {
-		for _, msg := range messages {
-			client.MessageChannel <- msg
-		}
-	}
-	client.MsgChannelKeepalive.Done()
-}
-
-func HandleSurvey(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
+// C2SSurvey implements the survey C2S Command.
+// Surveys are discarded.s
+func C2SSurvey(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
 	// Discard
 	return ResponseSuccess, nil
 }
 
-type FollowEvent struct {
+type followEvent struct {
 	User         string    `json:"u"`
 	Channel      string    `json:"c"`
 	NowFollowing bool      `json:"f"`
 	Timestamp    time.Time `json:"t"`
 }
 
-var FollowEvents []FollowEvent
-var FollowEventsLock sync.Mutex
+var followEvents []followEvent
 
-func HandleTrackFollow(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
+// followEventsLock is the lock for followEvents.
+var followEventsLock sync.Mutex
+
+// C2STrackFollow implements the `track_follow` C2S Command.
+// It adds the record to `followEvents`, which is submitted to the backend on a timer.
+func C2STrackFollow(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
 	channel, following, err := msg.ArgumentsAsStringAndBool()
 	if err != nil {
 		return
 	}
 	now := time.Now()
 
-	FollowEventsLock.Lock()
-	FollowEvents = append(FollowEvents, FollowEvent{client.TwitchUsername, channel, following, now})
-	FollowEventsLock.Unlock()
+	followEventsLock.Lock()
+	followEvents = append(followEvents, followEvent{client.TwitchUsername, channel, following, now})
+	followEventsLock.Unlock()
 
 	return ResponseSuccess, nil
 }
 
 // AggregateEmoteUsage is a map from emoteID to a map from chatroom name to usage count.
-var AggregateEmoteUsage map[int]map[string]int = make(map[int]map[string]int)
+var aggregateEmoteUsage = make(map[int]map[string]int)
 
 // AggregateEmoteUsageLock is the lock for AggregateEmoteUsage.
-var AggregateEmoteUsageLock sync.Mutex
+var aggregateEmoteUsageLock sync.Mutex
 
 // ErrNegativeEmoteUsage is emitted when the submitted emote usage is negative.
 var ErrNegativeEmoteUsage = errors.New("Emote usage count cannot be negative")
 
-func HandleEmoticonUses(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
-	// arguments is [1]map[emoteID]map[ChatroomName]float64
-
+// C2SEmoticonUses implements the `emoticon_uses` C2S Command.
+// msg.Arguments are in the JSON format of [1]map[emoteID]map[ChatroomName]float64.
+func C2SEmoticonUses(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
+	// if this panics, will be caught by callHandler
 	mapRoot := msg.Arguments.([]interface{})[0].(map[string]interface{})
 
+	// Validate: male suire
 	for strEmote, val1 := range mapRoot {
 		_, err = strconv.Atoi(strEmote)
 		if err != nil {
@@ -272,7 +250,7 @@ func HandleEmoticonUses(conn *websocket.Conn, client *ClientInfo, msg ClientMess
 		}
 		mapInner := val1.(map[string]interface{})
 		for _, val2 := range mapInner {
-			var count int = int(val2.(float64))
+			var count = int(val2.(float64))
 			if count <= 0 {
 				err = ErrNegativeEmoteUsage
 				return
@@ -280,8 +258,8 @@ func HandleEmoticonUses(conn *websocket.Conn, client *ClientInfo, msg ClientMess
 		}
 	}
 
-	AggregateEmoteUsageLock.Lock()
-	defer AggregateEmoteUsageLock.Unlock()
+	aggregateEmoteUsageLock.Lock()
+	defer aggregateEmoteUsageLock.Unlock()
 
 	for strEmote, val1 := range mapRoot {
 		var emoteID int
@@ -290,15 +268,15 @@ func HandleEmoticonUses(conn *websocket.Conn, client *ClientInfo, msg ClientMess
 			return
 		}
 
-		destMapInner, ok := AggregateEmoteUsage[emoteID]
+		destMapInner, ok := aggregateEmoteUsage[emoteID]
 		if !ok {
 			destMapInner = make(map[string]int)
-			AggregateEmoteUsage[emoteID] = destMapInner
+			aggregateEmoteUsage[emoteID] = destMapInner
 		}
 
 		mapInner := val1.(map[string]interface{})
 		for roomName, val2 := range mapInner {
-			var count int = int(val2.(float64))
+			var count = int(val2.(float64))
 			if count > 200 {
 				count = 200
 			}
@@ -309,22 +287,22 @@ func HandleEmoticonUses(conn *websocket.Conn, client *ClientInfo, msg ClientMess
 	return ResponseSuccess, nil
 }
 
-func sendAggregateData() {
+func aggregateDataSender() {
 	for {
 		time.Sleep(5 * time.Minute)
-		DoSendAggregateData()
+		doSendAggregateData()
 	}
 }
 
-func DoSendAggregateData() {
-	FollowEventsLock.Lock()
-	follows := FollowEvents
-	FollowEvents = nil
-	FollowEventsLock.Unlock()
-	AggregateEmoteUsageLock.Lock()
-	emoteUsage := AggregateEmoteUsage
-	AggregateEmoteUsage = make(map[int]map[string]int)
-	AggregateEmoteUsageLock.Unlock()
+func doSendAggregateData() {
+	followEventsLock.Lock()
+	follows := followEvents
+	followEvents = nil
+	followEventsLock.Unlock()
+	aggregateEmoteUsageLock.Lock()
+	emoteUsage := aggregateEmoteUsage
+	aggregateEmoteUsage = make(map[int]map[string]int)
+	aggregateEmoteUsageLock.Unlock()
 
 	reportForm := url.Values{}
 
@@ -362,27 +340,23 @@ func DoSendAggregateData() {
 	// done
 }
 
-type BunchedRequest struct {
+type bunchedRequest struct {
 	Command Command
 	Param   string
 }
 
-func BunchedRequestFromCM(msg *ClientMessage) BunchedRequest {
-	return BunchedRequest{Command: msg.Command, Param: msg.origArguments}
-}
-
-type CachedBunchedResponse struct {
+type cachedBunchedResponse struct {
 	Response  string
 	Timestamp time.Time
 }
-type BunchSubscriber struct {
+type bunchSubscriber struct {
 	Client    *ClientInfo
 	MessageID int
 }
 
-type BunchSubscriberList struct {
+type bunchSubscriberList struct {
 	sync.Mutex
-	Members []BunchSubscriber
+	Members []bunchSubscriber
 }
 
 type CacheStatus byte
@@ -393,50 +367,57 @@ const (
 	CacheStatusExpired
 )
 
-var PendingBunchedRequests map[BunchedRequest]*BunchSubscriberList = make(map[BunchedRequest]*BunchSubscriberList)
-var PendingBunchLock sync.Mutex
-var BunchCache map[BunchedRequest]CachedBunchedResponse = make(map[BunchedRequest]CachedBunchedResponse)
-var BunchCacheLock sync.RWMutex
-var BunchCacheCleanupSignal *sync.Cond = sync.NewCond(&BunchCacheLock)
-var BunchCacheLastCleanup time.Time
+var pendingBunchedRequests = make(map[bunchedRequest]*bunchSubscriberList)
+var pendingBunchLock sync.Mutex
+var bunchCache = make(map[bunchedRequest]cachedBunchedResponse)
+var bunchCacheLock sync.RWMutex
+var bunchCacheCleanupSignal = sync.NewCond(&bunchCacheLock)
+var bunchCacheLastCleanup time.Time
+
+func bunchedRequestFromCM(msg *ClientMessage) bunchedRequest {
+	return bunchedRequest{Command: msg.Command, Param: msg.origArguments}
+}
 
 func bunchCacheJanitor() {
 	go func() {
 		for {
 			time.Sleep(30 * time.Minute)
-			BunchCacheCleanupSignal.Signal()
+			bunchCacheCleanupSignal.Signal()
 		}
 	}()
 
-	BunchCacheLock.Lock()
+	bunchCacheLock.Lock()
 	for {
 		// Unlocks CachedBunchLock, waits for signal, re-locks
-		BunchCacheCleanupSignal.Wait()
+		bunchCacheCleanupSignal.Wait()
 
-		if BunchCacheLastCleanup.After(time.Now().Add(-1 * time.Second)) {
+		if bunchCacheLastCleanup.After(time.Now().Add(-1 * time.Second)) {
 			// skip if it's been less than 1 second
 			continue
 		}
 
 		// CachedBunchLock is held here
 		keepIfAfter := time.Now().Add(-5 * time.Minute)
-		for req, resp := range BunchCache {
+		for req, resp := range bunchCache {
 			if !resp.Timestamp.After(keepIfAfter) {
-				delete(BunchCache, req)
+				delete(bunchCache, req)
 			}
 		}
-		BunchCacheLastCleanup = time.Now()
+		bunchCacheLastCleanup = time.Now()
 		// Loop and Wait(), which re-locks
 	}
 }
 
-func HandleBunchedRemoteCommand(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
-	br := BunchedRequestFromCM(&msg)
+// C2SHandleBunchedCommand handles C2S Commands such as `get_link`.
+// It makes a request to the backend server for the data, but any other requests coming in while the first is pending also get the responses from the first one.
+// Additionally, results are cached.
+func C2SHandleBunchedCommand(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
+	br := bunchedRequestFromCM(&msg)
 
 	cacheStatus := func() byte {
-		BunchCacheLock.RLock()
-		defer BunchCacheLock.RUnlock()
-		bresp, ok := BunchCache[br]
+		bunchCacheLock.RLock()
+		defer bunchCacheLock.RUnlock()
+		bresp, ok := bunchCache[br]
 		if ok && bresp.Timestamp.After(time.Now().Add(-5*time.Minute)) {
 			client.MsgChannelKeepalive.Add(1)
 			go func() {
@@ -459,12 +440,12 @@ func HandleBunchedRemoteCommand(conn *websocket.Conn, client *ClientInfo, msg Cl
 		return ClientMessage{Command: AsyncResponseCommand}, nil
 	} else if cacheStatus == CacheStatusExpired {
 		// Wake up the lazy janitor
-		BunchCacheCleanupSignal.Signal()
+		bunchCacheCleanupSignal.Signal()
 	}
 
-	PendingBunchLock.Lock()
-	defer PendingBunchLock.Unlock()
-	list, ok := PendingBunchedRequests[br]
+	pendingBunchLock.Lock()
+	defer pendingBunchLock.Unlock()
+	list, ok := pendingBunchedRequests[br]
 	if ok {
 		list.Lock()
 		AddToSliceB(&list.Members, client, msg.MessageID)
@@ -473,9 +454,9 @@ func HandleBunchedRemoteCommand(conn *websocket.Conn, client *ClientInfo, msg Cl
 		return ClientMessage{Command: AsyncResponseCommand}, nil
 	}
 
-	PendingBunchedRequests[br] = &BunchSubscriberList{Members: []BunchSubscriber{{Client: client, MessageID: msg.MessageID}}}
+	pendingBunchedRequests[br] = &bunchSubscriberList{Members: []bunchSubscriber{{Client: client, MessageID: msg.MessageID}}}
 
-	go func(request BunchedRequest) {
+	go func(request bunchedRequest) {
 		respStr, err := SendRemoteCommandCached(string(request.Command), request.Param, AuthInfo{})
 
 		var msg ClientMessage
@@ -489,15 +470,15 @@ func HandleBunchedRemoteCommand(conn *websocket.Conn, client *ClientInfo, msg Cl
 		}
 
 		if err == nil {
-			BunchCacheLock.Lock()
-			BunchCache[request] = CachedBunchedResponse{Response: respStr, Timestamp: time.Now()}
-			BunchCacheLock.Unlock()
+			bunchCacheLock.Lock()
+			bunchCache[request] = cachedBunchedResponse{Response: respStr, Timestamp: time.Now()}
+			bunchCacheLock.Unlock()
 		}
 
-		PendingBunchLock.Lock()
-		bsl := PendingBunchedRequests[request]
-		delete(PendingBunchedRequests, request)
-		PendingBunchLock.Unlock()
+		pendingBunchLock.Lock()
+		bsl := pendingBunchedRequests[request]
+		delete(pendingBunchedRequests, request)
+		pendingBunchLock.Unlock()
 
 		bsl.Lock()
 		for _, member := range bsl.Members {
@@ -513,7 +494,7 @@ func HandleBunchedRemoteCommand(conn *websocket.Conn, client *ClientInfo, msg Cl
 	return ClientMessage{Command: AsyncResponseCommand}, nil
 }
 
-func HandleRemoteCommand(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
+func C2SHandleRemoteCommand(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
 	client.MsgChannelKeepalive.Add(1)
 	go doRemoteCommand(conn, msg, client)
 
diff --git a/socketserver/internal/server/handlecore.go b/socketserver/internal/server/handlecore.go
index e38c8d03..82ac446b 100644
--- a/socketserver/internal/server/handlecore.go
+++ b/socketserver/internal/server/handlecore.go
@@ -59,8 +59,8 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 	}
 	BannerHTML = bannerBytes
 
-	serveMux.HandleFunc("/", ServeWebsocketOrCatbag)
-	serveMux.HandleFunc("/drop_backlog", HBackendDropBacklog)
+	serveMux.HandleFunc("/", HTTPHandleRootURL)
+	serveMux.HandleFunc("/drop_backlog", HTTPBackendDropBacklog)
 	serveMux.HandleFunc("/uncached_pub", HBackendPublishRequest)
 	serveMux.HandleFunc("/cached_pub", HBackendUpdateAndPublish)
 
@@ -81,7 +81,7 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 	go backlogJanitor()
 	go bunchCacheJanitor()
 	go pubsubJanitor()
-	go sendAggregateData()
+	go aggregateDataSender()
 
 	go ircConnection()
 }
@@ -99,14 +99,16 @@ var SocketUpgrader = websocket.Upgrader{
 // Memes go here.
 var BannerHTML []byte
 
-func ServeWebsocketOrCatbag(w http.ResponseWriter, r *http.Request) {
+// HTTPHandleRootURL is the http.HandleFunc for requests on `/`.
+// It either uses the SocketUpgrader or writes out the BannerHTML.
+func HTTPHandleRootURL(w http.ResponseWriter, r *http.Request) {
 	if r.Header.Get("Connection") == "Upgrade" {
 		conn, err := SocketUpgrader.Upgrade(w, r, nil)
 		if err != nil {
 			fmt.Fprintf(w, "error: %v", err)
 			return
 		}
-		HandleSocketConnection(conn)
+		RunSocketConnection(conn)
 
 		return
 	} else {
@@ -114,26 +116,46 @@ func ServeWebsocketOrCatbag(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// Errors that get returned to the client.
-var ProtocolError error = errors.New("FFZ Socket protocol error.")
-var ProtocolErrorNegativeID error = errors.New("FFZ Socket protocol error: negative or zero message ID.")
-var ExpectedSingleString = errors.New("Error: Expected single string as arguments.")
-var ExpectedSingleInt = errors.New("Error: Expected single integer as arguments.")
-var ExpectedTwoStrings = errors.New("Error: Expected array of string, string as arguments.")
-var ExpectedStringAndInt = errors.New("Error: Expected array of string, int as arguments.")
-var ExpectedStringAndBool = errors.New("Error: Expected array of string, bool as arguments.")
-var ExpectedStringAndIntGotFloat = errors.New("Error: Second argument was a float, expected an integer.")
+// ErrProtocolGeneric is sent in a ErrorCommand Reply.
+var ErrProtocolGeneric error = errors.New("FFZ Socket protocol error.")
+// ErrProtocolNegativeMsgID is sent in a ErrorCommand Reply when a negative MessageID is received.
+var ErrProtocolNegativeMsgID error = errors.New("FFZ Socket protocol error: negative or zero message ID.")
+// ErrExpectedSingleString is sent in a ErrorCommand Reply when the Arguments are of the wrong type.
+var ErrExpectedSingleString = errors.New("Error: Expected single string as arguments.")
+// ErrExpectedSingleInt is sent in a ErrorCommand Reply when the Arguments are of the wrong type.
+var ErrExpectedSingleInt = errors.New("Error: Expected single integer as arguments.")
+// ErrExpectedTwoStrings is sent in a ErrorCommand Reply when the Arguments are of the wrong type.
+var ErrExpectedTwoStrings = errors.New("Error: Expected array of string, string as arguments.")
+// ErrExpectedStringAndBool is sent in a ErrorCommand Reply when the Arguments are of the wrong type.
+var ErrExpectedStringAndBool = errors.New("Error: Expected array of string, bool as arguments.")
+// ErrExpectedStringAndInt is sent in a ErrorCommand Reply when the Arguments are of the wrong type.
+var ErrExpectedStringAndInt = errors.New("Error: Expected array of string, int as arguments.")
+// ErrExpectedStringAndIntGotFloat is sent in a ErrorCommand Reply when the Arguments are of the wrong type.
+var ErrExpectedStringAndIntGotFloat = errors.New("Error: Second argument was a float, expected an integer.")
 
+// CloseGotBinaryMessage is the termination reason when the client sends a binary websocket frame.
 var CloseGotBinaryMessage = websocket.CloseError{Code: websocket.CloseUnsupportedData, Text: "got binary packet"}
+// CloseTimedOut is the termination reason when the client fails to send or respond to ping frames.
 var CloseTimedOut = websocket.CloseError{Code: websocket.CloseNoStatusReceived, Text: "no ping replies for 5 minutes"}
+// CloseFirstMessageNotHello is the termination reason
 var CloseFirstMessageNotHello = websocket.CloseError{
 	Text: "Error - the first message sent must be a 'hello'",
 	Code: websocket.ClosePolicyViolation,
 }
 
-// Handle a new websocket connection from a FFZ client.
-// This runs in a goroutine started by net/http.
-func HandleSocketConnection(conn *websocket.Conn) {
+// RunSocketConnection contains the main run loop of a websocket connection.
+
+// First, it sets up the channels, the ClientInfo object, and the pong frame handler.
+// It starts the reader goroutine pointing at the newly created channels.
+// The function then enters the run loop (a `for{select{}}`).
+// The run loop is broken when an object is received on errorChan, or if `hello` is not the first C2S Command.
+
+// After the run loop stops, the function launches a goroutine to drain
+// client.MessageChannel, signals the reader goroutine to stop, unsubscribes
+// from all pub/sub channels, waits on MsgChannelKeepalive (remember, the
+// messages are being drained), and finally closes client.MessageChannel
+// (which ends the drainer goroutine).
+func RunSocketConnection(conn *websocket.Conn) {
 	// websocket.Conn is a ReadWriteCloser
 
 	log.Println("Got socket connection from", conn.RemoteAddr())
@@ -201,7 +223,9 @@ func HandleSocketConnection(conn *websocket.Conn) {
 	}(_errorChan, _clientChan, stoppedChan)
 
 	conn.SetPongHandler(func(pongBody string) error {
+		client.Mutex.Lock()
 		client.pingCount = 0
+		client.Mutex.Unlock()
 		return nil
 	})
 
@@ -236,14 +260,17 @@ RunLoop:
 				break RunLoop
 			}
 
-			HandleCommand(conn, &client, msg)
+			DispatchC2SCommand(conn, &client, msg)
 
-		case smsg := <-serverMessageChan:
-			SendMessage(conn, smsg)
+		case msg := <-serverMessageChan:
+			SendMessage(conn, msg)
 
 		case <-time.After(1 * time.Minute):
+			client.Mutex.Lock()
 			client.pingCount++
-			if client.pingCount == 5 {
+			tooManyPings := client.pingCount == 5
+			client.Mutex.Unlock()
+			if tooManyPings {
 				CloseConnection(conn, &CloseTimedOut)
 				break RunLoop
 			} else {
@@ -280,20 +307,6 @@ func getDeadline() time.Time {
 	return time.Now().Add(1 * time.Minute)
 }
 
-func CallHandler(handler CommandHandler, conn *websocket.Conn, client *ClientInfo, cmsg ClientMessage) (rmsg ClientMessage, err error) {
-	defer func() {
-		if r := recover(); r != nil {
-			var ok bool
-			fmt.Print("[!] Error executing command", cmsg.Command, "--", r)
-			err, ok = r.(error)
-			if !ok {
-				err = fmt.Errorf("command handler: %v", r)
-			}
-		}
-	}()
-	return handler(conn, client, cmsg)
-}
-
 func CloseConnection(conn *websocket.Conn, closeMsg *websocket.CloseError) {
 	if closeMsg != &CloseFirstMessageNotHello {
 		log.Println("Terminating connection with", conn.RemoteAddr(), "-", closeMsg.Text)
@@ -323,11 +336,11 @@ func UnmarshalClientMessage(data []byte, payloadType int, v interface{}) (err er
 	// Message ID
 	spaceIdx = strings.IndexRune(dataStr, ' ')
 	if spaceIdx == -1 {
-		return ProtocolError
+		return ErrProtocolGeneric
 	}
 	messageID, err := strconv.Atoi(dataStr[:spaceIdx])
 	if messageID < -1 || messageID == 0 {
-		return ProtocolErrorNegativeID
+		return ErrProtocolNegativeMsgID
 	}
 
 	out.MessageID = messageID
@@ -397,23 +410,12 @@ func MarshalClientMessage(clientMessage interface{}) (payloadType int, data []by
 	return websocket.TextMessage, []byte(dataStr), nil
 }
 
-// Command handlers should use this to construct responses.
-func SuccessMessageFromString(arguments string) ClientMessage {
-	cm := ClientMessage{
-		MessageID:     -1, // filled by the select loop
-		Command:       SuccessCommand,
-		origArguments: arguments,
-	}
-	cm.parseOrigArguments()
-	return cm
-}
-
 // Convenience method: Parse the arguments of the ClientMessage as a single string.
 func (cm *ClientMessage) ArgumentsAsString() (string1 string, err error) {
 	var ok bool
 	string1, ok = cm.Arguments.(string)
 	if !ok {
-		err = ExpectedSingleString
+		err = ErrExpectedSingleString
 		return
 	} else {
 		return string1, nil
@@ -426,7 +428,7 @@ func (cm *ClientMessage) ArgumentsAsInt() (int1 int64, err error) {
 	var num float64
 	num, ok = cm.Arguments.(float64)
 	if !ok {
-		err = ExpectedSingleInt
+		err = ErrExpectedSingleInt
 		return
 	} else {
 		int1 = int64(num)
@@ -440,16 +442,16 @@ func (cm *ClientMessage) ArgumentsAsTwoStrings() (string1, string2 string, err e
 	var ary []interface{}
 	ary, ok = cm.Arguments.([]interface{})
 	if !ok {
-		err = ExpectedTwoStrings
+		err = ErrExpectedTwoStrings
 		return
 	} else {
 		if len(ary) != 2 {
-			err = ExpectedTwoStrings
+			err = ErrExpectedTwoStrings
 			return
 		}
 		string1, ok = ary[0].(string)
 		if !ok {
-			err = ExpectedTwoStrings
+			err = ErrExpectedTwoStrings
 			return
 		}
 		// clientID can be null
@@ -458,7 +460,7 @@ func (cm *ClientMessage) ArgumentsAsTwoStrings() (string1, string2 string, err e
 		}
 		string2, ok = ary[1].(string)
 		if !ok {
-			err = ExpectedTwoStrings
+			err = ErrExpectedTwoStrings
 			return
 		}
 		return string1, string2, nil
@@ -471,27 +473,27 @@ func (cm *ClientMessage) ArgumentsAsStringAndInt() (string1 string, int int64, e
 	var ary []interface{}
 	ary, ok = cm.Arguments.([]interface{})
 	if !ok {
-		err = ExpectedStringAndInt
+		err = ErrExpectedStringAndInt
 		return
 	} else {
 		if len(ary) != 2 {
-			err = ExpectedStringAndInt
+			err = ErrExpectedStringAndInt
 			return
 		}
 		string1, ok = ary[0].(string)
 		if !ok {
-			err = ExpectedStringAndInt
+			err = ErrExpectedStringAndInt
 			return
 		}
 		var num float64
 		num, ok = ary[1].(float64)
 		if !ok {
-			err = ExpectedStringAndInt
+			err = ErrExpectedStringAndInt
 			return
 		}
 		int = int64(num)
 		if float64(int) != num {
-			err = ExpectedStringAndIntGotFloat
+			err = ErrExpectedStringAndIntGotFloat
 			return
 		}
 		return string1, int, nil
@@ -504,21 +506,21 @@ func (cm *ClientMessage) ArgumentsAsStringAndBool() (str string, flag bool, err
 	var ary []interface{}
 	ary, ok = cm.Arguments.([]interface{})
 	if !ok {
-		err = ExpectedStringAndBool
+		err = ErrExpectedStringAndBool
 		return
 	} else {
 		if len(ary) != 2 {
-			err = ExpectedStringAndBool
+			err = ErrExpectedStringAndBool
 			return
 		}
 		str, ok = ary[0].(string)
 		if !ok {
-			err = ExpectedStringAndBool
+			err = ErrExpectedStringAndBool
 			return
 		}
 		flag, ok = ary[1].(bool)
 		if !ok {
-			err = ExpectedStringAndBool
+			err = ErrExpectedStringAndBool
 			return
 		}
 		return str, flag, nil
diff --git a/socketserver/internal/server/publisher.go b/socketserver/internal/server/publisher.go
index a90efb1c..d93618cf 100644
--- a/socketserver/internal/server/publisher.go
+++ b/socketserver/internal/server/publisher.go
@@ -117,7 +117,8 @@ var CachedGlobalMessages []TimestampedGlobalMessage
 var CachedChannelMessages []TimestampedMultichatMessage
 var CacheListsLock sync.RWMutex
 
-func DumpCache() {
+// DumpBacklogData drops all /cached_pub data.
+func DumpBacklogData() {
 	CachedLSMLock.Lock()
 	CachedLastMessages = make(map[Command]map[string]LastSavedMessage)
 	CachedLSMLock.Unlock()
@@ -132,6 +133,9 @@ func DumpCache() {
 	CacheListsLock.Unlock()
 }
 
+// SendBacklogForNewClient sends any backlog data relevant to a new client.
+// This should be done when the client sends a `ready` message.
+// This will only send data for CacheTypePersistent and CacheTypeLastOnly because those do not involve timestamps.
 func SendBacklogForNewClient(client *ClientInfo) {
 	client.Mutex.Lock() // reading CurrentChannels
 	PersistentLSMLock.RLock()
@@ -170,11 +174,13 @@ func SendBacklogForNewClient(client *ClientInfo) {
 	client.Mutex.Unlock()
 }
 
+// SendTimedBacklogMessages sends any once-off messages that the client may have missed while it was disconnected.
+// Effectively, this can only process CacheTypeTimestamps.
 func SendTimedBacklogMessages(client *ClientInfo, disconnectTime time.Time) {
 	client.Mutex.Lock() // reading CurrentChannels
 	CacheListsLock.RLock()
 
-	globIdx := FindFirstNewMessage(tgmarray(CachedGlobalMessages), disconnectTime)
+	globIdx := findFirstNewMessage(tgmarray(CachedGlobalMessages), disconnectTime)
 
 	if globIdx != -1 {
 		for i := globIdx; i < len(CachedGlobalMessages); i++ {
@@ -185,7 +191,7 @@ func SendTimedBacklogMessages(client *ClientInfo, disconnectTime time.Time) {
 		}
 	}
 
-	chanIdx := FindFirstNewMessage(tmmarray(CachedChannelMessages), disconnectTime)
+	chanIdx := findFirstNewMessage(tmmarray(CachedChannelMessages), disconnectTime)
 
 	if chanIdx != -1 {
 		for i := chanIdx; i < len(CachedChannelMessages); i++ {
@@ -217,20 +223,20 @@ func SendTimedBacklogMessages(client *ClientInfo, disconnectTime time.Time) {
 func backlogJanitor() {
 	for {
 		time.Sleep(1 * time.Hour)
-		CleanupTimedBacklogMessages()
+		cleanupTimedBacklogMessages()
 	}
 }
 
-func CleanupTimedBacklogMessages() {
+func cleanupTimedBacklogMessages() {
 	CacheListsLock.Lock()
 	oneHourAgo := time.Now().Add(-24 * time.Hour)
-	globIdx := FindFirstNewMessage(tgmarray(CachedGlobalMessages), oneHourAgo)
+	globIdx := findFirstNewMessage(tgmarray(CachedGlobalMessages), oneHourAgo)
 	if globIdx != -1 {
 		newGlobMsgs := make([]TimestampedGlobalMessage, len(CachedGlobalMessages)-globIdx)
 		copy(newGlobMsgs, CachedGlobalMessages[globIdx:])
 		CachedGlobalMessages = newGlobMsgs
 	}
-	chanIdx := FindFirstNewMessage(tmmarray(CachedChannelMessages), oneHourAgo)
+	chanIdx := findFirstNewMessage(tmmarray(CachedChannelMessages), oneHourAgo)
 	if chanIdx != -1 {
 		newChanMsgs := make([]TimestampedMultichatMessage, len(CachedChannelMessages)-chanIdx)
 		copy(newChanMsgs, CachedChannelMessages[chanIdx:])
@@ -239,7 +245,10 @@ func CleanupTimedBacklogMessages() {
 	CacheListsLock.Unlock()
 }
 
-func InsertionSort(ary sort.Interface) {
+// insertionSort implements insertion sort.
+// CacheTypeTimestamps should use insertion sort for O(N) average performance.
+// (The average case is the array is still sorted after insertion of the new item.)
+func insertionSort(ary sort.Interface) {
 	for i := 1; i < ary.Len(); i++ {
 		for j := i; j > 0 && ary.Less(j, j-1); j-- {
 			ary.Swap(j, j-1)
@@ -247,13 +256,12 @@ func InsertionSort(ary sort.Interface) {
 	}
 }
 
-type TimestampArray interface {
+type timestampArray interface {
 	Len() int
 	GetTime(int) time.Time
 }
 
-func FindFirstNewMessage(ary TimestampArray, disconnectTime time.Time) (idx int) {
-	// TODO needs tests
+func findFirstNewMessage(ary timestampArray, disconnectTime time.Time) (idx int) {
 	len := ary.Len()
 	i := len
 
@@ -304,14 +312,14 @@ func SaveLastMessage(which map[Command]map[string]LastSavedMessage, locker sync.
 func SaveGlobalMessage(cmd Command, timestamp time.Time, data string) {
 	CacheListsLock.Lock()
 	CachedGlobalMessages = append(CachedGlobalMessages, TimestampedGlobalMessage{timestamp, cmd, data})
-	InsertionSort(tgmarray(CachedGlobalMessages))
+	insertionSort(tgmarray(CachedGlobalMessages))
 	CacheListsLock.Unlock()
 }
 
 func SaveMultichanMessage(cmd Command, channels string, timestamp time.Time, data string) {
 	CacheListsLock.Lock()
 	CachedChannelMessages = append(CachedChannelMessages, TimestampedMultichatMessage{timestamp, strings.Split(channels, ","), cmd, data})
-	InsertionSort(tmmarray(CachedChannelMessages))
+	insertionSort(tmmarray(CachedChannelMessages))
 	CacheListsLock.Unlock()
 }
 
@@ -325,7 +333,7 @@ func GetCommandsOfType(match PushCommandCacheInfo) []Command {
 	return ret
 }
 
-func HBackendDropBacklog(w http.ResponseWriter, r *http.Request) {
+func HTTPBackendDropBacklog(w http.ResponseWriter, r *http.Request) {
 	r.ParseForm()
 	formData, err := UnsealRequest(r.Form)
 	if err != nil {
@@ -336,7 +344,7 @@ func HBackendDropBacklog(w http.ResponseWriter, r *http.Request) {
 
 	confirm := formData.Get("confirm")
 	if confirm == "1" {
-		DumpCache()
+		DumpBacklogData()
 	}
 }
 
diff --git a/socketserver/internal/server/publisher_test.go b/socketserver/internal/server/publisher_test.go
index 99ce4f5a..e3c0484f 100644
--- a/socketserver/internal/server/publisher_test.go
+++ b/socketserver/internal/server/publisher_test.go
@@ -11,7 +11,7 @@ func TestCleanupBacklogMessages(t *testing.T) {
 
 func TestFindFirstNewMessageEmpty(t *testing.T) {
 	CachedGlobalMessages = []TimestampedGlobalMessage{}
-	i := FindFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
+	i := findFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
 	if i != -1 {
 		t.Errorf("Expected -1, got %d", i)
 	}
@@ -20,7 +20,7 @@ func TestFindFirstNewMessageOneBefore(t *testing.T) {
 	CachedGlobalMessages = []TimestampedGlobalMessage{
 		{Timestamp: time.Unix(8, 0)},
 	}
-	i := FindFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
+	i := findFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
 	if i != -1 {
 		t.Errorf("Expected -1, got %d", i)
 	}
@@ -33,7 +33,7 @@ func TestFindFirstNewMessageSeveralBefore(t *testing.T) {
 		{Timestamp: time.Unix(4, 0)},
 		{Timestamp: time.Unix(5, 0)},
 	}
-	i := FindFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
+	i := findFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
 	if i != -1 {
 		t.Errorf("Expected -1, got %d", i)
 	}
@@ -51,7 +51,7 @@ func TestFindFirstNewMessageInMiddle(t *testing.T) {
 		{Timestamp: time.Unix(14, 0)},
 		{Timestamp: time.Unix(15, 0)},
 	}
-	i := FindFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
+	i := findFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
 	if i != 5 {
 		t.Errorf("Expected 5, got %d", i)
 	}
@@ -60,7 +60,7 @@ func TestFindFirstNewMessageOneAfter(t *testing.T) {
 	CachedGlobalMessages = []TimestampedGlobalMessage{
 		{Timestamp: time.Unix(15, 0)},
 	}
-	i := FindFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
+	i := findFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
 	if i != 0 {
 		t.Errorf("Expected 0, got %d", i)
 	}
@@ -73,7 +73,7 @@ func TestFindFirstNewMessageSeveralAfter(t *testing.T) {
 		{Timestamp: time.Unix(14, 0)},
 		{Timestamp: time.Unix(15, 0)},
 	}
-	i := FindFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
+	i := findFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
 	if i != 0 {
 		t.Errorf("Expected 0, got %d", i)
 	}
diff --git a/socketserver/internal/server/subscriptions_test.go b/socketserver/internal/server/subscriptions_test.go
index 9e87f504..290dcea9 100644
--- a/socketserver/internal/server/subscriptions_test.go
+++ b/socketserver/internal/server/subscriptions_test.go
@@ -134,7 +134,7 @@ func TGetUrls(testserver *httptest.Server) TURLs {
 }
 
 func TSetup(testserver **httptest.Server, urls *TURLs) {
-	DumpCache()
+	DumpBacklogData()
 
 	conf := &ConfigFile{
 		ServerID:     20,
diff --git a/socketserver/internal/server/types.go b/socketserver/internal/server/types.go
index 1088b0fd..7a3b2f99 100644
--- a/socketserver/internal/server/types.go
+++ b/socketserver/internal/server/types.go
@@ -98,10 +98,13 @@ type ClientInfo struct {
 	// Take out an Add() on this during a command if you need to use the MessageChannel later.
 	MsgChannelKeepalive sync.WaitGroup
 
-	// The number of pings sent without a response
+	// The number of pings sent without a response.
+	// Protected by Mutex
 	pingCount int
 }
 
+const usePendingSubscrptionsBacklog = false
+
 type tgmarray []TimestampedGlobalMessage
 type tmmarray []TimestampedMultichatMessage
 
@@ -178,9 +181,9 @@ func (bct *BacklogCacheType) UnmarshalJSON(data []byte) error {
 		*bct = CacheTypeInvalid
 		return nil
 	}
-	val := BacklogCacheTypeByName(str)
-	if val != CacheTypeInvalid {
-		*bct = val
+	newBct := BacklogCacheTypeByName(str)
+	if newBct != CacheTypeInvalid {
+		*bct = newBct
 		return nil
 	}
 	return ErrorUnrecognizedCacheType
@@ -234,9 +237,9 @@ func (mtt *MessageTargetType) UnmarshalJSON(data []byte) error {
 		*mtt = MsgTargetTypeInvalid
 		return nil
 	}
-	mtt := MessageTargetTypeByName(str)
-	if mtt != MsgTargetTypeInvalid {
-		*mtt = mtt
+	newMtt := MessageTargetTypeByName(str)
+	if newMtt != MsgTargetTypeInvalid {
+		*mtt = newMtt
 		return nil
 	}
 	return ErrorUnrecognizedTargetType
diff --git a/socketserver/internal/server/utils.go b/socketserver/internal/server/utils.go
index 11c9c77b..12c26ed0 100644
--- a/socketserver/internal/server/utils.go
+++ b/socketserver/internal/server/utils.go
@@ -160,8 +160,8 @@ func RemoveFromSliceC(ary *[]chan<- ClientMessage, val chan<- ClientMessage) boo
 	return true
 }
 
-func AddToSliceB(ary *[]BunchSubscriber, client *ClientInfo, mid int) bool {
-	newSub := BunchSubscriber{Client: client, MessageID: mid}
+func AddToSliceB(ary *[]bunchSubscriber, client *ClientInfo, mid int) bool {
+	newSub := bunchSubscriber{Client: client, MessageID: mid}
 	slice := *ary
 	for _, v := range slice {
 		if v == newSub {

From a3971a27bf816f706dd5ff7377d627f9db050b0a Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Mon, 16 Nov 2015 13:07:02 -0800
Subject: [PATCH 046/176] Replace log.Println() with Statistics

---
 socketserver/internal/server/commands.go   | 11 +++++----
 socketserver/internal/server/handlecore.go | 18 +++++++--------
 socketserver/internal/server/stats.go      | 26 ++++++++++++++++++++++
 socketserver/internal/server/utils.go      |  2 +-
 4 files changed, 42 insertions(+), 15 deletions(-)
 create mode 100644 socketserver/internal/server/stats.go

diff --git a/socketserver/internal/server/commands.go b/socketserver/internal/server/commands.go
index 8552f768..4dcebd2d 100644
--- a/socketserver/internal/server/commands.go
+++ b/socketserver/internal/server/commands.go
@@ -48,6 +48,9 @@ func DispatchC2SCommand(conn *websocket.Conn, client *ClientInfo, msg ClientMess
 		handler = C2SHandleRemoteCommand
 	}
 
+	Statistics.CommandsIssuedTotal++
+	Statistics.CommandsIssuedMap[msg.Command]++
+
 	response, err := callHandler(handler, conn, client, msg)
 
 	if err == nil {
@@ -308,7 +311,7 @@ func doSendAggregateData() {
 
 	followJSON, err := json.Marshal(follows)
 	if err != nil {
-		log.Print(err)
+		log.Println("error reporting aggregate data:", err)
 	} else {
 		reportForm.Set("follows", string(followJSON))
 	}
@@ -320,20 +323,20 @@ func doSendAggregateData() {
 	}
 	emoteJSON, err := json.Marshal(strEmoteUsage)
 	if err != nil {
-		log.Print(err)
+		log.Println("error reporting aggregate data:", err)
 	} else {
 		reportForm.Set("emotes", string(emoteJSON))
 	}
 
 	form, err := SealRequest(reportForm)
 	if err != nil {
-		log.Print(err)
+		log.Println("error reporting aggregate data:", err)
 		return
 	}
 
 	err = SendAggregatedData(form)
 	if err != nil {
-		log.Print(err)
+		log.Println("error reporting aggregate data:", err)
 		return
 	}
 
diff --git a/socketserver/internal/server/handlecore.go b/socketserver/internal/server/handlecore.go
index 82ac446b..b296f490 100644
--- a/socketserver/internal/server/handlecore.go
+++ b/socketserver/internal/server/handlecore.go
@@ -72,7 +72,7 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 	}
 	resp, err := backendHTTPClient.PostForm(announceStartupURL, announceForm)
 	if err != nil {
-		log.Println(err)
+		log.Println("could not announce startup to backend:", err)
 	} else {
 		resp.Body.Close()
 	}
@@ -158,7 +158,7 @@ var CloseFirstMessageNotHello = websocket.CloseError{
 func RunSocketConnection(conn *websocket.Conn) {
 	// websocket.Conn is a ReadWriteCloser
 
-	log.Println("Got socket connection from", conn.RemoteAddr())
+	Statistics.ClientConnectsTotal++
 
 	var _closer sync.Once
 	closer := func() {
@@ -210,9 +210,6 @@ func RunSocketConnection(conn *websocket.Conn) {
 		}
 
 		_, isClose := err.(*websocket.CloseError)
-		if err != io.EOF && !isClose {
-			log.Println("Error while reading from client:", err)
-		}
 		select {
 		case errorChan <- err:
 		case <-stoppedChan:
@@ -255,8 +252,8 @@ RunLoop:
 
 		case msg := <-clientChan:
 			if client.Version == "" && msg.Command != HelloCommand {
-				log.Println("error - first message wasn't hello from", conn.RemoteAddr(), "-", msg)
 				CloseConnection(conn, &CloseFirstMessageNotHello)
+				Statistics.FirstNotHelloDisconnects++
 				break RunLoop
 			}
 
@@ -300,7 +297,7 @@ RunLoop:
 	// Close the channel so the draining goroutine can finish, too.
 	close(_serverMessageChan)
 
-	log.Println("End socket connection from", conn.RemoteAddr())
+	Statistics.ClientDisconnectsTotal++
 }
 
 func getDeadline() time.Time {
@@ -308,9 +305,9 @@ func getDeadline() time.Time {
 }
 
 func CloseConnection(conn *websocket.Conn, closeMsg *websocket.CloseError) {
-	if closeMsg != &CloseFirstMessageNotHello {
-		log.Println("Terminating connection with", conn.RemoteAddr(), "-", closeMsg.Text)
-	}
+	Statistics.DisconnectCodes[closeMsg.Code]++
+	Statistics.DisconnectReasons[closeMsg.Text]++
+
 	conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(closeMsg.Code, closeMsg.Text), getDeadline())
 	conn.Close()
 }
@@ -324,6 +321,7 @@ func SendMessage(conn *websocket.Conn, msg ClientMessage) {
 	}
 	conn.SetWriteDeadline(getDeadline())
 	conn.WriteMessage(messageType, packet)
+	Statistics.MessagesSent++
 }
 
 // UnmarshalClientMessage unpacks websocket TextMessage into a ClientMessage provided in the `v` parameter.
diff --git a/socketserver/internal/server/stats.go b/socketserver/internal/server/stats.go
new file mode 100644
index 00000000..a917b160
--- /dev/null
+++ b/socketserver/internal/server/stats.go
@@ -0,0 +1,26 @@
+package server
+
+type StatsData struct {
+	ClientConnectsTotal int64
+	ClientDisconnectsTotal int64
+	FirstNotHelloDisconnects int64
+
+	DisconnectCodes map[int]int64
+	DisconnectReasons map[string]int64
+
+	CommandsIssuedTotal int64
+	CommandsIssuedMap map[Command]int64
+
+	MessagesSent int64
+}
+
+func newStatsData() *StatsData {
+	return &StatsData{
+		CommandsIssuedMap: make(map[Command]int64),
+		DisconnectCodes: make(map[int]int64),
+		DisconnectReasons: make(map[string]int64),
+	}
+}
+
+// Statistics is several variables that get incremented during normal operation of the server.
+var Statistics = newStatsData()
diff --git a/socketserver/internal/server/utils.go b/socketserver/internal/server/utils.go
index 12c26ed0..07866e4b 100644
--- a/socketserver/internal/server/utils.go
+++ b/socketserver/internal/server/utils.go
@@ -89,7 +89,7 @@ func UnsealRequest(form url.Values) (url.Values, error) {
 	retValues, err := url.ParseQuery(string(message))
 	if err != nil {
 		// Assume that the signature was accidentally correct but the contents were garbage
-		log.Print(err)
+		log.Println("Error unsealing request:", err)
 		return nil, ErrorInvalidSignature
 	}
 

From c38b9b00189a0879c63b163199ecd2681abc4a18 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Mon, 16 Nov 2015 13:25:25 -0800
Subject: [PATCH 047/176] Expose server statistics at /stats

---
 socketserver/internal/server/backend.go    |  2 +-
 socketserver/internal/server/handlecore.go | 17 ++++++++++--
 socketserver/internal/server/publisher.go  |  2 +-
 socketserver/internal/server/stats.go      | 32 ++++++++++++++++++----
 4 files changed, 43 insertions(+), 10 deletions(-)

diff --git a/socketserver/internal/server/backend.go b/socketserver/internal/server/backend.go
index c1241752..7d782a29 100644
--- a/socketserver/internal/server/backend.go
+++ b/socketserver/internal/server/backend.go
@@ -64,7 +64,7 @@ func getCacheKey(remoteCommand, data string) string {
 // The backend can POST here to publish a message to clients with no caching.
 // The POST arguments are `cmd`, `args`, `channel`, and `scope`.
 // The `scope` argument is required because no attempt is made to infer the scope from the command, unlike /cached_pub.
-func HBackendPublishRequest(w http.ResponseWriter, r *http.Request) {
+func HTTPBackendUncachedPublish(w http.ResponseWriter, r *http.Request) {
 	r.ParseForm()
 	formData, err := UnsealRequest(r.Form)
 	if err != nil {
diff --git a/socketserver/internal/server/handlecore.go b/socketserver/internal/server/handlecore.go
index b296f490..4548f682 100644
--- a/socketserver/internal/server/handlecore.go
+++ b/socketserver/internal/server/handlecore.go
@@ -60,9 +60,11 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 	BannerHTML = bannerBytes
 
 	serveMux.HandleFunc("/", HTTPHandleRootURL)
+	serveMux.HandleFunc("/stats", HTTPShowStatistics)
+
 	serveMux.HandleFunc("/drop_backlog", HTTPBackendDropBacklog)
-	serveMux.HandleFunc("/uncached_pub", HBackendPublishRequest)
-	serveMux.HandleFunc("/cached_pub", HBackendUpdateAndPublish)
+	serveMux.HandleFunc("/uncached_pub", HTTPBackendUncachedPublish)
+	serveMux.HandleFunc("/cached_pub", HTTPBackendCachedPublish)
 
 	announceForm, err := SealRequest(url.Values{
 		"startup": []string{"1"},
@@ -118,25 +120,34 @@ func HTTPHandleRootURL(w http.ResponseWriter, r *http.Request) {
 
 // ErrProtocolGeneric is sent in a ErrorCommand Reply.
 var ErrProtocolGeneric error = errors.New("FFZ Socket protocol error.")
+
 // ErrProtocolNegativeMsgID is sent in a ErrorCommand Reply when a negative MessageID is received.
 var ErrProtocolNegativeMsgID error = errors.New("FFZ Socket protocol error: negative or zero message ID.")
+
 // ErrExpectedSingleString is sent in a ErrorCommand Reply when the Arguments are of the wrong type.
 var ErrExpectedSingleString = errors.New("Error: Expected single string as arguments.")
+
 // ErrExpectedSingleInt is sent in a ErrorCommand Reply when the Arguments are of the wrong type.
 var ErrExpectedSingleInt = errors.New("Error: Expected single integer as arguments.")
+
 // ErrExpectedTwoStrings is sent in a ErrorCommand Reply when the Arguments are of the wrong type.
 var ErrExpectedTwoStrings = errors.New("Error: Expected array of string, string as arguments.")
+
 // ErrExpectedStringAndBool is sent in a ErrorCommand Reply when the Arguments are of the wrong type.
 var ErrExpectedStringAndBool = errors.New("Error: Expected array of string, bool as arguments.")
+
 // ErrExpectedStringAndInt is sent in a ErrorCommand Reply when the Arguments are of the wrong type.
 var ErrExpectedStringAndInt = errors.New("Error: Expected array of string, int as arguments.")
+
 // ErrExpectedStringAndIntGotFloat is sent in a ErrorCommand Reply when the Arguments are of the wrong type.
 var ErrExpectedStringAndIntGotFloat = errors.New("Error: Second argument was a float, expected an integer.")
 
 // CloseGotBinaryMessage is the termination reason when the client sends a binary websocket frame.
 var CloseGotBinaryMessage = websocket.CloseError{Code: websocket.CloseUnsupportedData, Text: "got binary packet"}
+
 // CloseTimedOut is the termination reason when the client fails to send or respond to ping frames.
 var CloseTimedOut = websocket.CloseError{Code: websocket.CloseNoStatusReceived, Text: "no ping replies for 5 minutes"}
+
 // CloseFirstMessageNotHello is the termination reason
 var CloseFirstMessageNotHello = websocket.CloseError{
 	Text: "Error - the first message sent must be a 'hello'",
@@ -305,7 +316,7 @@ func getDeadline() time.Time {
 }
 
 func CloseConnection(conn *websocket.Conn, closeMsg *websocket.CloseError) {
-	Statistics.DisconnectCodes[closeMsg.Code]++
+	Statistics.DisconnectCodes[strconv.Itoa(closeMsg.Code)]++
 	Statistics.DisconnectReasons[closeMsg.Text]++
 
 	conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(closeMsg.Code, closeMsg.Text), getDeadline())
diff --git a/socketserver/internal/server/publisher.go b/socketserver/internal/server/publisher.go
index d93618cf..9f9be8ff 100644
--- a/socketserver/internal/server/publisher.go
+++ b/socketserver/internal/server/publisher.go
@@ -351,7 +351,7 @@ func HTTPBackendDropBacklog(w http.ResponseWriter, r *http.Request) {
 // Publish a message to clients, and update the in-server cache for the message.
 // notes:
 // `scope` is implicit in the command
-func HBackendUpdateAndPublish(w http.ResponseWriter, r *http.Request) {
+func HTTPBackendCachedPublish(w http.ResponseWriter, r *http.Request) {
 	r.ParseForm()
 	formData, err := UnsealRequest(r.Form)
 	if err != nil {
diff --git a/socketserver/internal/server/stats.go b/socketserver/internal/server/stats.go
index a917b160..9630171b 100644
--- a/socketserver/internal/server/stats.go
+++ b/socketserver/internal/server/stats.go
@@ -1,26 +1,48 @@
 package server
 
+import (
+	"bytes"
+	"encoding/json"
+	"net/http"
+)
+
 type StatsData struct {
-	ClientConnectsTotal int64
-	ClientDisconnectsTotal int64
+	ClientConnectsTotal      int64
+	ClientDisconnectsTotal   int64
 	FirstNotHelloDisconnects int64
 
-	DisconnectCodes map[int]int64
+	DisconnectCodes   map[string]int64
 	DisconnectReasons map[string]int64
 
 	CommandsIssuedTotal int64
-	CommandsIssuedMap map[Command]int64
+	CommandsIssuedMap   map[Command]int64
 
 	MessagesSent int64
+
+	Version int
 }
 
+const StatsDataVersion = 1
+
 func newStatsData() *StatsData {
 	return &StatsData{
 		CommandsIssuedMap: make(map[Command]int64),
-		DisconnectCodes: make(map[int]int64),
+		DisconnectCodes:   make(map[string]int64),
 		DisconnectReasons: make(map[string]int64),
+		Version:           StatsDataVersion,
 	}
 }
 
 // Statistics is several variables that get incremented during normal operation of the server.
+// Its structure should be versioned as it is exposed via JSON.
 var Statistics = newStatsData()
+
+func HTTPShowStatistics(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+
+	jsonBytes, _ := json.Marshal(Statistics)
+	outBuf := bytes.NewBuffer(nil)
+	json.Indent(outBuf, jsonBytes, "", "\t")
+
+	outBuf.WriteTo(w)
+}

From 58dd0bd9ee8dda981750014b8ee817caff801745 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Mon, 16 Nov 2015 13:28:16 -0800
Subject: [PATCH 048/176] Fix the build (move out of internal)

---
 socketserver/cmd/ffzsocketserver/console.go              | 2 +-
 socketserver/cmd/ffzsocketserver/socketserver.go         | 2 +-
 socketserver/{internal => }/server/backend.go            | 0
 socketserver/{internal => }/server/backend_test.go       | 0
 socketserver/{internal => }/server/commands.go           | 0
 socketserver/{internal => }/server/handlecore.go         | 5 ++---
 socketserver/{internal => }/server/handlecore_test.go    | 0
 socketserver/{internal => }/server/irc.go                | 0
 socketserver/{internal => }/server/publisher.go          | 0
 socketserver/{internal => }/server/publisher_test.go     | 0
 socketserver/{internal => }/server/stats.go              | 0
 socketserver/{internal => }/server/subscriptions.go      | 0
 socketserver/{internal => }/server/subscriptions_test.go | 0
 socketserver/{internal => }/server/types.go              | 0
 socketserver/{internal => }/server/utils.go              | 0
 15 files changed, 4 insertions(+), 5 deletions(-)
 rename socketserver/{internal => }/server/backend.go (100%)
 rename socketserver/{internal => }/server/backend_test.go (100%)
 rename socketserver/{internal => }/server/commands.go (100%)
 rename socketserver/{internal => }/server/handlecore.go (99%)
 rename socketserver/{internal => }/server/handlecore_test.go (100%)
 rename socketserver/{internal => }/server/irc.go (100%)
 rename socketserver/{internal => }/server/publisher.go (100%)
 rename socketserver/{internal => }/server/publisher_test.go (100%)
 rename socketserver/{internal => }/server/stats.go (100%)
 rename socketserver/{internal => }/server/subscriptions.go (100%)
 rename socketserver/{internal => }/server/subscriptions_test.go (100%)
 rename socketserver/{internal => }/server/types.go (100%)
 rename socketserver/{internal => }/server/utils.go (100%)

diff --git a/socketserver/cmd/ffzsocketserver/console.go b/socketserver/cmd/ffzsocketserver/console.go
index 7da8e6fa..25b0921f 100644
--- a/socketserver/cmd/ffzsocketserver/console.go
+++ b/socketserver/cmd/ffzsocketserver/console.go
@@ -1,7 +1,7 @@
 package main
 
 import (
-	"bitbucket.org/stendec/frankerfacez/socketserver/internal/server"
+	"bitbucket.org/stendec/frankerfacez/socketserver/server"
 	"fmt"
 	"github.com/abiosoft/ishell"
 	"github.com/gorilla/websocket"
diff --git a/socketserver/cmd/ffzsocketserver/socketserver.go b/socketserver/cmd/ffzsocketserver/socketserver.go
index 67f982c5..2e80e1cb 100644
--- a/socketserver/cmd/ffzsocketserver/socketserver.go
+++ b/socketserver/cmd/ffzsocketserver/socketserver.go
@@ -1,7 +1,7 @@
 package main // import "bitbucket.org/stendec/frankerfacez/socketserver/cmd/ffzsocketserver"
 
 import (
-	"bitbucket.org/stendec/frankerfacez/socketserver/internal/server"
+	"bitbucket.org/stendec/frankerfacez/socketserver/server"
 	"encoding/json"
 	"flag"
 	"fmt"
diff --git a/socketserver/internal/server/backend.go b/socketserver/server/backend.go
similarity index 100%
rename from socketserver/internal/server/backend.go
rename to socketserver/server/backend.go
diff --git a/socketserver/internal/server/backend_test.go b/socketserver/server/backend_test.go
similarity index 100%
rename from socketserver/internal/server/backend_test.go
rename to socketserver/server/backend_test.go
diff --git a/socketserver/internal/server/commands.go b/socketserver/server/commands.go
similarity index 100%
rename from socketserver/internal/server/commands.go
rename to socketserver/server/commands.go
diff --git a/socketserver/internal/server/handlecore.go b/socketserver/server/handlecore.go
similarity index 99%
rename from socketserver/internal/server/handlecore.go
rename to socketserver/server/handlecore.go
index 4548f682..fd3c8c6f 100644
--- a/socketserver/internal/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -1,4 +1,4 @@
-package server // import "bitbucket.org/stendec/frankerfacez/socketserver/internal/server"
+package server // import "bitbucket.org/stendec/frankerfacez/socketserver/server"
 
 import (
 	"encoding/json"
@@ -220,14 +220,13 @@ func RunSocketConnection(conn *websocket.Conn) {
 			}
 		}
 
-		_, isClose := err.(*websocket.CloseError)
 		select {
 		case errorChan <- err:
 		case <-stoppedChan:
 		}
 		close(errorChan)
 		close(clientChan)
-		// exit
+		// exit goroutine
 	}(_errorChan, _clientChan, stoppedChan)
 
 	conn.SetPongHandler(func(pongBody string) error {
diff --git a/socketserver/internal/server/handlecore_test.go b/socketserver/server/handlecore_test.go
similarity index 100%
rename from socketserver/internal/server/handlecore_test.go
rename to socketserver/server/handlecore_test.go
diff --git a/socketserver/internal/server/irc.go b/socketserver/server/irc.go
similarity index 100%
rename from socketserver/internal/server/irc.go
rename to socketserver/server/irc.go
diff --git a/socketserver/internal/server/publisher.go b/socketserver/server/publisher.go
similarity index 100%
rename from socketserver/internal/server/publisher.go
rename to socketserver/server/publisher.go
diff --git a/socketserver/internal/server/publisher_test.go b/socketserver/server/publisher_test.go
similarity index 100%
rename from socketserver/internal/server/publisher_test.go
rename to socketserver/server/publisher_test.go
diff --git a/socketserver/internal/server/stats.go b/socketserver/server/stats.go
similarity index 100%
rename from socketserver/internal/server/stats.go
rename to socketserver/server/stats.go
diff --git a/socketserver/internal/server/subscriptions.go b/socketserver/server/subscriptions.go
similarity index 100%
rename from socketserver/internal/server/subscriptions.go
rename to socketserver/server/subscriptions.go
diff --git a/socketserver/internal/server/subscriptions_test.go b/socketserver/server/subscriptions_test.go
similarity index 100%
rename from socketserver/internal/server/subscriptions_test.go
rename to socketserver/server/subscriptions_test.go
diff --git a/socketserver/internal/server/types.go b/socketserver/server/types.go
similarity index 100%
rename from socketserver/internal/server/types.go
rename to socketserver/server/types.go
diff --git a/socketserver/internal/server/utils.go b/socketserver/server/utils.go
similarity index 100%
rename from socketserver/internal/server/utils.go
rename to socketserver/server/utils.go

From d3fa1e68948649dafecc75845902fc94d8c9dfbf Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Mon, 16 Nov 2015 14:07:25 -0800
Subject: [PATCH 049/176] revert to relative imports

---
 socketserver/cmd/ffzsocketserver/console.go      | 2 +-
 socketserver/cmd/ffzsocketserver/socketserver.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/socketserver/cmd/ffzsocketserver/console.go b/socketserver/cmd/ffzsocketserver/console.go
index 25b0921f..146468b2 100644
--- a/socketserver/cmd/ffzsocketserver/console.go
+++ b/socketserver/cmd/ffzsocketserver/console.go
@@ -1,7 +1,7 @@
 package main
 
 import (
-	"bitbucket.org/stendec/frankerfacez/socketserver/server"
+	"../../server"
 	"fmt"
 	"github.com/abiosoft/ishell"
 	"github.com/gorilla/websocket"
diff --git a/socketserver/cmd/ffzsocketserver/socketserver.go b/socketserver/cmd/ffzsocketserver/socketserver.go
index 2e80e1cb..31559502 100644
--- a/socketserver/cmd/ffzsocketserver/socketserver.go
+++ b/socketserver/cmd/ffzsocketserver/socketserver.go
@@ -1,7 +1,7 @@
 package main // import "bitbucket.org/stendec/frankerfacez/socketserver/cmd/ffzsocketserver"
 
 import (
-	"bitbucket.org/stendec/frankerfacez/socketserver/server"
+	"../../server"
 	"encoding/json"
 	"flag"
 	"fmt"

From 71f08f3c53394bbc63709b26f2036ed6a4410f79 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Mon, 16 Nov 2015 14:30:09 -0800
Subject: [PATCH 050/176] Implement version comparisons

---
 socketserver/server/commands.go   | 21 ++++++++++++---
 socketserver/server/handlecore.go |  2 +-
 socketserver/server/types.go      | 43 +++++++++++++++++++++++++++++--
 3 files changed, 59 insertions(+), 7 deletions(-)

diff --git a/socketserver/server/commands.go b/socketserver/server/commands.go
index 4dcebd2d..d9df3fd2 100644
--- a/socketserver/server/commands.go
+++ b/socketserver/server/commands.go
@@ -84,6 +84,8 @@ func callHandler(handler CommandHandler, conn *websocket.Conn, client *ClientInf
 	return handler(conn, client, cmsg)
 }
 
+var lastVersionWithoutReplyWithServerTime = VersionFromString("ffz_3.5.78")
+
 // C2SHello implements the `hello` C2S Command.
 // It calls SubscribeGlobal() and SubscribeDefaults() with the client, and fills out ClientInfo.Version and ClientInfo.ClientID.
 func C2SHello(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
@@ -92,7 +94,9 @@ func C2SHello(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg
 		return
 	}
 
-	client.Version = version
+	client.VersionString = version
+	client.Version = VersionFromString(version)
+
 	client.ClientID = uuid.FromStringOrNil(clientID)
 	if client.ClientID == uuid.Nil {
 		client.ClientID = uuid.NewV4()
@@ -101,9 +105,18 @@ func C2SHello(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg
 	SubscribeGlobal(client)
 	SubscribeDefaults(client)
 
-	return ClientMessage{
-		Arguments: client.ClientID.String(),
-	}, nil
+	if client.Version.After(lastVersionWithoutReplyWithServerTime) {
+		return ClientMessage{
+			Arguments: []interface{}{
+				client.ClientID.String(),
+				time.Now().Unix(),
+			},
+		}
+	} else {
+		return ClientMessage{
+			Arguments: client.ClientID.String(),
+		}, nil
+	}
 }
 
 func C2SReady(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index fd3c8c6f..4157a8fa 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -261,7 +261,7 @@ RunLoop:
 			break RunLoop
 
 		case msg := <-clientChan:
-			if client.Version == "" && msg.Command != HelloCommand {
+			if client.VersionString == "" && msg.Command != HelloCommand {
 				CloseConnection(conn, &CloseFirstMessageNotHello)
 				Statistics.FirstNotHelloDisconnects++
 				break RunLoop
diff --git a/socketserver/server/types.go b/socketserver/server/types.go
index 7a3b2f99..8dbf5ca2 100644
--- a/socketserver/server/types.go
+++ b/socketserver/server/types.go
@@ -6,6 +6,7 @@ import (
 	"net"
 	"sync"
 	"time"
+	"fmt"
 )
 
 const CryptoBoxKeyLength = 32
@@ -55,14 +56,22 @@ type AuthInfo struct {
 	UsernameValidated bool
 }
 
+type ClientVersion struct {
+	Major int
+	Minor int
+	Revision int
+}
+
 type ClientInfo struct {
 	// The client ID.
 	// This must be written once by the owning goroutine before the struct is passed off to any other goroutines.
 	ClientID uuid.UUID
 
-	// The client's version.
+	// The client's literal version string.
 	// This must be written once by the owning goroutine before the struct is passed off to any other goroutines.
-	Version string
+	VersionString string
+
+	Version ClientVersion
 
 	// This mutex protects writable data in this struct.
 	// If it seems to be a performance problem, we can split this.
@@ -103,6 +112,36 @@ type ClientInfo struct {
 	pingCount int
 }
 
+func VersionFromString(v string) ClientVersion {
+	var cv ClientVersion
+	fmt.Sscanf(v, "ffz_%d.%d.%d", &cv.Major, &cv.Minor, &cv.Revision)
+	return cv
+}
+
+func (cv *ClientVersion) After(cv2 *ClientVersion) bool {
+	if cv.Major > cv2.Major {
+		return true
+	} else if cv.Major < cv2.Major {
+		return false
+	}
+	if cv.Minor > cv2.Minor {
+		return true
+	} else if cv.Minor < cv2.Minor {
+		return false
+	}
+	if cv.Revision > cv2.Revision {
+		return true
+	} else if cv.Revision < cv2.Revision {
+		return false
+	}
+
+	return false // equal
+}
+
+func (cv *ClientVersion) Equal(cv2 *ClientVersion) bool {
+	return cv.Major == cv2.Major && cv.Minor == cv2.Minor && cv.Revision == cv2.Revision
+}
+
 const usePendingSubscrptionsBacklog = false
 
 type tgmarray []TimestampedGlobalMessage

From c63a93679b2b8cc02da877a0537d55327c324037 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Mon, 16 Nov 2015 14:48:24 -0800
Subject: [PATCH 051/176] Use UnixNano() so we have millis

---
 socketserver/server/commands.go | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/socketserver/server/commands.go b/socketserver/server/commands.go
index d9df3fd2..23ac35d3 100644
--- a/socketserver/server/commands.go
+++ b/socketserver/server/commands.go
@@ -105,13 +105,14 @@ func C2SHello(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg
 	SubscribeGlobal(client)
 	SubscribeDefaults(client)
 
-	if client.Version.After(lastVersionWithoutReplyWithServerTime) {
+	if client.Version.After(&lastVersionWithoutReplyWithServerTime) {
+		jsTime := float64(time.Now().UnixNano()) / (1000 * 1000)
 		return ClientMessage{
 			Arguments: []interface{}{
 				client.ClientID.String(),
-				time.Now().Unix(),
+				jsTime,
 			},
-		}
+		}, nil
 	} else {
 		return ClientMessage{
 			Arguments: client.ClientID.String(),
@@ -120,10 +121,10 @@ func C2SHello(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg
 }
 
 func C2SReady(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
-	disconnectAt, err := msg.ArgumentsAsInt()
-	if err != nil {
-		return
-	}
+//	disconnectAt, err := msg.ArgumentsAsInt()
+//	if err != nil {
+//		return
+//	}
 
 	client.Mutex.Lock()
 	if client.MakePendingRequests != nil {
@@ -141,9 +142,9 @@ func C2SReady(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg
 	go func() {
 		client.MessageChannel <- ClientMessage{MessageID: msg.MessageID, Command: SuccessCommand}
 		SendBacklogForNewClient(client)
-		if disconnectAt != 0 {
-			SendTimedBacklogMessages(client, time.Unix(disconnectAt, 0))
-		}
+//		if disconnectAt != 0 {
+//			SendTimedBacklogMessages(client, time.Unix(disconnectAt, 0))
+//		}
 		client.MsgChannelKeepalive.Done()
 	}()
 	return ClientMessage{Command: AsyncResponseCommand}, nil

From 30bc90b1529d1da3274cf3a7f2f9cc083f9af167 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Mon, 16 Nov 2015 14:58:42 -0800
Subject: [PATCH 052/176] are we a victim of rounding?

---
 socketserver/server/commands.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/socketserver/server/commands.go b/socketserver/server/commands.go
index 23ac35d3..fd27e5af 100644
--- a/socketserver/server/commands.go
+++ b/socketserver/server/commands.go
@@ -106,7 +106,7 @@ func C2SHello(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg
 	SubscribeDefaults(client)
 
 	if client.Version.After(&lastVersionWithoutReplyWithServerTime) {
-		jsTime := float64(time.Now().UnixNano()) / (1000 * 1000)
+		jsTime := float64(time.Now().UnixNano() / 1000) / 1000
 		return ClientMessage{
 			Arguments: []interface{}{
 				client.ClientID.String(),

From 7a6b9e66abd30cc033cd95157f04a57d51ca86c2 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Mon, 16 Nov 2015 15:23:27 -0800
Subject: [PATCH 053/176] Add ping command

---
 socketserver/server/commands.go | 51 +++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 22 deletions(-)

diff --git a/socketserver/server/commands.go b/socketserver/server/commands.go
index fd27e5af..ab2055a4 100644
--- a/socketserver/server/commands.go
+++ b/socketserver/server/commands.go
@@ -22,6 +22,7 @@ type CommandHandler func(*websocket.Conn, *ClientInfo, ClientMessage) (ClientMes
 
 var commandHandlers = map[Command]CommandHandler{
 	HelloCommand: C2SHello,
+	"ping":       C2SPing,
 	"setuser":    C2SSetUser,
 	"ready":      C2SReady,
 
@@ -120,6 +121,34 @@ func C2SHello(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg
 	}
 }
 
+func C2SPing(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
+	return ClientMessage{
+		Arguments: float64(time.Now().UnixNano() / 1000) / 1000,
+	}
+}
+
+func C2SSetUser(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
+	username, err := msg.ArgumentsAsString()
+	if err != nil {
+		return
+	}
+
+	client.Mutex.Lock()
+	client.TwitchUsername = username
+	client.UsernameValidated = false
+	client.Mutex.Unlock()
+
+	if Configuration.SendAuthToNewClients {
+		client.MsgChannelKeepalive.Add(1)
+		go client.StartAuthorization(func(_ *ClientInfo, _ bool) {
+			client.MsgChannelKeepalive.Done()
+		})
+
+	}
+
+	return ResponseSuccess, nil
+}
+
 func C2SReady(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
 //	disconnectAt, err := msg.ArgumentsAsInt()
 //	if err != nil {
@@ -150,28 +179,6 @@ func C2SReady(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg
 	return ClientMessage{Command: AsyncResponseCommand}, nil
 }
 
-func C2SSetUser(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
-	username, err := msg.ArgumentsAsString()
-	if err != nil {
-		return
-	}
-
-	client.Mutex.Lock()
-	client.TwitchUsername = username
-	client.UsernameValidated = false
-	client.Mutex.Unlock()
-
-	if Configuration.SendAuthToNewClients {
-		client.MsgChannelKeepalive.Add(1)
-		go client.StartAuthorization(func(_ *ClientInfo, _ bool) {
-			client.MsgChannelKeepalive.Done()
-		})
-
-	}
-
-	return ResponseSuccess, nil
-}
-
 func C2SSubscribe(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
 	channel, err := msg.ArgumentsAsString()
 

From c42a0fb171be72f6546a6c2d82180e1fd82c33d3 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Mon, 16 Nov 2015 16:56:27 -0800
Subject: [PATCH 054/176] not enough arguments to retur

---
 socketserver/server/commands.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/socketserver/server/commands.go b/socketserver/server/commands.go
index ab2055a4..4f22feee 100644
--- a/socketserver/server/commands.go
+++ b/socketserver/server/commands.go
@@ -124,7 +124,7 @@ func C2SHello(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg
 func C2SPing(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
 	return ClientMessage{
 		Arguments: float64(time.Now().UnixNano() / 1000) / 1000,
-	}
+	}, nil
 }
 
 func C2SSetUser(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {

From ab14cdac725445c81647a2041e0fca8feef4c786 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Mon, 16 Nov 2015 20:26:59 -0800
Subject: [PATCH 055/176] Fix disconnect reason statistics

---
 socketserver/server/handlecore.go | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 4157a8fa..2a901f86 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -316,7 +316,13 @@ func getDeadline() time.Time {
 
 func CloseConnection(conn *websocket.Conn, closeMsg *websocket.CloseError) {
 	Statistics.DisconnectCodes[strconv.Itoa(closeMsg.Code)]++
-	Statistics.DisconnectReasons[closeMsg.Text]++
+	closeTxt := closeMsg.Text
+	if strings.Contains(closeTxt, "read: connection reset by peer") {
+		closeTxt = "read: connection reset by peer"
+	} else if closeMsg.Code == 1001 {
+		closeTxt = "clean shutdown"
+	}
+	Statistics.DisconnectReasons[closeTxt]++
 
 	conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(closeMsg.Code, closeMsg.Text), getDeadline())
 	conn.Close()

From c55b5fb277c80f8c816f50dbdd8d1fd7da1e4631 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Mon, 16 Nov 2015 20:35:03 -0800
Subject: [PATCH 056/176] update to /stats

---
 socketserver/server/commands.go   | 18 +++++++++---------
 socketserver/server/handlecore.go |  3 ++-
 socketserver/server/stats.go      |  7 ++++---
 socketserver/server/types.go      |  6 +++---
 4 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/socketserver/server/commands.go b/socketserver/server/commands.go
index 4f22feee..2cf2e29a 100644
--- a/socketserver/server/commands.go
+++ b/socketserver/server/commands.go
@@ -107,7 +107,7 @@ func C2SHello(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg
 	SubscribeDefaults(client)
 
 	if client.Version.After(&lastVersionWithoutReplyWithServerTime) {
-		jsTime := float64(time.Now().UnixNano() / 1000) / 1000
+		jsTime := float64(time.Now().UnixNano()/1000) / 1000
 		return ClientMessage{
 			Arguments: []interface{}{
 				client.ClientID.String(),
@@ -123,7 +123,7 @@ func C2SHello(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg
 
 func C2SPing(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
 	return ClientMessage{
-		Arguments: float64(time.Now().UnixNano() / 1000) / 1000,
+		Arguments: float64(time.Now().UnixNano()/1000) / 1000,
 	}, nil
 }
 
@@ -150,10 +150,10 @@ func C2SSetUser(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rm
 }
 
 func C2SReady(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
-//	disconnectAt, err := msg.ArgumentsAsInt()
-//	if err != nil {
-//		return
-//	}
+	//	disconnectAt, err := msg.ArgumentsAsInt()
+	//	if err != nil {
+	//		return
+	//	}
 
 	client.Mutex.Lock()
 	if client.MakePendingRequests != nil {
@@ -171,9 +171,9 @@ func C2SReady(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg
 	go func() {
 		client.MessageChannel <- ClientMessage{MessageID: msg.MessageID, Command: SuccessCommand}
 		SendBacklogForNewClient(client)
-//		if disconnectAt != 0 {
-//			SendTimedBacklogMessages(client, time.Unix(disconnectAt, 0))
-//		}
+		//		if disconnectAt != 0 {
+		//			SendTimedBacklogMessages(client, time.Unix(disconnectAt, 0))
+		//		}
 		client.MsgChannelKeepalive.Done()
 	}()
 	return ClientMessage{Command: AsyncResponseCommand}, nil
diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 2a901f86..e19f93e4 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -170,6 +170,7 @@ func RunSocketConnection(conn *websocket.Conn) {
 	// websocket.Conn is a ReadWriteCloser
 
 	Statistics.ClientConnectsTotal++
+	Statistics.CurrentClientCount++
 
 	var _closer sync.Once
 	closer := func() {
@@ -263,7 +264,6 @@ RunLoop:
 		case msg := <-clientChan:
 			if client.VersionString == "" && msg.Command != HelloCommand {
 				CloseConnection(conn, &CloseFirstMessageNotHello)
-				Statistics.FirstNotHelloDisconnects++
 				break RunLoop
 			}
 
@@ -308,6 +308,7 @@ RunLoop:
 	close(_serverMessageChan)
 
 	Statistics.ClientDisconnectsTotal++
+	Statistics.CurrentClientCount--
 }
 
 func getDeadline() time.Time {
diff --git a/socketserver/server/stats.go b/socketserver/server/stats.go
index 9630171b..5c88c746 100644
--- a/socketserver/server/stats.go
+++ b/socketserver/server/stats.go
@@ -7,9 +7,10 @@ import (
 )
 
 type StatsData struct {
-	ClientConnectsTotal      int64
-	ClientDisconnectsTotal   int64
-	FirstNotHelloDisconnects int64
+	CurrentClientCount int64
+
+	ClientConnectsTotal    int64
+	ClientDisconnectsTotal int64
 
 	DisconnectCodes   map[string]int64
 	DisconnectReasons map[string]int64
diff --git a/socketserver/server/types.go b/socketserver/server/types.go
index 8dbf5ca2..2eecea38 100644
--- a/socketserver/server/types.go
+++ b/socketserver/server/types.go
@@ -2,11 +2,11 @@ package server
 
 import (
 	"encoding/json"
+	"fmt"
 	"github.com/satori/go.uuid"
 	"net"
 	"sync"
 	"time"
-	"fmt"
 )
 
 const CryptoBoxKeyLength = 32
@@ -57,8 +57,8 @@ type AuthInfo struct {
 }
 
 type ClientVersion struct {
-	Major int
-	Minor int
+	Major    int
+	Minor    int
 	Revision int
 }
 

From 56b80078e516579f9d91f72217da5f48fd860817 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Mon, 16 Nov 2015 20:36:50 -0800
Subject: [PATCH 057/176] more /stats updates

---
 socketserver/server/stats.go | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/socketserver/server/stats.go b/socketserver/server/stats.go
index 5c88c746..00cfb3de 100644
--- a/socketserver/server/stats.go
+++ b/socketserver/server/stats.go
@@ -7,20 +7,22 @@ import (
 )
 
 type StatsData struct {
+	Version int
+
 	CurrentClientCount int64
 
 	ClientConnectsTotal    int64
 	ClientDisconnectsTotal int64
 
 	DisconnectCodes   map[string]int64
-	DisconnectReasons map[string]int64
 
 	CommandsIssuedTotal int64
 	CommandsIssuedMap   map[Command]int64
 
 	MessagesSent int64
 
-	Version int
+	// DisconnectReasons is at the bottom because it has indeterminate size
+	DisconnectReasons map[string]int64
 }
 
 const StatsDataVersion = 1

From cc98c05472e4505e72cc375abe5d0487ad4adf90 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Mon, 16 Nov 2015 21:15:29 -0800
Subject: [PATCH 058/176] update /stats - include cpu & mem usage

---
 socketserver/server/stats.go          | 89 +++++++++++++++++++++------
 socketserver/server/tickspersecond.go |  9 +++
 2 files changed, 79 insertions(+), 19 deletions(-)
 create mode 100644 socketserver/server/tickspersecond.go

diff --git a/socketserver/server/stats.go b/socketserver/server/stats.go
index 00cfb3de..e8b0e28c 100644
--- a/socketserver/server/stats.go
+++ b/socketserver/server/stats.go
@@ -4,45 +4,96 @@ import (
 	"bytes"
 	"encoding/json"
 	"net/http"
+	"runtime"
+	"time"
+
+	linuxproc "github.com/c9s/goprocinfo/linux"
 )
 
 type StatsData struct {
 	Version int
 
-	CurrentClientCount int64
+	CurrentClientCount uint64
 
-	ClientConnectsTotal    int64
-	ClientDisconnectsTotal int64
+	ClientConnectsTotal    uint64
+	ClientDisconnectsTotal uint64
 
-	DisconnectCodes   map[string]int64
+	DisconnectCodes map[string]uint64
 
-	CommandsIssuedTotal int64
-	CommandsIssuedMap   map[Command]int64
+	CommandsIssuedTotal uint64
+	CommandsIssuedMap   map[Command]uint64
 
-	MessagesSent int64
+	MessagesSent uint64
+
+	CachedStatsLastUpdate time.Time
+
+	MemoryInUse    uint64
+	MemoryRSS int64
+
+	CpuUsagePct float64
 
 	// DisconnectReasons is at the bottom because it has indeterminate size
-	DisconnectReasons map[string]int64
-}
-
-const StatsDataVersion = 1
-
-func newStatsData() *StatsData {
-	return &StatsData{
-		CommandsIssuedMap: make(map[Command]int64),
-		DisconnectCodes:   make(map[string]int64),
-		DisconnectReasons: make(map[string]int64),
-		Version:           StatsDataVersion,
-	}
+	DisconnectReasons map[string]uint64
 }
 
 // Statistics is several variables that get incremented during normal operation of the server.
 // Its structure should be versioned as it is exposed via JSON.
 var Statistics = newStatsData()
 
+const StatsDataVersion = 2
+const pageSize = 4096
+
+var cpuUsage struct {
+	UserTime uint64
+	SysTime uint64
+}
+
+func newStatsData() *StatsData {
+	return &StatsData{
+		CommandsIssuedMap: make(map[Command]uint64),
+		DisconnectCodes:   make(map[string]uint64),
+		DisconnectReasons: make(map[string]uint64),
+		Version:           StatsDataVersion,
+	}
+}
+
+func updateStatsIfNeeded() {
+	if time.Now().Add(-2 * time.Second).After(Statistics.CachedStatsLastUpdate) {
+		updatePeriodicStats()
+	}
+}
+
+func updatePeriodicStats() {
+	nowUpdate := time.Now()
+	timeDiff := nowUpdate.Sub(Statistics.CachedStatsLastUpdate)
+	Statistics.CachedStatsLastUpdate = nowUpdate
+
+	{
+		m := runtime.MemStats{}
+		runtime.ReadMemStats(&m)
+
+		Statistics.MemoryInUse = m.Alloc
+	}
+
+	{
+		pstat, err := linuxproc.ReadProcessStat("/proc/self/stat")
+		if err == nil {
+			userTicks := pstat.Utime - cpuUsage.UserTime
+			sysTicks := pstat.Stime - cpuUsage.SysTime
+			cpuUsage.UserTime = pstat.Utime
+			cpuUsage.SysTime = pstat.Stime
+
+			Statistics.CpuUsagePct = 100 * float64(userTicks + sysTicks) / (timeDiff.Seconds() * float64(ticksPerSecond))
+			Statistics.MemoryRSS = pstat.Rss * pageSize
+		}
+	}
+}
+
 func HTTPShowStatistics(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json")
 
+	updateStatsIfNeeded()
+
 	jsonBytes, _ := json.Marshal(Statistics)
 	outBuf := bytes.NewBuffer(nil)
 	json.Indent(outBuf, jsonBytes, "", "\t")
diff --git a/socketserver/server/tickspersecond.go b/socketserver/server/tickspersecond.go
new file mode 100644
index 00000000..b545740d
--- /dev/null
+++ b/socketserver/server/tickspersecond.go
@@ -0,0 +1,9 @@
+package server
+
+// #include <unistd.h>
+// long get_ticks_per_second() {
+//   return sysconf(_SC_CLK_TCK);
+// }
+import "C"
+
+var ticksPerSecond = int(C.get_ticks_per_second())

From 13b3f0f68ae4e3ea0399b1c09ffb0a786228f7e9 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Mon, 16 Nov 2015 21:57:18 -0800
Subject: [PATCH 059/176] more /stats updates

---
 socketserver/server/commands.go |  5 +++++
 socketserver/server/stats.go    | 35 +++++++++++++++++++++++++--------
 2 files changed, 32 insertions(+), 8 deletions(-)

diff --git a/socketserver/server/commands.go b/socketserver/server/commands.go
index 2cf2e29a..958939c5 100644
--- a/socketserver/server/commands.go
+++ b/socketserver/server/commands.go
@@ -285,6 +285,8 @@ func C2SEmoticonUses(conn *websocket.Conn, client *ClientInfo, msg ClientMessage
 	aggregateEmoteUsageLock.Lock()
 	defer aggregateEmoteUsageLock.Unlock()
 
+	var total int
+
 	for strEmote, val1 := range mapRoot {
 		var emoteID int
 		emoteID, err = strconv.Atoi(strEmote)
@@ -305,9 +307,12 @@ func C2SEmoticonUses(conn *websocket.Conn, client *ClientInfo, msg ClientMessage
 				count = 200
 			}
 			destMapInner[roomName] += count
+			total += count
 		}
 	}
 
+	Statistics.EmotesReportedTotal += uint64(total)
+
 	return ResponseSuccess, nil
 }
 
diff --git a/socketserver/server/stats.go b/socketserver/server/stats.go
index e8b0e28c..d22634c0 100644
--- a/socketserver/server/stats.go
+++ b/socketserver/server/stats.go
@@ -12,9 +12,19 @@ import (
 
 type StatsData struct {
 	Version int
+	CachedStatsLastUpdate time.Time
 
 	CurrentClientCount uint64
 
+	PubSubChannelCount int
+
+	MemoryInUse    uint64
+	MemoryRSS uint64
+
+	MemoryPerClient uint64
+
+	CpuUsagePct float64
+
 	ClientConnectsTotal    uint64
 	ClientDisconnectsTotal uint64
 
@@ -25,12 +35,7 @@ type StatsData struct {
 
 	MessagesSent uint64
 
-	CachedStatsLastUpdate time.Time
-
-	MemoryInUse    uint64
-	MemoryRSS int64
-
-	CpuUsagePct float64
+	EmotesReportedTotal uint64
 
 	// DisconnectReasons is at the bottom because it has indeterminate size
 	DisconnectReasons map[string]uint64
@@ -38,9 +43,12 @@ type StatsData struct {
 
 // Statistics is several variables that get incremented during normal operation of the server.
 // Its structure should be versioned as it is exposed via JSON.
+//
+// Note as to threaded access - this is soft/fun data and not critical to data integrity.
+// I don't really care.
 var Statistics = newStatsData()
 
-const StatsDataVersion = 2
+const StatsDataVersion = 3
 const pageSize = 4096
 
 var cpuUsage struct {
@@ -84,9 +92,20 @@ func updatePeriodicStats() {
 			cpuUsage.SysTime = pstat.Stime
 
 			Statistics.CpuUsagePct = 100 * float64(userTicks + sysTicks) / (timeDiff.Seconds() * float64(ticksPerSecond))
-			Statistics.MemoryRSS = pstat.Rss * pageSize
+			Statistics.MemoryRSS = uint64(pstat.Rss * pageSize)
+			Statistics.MemoryPerClient = Statistics.MemoryRSS / Statistics.CurrentClientCount
 		}
 	}
+
+	{
+		ChatSubscriptionLock.RLock()
+		Statistics.PubSubChannelCount = len(ChatSubscriptionInfo)
+		ChatSubscriptionLock.RUnlock()
+
+		GlobalSubscriptionInfo.RLock()
+		Statistics.CurrentClientCount = uint64(len(GlobalSubscriptionInfo.Members))
+		GlobalSubscriptionInfo.RUnlock()
+	}
 }
 
 func HTTPShowStatistics(w http.ResponseWriter, r *http.Request) {

From bedd14ab4e7568b76857ff1c2871aa7e09a91c68 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Mon, 16 Nov 2015 22:07:43 -0800
Subject: [PATCH 060/176] Switch twitch_emote to bunched

---
 socketserver/server/commands.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/socketserver/server/commands.go b/socketserver/server/commands.go
index 958939c5..d2f69c8b 100644
--- a/socketserver/server/commands.go
+++ b/socketserver/server/commands.go
@@ -33,7 +33,7 @@ var commandHandlers = map[Command]CommandHandler{
 	"emoticon_uses": C2SEmoticonUses,
 	"survey":        C2SSurvey,
 
-	"twitch_emote":          C2SHandleRemoteCommand,
+	"twitch_emote":          C2SHandleBunchedCommand,
 	"get_link":              C2SHandleBunchedCommand,
 	"get_display_name":      C2SHandleBunchedCommand,
 	"update_follow_buttons": C2SHandleRemoteCommand,

From 4c3a4d9709bea34d05f5b018f2e8a5dfaa87c360 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Mon, 16 Nov 2015 22:10:55 -0800
Subject: [PATCH 061/176] add user_history command

---
 socketserver/server/commands.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/socketserver/server/commands.go b/socketserver/server/commands.go
index d2f69c8b..57461278 100644
--- a/socketserver/server/commands.go
+++ b/socketserver/server/commands.go
@@ -38,6 +38,7 @@ var commandHandlers = map[Command]CommandHandler{
 	"get_display_name":      C2SHandleBunchedCommand,
 	"update_follow_buttons": C2SHandleRemoteCommand,
 	"chat_history":          C2SHandleRemoteCommand,
+	"user_history":          C2SHandleRemoteCommand,
 }
 
 // DispatchC2SCommand handles a C2S Command in the provided ClientMessage.

From 5dc54bbd2ebb910f3678a76326f5acc58abcff8b Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Mon, 16 Nov 2015 22:18:36 -0800
Subject: [PATCH 062/176] Fix tests (by calling Skip() lmao)

---
 socketserver/server/subscriptions_test.go | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/socketserver/server/subscriptions_test.go b/socketserver/server/subscriptions_test.go
index 290dcea9..9a31b95c 100644
--- a/socketserver/server/subscriptions_test.go
+++ b/socketserver/server/subscriptions_test.go
@@ -136,11 +136,7 @@ func TGetUrls(testserver *httptest.Server) TURLs {
 func TSetup(testserver **httptest.Server, urls *TURLs) {
 	DumpBacklogData()
 
-	conf := &ConfigFile{
-		ServerID:     20,
-		UseSSL:       false,
-		SocketOrigin: "localhost:2002",
-		BannerHTML: `
+	ioutil.WriteFile("index.html", []byte(`
 <!DOCTYPE html>
 <title>CatBag</title>
 <link rel="stylesheet" href="//cdn.frankerfacez.com/script/catbag.css">
@@ -153,13 +149,16 @@ func TSetup(testserver **httptest.Server, urls *TURLs) {
 	A <a href="http://www.frankerfacez.com/">FrankerFaceZ</a> Service
 	&mdash; CatBag by <a href="http://www.twitch.tv/wolsk">Wolsk</a>
 </div>
-</div>
-`,
+</div>`), 0600)
+	conf := &ConfigFile{
+		ServerID:     20,
+		UseSSL:       false,
+		SocketOrigin: "localhost:2002",
 		OurPublicKey:     []byte{176, 149, 72, 209, 35, 42, 110, 220, 22, 236, 212, 129, 213, 199, 1, 227, 185, 167, 150, 159, 117, 202, 164, 100, 9, 107, 45, 141, 122, 221, 155, 73},
 		OurPrivateKey:    []byte{247, 133, 147, 194, 70, 240, 211, 216, 223, 16, 241, 253, 120, 14, 198, 74, 237, 180, 89, 33, 146, 146, 140, 58, 88, 160, 2, 246, 112, 35, 239, 87},
 		BackendPublicKey: []byte{19, 163, 37, 157, 50, 139, 193, 85, 229, 47, 166, 21, 153, 231, 31, 133, 41, 158, 8, 53, 73, 0, 113, 91, 13, 181, 131, 248, 176, 18, 1, 107},
 	}
-	gconfig = conf
+	Configuration = conf
 	setupBackend(conf)
 
 	if testserver != nil {
@@ -214,6 +213,8 @@ func TestSubscriptionAndPublish(t *testing.T) {
 	// msg 3: chEmpty
 	// msg 4: global
 
+	t.SkipNow()
+
 	// Client 1
 	conn, resp, err = websocket.DefaultDialer.Dial(urls.Websocket, http.Header{})
 	if err != nil {

From 532cd0e2ce7f7ff6f0553c446fe3bad76181e049 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Mon, 16 Nov 2015 22:34:55 -0800
Subject: [PATCH 063/176] Fix tests without skipping

---
 socketserver/server/handlecore.go         |  4 +++-
 socketserver/server/subscriptions_test.go | 22 +++++++++++++---------
 2 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index e19f93e4..0e0decb8 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -88,12 +88,14 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 	go ircConnection()
 }
 
+const TwitchDotTv = "http://www.twitch.tv"
+
 // SocketUpgrader is the websocket.Upgrader currently in use.
 var SocketUpgrader = websocket.Upgrader{
 	ReadBufferSize:  1024,
 	WriteBufferSize: 1024,
 	CheckOrigin: func(r *http.Request) bool {
-		return r.Header.Get("Origin") == "http://www.twitch.tv"
+		return r.Header.Get("Origin") == TwitchDotTv
 	},
 }
 
diff --git a/socketserver/server/subscriptions_test.go b/socketserver/server/subscriptions_test.go
index 9a31b95c..f404654e 100644
--- a/socketserver/server/subscriptions_test.go
+++ b/socketserver/server/subscriptions_test.go
@@ -128,8 +128,8 @@ func TGetUrls(testserver *httptest.Server) TURLs {
 	return TURLs{
 		Websocket:  fmt.Sprintf("ws://%s/", addr),
 		Origin:     fmt.Sprintf("http://%s", addr),
-		PubMsg:     fmt.Sprintf("http://%s/pub_msg", addr),
-		SavePubMsg: fmt.Sprintf("http://%s/update_and_pub", addr),
+		PubMsg:     fmt.Sprintf("http://%s/uncached_pub", addr),
+		SavePubMsg: fmt.Sprintf("http://%s/cached_pub", addr),
 	}
 }
 
@@ -204,6 +204,9 @@ func TestSubscriptionAndPublish(t *testing.T) {
 	var resp *http.Response
 	var err error
 
+	var headers http.Header = make(http.Header)
+	headers.Set("Origin", TwitchDotTv)
+
 	// client 1: sub ch1, ch2
 	// client 2: sub ch1, ch3
 	// client 3: sub none
@@ -213,10 +216,8 @@ func TestSubscriptionAndPublish(t *testing.T) {
 	// msg 3: chEmpty
 	// msg 4: global
 
-	t.SkipNow()
-
 	// Client 1
-	conn, resp, err = websocket.DefaultDialer.Dial(urls.Websocket, http.Header{})
+	conn, resp, err = websocket.DefaultDialer.Dial(urls.Websocket, headers)
 	if err != nil {
 		t.Error(err)
 		return
@@ -245,7 +246,7 @@ func TestSubscriptionAndPublish(t *testing.T) {
 	}(conn)
 
 	// Client 2
-	conn, resp, err = websocket.DefaultDialer.Dial(urls.Websocket, http.Header{})
+	conn, resp, err = websocket.DefaultDialer.Dial(urls.Websocket, headers)
 	if err != nil {
 		t.Error(err)
 		return
@@ -274,7 +275,7 @@ func TestSubscriptionAndPublish(t *testing.T) {
 	}(conn)
 
 	// Client 3
-	conn, resp, err = websocket.DefaultDialer.Dial(urls.Websocket, http.Header{})
+	conn, resp, err = websocket.DefaultDialer.Dial(urls.Websocket, headers)
 	if err != nil {
 		t.Error(err)
 		return
@@ -346,7 +347,7 @@ func TestSubscriptionAndPublish(t *testing.T) {
 	}
 
 	// Start client 4
-	conn, resp, err = websocket.DefaultDialer.Dial(urls.Websocket, http.Header{})
+	conn, resp, err = websocket.DefaultDialer.Dial(urls.Websocket, headers)
 	if err != nil {
 		t.Error(err)
 		return
@@ -407,9 +408,12 @@ func BenchmarkUserSubscriptionSinglePublish(b *testing.B) {
 	TSetup(&server, &urls)
 	defer unsubscribeAllClients()
 
+	var headers http.Header = make(http.Header)
+	headers.Set("Origin", TwitchDotTv)
+
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		conn, _, err := websocket.DefaultDialer.Dial(urls.Websocket, http.Header{})
+		conn, _, err := websocket.DefaultDialer.Dial(urls.Websocket, headers)
 		if err != nil {
 			b.Error(err)
 			break

From 6cadcffd5cb0d870409fe640ab8fa7ad8df1b95c Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Mon, 16 Nov 2015 22:46:19 -0800
Subject: [PATCH 064/176] Add uptime to /stats

---
 socketserver/server/stats.go | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/socketserver/server/stats.go b/socketserver/server/stats.go
index d22634c0..b5012f14 100644
--- a/socketserver/server/stats.go
+++ b/socketserver/server/stats.go
@@ -12,6 +12,8 @@ import (
 
 type StatsData struct {
 	Version int
+	StartTime time.Time
+	Uptime time.Duration
 	CachedStatsLastUpdate time.Time
 
 	CurrentClientCount uint64
@@ -58,6 +60,7 @@ var cpuUsage struct {
 
 func newStatsData() *StatsData {
 	return &StatsData{
+		StartTime:         time.Now(),
 		CommandsIssuedMap: make(map[Command]uint64),
 		DisconnectCodes:   make(map[string]uint64),
 		DisconnectReasons: make(map[string]uint64),
@@ -106,6 +109,10 @@ func updatePeriodicStats() {
 		Statistics.CurrentClientCount = uint64(len(GlobalSubscriptionInfo.Members))
 		GlobalSubscriptionInfo.RUnlock()
 	}
+
+	{
+		Statistics.Uptime = nowUpdate.Sub(Statistics.StartTime)
+	}
 }
 
 func HTTPShowStatistics(w http.ResponseWriter, r *http.Request) {

From 976a2593eb7652d8c7c976497adfaeace5ba8508 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Mon, 16 Nov 2015 22:56:40 -0800
Subject: [PATCH 065/176] add build stamp to /stats

---
 socketserver/cmd/ffzsocketserver/socketserver.go |  4 ++++
 socketserver/server/stats.go                     | 15 ++++++++++++---
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/socketserver/cmd/ffzsocketserver/socketserver.go b/socketserver/cmd/ffzsocketserver/socketserver.go
index 31559502..47da3871 100644
--- a/socketserver/cmd/ffzsocketserver/socketserver.go
+++ b/socketserver/cmd/ffzsocketserver/socketserver.go
@@ -14,6 +14,9 @@ import (
 var configFilename = flag.String("config", "config.json", "Configuration file, including the keypairs for the NaCl crypto library, for communicating with the backend.")
 var flagGenerateKeys = flag.Bool("genkeys", false, "Generate NaCl keys instead of serving requests.\nArguments: [int serverId] [base64 backendPublic]\nThe backend public key can either be specified in base64 on the command line, or put in the json file later.")
 
+var BuildTime string = "build not stamped"
+var BuildHash string = "build not stamped"
+
 func main() {
 	flag.Parse()
 
@@ -50,6 +53,7 @@ func main() {
 	}
 
 	server.SetupServerAndHandle(conf, nil)
+	server.SetBuildStamp(BuildTime, BuildHash)
 
 	go commandLineConsole()
 
diff --git a/socketserver/server/stats.go b/socketserver/server/stats.go
index b5012f14..25173db7 100644
--- a/socketserver/server/stats.go
+++ b/socketserver/server/stats.go
@@ -11,9 +11,13 @@ import (
 )
 
 type StatsData struct {
-	Version int
+	StatsDataVersion int
+
 	StartTime time.Time
 	Uptime time.Duration
+	BuildTime string
+	BuildHash string
+
 	CachedStatsLastUpdate time.Time
 
 	CurrentClientCount uint64
@@ -50,7 +54,7 @@ type StatsData struct {
 // I don't really care.
 var Statistics = newStatsData()
 
-const StatsDataVersion = 3
+const StatsDataVersion = 4
 const pageSize = 4096
 
 var cpuUsage struct {
@@ -64,10 +68,15 @@ func newStatsData() *StatsData {
 		CommandsIssuedMap: make(map[Command]uint64),
 		DisconnectCodes:   make(map[string]uint64),
 		DisconnectReasons: make(map[string]uint64),
-		Version:           StatsDataVersion,
+		StatsDataVersion:  StatsDataVersion,
 	}
 }
 
+func SetBuildStamp(buildTime, buildHash string) {
+	Statistics.BuildTime = buildTime
+	Statistics.BuildHash = buildHash
+}
+
 func updateStatsIfNeeded() {
 	if time.Now().Add(-2 * time.Second).After(Statistics.CachedStatsLastUpdate) {
 		updatePeriodicStats()

From 3ad59fb47eb107404475a79e06de2ede7d36cc1a Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Mon, 16 Nov 2015 23:15:38 -0800
Subject: [PATCH 066/176] format the uptime >.>

---
 socketserver/server/stats.go              | 12 ++++++------
 socketserver/server/subscriptions_test.go |  6 +++---
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/socketserver/server/stats.go b/socketserver/server/stats.go
index 25173db7..dd4f4b55 100644
--- a/socketserver/server/stats.go
+++ b/socketserver/server/stats.go
@@ -14,7 +14,7 @@ type StatsData struct {
 	StatsDataVersion int
 
 	StartTime time.Time
-	Uptime time.Duration
+	Uptime    string
 	BuildTime string
 	BuildHash string
 
@@ -24,8 +24,8 @@ type StatsData struct {
 
 	PubSubChannelCount int
 
-	MemoryInUse    uint64
-	MemoryRSS uint64
+	MemoryInUse uint64
+	MemoryRSS   uint64
 
 	MemoryPerClient uint64
 
@@ -59,7 +59,7 @@ const pageSize = 4096
 
 var cpuUsage struct {
 	UserTime uint64
-	SysTime uint64
+	SysTime  uint64
 }
 
 func newStatsData() *StatsData {
@@ -103,7 +103,7 @@ func updatePeriodicStats() {
 			cpuUsage.UserTime = pstat.Utime
 			cpuUsage.SysTime = pstat.Stime
 
-			Statistics.CpuUsagePct = 100 * float64(userTicks + sysTicks) / (timeDiff.Seconds() * float64(ticksPerSecond))
+			Statistics.CpuUsagePct = 100 * float64(userTicks+sysTicks) / (timeDiff.Seconds() * float64(ticksPerSecond))
 			Statistics.MemoryRSS = uint64(pstat.Rss * pageSize)
 			Statistics.MemoryPerClient = Statistics.MemoryRSS / Statistics.CurrentClientCount
 		}
@@ -120,7 +120,7 @@ func updatePeriodicStats() {
 	}
 
 	{
-		Statistics.Uptime = nowUpdate.Sub(Statistics.StartTime)
+		Statistics.Uptime = nowUpdate.Sub(Statistics.StartTime).String()
 	}
 }
 
diff --git a/socketserver/server/subscriptions_test.go b/socketserver/server/subscriptions_test.go
index f404654e..5d7adba3 100644
--- a/socketserver/server/subscriptions_test.go
+++ b/socketserver/server/subscriptions_test.go
@@ -151,9 +151,9 @@ func TSetup(testserver **httptest.Server, urls *TURLs) {
 </div>
 </div>`), 0600)
 	conf := &ConfigFile{
-		ServerID:     20,
-		UseSSL:       false,
-		SocketOrigin: "localhost:2002",
+		ServerID:         20,
+		UseSSL:           false,
+		SocketOrigin:     "localhost:2002",
 		OurPublicKey:     []byte{176, 149, 72, 209, 35, 42, 110, 220, 22, 236, 212, 129, 213, 199, 1, 227, 185, 167, 150, 159, 117, 202, 164, 100, 9, 107, 45, 141, 122, 221, 155, 73},
 		OurPrivateKey:    []byte{247, 133, 147, 194, 70, 240, 211, 216, 223, 16, 241, 253, 120, 14, 198, 74, 237, 180, 89, 33, 146, 146, 140, 58, 88, 160, 2, 246, 112, 35, 239, 87},
 		BackendPublicKey: []byte{19, 163, 37, 157, 50, 139, 193, 85, 229, 47, 166, 21, 153, 231, 31, 133, 41, 158, 8, 53, 73, 0, 113, 91, 13, 181, 131, 248, 176, 18, 1, 107},

From 0c1d6ed725a7040b2f1f8bec64d1f9ad619e7601 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Tue, 17 Nov 2015 00:28:42 -0800
Subject: [PATCH 067/176] Protect against filled send buffers

---
 socketserver/server/handlecore.go | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 0e0decb8..0d8499d6 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -150,12 +150,18 @@ var CloseGotBinaryMessage = websocket.CloseError{Code: websocket.CloseUnsupporte
 // CloseTimedOut is the termination reason when the client fails to send or respond to ping frames.
 var CloseTimedOut = websocket.CloseError{Code: websocket.CloseNoStatusReceived, Text: "no ping replies for 5 minutes"}
 
+// CloseTooManyBufferedMessages is the termination reason when the sending thread buffers too many messages.
+var CloseTooManyBufferedMessages = websocket.CloseError{Code: websocket.CloseMessageTooBig, Text: "too many pending messages"}
+
 // CloseFirstMessageNotHello is the termination reason
 var CloseFirstMessageNotHello = websocket.CloseError{
 	Text: "Error - the first message sent must be a 'hello'",
 	Code: websocket.ClosePolicyViolation,
 }
 
+const sendMessageBufferLength = 125
+const sendMessageAbortLength = 50
+
 // RunSocketConnection contains the main run loop of a websocket connection.
 
 // First, it sets up the channels, the ClientInfo object, and the pong frame handler.
@@ -185,7 +191,7 @@ func RunSocketConnection(conn *websocket.Conn) {
 	defer closer()
 
 	_clientChan := make(chan ClientMessage)
-	_serverMessageChan := make(chan ClientMessage)
+	_serverMessageChan := make(chan ClientMessage, sendMessageBufferLength)
 	_errorChan := make(chan error)
 	stoppedChan := make(chan struct{})
 
@@ -272,6 +278,9 @@ RunLoop:
 			DispatchC2SCommand(conn, &client, msg)
 
 		case msg := <-serverMessageChan:
+			if len(serverMessageChan) > sendMessageAbortLength {
+				CloseConnection(conn, &CloseTooManyBufferedMessages)
+			}
 			SendMessage(conn, msg)
 
 		case <-time.After(1 * time.Minute):

From da790fbaa61168b511ff088c86b05ef79412113d Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Tue, 17 Nov 2015 11:01:42 -0800
Subject: [PATCH 068/176] Refuse connections if not enough RAM

---
 socketserver/server/backend.go    |  7 ++++---
 socketserver/server/handlecore.go |  7 +++++++
 socketserver/server/stats.go      | 26 ++++++++++++++++++++++++++
 socketserver/server/types.go      |  3 +++
 4 files changed, 40 insertions(+), 3 deletions(-)

diff --git a/socketserver/server/backend.go b/socketserver/server/backend.go
index 7d782a29..e456c9df 100644
--- a/socketserver/server/backend.go
+++ b/socketserver/server/backend.go
@@ -307,9 +307,10 @@ func httpError(statusCode int) error {
 func GenerateKeys(outputFile, serverID, theirPublicStr string) {
 	var err error
 	output := ConfigFile{
-		ListenAddr:   "0.0.0.0:8001",
-		SocketOrigin: "localhost:8001",
-		BackendURL:   "http://localhost:8002/ffz",
+		ListenAddr:     "0.0.0.0:8001",
+		SocketOrigin:   "localhost:8001",
+		BackendURL:     "http://localhost:8002/ffz",
+		MinMemoryBytes: 1024 * 1024 * 24,
 	}
 
 	output.ServerID, err = strconv.Atoi(serverID)
diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 0d8499d6..5cb9584a 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -107,6 +107,13 @@ var BannerHTML []byte
 // It either uses the SocketUpgrader or writes out the BannerHTML.
 func HTTPHandleRootURL(w http.ResponseWriter, r *http.Request) {
 	if r.Header.Get("Connection") == "Upgrade" {
+		updateSysMem()
+
+		if Statistics.SysMemTotal-Statistics.SysMemFree < Configuration.MinMemoryBytes {
+			w.WriteHeader(503)
+			return
+		}
+
 		conn, err := SocketUpgrader.Upgrade(w, r, nil)
 		if err != nil {
 			fmt.Fprintf(w, "error: %v", err)
diff --git a/socketserver/server/stats.go b/socketserver/server/stats.go
index dd4f4b55..fe692a00 100644
--- a/socketserver/server/stats.go
+++ b/socketserver/server/stats.go
@@ -8,6 +8,7 @@ import (
 	"time"
 
 	linuxproc "github.com/c9s/goprocinfo/linux"
+	"sync"
 )
 
 type StatsData struct {
@@ -24,6 +25,8 @@ type StatsData struct {
 
 	PubSubChannelCount int
 
+	SysMemTotal uint64
+	SysMemFree  uint64
 	MemoryInUse uint64
 	MemoryRSS   uint64
 
@@ -72,6 +75,7 @@ func newStatsData() *StatsData {
 	}
 }
 
+// SetBuildStamp should be called from the main package to identify the git build hash and build time.
 func SetBuildStamp(buildTime, buildHash string) {
 	Statistics.BuildTime = buildTime
 	Statistics.BuildHash = buildHash
@@ -107,6 +111,7 @@ func updatePeriodicStats() {
 			Statistics.MemoryRSS = uint64(pstat.Rss * pageSize)
 			Statistics.MemoryPerClient = Statistics.MemoryRSS / Statistics.CurrentClientCount
 		}
+		updateSysMem()
 	}
 
 	{
@@ -124,6 +129,27 @@ func updatePeriodicStats() {
 	}
 }
 
+var sysMemLastUpdate time.Time
+var sysMemUpdateLock sync.Mutex
+
+func updateSysMem() {
+	if time.Now().Add(-2 * time.Second).After(sysMemLastUpdate) {
+		sysMemUpdateLock.Lock()
+		defer sysMemUpdateLock.Unlock()
+		if !time.Now().Add(-2 * time.Second).After(sysMemLastUpdate) {
+			return
+		}
+	} else {
+		return
+	}
+	sysMemLastUpdate = time.Now()
+	memInfo, err := linuxproc.ReadMemInfo("/proc/meminfo")
+	if err == nil {
+		Statistics.SysMemTotal = memInfo.MemTotal
+		Statistics.SysMemFree = memInfo.MemAvailable
+	}
+}
+
 func HTTPShowStatistics(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json")
 
diff --git a/socketserver/server/types.go b/socketserver/server/types.go
index 2eecea38..e851961b 100644
--- a/socketserver/server/types.go
+++ b/socketserver/server/types.go
@@ -20,6 +20,9 @@ type ConfigFile struct {
 	// URL to the backend server
 	BackendURL string
 
+	// Minimum memory to accept a new connection
+	MinMemoryBytes uint64
+
 	// SSL/TLS
 	UseSSL             bool
 	SSLCertificateFile string

From 096fe787b72c129544fe744134caeb0175033274 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Tue, 17 Nov 2015 11:04:10 -0800
Subject: [PATCH 069/176] Set min memory to 24mb if blank

---
 socketserver/server/backend.go    |  2 +-
 socketserver/server/handlecore.go | 11 +++++++++--
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/socketserver/server/backend.go b/socketserver/server/backend.go
index e456c9df..e43d6448 100644
--- a/socketserver/server/backend.go
+++ b/socketserver/server/backend.go
@@ -310,7 +310,7 @@ func GenerateKeys(outputFile, serverID, theirPublicStr string) {
 		ListenAddr:     "0.0.0.0:8001",
 		SocketOrigin:   "localhost:8001",
 		BackendURL:     "http://localhost:8002/ffz",
-		MinMemoryBytes: 1024 * 1024 * 24,
+		MinMemoryBytes: defaultMinMemory,
 	}
 
 	output.ServerID, err = strconv.Atoi(serverID)
diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 5cb9584a..fe9a4a2a 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -35,6 +35,11 @@ const AuthorizeCommand Command = "do_authorize"
 // on a goroutine over the ClientInfo.MessageChannel and should not be delivered immediately.
 const AsyncResponseCommand Command = "_async"
 
+const defaultMinMemory = 1024 * 1024 * 24
+
+// TwitchDotTv is the http origin for twitch.tv.
+const TwitchDotTv = "http://www.twitch.tv"
+
 // ResponseSuccess is a Reply ClientMessage with the MessageID not yet filled out.
 var ResponseSuccess = ClientMessage{Command: SuccessCommand}
 
@@ -47,6 +52,10 @@ var Configuration *ConfigFile
 func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 	Configuration = config
 
+	if config.MinMemoryBytes == 0 {
+		config.MinMemoryBytes = defaultMinMemory
+	}
+
 	setupBackend(config)
 
 	if serveMux == nil {
@@ -88,8 +97,6 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 	go ircConnection()
 }
 
-const TwitchDotTv = "http://www.twitch.tv"
-
 // SocketUpgrader is the websocket.Upgrader currently in use.
 var SocketUpgrader = websocket.Upgrader{
 	ReadBufferSize:  1024,

From 9b0597ca8245373e53cca891024559c1574552e1 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Tue, 17 Nov 2015 11:11:14 -0800
Subject: [PATCH 070/176] Those numbers are in kilobytes.

---
 socketserver/server/backend.go    |  8 ++++----
 socketserver/server/handlecore.go |  8 ++++----
 socketserver/server/stats.go      | 14 +++++++-------
 socketserver/server/types.go      |  2 +-
 4 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/socketserver/server/backend.go b/socketserver/server/backend.go
index e43d6448..8c96fd51 100644
--- a/socketserver/server/backend.go
+++ b/socketserver/server/backend.go
@@ -307,10 +307,10 @@ func httpError(statusCode int) error {
 func GenerateKeys(outputFile, serverID, theirPublicStr string) {
 	var err error
 	output := ConfigFile{
-		ListenAddr:     "0.0.0.0:8001",
-		SocketOrigin:   "localhost:8001",
-		BackendURL:     "http://localhost:8002/ffz",
-		MinMemoryBytes: defaultMinMemory,
+		ListenAddr:      "0.0.0.0:8001",
+		SocketOrigin:    "localhost:8001",
+		BackendURL:      "http://localhost:8002/ffz",
+		MinMemoryKBytes: defaultMinMemoryKB,
 	}
 
 	output.ServerID, err = strconv.Atoi(serverID)
diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index fe9a4a2a..dad4de2b 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -35,7 +35,7 @@ const AuthorizeCommand Command = "do_authorize"
 // on a goroutine over the ClientInfo.MessageChannel and should not be delivered immediately.
 const AsyncResponseCommand Command = "_async"
 
-const defaultMinMemory = 1024 * 1024 * 24
+const defaultMinMemoryKB = 1024 * 24
 
 // TwitchDotTv is the http origin for twitch.tv.
 const TwitchDotTv = "http://www.twitch.tv"
@@ -52,8 +52,8 @@ var Configuration *ConfigFile
 func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 	Configuration = config
 
-	if config.MinMemoryBytes == 0 {
-		config.MinMemoryBytes = defaultMinMemory
+	if config.MinMemoryKBytes == 0 {
+		config.MinMemoryKBytes = defaultMinMemoryKB
 	}
 
 	setupBackend(config)
@@ -116,7 +116,7 @@ func HTTPHandleRootURL(w http.ResponseWriter, r *http.Request) {
 	if r.Header.Get("Connection") == "Upgrade" {
 		updateSysMem()
 
-		if Statistics.SysMemTotal-Statistics.SysMemFree < Configuration.MinMemoryBytes {
+		if Statistics.SysMemTotalKB-Statistics.SysMemFreeKB < Configuration.MinMemoryKBytes {
 			w.WriteHeader(503)
 			return
 		}
diff --git a/socketserver/server/stats.go b/socketserver/server/stats.go
index fe692a00..9fe89978 100644
--- a/socketserver/server/stats.go
+++ b/socketserver/server/stats.go
@@ -5,10 +5,10 @@ import (
 	"encoding/json"
 	"net/http"
 	"runtime"
+	"sync"
 	"time"
 
 	linuxproc "github.com/c9s/goprocinfo/linux"
-	"sync"
 )
 
 type StatsData struct {
@@ -25,10 +25,10 @@ type StatsData struct {
 
 	PubSubChannelCount int
 
-	SysMemTotal uint64
-	SysMemFree  uint64
-	MemoryInUse uint64
-	MemoryRSS   uint64
+	SysMemTotalKB uint64
+	SysMemFreeKB  uint64
+	MemoryInUse   uint64
+	MemoryRSS     uint64
 
 	MemoryPerClient uint64
 
@@ -145,8 +145,8 @@ func updateSysMem() {
 	sysMemLastUpdate = time.Now()
 	memInfo, err := linuxproc.ReadMemInfo("/proc/meminfo")
 	if err == nil {
-		Statistics.SysMemTotal = memInfo.MemTotal
-		Statistics.SysMemFree = memInfo.MemAvailable
+		Statistics.SysMemTotalKB = memInfo.MemTotal
+		Statistics.SysMemFreeKB = memInfo.MemAvailable
 	}
 }
 
diff --git a/socketserver/server/types.go b/socketserver/server/types.go
index e851961b..a3358028 100644
--- a/socketserver/server/types.go
+++ b/socketserver/server/types.go
@@ -21,7 +21,7 @@ type ConfigFile struct {
 	BackendURL string
 
 	// Minimum memory to accept a new connection
-	MinMemoryBytes uint64
+	MinMemoryKBytes uint64
 
 	// SSL/TLS
 	UseSSL             bool

From 0229274a6c408cf443422e74495587ef707c25a2 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Tue, 17 Nov 2015 12:16:46 -0800
Subject: [PATCH 071/176] Add units to memory figures

---
 socketserver/server/stats.go | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/socketserver/server/stats.go b/socketserver/server/stats.go
index 9fe89978..0b9a8a30 100644
--- a/socketserver/server/stats.go
+++ b/socketserver/server/stats.go
@@ -27,10 +27,10 @@ type StatsData struct {
 
 	SysMemTotalKB uint64
 	SysMemFreeKB  uint64
-	MemoryInUse   uint64
-	MemoryRSS     uint64
+	MemoryInUseKB uint64
+	MemoryRSSKB   uint64
 
-	MemoryPerClient uint64
+	MemPerClientBytes uint64
 
 	CpuUsagePct float64
 
@@ -96,7 +96,7 @@ func updatePeriodicStats() {
 		m := runtime.MemStats{}
 		runtime.ReadMemStats(&m)
 
-		Statistics.MemoryInUse = m.Alloc
+		Statistics.MemoryInUseKB = m.Alloc / 1024
 	}
 
 	{
@@ -108,8 +108,8 @@ func updatePeriodicStats() {
 			cpuUsage.SysTime = pstat.Stime
 
 			Statistics.CpuUsagePct = 100 * float64(userTicks+sysTicks) / (timeDiff.Seconds() * float64(ticksPerSecond))
-			Statistics.MemoryRSS = uint64(pstat.Rss * pageSize)
-			Statistics.MemoryPerClient = Statistics.MemoryRSS / Statistics.CurrentClientCount
+			Statistics.MemoryRSSKB = uint64(pstat.Rss * pageSize / 1024)
+			Statistics.MemPerClientBytes = (Statistics.MemoryRSSKB * 1024) / Statistics.CurrentClientCount
 		}
 		updateSysMem()
 	}

From 121298af0e2d9749c1f235266e468c308f3fe7e2 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Tue, 17 Nov 2015 13:36:34 -0800
Subject: [PATCH 072/176] Add LE support, listen on HTTPS and HTTP

---
 .../cmd/ffzsocketserver/socketserver.go       | 20 +++++++++----------
 socketserver/server/handlecore.go             |  1 +
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/socketserver/cmd/ffzsocketserver/socketserver.go b/socketserver/cmd/ffzsocketserver/socketserver.go
index 47da3871..5432f3a3 100644
--- a/socketserver/cmd/ffzsocketserver/socketserver.go
+++ b/socketserver/cmd/ffzsocketserver/socketserver.go
@@ -47,25 +47,25 @@ func main() {
 		Addr: conf.ListenAddr,
 	}
 
-	logFile, err := os.OpenFile("output.log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
-	if err != nil {
-		log.Fatal("Could not create logfile: ", err)
-	}
+	//	logFile, err := os.OpenFile("output.log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
+	//	if err != nil {
+	//		log.Fatal("Could not create logfile: ", err)
+	//	}
 
 	server.SetupServerAndHandle(conf, nil)
 	server.SetBuildStamp(BuildTime, BuildHash)
 
 	go commandLineConsole()
 
-	log.SetOutput(logFile)
-
 	if conf.UseSSL {
-		err = httpServer.ListenAndServeTLS(conf.SSLCertificateFile, conf.SSLKeyFile)
-	} else {
-		err = httpServer.ListenAndServe()
+		go func() {
+			if err := httpServer.ListenAndServeTLS(conf.SSLCertificateFile, conf.SSLKeyFile); err != nil {
+				log.Fatal("ListenAndServeTLS: ", err)
+			}
+		}()
 	}
 
-	if err != nil {
+	if err = httpServer.ListenAndServe(); err != nil {
 		log.Fatal("ListenAndServe: ", err)
 	}
 }
diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index dad4de2b..78329241 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -70,6 +70,7 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 
 	serveMux.HandleFunc("/", HTTPHandleRootURL)
 	serveMux.HandleFunc("/stats", HTTPShowStatistics)
+	serveMux.Handle("/.well-known", http.FileServer(http.FileSystem(http.Dir("/tmp/letsencrypt/.well-known"))))
 
 	serveMux.HandleFunc("/drop_backlog", HTTPBackendDropBacklog)
 	serveMux.HandleFunc("/uncached_pub", HTTPBackendUncachedPublish)

From 3e956ba456ac76599cb180f76150fdf0f12fb4ca Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Tue, 17 Nov 2015 13:44:52 -0800
Subject: [PATCH 073/176] Fix the LE handler, serve 404s

---
 socketserver/server/handlecore.go | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 78329241..c0527d14 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -69,8 +69,8 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 	BannerHTML = bannerBytes
 
 	serveMux.HandleFunc("/", HTTPHandleRootURL)
+	serveMux.Handle("/.well-known/", http.FileServer(http.FileSystem(http.Dir("/tmp/letsencrypt/"))))
 	serveMux.HandleFunc("/stats", HTTPShowStatistics)
-	serveMux.Handle("/.well-known", http.FileServer(http.FileSystem(http.Dir("/tmp/letsencrypt/.well-known"))))
 
 	serveMux.HandleFunc("/drop_backlog", HTTPBackendDropBacklog)
 	serveMux.HandleFunc("/uncached_pub", HTTPBackendUncachedPublish)
@@ -114,6 +114,11 @@ var BannerHTML []byte
 // HTTPHandleRootURL is the http.HandleFunc for requests on `/`.
 // It either uses the SocketUpgrader or writes out the BannerHTML.
 func HTTPHandleRootURL(w http.ResponseWriter, r *http.Request) {
+	if r.URL.Path != "/" {
+		http.NotFound(w, r)
+		fmt.Println(404)
+		return
+	}
 	if r.Header.Get("Connection") == "Upgrade" {
 		updateSysMem()
 

From 3802dea35c87d5d437d24252ad4d8d1b71fb981d Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Tue, 17 Nov 2015 19:53:58 -0800
Subject: [PATCH 074/176] refactor CloseConnection

---
 socketserver/server/handlecore.go | 34 ++++++++++++++++++-------------
 socketserver/server/types.go      | 12 +++++++++++
 2 files changed, 32 insertions(+), 14 deletions(-)

diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index c0527d14..31bbe9d3 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -14,6 +14,7 @@ import (
 	"strings"
 	"sync"
 	"time"
+	"sync/atomic"
 )
 
 // SuccessCommand is a Reply Command to indicate success in reply to a C2S Command.
@@ -197,8 +198,8 @@ const sendMessageAbortLength = 50
 func RunSocketConnection(conn *websocket.Conn) {
 	// websocket.Conn is a ReadWriteCloser
 
-	Statistics.ClientConnectsTotal++
-	Statistics.CurrentClientCount++
+	atomic.AddUint64(&Statistics.ClientConnectsTotal, 1)
+	atomic.AddUint64(&Statistics.CurrentClientCount, 1)
 
 	var _closer sync.Once
 	closer := func() {
@@ -271,27 +272,30 @@ func RunSocketConnection(conn *websocket.Conn) {
 
 	// All set up, now enter the work loop
 
+	var closeReason websocket.CloseError
+
 RunLoop:
 	for {
 		select {
 		case err := <-errorChan:
 			if err == io.EOF {
-				conn.Close() // no need to send a close frame :)
-				break RunLoop
+				closeReason = websocket.CloseError{
+					Code: websocket.CloseGoingAway,
+					Text: err.Error(),
+				}
 			} else if closeMsg, isClose := err.(*websocket.CloseError); isClose {
-				CloseConnection(conn, closeMsg)
+				closeReason = *closeMsg
 			} else {
-				CloseConnection(conn, &websocket.CloseError{
+				closeReason = websocket.CloseError{
 					Code: websocket.CloseInternalServerErr,
 					Text: err.Error(),
-				})
+				}
 			}
-
 			break RunLoop
 
 		case msg := <-clientChan:
 			if client.VersionString == "" && msg.Command != HelloCommand {
-				CloseConnection(conn, &CloseFirstMessageNotHello)
+				closeReason = CloseFirstMessageNotHello
 				break RunLoop
 			}
 
@@ -299,7 +303,8 @@ RunLoop:
 
 		case msg := <-serverMessageChan:
 			if len(serverMessageChan) > sendMessageAbortLength {
-				CloseConnection(conn, &CloseTooManyBufferedMessages)
+				closeReason = CloseTooManyBufferedMessages
+				break RunLoop
 			}
 			SendMessage(conn, msg)
 
@@ -309,7 +314,7 @@ RunLoop:
 			tooManyPings := client.pingCount == 5
 			client.Mutex.Unlock()
 			if tooManyPings {
-				CloseConnection(conn, &CloseTimedOut)
+				closeReason = CloseTimedOut
 				break RunLoop
 			} else {
 				conn.WriteControl(websocket.PingMessage, []byte(strconv.FormatInt(time.Now().Unix(), 10)), getDeadline())
@@ -318,6 +323,7 @@ RunLoop:
 	}
 
 	// Exit
+	CloseConnection(conn, closeReason)
 
 	// Launch message draining goroutine - we aren't out of the pub/sub records
 	go func() {
@@ -338,15 +344,15 @@ RunLoop:
 	// Close the channel so the draining goroutine can finish, too.
 	close(_serverMessageChan)
 
-	Statistics.ClientDisconnectsTotal++
-	Statistics.CurrentClientCount--
+	atomic.AddUint64(&Statistics.ClientDisconnectsTotal, 1)
+	atomic.AddUint64(&Statistics.CurrentClientCount, ^uint64(0))
 }
 
 func getDeadline() time.Time {
 	return time.Now().Add(1 * time.Minute)
 }
 
-func CloseConnection(conn *websocket.Conn, closeMsg *websocket.CloseError) {
+func CloseConnection(conn *websocket.Conn, closeMsg websocket.CloseError) {
 	Statistics.DisconnectCodes[strconv.Itoa(closeMsg.Code)]++
 	closeTxt := closeMsg.Text
 	if strings.Contains(closeTxt, "read: connection reset by peer") {
diff --git a/socketserver/server/types.go b/socketserver/server/types.go
index a3358028..1910c889 100644
--- a/socketserver/server/types.go
+++ b/socketserver/server/types.go
@@ -28,6 +28,11 @@ type ConfigFile struct {
 	SSLCertificateFile string
 	SSLKeyFile         string
 
+	UseElasticSearch bool
+	ESServer         string
+	ESIndexPrefix    string
+	ESHostName       string
+
 	// Nacl keys
 	OurPrivateKey    []byte
 	OurPublicKey     []byte
@@ -115,6 +120,13 @@ type ClientInfo struct {
 	pingCount int
 }
 
+type esReportBasic struct {
+	Timestamp time.Time
+	Host string
+}
+type esDisconnectReport struct {
+}
+
 func VersionFromString(v string) ClientVersion {
 	var cv ClientVersion
 	fmt.Sscanf(v, "ffz_%d.%d.%d", &cv.Major, &cv.Minor, &cv.Revision)

From 00175cad39cabbc6e99c1166d838267abebee974 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 18 Nov 2015 09:07:34 -0800
Subject: [PATCH 075/176] change global sub to ClientInfo, remove ip output

---
 socketserver/server/handlecore.go    |  5 ++++-
 socketserver/server/stats.go         |  6 +++---
 socketserver/server/subscriptions.go | 32 ++++++++++++++++------------
 socketserver/server/utils.go         | 32 ++++++++++++++++++++++++++++
 4 files changed, 57 insertions(+), 18 deletions(-)

diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 31bbe9d3..a8272b71 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -353,13 +353,16 @@ func getDeadline() time.Time {
 }
 
 func CloseConnection(conn *websocket.Conn, closeMsg websocket.CloseError) {
-	Statistics.DisconnectCodes[strconv.Itoa(closeMsg.Code)]++
 	closeTxt := closeMsg.Text
 	if strings.Contains(closeTxt, "read: connection reset by peer") {
 		closeTxt = "read: connection reset by peer"
+	} else if strings.Contains(closeTxt, "use of closed network connection") {
+		closeTxt = "read: use of closed network connection"
 	} else if closeMsg.Code == 1001 {
 		closeTxt = "clean shutdown"
 	}
+	// todo kibana cannot analyze these
+	Statistics.DisconnectCodes[strconv.Itoa(closeMsg.Code)]++
 	Statistics.DisconnectReasons[closeTxt]++
 
 	conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(closeMsg.Code, closeMsg.Text), getDeadline())
diff --git a/socketserver/server/stats.go b/socketserver/server/stats.go
index 0b9a8a30..34d0db25 100644
--- a/socketserver/server/stats.go
+++ b/socketserver/server/stats.go
@@ -119,9 +119,9 @@ func updatePeriodicStats() {
 		Statistics.PubSubChannelCount = len(ChatSubscriptionInfo)
 		ChatSubscriptionLock.RUnlock()
 
-		GlobalSubscriptionInfo.RLock()
-		Statistics.CurrentClientCount = uint64(len(GlobalSubscriptionInfo.Members))
-		GlobalSubscriptionInfo.RUnlock()
+		GlobalSubscriptionLock.RLock()
+		Statistics.CurrentClientCount = uint64(len(GlobalSubscriptionInfo))
+		GlobalSubscriptionLock.RUnlock()
 	}
 
 	{
diff --git a/socketserver/server/subscriptions.go b/socketserver/server/subscriptions.go
index 35b6020b..c9e85a1c 100644
--- a/socketserver/server/subscriptions.go
+++ b/socketserver/server/subscriptions.go
@@ -16,7 +16,8 @@ type SubscriberList struct {
 
 var ChatSubscriptionInfo map[string]*SubscriberList = make(map[string]*SubscriberList)
 var ChatSubscriptionLock sync.RWMutex
-var GlobalSubscriptionInfo SubscriberList
+var GlobalSubscriptionInfo []*ClientInfo
+var GlobalSubscriptionLock sync.RWMutex
 
 func SubscribeChannel(client *ClientInfo, channelName string) {
 	ChatSubscriptionLock.RLock()
@@ -29,9 +30,9 @@ func SubscribeDefaults(client *ClientInfo) {
 }
 
 func SubscribeGlobal(client *ClientInfo) {
-	GlobalSubscriptionInfo.Lock()
-	AddToSliceC(&GlobalSubscriptionInfo.Members, client.MessageChannel)
-	GlobalSubscriptionInfo.Unlock()
+	GlobalSubscriptionLock.Lock()
+	AddToSliceCl(&GlobalSubscriptionInfo, client)
+	GlobalSubscriptionLock.Unlock()
 }
 
 func PublishToChannel(channel string, msg ClientMessage) (count int) {
@@ -75,12 +76,15 @@ func PublishToMultiple(channels []string, msg ClientMessage) (count int) {
 }
 
 func PublishToAll(msg ClientMessage) (count int) {
-	GlobalSubscriptionInfo.RLock()
-	for _, msgChan := range GlobalSubscriptionInfo.Members {
-		msgChan <- msg
+	GlobalSubscriptionLock.RLock()
+	for _, client := range GlobalSubscriptionInfo {
+		select {
+		case client.MessageChannel <- msg:
+		case <-client.MsgChannelIsDone:
+		}
 		count++
 	}
-	GlobalSubscriptionInfo.RUnlock()
+	GlobalSubscriptionLock.RUnlock()
 	return
 }
 
@@ -106,9 +110,9 @@ func UnsubscribeAll(client *ClientInfo) {
 	client.PendingSubscriptionsBacklog = nil
 	client.Mutex.Unlock()
 
-	GlobalSubscriptionInfo.Lock()
-	RemoveFromSliceC(&GlobalSubscriptionInfo.Members, client.MessageChannel)
-	GlobalSubscriptionInfo.Unlock()
+	GlobalSubscriptionLock.Lock()
+	RemoveFromSliceCl(&GlobalSubscriptionInfo, client)
+	GlobalSubscriptionLock.Unlock()
 
 	ChatSubscriptionLock.RLock()
 	client.Mutex.Lock()
@@ -126,9 +130,9 @@ func UnsubscribeAll(client *ClientInfo) {
 }
 
 func unsubscribeAllClients() {
-	GlobalSubscriptionInfo.Lock()
-	GlobalSubscriptionInfo.Members = nil
-	GlobalSubscriptionInfo.Unlock()
+	GlobalSubscriptionLock.Lock()
+	GlobalSubscriptionInfo = nil
+	GlobalSubscriptionLock.Unlock()
 	ChatSubscriptionLock.Lock()
 	ChatSubscriptionInfo = make(map[string]*SubscriberList)
 	ChatSubscriptionLock.Unlock()
diff --git a/socketserver/server/utils.go b/socketserver/server/utils.go
index 07866e4b..258acda1 100644
--- a/socketserver/server/utils.go
+++ b/socketserver/server/utils.go
@@ -160,6 +160,38 @@ func RemoveFromSliceC(ary *[]chan<- ClientMessage, val chan<- ClientMessage) boo
 	return true
 }
 
+func AddToSliceCl(ary *[]*ClientInfo, val *ClientInfo) bool {
+	slice := *ary
+	for _, v := range slice {
+		if v == val {
+			return false
+		}
+	}
+
+	slice = append(slice, val)
+	*ary = slice
+	return true
+}
+
+func RemoveFromSliceCl(ary *[]*ClientInfo, val *ClientInfo) bool {
+	slice := *ary
+	var idx int = -1
+	for i, v := range slice {
+		if v == val {
+			idx = i
+			break
+		}
+	}
+	if idx == -1 {
+		return false
+	}
+
+	slice[idx] = slice[len(slice)-1]
+	slice = slice[:len(slice)-1]
+	*ary = slice
+	return true
+}
+
 func AddToSliceB(ary *[]bunchSubscriber, client *ClientInfo, mid int) bool {
 	newSub := bunchSubscriber{Client: client, MessageID: mid}
 	slice := *ary

From 3805fa1b6697cc62f50fe1229abeb38109514207 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 18 Nov 2015 18:33:20 -0800
Subject: [PATCH 076/176] Code cleanup: Remove single-target, timestamp-cache

---
 socketserver/server/backend.go            |  44 +------
 socketserver/server/commands.go           |   3 -
 socketserver/server/handlecore.go         |   3 +-
 socketserver/server/publisher.go          | 152 ++--------------------
 socketserver/server/subscriptions_test.go |   6 +-
 socketserver/server/types.go              |  36 +----
 6 files changed, 16 insertions(+), 228 deletions(-)

diff --git a/socketserver/server/backend.go b/socketserver/server/backend.go
index 8c96fd51..f5b67c37 100644
--- a/socketserver/server/backend.go
+++ b/socketserver/server/backend.go
@@ -6,7 +6,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/gorilla/websocket"
 	"github.com/pmylund/go-cache"
 	"golang.org/x/crypto/nacl/box"
 	"io/ioutil"
@@ -23,7 +22,6 @@ var backendHTTPClient http.Client
 var backendURL string
 var responseCache *cache.Cache
 
-var getBacklogURL string
 var postStatisticsURL string
 var addTopicURL string
 var announceStartupURL string
@@ -41,7 +39,6 @@ func setupBackend(config *ConfigFile) {
 	}
 	responseCache = cache.New(60*time.Second, 120*time.Second)
 
-	getBacklogURL = fmt.Sprintf("%s/backlog", backendURL)
 	postStatisticsURL = fmt.Sprintf("%s/stats", backendURL)
 	addTopicURL = fmt.Sprintf("%s/topics", backendURL)
 	announceStartupURL = fmt.Sprintf("%s/startup", backendURL)
@@ -96,18 +93,16 @@ func HTTPBackendUncachedPublish(w http.ResponseWriter, r *http.Request) {
 	var count int
 
 	switch target {
-	case MsgTargetTypeSingle:
-	// TODO
 	case MsgTargetTypeChat:
 		count = PublishToChannel(channel, cm)
 	case MsgTargetTypeMultichat:
 		count = PublishToMultiple(strings.Split(channel, ","), cm)
 	case MsgTargetTypeGlobal:
 		count = PublishToAll(cm)
-	case MsgTargetTypeInvalid:
+	case MsgTargetTypeInvalid: fallthrough
 	default:
 		w.WriteHeader(422)
-		fmt.Fprint(w, "Invalid 'scope'. must be single, chat, multichat, channel, or global")
+		fmt.Fprint(w, "Invalid 'scope'. must be chat, multichat, channel, or global")
 		return
 	}
 	fmt.Fprint(w, count)
@@ -204,41 +199,6 @@ func SendAggregatedData(sealedForm url.Values) error {
 	return resp.Body.Close()
 }
 
-// FetchBacklogData makes a request to the backend for backlog data on a set of pub/sub topics.
-// TODO scrap this, replaced by /cached_pub
-func FetchBacklogData(chatSubs []string) ([]ClientMessage, error) {
-	formData := url.Values{
-		"subs": chatSubs,
-	}
-
-	sealedForm, err := SealRequest(formData)
-	if err != nil {
-		return nil, err
-	}
-
-	resp, err := backendHTTPClient.PostForm(getBacklogURL, sealedForm)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode != 200 {
-		return nil, httpError(resp.StatusCode)
-	}
-	dec := json.NewDecoder(resp.Body)
-	var messageStrings []string
-	err = dec.Decode(messageStrings)
-	if err != nil {
-		return nil, err
-	}
-
-	var messages = make([]ClientMessage, len(messageStrings))
-	for i, str := range messageStrings {
-		UnmarshalClientMessage([]byte(str), websocket.TextMessage, &messages[i])
-	}
-
-	return messages, nil
-}
-
 // ErrBackendNotOK indicates that the backend replied with something other than the string "ok".
 type ErrBackendNotOK struct {
 	Response string
diff --git a/socketserver/server/commands.go b/socketserver/server/commands.go
index 57461278..b68d4ebf 100644
--- a/socketserver/server/commands.go
+++ b/socketserver/server/commands.go
@@ -172,9 +172,6 @@ func C2SReady(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg
 	go func() {
 		client.MessageChannel <- ClientMessage{MessageID: msg.MessageID, Command: SuccessCommand}
 		SendBacklogForNewClient(client)
-		//		if disconnectAt != 0 {
-		//			SendTimedBacklogMessages(client, time.Unix(disconnectAt, 0))
-		//		}
 		client.MsgChannelKeepalive.Done()
 	}()
 	return ClientMessage{Command: AsyncResponseCommand}, nil
diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index a8272b71..7ab17b1e 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -13,8 +13,8 @@ import (
 	"strconv"
 	"strings"
 	"sync"
-	"time"
 	"sync/atomic"
+	"time"
 )
 
 // SuccessCommand is a Reply Command to indicate success in reply to a C2S Command.
@@ -91,7 +91,6 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 	}
 
 	go authorizationJanitor()
-	go backlogJanitor()
 	go bunchCacheJanitor()
 	go pubsubJanitor()
 	go aggregateDataSender()
diff --git a/socketserver/server/publisher.go b/socketserver/server/publisher.go
index 9f9be8ff..b6e9a79a 100644
--- a/socketserver/server/publisher.go
+++ b/socketserver/server/publisher.go
@@ -6,7 +6,6 @@ import (
 	"net/http"
 	"sort"
 	"strconv"
-	"strings"
 	"sync"
 	"time"
 )
@@ -16,32 +15,18 @@ type PushCommandCacheInfo struct {
 	Target  MessageTargetType
 }
 
-// this value is just docs right now
-var ServerInitiatedCommands = map[Command]PushCommandCacheInfo{
-	/// Global updates & notices
-	"update_news": {CacheTypeTimestamps, MsgTargetTypeGlobal}, // timecache:global
-	"message":     {CacheTypeTimestamps, MsgTargetTypeGlobal}, // timecache:global
-	"reload_ff":   {CacheTypeTimestamps, MsgTargetTypeGlobal}, // timecache:global
-
-	/// Emote updates
-	"reload_badges": {CacheTypeTimestamps, MsgTargetTypeGlobal},    // timecache:global
-	"set_badge":     {CacheTypeTimestamps, MsgTargetTypeMultichat}, // timecache:multichat
-	"reload_set":    {},                                            // timecache:multichat
-	"load_set":      {},                                            // TODO what are the semantics of this?
-
-	/// User auth
-	"do_authorize": {CacheTypeNever, MsgTargetTypeSingle}, // nocache:single
-
+// S2CCommandsCacheInfo details what the behavior is of each command that can be sent to /cached_pub.
+var S2CCommandsCacheInfo = map[Command]PushCommandCacheInfo{
 	/// Channel data
 	// follow_sets: extra emote sets included in the chat
 	// follow_buttons: extra follow buttons below the stream
-	"follow_sets":    {CacheTypePersistent, MsgTargetTypeChat}, // mustcache:chat
-	"follow_buttons": {CacheTypePersistent, MsgTargetTypeChat}, // mustcache:watching
-	"srl_race":       {CacheTypeLastOnly, MsgTargetTypeChat},   // cachelast:watching
+	"follow_sets":    {CacheTypePersistent, MsgTargetTypeChat},
+	"follow_buttons": {CacheTypePersistent, MsgTargetTypeChat},
+	"srl_race":       {CacheTypeLastOnly, MsgTargetTypeChat},
 
 	/// Chatter/viewer counts
-	"chatters": {CacheTypeLastOnly, MsgTargetTypeChat}, // cachelast:watching
-	"viewers":  {CacheTypeLastOnly, MsgTargetTypeChat}, // cachelast:watching
+	"chatters": {CacheTypeLastOnly, MsgTargetTypeChat},
+	"viewers":  {CacheTypeLastOnly, MsgTargetTypeChat},
 }
 
 type BacklogCacheType int
@@ -51,10 +36,6 @@ const (
 	CacheTypeInvalid BacklogCacheType = iota
 	// This message cannot be cached.
 	CacheTypeNever
-	// Save the last 24 hours of this message.
-	// If a client indicates that it has reconnected, replay the messages sent after the disconnect.
-	// Do not replay if the client indicates that this is a firstload.
-	CacheTypeTimestamps
 	// Save only the last copy of this message, and always send it when the backlog is requested.
 	CacheTypeLastOnly
 	// Save this backlog data to disk with its timestamp.
@@ -67,8 +48,6 @@ type MessageTargetType int
 const (
 	// This is not a message target.
 	MsgTargetTypeInvalid MessageTargetType = iota
-	// This message is targeted to a single TODO(user or connection)
-	MsgTargetTypeSingle
 	// This message is targeted to all users in a chat
 	MsgTargetTypeChat
 	// This message is targeted to all users in multiple chats
@@ -85,19 +64,6 @@ var ErrorUnrecognizedCacheType = errors.New("Invalid value for cachetype")
 // Returned by MessageTargetType.UnmarshalJSON()
 var ErrorUnrecognizedTargetType = errors.New("Invalid value for message target")
 
-type TimestampedGlobalMessage struct {
-	Timestamp time.Time
-	Command   Command
-	Data      string
-}
-
-type TimestampedMultichatMessage struct {
-	Timestamp time.Time
-	Channels  []string
-	Command   Command
-	Data      string
-}
-
 type LastSavedMessage struct {
 	Timestamp time.Time
 	Data      string
@@ -113,10 +79,6 @@ var CachedLSMLock sync.RWMutex
 var PersistentLastMessages map[Command]map[string]LastSavedMessage
 var PersistentLSMLock sync.RWMutex
 
-var CachedGlobalMessages []TimestampedGlobalMessage
-var CachedChannelMessages []TimestampedMultichatMessage
-var CacheListsLock sync.RWMutex
-
 // DumpBacklogData drops all /cached_pub data.
 func DumpBacklogData() {
 	CachedLSMLock.Lock()
@@ -126,11 +88,6 @@ func DumpBacklogData() {
 	PersistentLSMLock.Lock()
 	PersistentLastMessages = make(map[Command]map[string]LastSavedMessage)
 	PersistentLSMLock.Unlock()
-
-	CacheListsLock.Lock()
-	CachedGlobalMessages = make(tgmarray, 0)
-	CachedChannelMessages = make(tmmarray, 0)
-	CacheListsLock.Unlock()
 }
 
 // SendBacklogForNewClient sends any backlog data relevant to a new client.
@@ -174,77 +131,6 @@ func SendBacklogForNewClient(client *ClientInfo) {
 	client.Mutex.Unlock()
 }
 
-// SendTimedBacklogMessages sends any once-off messages that the client may have missed while it was disconnected.
-// Effectively, this can only process CacheTypeTimestamps.
-func SendTimedBacklogMessages(client *ClientInfo, disconnectTime time.Time) {
-	client.Mutex.Lock() // reading CurrentChannels
-	CacheListsLock.RLock()
-
-	globIdx := findFirstNewMessage(tgmarray(CachedGlobalMessages), disconnectTime)
-
-	if globIdx != -1 {
-		for i := globIdx; i < len(CachedGlobalMessages); i++ {
-			item := CachedGlobalMessages[i]
-			msg := ClientMessage{MessageID: -1, Command: item.Command, origArguments: item.Data}
-			msg.parseOrigArguments()
-			client.MessageChannel <- msg
-		}
-	}
-
-	chanIdx := findFirstNewMessage(tmmarray(CachedChannelMessages), disconnectTime)
-
-	if chanIdx != -1 {
-		for i := chanIdx; i < len(CachedChannelMessages); i++ {
-			item := CachedChannelMessages[i]
-			var send bool
-			for _, channel := range item.Channels {
-				for _, matchChannel := range client.CurrentChannels {
-					if channel == matchChannel {
-						send = true
-						break
-					}
-				}
-				if send {
-					break
-				}
-			}
-			if send {
-				msg := ClientMessage{MessageID: -1, Command: item.Command, origArguments: item.Data}
-				msg.parseOrigArguments()
-				client.MessageChannel <- msg
-			}
-		}
-	}
-
-	CacheListsLock.RUnlock()
-	client.Mutex.Unlock()
-}
-
-func backlogJanitor() {
-	for {
-		time.Sleep(1 * time.Hour)
-		cleanupTimedBacklogMessages()
-	}
-}
-
-func cleanupTimedBacklogMessages() {
-	CacheListsLock.Lock()
-	oneHourAgo := time.Now().Add(-24 * time.Hour)
-	globIdx := findFirstNewMessage(tgmarray(CachedGlobalMessages), oneHourAgo)
-	if globIdx != -1 {
-		newGlobMsgs := make([]TimestampedGlobalMessage, len(CachedGlobalMessages)-globIdx)
-		copy(newGlobMsgs, CachedGlobalMessages[globIdx:])
-		CachedGlobalMessages = newGlobMsgs
-	}
-	chanIdx := findFirstNewMessage(tmmarray(CachedChannelMessages), oneHourAgo)
-	if chanIdx != -1 {
-		newChanMsgs := make([]TimestampedMultichatMessage, len(CachedChannelMessages)-chanIdx)
-		copy(newChanMsgs, CachedChannelMessages[chanIdx:])
-		CachedChannelMessages = newChanMsgs
-	}
-	CacheListsLock.Unlock()
-}
-
 // insertionSort implements insertion sort.
 // CacheTypeTimestamps should use insertion sort for O(N) average performance.
 // (The average case is the array is still sorted after insertion of the new item.)
@@ -309,23 +195,9 @@ func SaveLastMessage(which map[Command]map[string]LastSavedMessage, locker sync.
 	}
 }
 
-func SaveGlobalMessage(cmd Command, timestamp time.Time, data string) {
-	CacheListsLock.Lock()
-	CachedGlobalMessages = append(CachedGlobalMessages, TimestampedGlobalMessage{timestamp, cmd, data})
-	insertionSort(tgmarray(CachedGlobalMessages))
-	CacheListsLock.Unlock()
-}
-
-func SaveMultichanMessage(cmd Command, channels string, timestamp time.Time, data string) {
-	CacheListsLock.Lock()
-	CachedChannelMessages = append(CachedChannelMessages, TimestampedMultichatMessage{timestamp, strings.Split(channels, ","), cmd, data})
-	insertionSort(tmmarray(CachedChannelMessages))
-	CacheListsLock.Unlock()
-}
-
 func GetCommandsOfType(match PushCommandCacheInfo) []Command {
 	var ret []Command
-	for cmd, info := range ServerInitiatedCommands {
+	for cmd, info := range S2CCommandsCacheInfo {
 		if info == match {
 			ret = append(ret, cmd)
 		}
@@ -371,7 +243,7 @@ func HTTPBackendCachedPublish(w http.ResponseWriter, r *http.Request) {
 		fmt.Fprintf(w, "error parsing time: %v", err)
 	}
 
-	cacheinfo, ok := ServerInitiatedCommands[cmd]
+	cacheinfo, ok := S2CCommandsCacheInfo[cmd]
 	if !ok {
 		w.WriteHeader(422)
 		fmt.Fprintf(w, "Caching semantics unknown for command '%s'. Post to /addcachedcommand first.", cmd)
@@ -388,12 +260,6 @@ func HTTPBackendCachedPublish(w http.ResponseWriter, r *http.Request) {
 	} else if cacheinfo.Caching == CacheTypePersistent && cacheinfo.Target == MsgTargetTypeChat {
 		SaveLastMessage(PersistentLastMessages, &PersistentLSMLock, cmd, channel, timestamp, json, deleteMode)
 		count = PublishToChannel(channel, msg)
-	} else if cacheinfo.Caching == CacheTypeTimestamps && cacheinfo.Target == MsgTargetTypeMultichat {
-		SaveMultichanMessage(cmd, channel, timestamp, json)
-		count = PublishToMultiple(strings.Split(channel, ","), msg)
-	} else if cacheinfo.Caching == CacheTypeTimestamps && cacheinfo.Target == MsgTargetTypeGlobal {
-		SaveGlobalMessage(cmd, timestamp, json)
-		count = PublishToAll(msg)
 	}
 
 	w.Write([]byte(strconv.Itoa(count)))
diff --git a/socketserver/server/subscriptions_test.go b/socketserver/server/subscriptions_test.go
index 5d7adba3..c798ca2d 100644
--- a/socketserver/server/subscriptions_test.go
+++ b/socketserver/server/subscriptions_test.go
@@ -190,9 +190,9 @@ func TestSubscriptionAndPublish(t *testing.T) {
 	const TestData3 = false
 	var TestData4 = []interface{}{"str1", "str2", "str3"}
 
-	ServerInitiatedCommands[TestCommandChan] = PushCommandCacheInfo{CacheTypeLastOnly, MsgTargetTypeChat}
-	ServerInitiatedCommands[TestCommandMulti] = PushCommandCacheInfo{CacheTypeTimestamps, MsgTargetTypeMultichat}
-	ServerInitiatedCommands[TestCommandGlobal] = PushCommandCacheInfo{CacheTypeTimestamps, MsgTargetTypeGlobal}
+	S2CCommandsCacheInfo[TestCommandChan] = PushCommandCacheInfo{CacheTypeLastOnly, MsgTargetTypeChat}
+	S2CCommandsCacheInfo[TestCommandMulti] = PushCommandCacheInfo{CacheTypeTimestamps, MsgTargetTypeMultichat}
+	S2CCommandsCacheInfo[TestCommandGlobal] = PushCommandCacheInfo{CacheTypeTimestamps, MsgTargetTypeGlobal}
 
 	var server *httptest.Server
 	var urls TURLs
diff --git a/socketserver/server/types.go b/socketserver/server/types.go
index 1910c889..bc660aed 100644
--- a/socketserver/server/types.go
+++ b/socketserver/server/types.go
@@ -122,7 +122,7 @@ type ClientInfo struct {
 
 type esReportBasic struct {
 	Timestamp time.Time
-	Host string
+	Host      string
 }
 type esDisconnectReport struct {
 }
@@ -159,42 +159,12 @@ func (cv *ClientVersion) Equal(cv2 *ClientVersion) bool {
 
 const usePendingSubscrptionsBacklog = false
 
-type tgmarray []TimestampedGlobalMessage
-type tmmarray []TimestampedMultichatMessage
-
-func (ta tgmarray) Len() int {
-	return len(ta)
-}
-func (ta tgmarray) Less(i, j int) bool {
-	return ta[i].Timestamp.Before(ta[j].Timestamp)
-}
-func (ta tgmarray) Swap(i, j int) {
-	ta[i], ta[j] = ta[j], ta[i]
-}
-func (ta tgmarray) GetTime(i int) time.Time {
-	return ta[i].Timestamp
-}
-func (ta tmmarray) Len() int {
-	return len(ta)
-}
-func (ta tmmarray) Less(i, j int) bool {
-	return ta[i].Timestamp.Before(ta[j].Timestamp)
-}
-func (ta tmmarray) Swap(i, j int) {
-	ta[i], ta[j] = ta[j], ta[i]
-}
-func (ta tmmarray) GetTime(i int) time.Time {
-	return ta[i].Timestamp
-}
-
 func (bct BacklogCacheType) Name() string {
 	switch bct {
 	case CacheTypeInvalid:
 		return ""
 	case CacheTypeNever:
 		return "never"
-	case CacheTypeTimestamps:
-		return "timed"
 	case CacheTypeLastOnly:
 		return "last"
 	case CacheTypePersistent:
@@ -205,7 +175,6 @@ func (bct BacklogCacheType) Name() string {
 
 var CacheTypesByName = map[string]BacklogCacheType{
 	"never":   CacheTypeNever,
-	"timed":   CacheTypeTimestamps,
 	"last":    CacheTypeLastOnly,
 	"persist": CacheTypePersistent,
 }
@@ -247,8 +216,6 @@ func (mtt MessageTargetType) Name() string {
 	switch mtt {
 	case MsgTargetTypeInvalid:
 		return ""
-	case MsgTargetTypeSingle:
-		return "single"
 	case MsgTargetTypeChat:
 		return "chat"
 	case MsgTargetTypeMultichat:
@@ -260,7 +227,6 @@ func (mtt MessageTargetType) Name() string {
 }
 
 var TargetTypesByName = map[string]MessageTargetType{
-	"single":    MsgTargetTypeSingle,
 	"chat":      MsgTargetTypeChat,
 	"multichat": MsgTargetTypeMultichat,
 	"global":    MsgTargetTypeGlobal,

From 75bee4fee570bf56bd522970367479720cf1129f Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Thu, 19 Nov 2015 16:31:10 -0800
Subject: [PATCH 077/176] Oops, forgot to remove test for deleted code

---
 socketserver/server/publisher_test.go | 80 ---------------------------
 1 file changed, 80 deletions(-)
 delete mode 100644 socketserver/server/publisher_test.go

diff --git a/socketserver/server/publisher_test.go b/socketserver/server/publisher_test.go
deleted file mode 100644
index e3c0484f..00000000
--- a/socketserver/server/publisher_test.go
+++ /dev/null
@@ -1,80 +0,0 @@
-package server
-
-import (
-	"testing"
-	"time"
-)
-
-func TestCleanupBacklogMessages(t *testing.T) {
-
-}
-
-func TestFindFirstNewMessageEmpty(t *testing.T) {
-	CachedGlobalMessages = []TimestampedGlobalMessage{}
-	i := findFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
-	if i != -1 {
-		t.Errorf("Expected -1, got %d", i)
-	}
-}
-func TestFindFirstNewMessageOneBefore(t *testing.T) {
-	CachedGlobalMessages = []TimestampedGlobalMessage{
-		{Timestamp: time.Unix(8, 0)},
-	}
-	i := findFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
-	if i != -1 {
-		t.Errorf("Expected -1, got %d", i)
-	}
-}
-func TestFindFirstNewMessageSeveralBefore(t *testing.T) {
-	CachedGlobalMessages = []TimestampedGlobalMessage{
-		{Timestamp: time.Unix(1, 0)},
-		{Timestamp: time.Unix(2, 0)},
-		{Timestamp: time.Unix(3, 0)},
-		{Timestamp: time.Unix(4, 0)},
-		{Timestamp: time.Unix(5, 0)},
-	}
-	i := findFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
-	if i != -1 {
-		t.Errorf("Expected -1, got %d", i)
-	}
-}
-func TestFindFirstNewMessageInMiddle(t *testing.T) {
-	CachedGlobalMessages = []TimestampedGlobalMessage{
-		{Timestamp: time.Unix(1, 0)},
-		{Timestamp: time.Unix(2, 0)},
-		{Timestamp: time.Unix(3, 0)},
-		{Timestamp: time.Unix(4, 0)},
-		{Timestamp: time.Unix(5, 0)},
-		{Timestamp: time.Unix(11, 0)},
-		{Timestamp: time.Unix(12, 0)},
-		{Timestamp: time.Unix(13, 0)},
-		{Timestamp: time.Unix(14, 0)},
-		{Timestamp: time.Unix(15, 0)},
-	}
-	i := findFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
-	if i != 5 {
-		t.Errorf("Expected 5, got %d", i)
-	}
-}
-func TestFindFirstNewMessageOneAfter(t *testing.T) {
-	CachedGlobalMessages = []TimestampedGlobalMessage{
-		{Timestamp: time.Unix(15, 0)},
-	}
-	i := findFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
-	if i != 0 {
-		t.Errorf("Expected 0, got %d", i)
-	}
-}
-func TestFindFirstNewMessageSeveralAfter(t *testing.T) {
-	CachedGlobalMessages = []TimestampedGlobalMessage{
-		{Timestamp: time.Unix(11, 0)},
-		{Timestamp: time.Unix(12, 0)},
-		{Timestamp: time.Unix(13, 0)},
-		{Timestamp: time.Unix(14, 0)},
-		{Timestamp: time.Unix(15, 0)},
-	}
-	i := findFirstNewMessage(tgmarray(CachedGlobalMessages), time.Unix(10, 0))
-	if i != 0 {
-		t.Errorf("Expected 0, got %d", i)
-	}
-}

From 3c3791579f055e0c666ed774cc7dfb4b182ec9a8 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Thu, 19 Nov 2015 16:38:07 -0800
Subject: [PATCH 078/176] Fix tests for deleted code

---
 socketserver/server/publisher.go          |  7 +++
 socketserver/server/subscriptions_test.go | 52 +++++++++++++++++------
 2 files changed, 45 insertions(+), 14 deletions(-)

diff --git a/socketserver/server/publisher.go b/socketserver/server/publisher.go
index b6e9a79a..83aa8b87 100644
--- a/socketserver/server/publisher.go
+++ b/socketserver/server/publisher.go
@@ -8,6 +8,7 @@ import (
 	"strconv"
 	"sync"
 	"time"
+	"strings"
 )
 
 type PushCommandCacheInfo struct {
@@ -260,6 +261,12 @@ func HTTPBackendCachedPublish(w http.ResponseWriter, r *http.Request) {
 	} else if cacheinfo.Caching == CacheTypePersistent && cacheinfo.Target == MsgTargetTypeChat {
 		SaveLastMessage(PersistentLastMessages, &PersistentLSMLock, cmd, channel, timestamp, json, deleteMode)
 		count = PublishToChannel(channel, msg)
+	} else if cacheinfo.Caching == CacheTypeLastOnly && cacheinfo.Target == MsgTargetTypeMultichat {
+		channels := strings.Split(channel, ",")
+		for _, channel := range channels {
+			SaveLastMessage(CachedLastMessages, &CachedLSMLock, cmd, channel, timestamp, json, deleteMode)
+		}
+		count = PublishToMultiple(channels, msg)
 	}
 
 	w.Write([]byte(strconv.Itoa(count)))
diff --git a/socketserver/server/subscriptions_test.go b/socketserver/server/subscriptions_test.go
index c798ca2d..f52ffdbf 100644
--- a/socketserver/server/subscriptions_test.go
+++ b/socketserver/server/subscriptions_test.go
@@ -93,7 +93,31 @@ func TSealForSavePubMsg(tb testing.TB, cmd Command, channel string, arguments in
 	return sealed, nil
 }
 
-func TCheckResponse(tb testing.TB, resp *http.Response, expected string) bool {
+func TSealForUncachedPubMsg(tb testing.TB, cmd Command, channel string, arguments interface{}, scope MessageTargetType, deleteMode bool) (url.Values, error) {
+	form := url.Values{}
+	form.Set("cmd", string(cmd))
+	argsBytes, err := json.Marshal(arguments)
+	if err != nil {
+		tb.Error(err)
+		return nil, err
+	}
+	form.Set("args", string(argsBytes))
+	form.Set("channel", channel)
+	if deleteMode {
+		form.Set("delete", "1")
+	}
+	form.Set("time", time.Now().Format(time.UnixDate))
+	form.Set("scope", scope.String())
+
+	sealed, err := SealRequest(form)
+	if err != nil {
+		tb.Error(err)
+		return nil, err
+	}
+	return sealed, nil
+}
+
+func TCheckResponse(tb testing.TB, resp *http.Response, expected string, desc string) bool {
 	var failed bool
 	respBytes, err := ioutil.ReadAll(resp.Body)
 	resp.Body.Close()
@@ -110,7 +134,7 @@ func TCheckResponse(tb testing.TB, resp *http.Response, expected string) bool {
 	}
 
 	if respStr != expected {
-		tb.Errorf("Got wrong response from server. Expected: '%s' Got: '%s'", expected, respStr)
+		tb.Errorf("Got wrong response from server. %s Expected: '%s' Got: '%s'", desc, expected, respStr)
 		failed = true
 	}
 	return !failed
@@ -119,8 +143,8 @@ func TCheckResponse(tb testing.TB, resp *http.Response, expected string) bool {
 type TURLs struct {
 	Websocket  string
 	Origin     string
-	PubMsg     string
-	SavePubMsg string // update_and_pub
+	UncachedPubMsg string // uncached_pub
+	SavePubMsg string // cached_pub
 }
 
 func TGetUrls(testserver *httptest.Server) TURLs {
@@ -128,7 +152,7 @@ func TGetUrls(testserver *httptest.Server) TURLs {
 	return TURLs{
 		Websocket:  fmt.Sprintf("ws://%s/", addr),
 		Origin:     fmt.Sprintf("http://%s", addr),
-		PubMsg:     fmt.Sprintf("http://%s/uncached_pub", addr),
+		UncachedPubMsg:     fmt.Sprintf("http://%s/uncached_pub", addr),
 		SavePubMsg: fmt.Sprintf("http://%s/cached_pub", addr),
 	}
 }
@@ -191,8 +215,8 @@ func TestSubscriptionAndPublish(t *testing.T) {
 	var TestData4 = []interface{}{"str1", "str2", "str3"}
 
 	S2CCommandsCacheInfo[TestCommandChan] = PushCommandCacheInfo{CacheTypeLastOnly, MsgTargetTypeChat}
-	S2CCommandsCacheInfo[TestCommandMulti] = PushCommandCacheInfo{CacheTypeTimestamps, MsgTargetTypeMultichat}
-	S2CCommandsCacheInfo[TestCommandGlobal] = PushCommandCacheInfo{CacheTypeTimestamps, MsgTargetTypeGlobal}
+	S2CCommandsCacheInfo[TestCommandMulti] = PushCommandCacheInfo{CacheTypeLastOnly, MsgTargetTypeMultichat}
+	S2CCommandsCacheInfo[TestCommandGlobal] = PushCommandCacheInfo{CacheTypeLastOnly, MsgTargetTypeGlobal}
 
 	var server *httptest.Server
 	var urls TURLs
@@ -214,7 +238,7 @@ func TestSubscriptionAndPublish(t *testing.T) {
 	// msg 1: ch1
 	// msg 2: ch2, ch3
 	// msg 3: chEmpty
-	// msg 4: global
+	// msg 4: global uncached
 
 	// Client 1
 	conn, resp, err = websocket.DefaultDialer.Dial(urls.Websocket, headers)
@@ -309,7 +333,7 @@ func TestSubscriptionAndPublish(t *testing.T) {
 		t.FailNow()
 	}
 	resp, err = http.PostForm(urls.SavePubMsg, form)
-	if !TCheckResponse(t, resp, strconv.Itoa(2)) {
+	if !TCheckResponse(t, resp, strconv.Itoa(2), "pub msg 1") {
 		t.FailNow()
 	}
 
@@ -320,7 +344,7 @@ func TestSubscriptionAndPublish(t *testing.T) {
 		t.FailNow()
 	}
 	resp, err = http.PostForm(urls.SavePubMsg, form)
-	if !TCheckResponse(t, resp, strconv.Itoa(2)) {
+	if !TCheckResponse(t, resp, strconv.Itoa(2), "pub msg 2") {
 		t.FailNow()
 	}
 
@@ -331,18 +355,18 @@ func TestSubscriptionAndPublish(t *testing.T) {
 		t.FailNow()
 	}
 	resp, err = http.PostForm(urls.SavePubMsg, form)
-	if !TCheckResponse(t, resp, strconv.Itoa(0)) {
+	if !TCheckResponse(t, resp, strconv.Itoa(0), "pub msg 3") {
 		t.FailNow()
 	}
 
 	// Publish message 4 - should go to clients 1, 2, 3
 
-	form, err = TSealForSavePubMsg(t, TestCommandGlobal, "", TestData4, false)
+	form, err = TSealForUncachedPubMsg(t, TestCommandGlobal, "", TestData4, MsgTargetTypeGlobal, false)
 	if err != nil {
 		t.FailNow()
 	}
-	resp, err = http.PostForm(urls.SavePubMsg, form)
-	if !TCheckResponse(t, resp, strconv.Itoa(3)) {
+	resp, err = http.PostForm(urls.UncachedPubMsg, form)
+	if !TCheckResponse(t, resp, strconv.Itoa(3), "pub msg 4") {
 		t.FailNow()
 	}
 

From cf238bf650b9a920785fe8b3280d80da01aceb74 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Thu, 19 Nov 2015 16:39:47 -0800
Subject: [PATCH 079/176] lol guess i missed one

---
 socketserver/cmd/ffzsocketserver/console.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/socketserver/cmd/ffzsocketserver/console.go b/socketserver/cmd/ffzsocketserver/console.go
index 146468b2..793d73d0 100644
--- a/socketserver/cmd/ffzsocketserver/console.go
+++ b/socketserver/cmd/ffzsocketserver/console.go
@@ -19,9 +19,9 @@ func commandLineConsole() {
 	})
 
 	shell.Register("clientcount", func(args ...string) (string, error) {
-		server.GlobalSubscriptionInfo.RLock()
-		count := len(server.GlobalSubscriptionInfo.Members)
-		server.GlobalSubscriptionInfo.RUnlock()
+		server.GlobalSubscriptionLock.RLock()
+		count := len(server.GlobalSubscriptionInfo)
+		server.GlobalSubscriptionLock.RUnlock()
 		return fmt.Sprintln(count, "clients connected"), nil
 	})
 

From 62c96594301fd076ca7aa251ab34c1018ae443e3 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Thu, 19 Nov 2015 16:55:03 -0800
Subject: [PATCH 080/176] Close the connection if we get corrupted commands.

---
 socketserver/server/backend.go            |  3 ++-
 socketserver/server/handlecore.go         | 13 +++++++++++++
 socketserver/server/publisher.go          |  2 +-
 socketserver/server/subscriptions_test.go | 14 +++++++-------
 4 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/socketserver/server/backend.go b/socketserver/server/backend.go
index f5b67c37..b0927f86 100644
--- a/socketserver/server/backend.go
+++ b/socketserver/server/backend.go
@@ -99,7 +99,8 @@ func HTTPBackendUncachedPublish(w http.ResponseWriter, r *http.Request) {
 		count = PublishToMultiple(strings.Split(channel, ","), cm)
 	case MsgTargetTypeGlobal:
 		count = PublishToAll(cm)
-	case MsgTargetTypeInvalid: fallthrough
+	case MsgTargetTypeInvalid:
+		fallthrough
 	default:
 		w.WriteHeader(422)
 		fmt.Fprint(w, "Invalid 'scope'. must be chat, multichat, channel, or global")
diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 7ab17b1e..4200b758 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -15,6 +15,7 @@ import (
 	"sync"
 	"sync/atomic"
 	"time"
+	"unicode/utf8"
 )
 
 // SuccessCommand is a Reply Command to indicate success in reply to a C2S Command.
@@ -179,6 +180,11 @@ var CloseFirstMessageNotHello = websocket.CloseError{
 	Code: websocket.ClosePolicyViolation,
 }
 
+var CloseNonUTF8Data = websocket.CloseError{
+	Code: 4001,
+	Text: "Non UTF8 data recieved. Network corruption likely.",
+}
+
 const sendMessageBufferLength = 125
 const sendMessageAbortLength = 50
 
@@ -298,6 +304,13 @@ RunLoop:
 				break RunLoop
 			}
 
+			for _, char := range msg.Command {
+				if char == utf8.RuneError {
+					closeReason = CloseNonUTF8Data
+					break RunLoop
+				}
+			}
+
 			DispatchC2SCommand(conn, &client, msg)
 
 		case msg := <-serverMessageChan:
diff --git a/socketserver/server/publisher.go b/socketserver/server/publisher.go
index 83aa8b87..4ec4dbf3 100644
--- a/socketserver/server/publisher.go
+++ b/socketserver/server/publisher.go
@@ -6,9 +6,9 @@ import (
 	"net/http"
 	"sort"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
-	"strings"
 )
 
 type PushCommandCacheInfo struct {
diff --git a/socketserver/server/subscriptions_test.go b/socketserver/server/subscriptions_test.go
index f52ffdbf..a9a09ac0 100644
--- a/socketserver/server/subscriptions_test.go
+++ b/socketserver/server/subscriptions_test.go
@@ -141,19 +141,19 @@ func TCheckResponse(tb testing.TB, resp *http.Response, expected string, desc st
 }
 
 type TURLs struct {
-	Websocket  string
-	Origin     string
+	Websocket      string
+	Origin         string
 	UncachedPubMsg string // uncached_pub
-	SavePubMsg string // cached_pub
+	SavePubMsg     string // cached_pub
 }
 
 func TGetUrls(testserver *httptest.Server) TURLs {
 	addr := testserver.Listener.Addr().String()
 	return TURLs{
-		Websocket:  fmt.Sprintf("ws://%s/", addr),
-		Origin:     fmt.Sprintf("http://%s", addr),
-		UncachedPubMsg:     fmt.Sprintf("http://%s/uncached_pub", addr),
-		SavePubMsg: fmt.Sprintf("http://%s/cached_pub", addr),
+		Websocket:      fmt.Sprintf("ws://%s/", addr),
+		Origin:         fmt.Sprintf("http://%s", addr),
+		UncachedPubMsg: fmt.Sprintf("http://%s/uncached_pub", addr),
+		SavePubMsg:     fmt.Sprintf("http://%s/cached_pub", addr),
 	}
 }
 

From 38972364fb66d517b47409c21dd32a14d328a25c Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Thu, 19 Nov 2015 17:49:48 -0800
Subject: [PATCH 081/176] Close server on SIGUSR1, 'kickclients' for
 rebalancing

---
 socketserver/cmd/ffzsocketserver/console.go | 36 +++++++++++++
 socketserver/server/handlecore.go           | 59 ++++++++++++++++++---
 socketserver/server/subscriptions.go        |  4 ++
 3 files changed, 93 insertions(+), 6 deletions(-)

diff --git a/socketserver/cmd/ffzsocketserver/console.go b/socketserver/cmd/ffzsocketserver/console.go
index 793d73d0..21da6e4c 100644
--- a/socketserver/cmd/ffzsocketserver/console.go
+++ b/socketserver/cmd/ffzsocketserver/console.go
@@ -7,6 +7,7 @@ import (
 	"github.com/gorilla/websocket"
 	"runtime"
 	"strings"
+	"strconv"
 )
 
 func commandLineConsole() {
@@ -88,6 +89,41 @@ func commandLineConsole() {
 		return "Usage: authorizeeveryone [ on | off ]", nil
 	})
 
+	shell.Register("kickclients", func(args ...string) (string, error) {
+		if len(args) == 0 {
+			return "Please enter either a count or a fraction of clients to kick.", nil
+		}
+		input, err := strconv.ParseFloat(args[0], 64)
+		if err != nil {
+			return "Argument must be a number", err
+		}
+		var count int
+		if input >= 1 {
+			count = int(input)
+		} else {
+			server.GlobalSubscriptionLock.RLock()
+			count = int(float64(len(server.GlobalSubscriptionInfo)) * input)
+			server.GlobalSubscriptionLock.RUnlock()
+		}
+
+		msg := server.ClientMessage{ Arguments: &server.CloseRebalance }
+		server.GlobalSubscriptionLock.RLock()
+		defer server.GlobalSubscriptionLock.RUnlock()
+
+		kickCount := 0
+		for i, cl := range server.GlobalSubscriptionInfo {
+			if i >= count {
+				break
+			}
+			select {
+			case cl.MessageChannel <- msg:
+			case <-cl.MsgChannelIsDone:
+			}
+			kickCount++
+		}
+		return fmt.Sprintf("Kicked %d clients", kickCount), nil
+	})
+
 	shell.Register("panic", func(args ...string) (string, error) {
 		go func() {
 			panic("requested panic")
diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 4200b758..46c8adfd 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -16,6 +16,9 @@ import (
 	"sync/atomic"
 	"time"
 	"unicode/utf8"
+	"os"
+	"os/signal"
+	"syscall"
 )
 
 // SuccessCommand is a Reply Command to indicate success in reply to a C2S Command.
@@ -97,6 +100,20 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 	go aggregateDataSender()
 
 	go ircConnection()
+	go shutdownHandler()
+}
+
+func shutdownHandler() {
+	ch := make(chan os.Signal)
+	signal.Notify(ch, syscall.SIGUSR1)
+	<-ch
+	log.Println("Shutting down...")
+
+	StopAcceptingConnections = true
+	close(StopAcceptingConnectionsCh)
+
+	time.Sleep(1*time.Second)
+	os.Exit(0)
 }
 
 // SocketUpgrader is the websocket.Upgrader currently in use.
@@ -112,6 +129,10 @@ var SocketUpgrader = websocket.Upgrader{
 // Memes go here.
 var BannerHTML []byte
 
+// StopAcceptingConnections is closed while the server is shutting down.
+var StopAcceptingConnectionsCh = make(chan struct{})
+var StopAcceptingConnections = false
+
 // HTTPHandleRootURL is the http.HandleFunc for requests on `/`.
 // It either uses the SocketUpgrader or writes out the BannerHTML.
 func HTTPHandleRootURL(w http.ResponseWriter, r *http.Request) {
@@ -120,6 +141,14 @@ func HTTPHandleRootURL(w http.ResponseWriter, r *http.Request) {
 		fmt.Println(404)
 		return
 	}
+
+	// racy, but should be ok?
+	if StopAcceptingConnections {
+		w.WriteHeader(503)
+		fmt.Fprint(w, "server is shutting down")
+		return
+	}
+
 	if r.Header.Get("Connection") == "Upgrade" {
 		updateSysMem()
 
@@ -165,6 +194,12 @@ var ErrExpectedStringAndInt = errors.New("Error: Expected array of string, int a
 // ErrExpectedStringAndIntGotFloat is sent in a ErrorCommand Reply when the Arguments are of the wrong type.
 var ErrExpectedStringAndIntGotFloat = errors.New("Error: Second argument was a float, expected an integer.")
 
+// CloseGoingAway is sent when the server is restarting.
+var CloseGoingAway = websocket.CloseError{Code: websocket.CloseGoingAway, Text: "server restarting"}
+
+// CloseRebalance is sent when the server has too many clients and needs to shunt some to another server.
+var CloseRebalance = websocket.CloseError{Code: websocket.CloseGoingAway, Text: "kicked for rebalancing, please select a new server"}
+
 // CloseGotBinaryMessage is the termination reason when the client sends a binary websocket frame.
 var CloseGotBinaryMessage = websocket.CloseError{Code: websocket.CloseUnsupportedData, Text: "got binary packet"}
 
@@ -232,6 +267,10 @@ func RunSocketConnection(conn *websocket.Conn) {
 		var messageType int
 		var packet []byte
 		var err error
+
+		defer close(errorChan)
+		defer close(clientChan)
+
 		for ; err == nil; messageType, packet, err = conn.ReadMessage() {
 			if messageType == websocket.BinaryMessage {
 				err = &CloseGotBinaryMessage
@@ -249,8 +288,6 @@ func RunSocketConnection(conn *websocket.Conn) {
 			select {
 			case clientChan <- msg:
 			case <-stoppedChan:
-				close(errorChan)
-				close(clientChan)
 				return
 			}
 		}
@@ -259,8 +296,6 @@ func RunSocketConnection(conn *websocket.Conn) {
 		case errorChan <- err:
 		case <-stoppedChan:
 		}
-		close(errorChan)
-		close(clientChan)
 		// exit goroutine
 	}(_errorChan, _clientChan, stoppedChan)
 
@@ -318,6 +353,10 @@ RunLoop:
 				closeReason = CloseTooManyBufferedMessages
 				break RunLoop
 			}
+			if cls, ok := msg.Arguments.(*websocket.CloseError); ok {
+				closeReason = *cls
+				break RunLoop
+			}
 			SendMessage(conn, msg)
 
 		case <-time.After(1 * time.Minute):
@@ -331,6 +370,10 @@ RunLoop:
 			} else {
 				conn.WriteControl(websocket.PingMessage, []byte(strconv.FormatInt(time.Now().Unix(), 10)), getDeadline())
 			}
+
+		case <-StopAcceptingConnectionsCh:
+			closeReason = CloseGoingAway
+			break RunLoop
 		}
 	}
 
@@ -343,6 +386,7 @@ RunLoop:
 		}
 	}()
 
+	// Closes client.MsgChannelIsDone and also stops the reader thread
 	close(stoppedChan)
 
 	// Stop getting messages...
@@ -356,8 +400,11 @@ RunLoop:
 	// Close the channel so the draining goroutine can finish, too.
 	close(_serverMessageChan)
 
-	atomic.AddUint64(&Statistics.ClientDisconnectsTotal, 1)
-	atomic.AddUint64(&Statistics.CurrentClientCount, ^uint64(0))
+	if !StopAcceptingConnections {
+		// Don't perform high contention operations when server is closing
+		atomic.AddUint64(&Statistics.ClientDisconnectsTotal, 1)
+		atomic.AddUint64(&Statistics.CurrentClientCount, ^uint64(0))
+	}
 }
 
 func getDeadline() time.Time {
diff --git a/socketserver/server/subscriptions.go b/socketserver/server/subscriptions.go
index c9e85a1c..3a139165 100644
--- a/socketserver/server/subscriptions.go
+++ b/socketserver/server/subscriptions.go
@@ -105,6 +105,10 @@ func UnsubscribeSingleChat(client *ClientInfo, channelName string) {
 //   - write lock to SubscriptionInfos
 //   - write lock to ClientInfo
 func UnsubscribeAll(client *ClientInfo) {
+	if StopAcceptingConnections {
+		return // no need to remove from a high-contention list when the server is closing
+	}
+
 	client.Mutex.Lock()
 	client.PendingSubscriptionsBacklog = nil
 	client.PendingSubscriptionsBacklog = nil

From 6f13ea360cd0e887a2c6572c2bdf162c69aee26d Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sat, 21 Nov 2015 12:10:01 -0800
Subject: [PATCH 082/176] Add BackendVerifyFails to /stats to count request
 spam

---
 socketserver/server/stats.go | 4 +++-
 socketserver/server/utils.go | 7 ++++---
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/socketserver/server/stats.go b/socketserver/server/stats.go
index 34d0db25..a64a315b 100644
--- a/socketserver/server/stats.go
+++ b/socketserver/server/stats.go
@@ -46,6 +46,8 @@ type StatsData struct {
 
 	EmotesReportedTotal uint64
 
+	BackendVerifyFails uint64
+
 	// DisconnectReasons is at the bottom because it has indeterminate size
 	DisconnectReasons map[string]uint64
 }
@@ -57,7 +59,7 @@ type StatsData struct {
 // I don't really care.
 var Statistics = newStatsData()
 
-const StatsDataVersion = 4
+const StatsDataVersion = 5
 const pageSize = 4096
 
 var cpuUsage struct {
diff --git a/socketserver/server/utils.go b/socketserver/server/utils.go
index 258acda1..2cfa6a37 100644
--- a/socketserver/server/utils.go
+++ b/socketserver/server/utils.go
@@ -6,7 +6,6 @@ import (
 	"encoding/base64"
 	"errors"
 	"golang.org/x/crypto/nacl/box"
-	"log"
 	"net/url"
 	"strconv"
 	"strings"
@@ -70,9 +69,11 @@ func UnsealRequest(form url.Values) (url.Values, error) {
 	dec := base64.NewDecoder(base64.URLEncoding, strings.NewReader(nonceString))
 	count, err := dec.Read(nonce[:])
 	if err != nil {
+		Statistics.BackendVerifyFails++
 		return nil, err
 	}
 	if count != 24 {
+		Statistics.BackendVerifyFails++
 		return nil, ErrorShortNonce
 	}
 
@@ -83,13 +84,13 @@ func UnsealRequest(form url.Values) (url.Values, error) {
 
 	message, ok := box.OpenAfterPrecomputation(nil, cipherBuffer.Bytes(), &nonce, &backendSharedKey)
 	if !ok {
+		Statistics.BackendVerifyFails++
 		return nil, ErrorInvalidSignature
 	}
 
 	retValues, err := url.ParseQuery(string(message))
 	if err != nil {
-		// Assume that the signature was accidentally correct but the contents were garbage
-		log.Println("Error unsealing request:", err)
+		Statistics.BackendVerifyFails++
 		return nil, ErrorInvalidSignature
 	}
 

From 6a5bcdca4a12c92b1c408235341bf34b85b6e3bb Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Mon, 23 Nov 2015 16:08:22 -0800
Subject: [PATCH 083/176] Oops, SSL needs a different listen address..

---
 socketserver/cmd/ffzsocketserver/socketserver.go | 10 +++-------
 socketserver/server/backend.go                   |  1 +
 socketserver/server/types.go                     |  1 +
 3 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/socketserver/cmd/ffzsocketserver/socketserver.go b/socketserver/cmd/ffzsocketserver/socketserver.go
index 5432f3a3..fedcdf80 100644
--- a/socketserver/cmd/ffzsocketserver/socketserver.go
+++ b/socketserver/cmd/ffzsocketserver/socketserver.go
@@ -43,29 +43,25 @@ func main() {
 		log.Fatal(err)
 	}
 
-	httpServer := &http.Server{
-		Addr: conf.ListenAddr,
-	}
-
 	//	logFile, err := os.OpenFile("output.log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
 	//	if err != nil {
 	//		log.Fatal("Could not create logfile: ", err)
 	//	}
 
-	server.SetupServerAndHandle(conf, nil)
+	server.SetupServerAndHandle(conf, http.DefaultServeMux)
 	server.SetBuildStamp(BuildTime, BuildHash)
 
 	go commandLineConsole()
 
 	if conf.UseSSL {
 		go func() {
-			if err := httpServer.ListenAndServeTLS(conf.SSLCertificateFile, conf.SSLKeyFile); err != nil {
+			if err := http.ListenAndServeTLS(conf.SSLListenAddr, conf.SSLCertificateFile, conf.SSLKeyFile, http.DefaultServeMux); err != nil {
 				log.Fatal("ListenAndServeTLS: ", err)
 			}
 		}()
 	}
 
-	if err = httpServer.ListenAndServe(); err != nil {
+	if err = http.ListenAndServe(conf.ListenAddr, http.DefaultServeMux); err != nil {
 		log.Fatal("ListenAndServe: ", err)
 	}
 }
diff --git a/socketserver/server/backend.go b/socketserver/server/backend.go
index b0927f86..73fa03aa 100644
--- a/socketserver/server/backend.go
+++ b/socketserver/server/backend.go
@@ -269,6 +269,7 @@ func GenerateKeys(outputFile, serverID, theirPublicStr string) {
 	var err error
 	output := ConfigFile{
 		ListenAddr:      "0.0.0.0:8001",
+		SSLListenAddr:      "0.0.0.0:443",
 		SocketOrigin:    "localhost:8001",
 		BackendURL:      "http://localhost:8002/ffz",
 		MinMemoryKBytes: defaultMinMemoryKB,
diff --git a/socketserver/server/types.go b/socketserver/server/types.go
index bc660aed..2fe42b8a 100644
--- a/socketserver/server/types.go
+++ b/socketserver/server/types.go
@@ -15,6 +15,7 @@ type ConfigFile struct {
 	// Numeric server id known to the backend
 	ServerID   int
 	ListenAddr string
+	SSLListenAddr string
 	// Hostname of the socket server
 	SocketOrigin string
 	// URL to the backend server

From de6e671bdb1d045a370e3eacb23e5c09fc8fafeb Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Mon, 23 Nov 2015 23:34:57 -0800
Subject: [PATCH 084/176] Better standards compliance with codes

---
 socketserver/server/handlecore.go | 4 ++--
 socketserver/server/publisher.go  | 9 ++++++---
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 46c8adfd..2871159f 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -204,7 +204,7 @@ var CloseRebalance = websocket.CloseError{Code: websocket.CloseGoingAway, Text:
 var CloseGotBinaryMessage = websocket.CloseError{Code: websocket.CloseUnsupportedData, Text: "got binary packet"}
 
 // CloseTimedOut is the termination reason when the client fails to send or respond to ping frames.
-var CloseTimedOut = websocket.CloseError{Code: websocket.CloseNoStatusReceived, Text: "no ping replies for 5 minutes"}
+var CloseTimedOut = websocket.CloseError{Code: 3003, Text: "no ping replies for 5 minutes"}
 
 // CloseTooManyBufferedMessages is the termination reason when the sending thread buffers too many messages.
 var CloseTooManyBufferedMessages = websocket.CloseError{Code: websocket.CloseMessageTooBig, Text: "too many pending messages"}
@@ -216,7 +216,7 @@ var CloseFirstMessageNotHello = websocket.CloseError{
 }
 
 var CloseNonUTF8Data = websocket.CloseError{
-	Code: 4001,
+	Code: websocket.CloseUnsupportedData,
 	Text: "Non UTF8 data recieved. Network corruption likely.",
 }
 
diff --git a/socketserver/server/publisher.go b/socketserver/server/publisher.go
index 4ec4dbf3..cad5b62a 100644
--- a/socketserver/server/publisher.go
+++ b/socketserver/server/publisher.go
@@ -96,13 +96,17 @@ func DumpBacklogData() {
 // This will only send data for CacheTypePersistent and CacheTypeLastOnly because those do not involve timestamps.
 func SendBacklogForNewClient(client *ClientInfo) {
 	client.Mutex.Lock() // reading CurrentChannels
+	curChannels := make([]string, len(client.CurrentChannels))
+	copy(curChannels, client.CurrentChannels)
+	client.Mutex.Unlock()
+
 	PersistentLSMLock.RLock()
 	for _, cmd := range GetCommandsOfType(PushCommandCacheInfo{CacheTypePersistent, MsgTargetTypeChat}) {
 		chanMap := CachedLastMessages[cmd]
 		if chanMap == nil {
 			continue
 		}
-		for _, channel := range client.CurrentChannels {
+		for _, channel := range curChannels {
 			msg, ok := chanMap[channel]
 			if ok {
 				msg := ClientMessage{MessageID: -1, Command: cmd, origArguments: msg.Data}
@@ -119,7 +123,7 @@ func SendBacklogForNewClient(client *ClientInfo) {
 		if chanMap == nil {
 			continue
 		}
-		for _, channel := range client.CurrentChannels {
+		for _, channel := range curChannels {
 			msg, ok := chanMap[channel]
 			if ok {
 				msg := ClientMessage{MessageID: -1, Command: cmd, origArguments: msg.Data}
@@ -129,7 +133,6 @@ func SendBacklogForNewClient(client *ClientInfo) {
 		}
 	}
 	CachedLSMLock.RUnlock()
-	client.Mutex.Unlock()
 }
 
 // insertionSort implements insertion sort.

From 0aaa1f87009d935290dcbaa76630eac68475efa9 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 2 Dec 2015 17:20:52 -0800
Subject: [PATCH 085/176] Add architecgture design document

---
 socketserver/SocketServerDesign.svg | 1057 +++++++++++++++++++++++++++
 1 file changed, 1057 insertions(+)
 create mode 100644 socketserver/SocketServerDesign.svg

diff --git a/socketserver/SocketServerDesign.svg b/socketserver/SocketServerDesign.svg
new file mode 100644
index 00000000..b635d0ba
--- /dev/null
+++ b/socketserver/SocketServerDesign.svg
@@ -0,0 +1,1057 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="210mm"
+   height="297mm"
+   viewBox="0 0 744.09448819 1052.3622047"
+   id="svg2"
+   version="1.1"
+   inkscape:version="0.91 r13725"
+   sodipodi:docname="SocketServerDesign.svg">
+  <defs
+     id="defs4">
+    <marker
+       inkscape:stockid="Arrow1Lstart"
+       orient="auto"
+       refY="0.0"
+       refX="0.0"
+       id="marker23417"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path23419"
+         d="M 0.0,0.0 L 5.0,-5.0 L -12.5,0.0 L 5.0,5.0 L 0.0,0.0 z "
+         style="fill-rule:evenodd;stroke:#c87137;stroke-width:1pt;stroke-opacity:1;fill:#c87137;fill-opacity:1"
+         transform="scale(0.8) translate(12.5,0)" />
+    </marker>
+    <marker
+       inkscape:isstock="true"
+       style="overflow:visible"
+       id="marker23251"
+       refX="0.0"
+       refY="0.0"
+       orient="auto"
+       inkscape:stockid="Arrow1Lstart"
+       inkscape:collect="always">
+      <path
+         transform="scale(0.8) translate(12.5,0)"
+         style="fill-rule:evenodd;stroke:#c87137;stroke-width:1pt;stroke-opacity:1;fill:#c87137;fill-opacity:1"
+         d="M 0.0,0.0 L 5.0,-5.0 L -12.5,0.0 L 5.0,5.0 L 0.0,0.0 z "
+         id="path23253" />
+    </marker>
+    <marker
+       inkscape:isstock="true"
+       style="overflow:visible"
+       id="marker23127"
+       refX="0.0"
+       refY="0.0"
+       orient="auto"
+       inkscape:stockid="Arrow1Lstart">
+      <path
+         transform="scale(0.8) translate(12.5,0)"
+         style="fill-rule:evenodd;stroke:#008000;stroke-width:1pt;stroke-opacity:1;fill:#000000;fill-opacity:1"
+         d="M 0.0,0.0 L 5.0,-5.0 L -12.5,0.0 L 5.0,5.0 L 0.0,0.0 z "
+         id="path23129" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Lstart"
+       orient="auto"
+       refY="0.0"
+       refX="0.0"
+       id="Arrow1Lstart"
+       style="overflow:visible"
+       inkscape:isstock="true"
+       inkscape:collect="always">
+      <path
+         id="path4394"
+         d="M 0.0,0.0 L 5.0,-5.0 L -12.5,0.0 L 5.0,5.0 L 0.0,0.0 z "
+         style="fill-rule:evenodd;stroke:#008000;stroke-width:1pt;stroke-opacity:1;fill:#000000;fill-opacity:1"
+         transform="scale(0.8) translate(12.5,0)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Lend"
+       orient="auto"
+       refY="0.0"
+       refX="0.0"
+       id="marker22805"
+       style="overflow:visible;"
+       inkscape:isstock="true">
+      <path
+         id="path22807"
+         d="M 0.0,0.0 L 5.0,-5.0 L -12.5,0.0 L 5.0,5.0 L 0.0,0.0 z "
+         style="fill-rule:evenodd;stroke:#008000;stroke-width:1pt;stroke-opacity:1;fill:#000000;fill-opacity:1"
+         transform="scale(0.8) rotate(180) translate(12.5,0)" />
+    </marker>
+    <marker
+       inkscape:isstock="true"
+       style="overflow:visible;"
+       id="marker6763"
+       refX="0.0"
+       refY="0.0"
+       orient="auto"
+       inkscape:stockid="Arrow1Lend"
+       inkscape:collect="always">
+      <path
+         transform="scale(0.8) rotate(180) translate(12.5,0)"
+         style="fill-rule:evenodd;stroke:#008000;stroke-width:1pt;stroke-opacity:1;fill:#008000;fill-opacity:1"
+         d="M 0.0,0.0 L 5.0,-5.0 L -12.5,0.0 L 5.0,5.0 L 0.0,0.0 z "
+         id="path6765" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Lend"
+       orient="auto"
+       refY="0.0"
+       refX="0.0"
+       id="marker6717"
+       style="overflow:visible;"
+       inkscape:isstock="true">
+      <path
+         id="path6719"
+         d="M 0.0,0.0 L 5.0,-5.0 L -12.5,0.0 L 5.0,5.0 L 0.0,0.0 z "
+         style="fill-rule:evenodd;stroke:#000000;stroke-width:1pt;stroke-opacity:1;fill:#000000;fill-opacity:1"
+         transform="scale(0.8) rotate(180) translate(12.5,0)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Lend"
+       orient="auto"
+       refY="0.0"
+       refX="0.0"
+       id="Arrow1Lend"
+       style="overflow:visible;"
+       inkscape:isstock="true"
+       inkscape:collect="always">
+      <path
+         id="path4397"
+         d="M 0.0,0.0 L 5.0,-5.0 L -12.5,0.0 L 5.0,5.0 L 0.0,0.0 z "
+         style="fill-rule:evenodd;stroke:#008000;stroke-width:1pt;stroke-opacity:1;fill:#008000;fill-opacity:1"
+         transform="scale(0.8) rotate(180) translate(12.5,0)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Lend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Lend-9"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4397-0"
+         d="M 0,0 5,-5 -12.5,0 5,5 0,0 Z"
+         style="fill:#008000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:1pt;stroke-opacity:1"
+         transform="matrix(-0.8,0,0,-0.8,-10,0)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Lend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Lend-2"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4397-9"
+         d="M 0,0 5,-5 -12.5,0 5,5 0,0 Z"
+         style="fill:#008000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:1pt;stroke-opacity:1"
+         transform="matrix(-0.8,0,0,-0.8,-10,0)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Lend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Lend-9-5"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4397-0-5"
+         d="M 0,0 5,-5 -12.5,0 5,5 0,0 Z"
+         style="fill:#008000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:1pt;stroke-opacity:1"
+         transform="matrix(-0.8,0,0,-0.8,-10,0)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Lend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Lend-9-5-3"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4397-0-5-4"
+         d="M 0,0 5,-5 -12.5,0 5,5 0,0 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1pt;stroke-opacity:1"
+         transform="matrix(-0.8,0,0,-0.8,-10,0)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Lend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Lend-9-5-3-1"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4397-0-5-4-1"
+         d="M 0,0 5,-5 -12.5,0 5,5 0,0 Z"
+         style="fill:#008000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:1pt;stroke-opacity:1"
+         transform="matrix(-0.8,0,0,-0.8,-10,0)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Lend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Lend-9-5-3-1-7"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4397-0-5-4-1-4"
+         d="M 0,0 5,-5 -12.5,0 5,5 0,0 Z"
+         style="fill:#008000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:1pt;stroke-opacity:1"
+         transform="matrix(-0.8,0,0,-0.8,-10,0)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Lend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Lend-9-5-3-1-7-2"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4397-0-5-4-1-4-4"
+         d="M 0,0 5,-5 -12.5,0 5,5 0,0 Z"
+         style="fill:#008000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:1pt;stroke-opacity:1"
+         transform="matrix(-0.8,0,0,-0.8,-10,0)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Lend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Lend-9-5-3-1-7-4"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4397-0-5-4-1-4-8"
+         d="M 0,0 5,-5 -12.5,0 5,5 0,0 Z"
+         style="fill:#008000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:1pt;stroke-opacity:1"
+         transform="matrix(-0.8,0,0,-0.8,-10,0)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Lend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Lend-9-5-3-1-7-6"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4397-0-5-4-1-4-7"
+         d="M 0,0 5,-5 -12.5,0 5,5 0,0 Z"
+         style="fill:#008000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:1pt;stroke-opacity:1"
+         transform="matrix(-0.8,0,0,-0.8,-10,0)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Lend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Lend-9-5-3-1-7-2-5"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4397-0-5-4-1-4-4-3"
+         d="M 0,0 5,-5 -12.5,0 5,5 0,0 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:1pt;stroke-opacity:1"
+         transform="matrix(-0.8,0,0,-0.8,-10,0)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Lend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Lend-9-5-3-1-7-2-5-2"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4397-0-5-4-1-4-4-3-0"
+         d="M 0,0 5,-5 -12.5,0 5,5 0,0 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:1pt;stroke-opacity:1"
+         transform="matrix(-0.8,0,0,-0.8,-10,0)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Lend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Lend-9-5-3-1-7-2-5-6"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4397-0-5-4-1-4-4-3-9"
+         d="M 0,0 5,-5 -12.5,0 5,5 0,0 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:1pt;stroke-opacity:1"
+         transform="matrix(-0.8,0,0,-0.8,-10,0)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Lend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Lend-9-5-3-1-7-2-5-2-2"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4397-0-5-4-1-4-4-3-0-1"
+         d="M 0,0 5,-5 -12.5,0 5,5 0,0 Z"
+         style="fill:#c87137;fill-opacity:1;fill-rule:evenodd;stroke:#c87137;stroke-width:1pt;stroke-opacity:1"
+         transform="matrix(-0.8,0,0,-0.8,-10,0)" />
+    </marker>
+  </defs>
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="1"
+     inkscape:cx="361.33859"
+     inkscape:cy="571.17255"
+     inkscape:document-units="px"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     inkscape:window-width="1920"
+     inkscape:window-height="1031"
+     inkscape:window-x="0"
+     inkscape:window-y="25"
+     inkscape:window-maximized="1" />
+  <metadata
+     id="metadata7">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1">
+    <g
+       id="g4150">
+      <ellipse
+         ry="37.880722"
+         rx="36.870567"
+         cy="729.1134"
+         cx="48.467022"
+         id="path3336"
+         style="fill:#6441a5;fill-opacity:0.34901962;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <text
+         sodipodi:linespacing="125%"
+         id="text4138"
+         y="734.16418"
+         x="24.72843"
+         style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         xml:space="preserve"><tspan
+           style="font-size:16.25px"
+           y="734.16418"
+           x="24.72843"
+           id="tspan4140"
+           sodipodi:role="line">Client</tspan></text>
+    </g>
+    <g
+       transform="translate(82.96155,71.82023)"
+       id="g4150-0">
+      <ellipse
+         ry="37.880722"
+         rx="36.870567"
+         cy="729.1134"
+         cx="48.467022"
+         id="path3336-9"
+         style="fill:#6441a5;fill-opacity:0.34901961;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <text
+         sodipodi:linespacing="125%"
+         id="text4138-6"
+         y="734.16418"
+         x="24.72843"
+         style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         xml:space="preserve"><tspan
+           style="font-size:16.25px"
+           y="734.16418"
+           x="24.72843"
+           id="tspan4140-3"
+           sodipodi:role="line">Client</tspan></text>
+    </g>
+    <g
+       transform="translate(142.24726,-13.17977)"
+       id="g4150-6">
+      <ellipse
+         ry="37.880722"
+         rx="36.870567"
+         cy="729.1134"
+         cx="48.467022"
+         id="path3336-8"
+         style="fill:#6441a5;fill-opacity:0.34901961;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <text
+         sodipodi:linespacing="125%"
+         id="text4138-0"
+         y="734.16418"
+         x="24.72843"
+         style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         xml:space="preserve"><tspan
+           style="font-size:16.25px"
+           y="734.16418"
+           x="24.72843"
+           id="tspan4140-7"
+           sodipodi:role="line">Client</tspan></text>
+    </g>
+    <g
+       transform="translate(228.67583,80.391658)"
+       id="g4150-61">
+      <ellipse
+         ry="37.880722"
+         rx="36.870567"
+         cy="729.1134"
+         cx="48.467022"
+         id="path3336-6"
+         style="fill:#6441a5;fill-opacity:0.34901961;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <text
+         sodipodi:linespacing="125%"
+         id="text4138-9"
+         y="734.16418"
+         x="24.72843"
+         style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         xml:space="preserve"><tspan
+           style="font-size:16.25px"
+           y="734.16418"
+           x="24.72843"
+           id="tspan4140-9"
+           sodipodi:role="line">Client</tspan></text>
+    </g>
+    <g
+       transform="translate(300.1044,-11.751199)"
+       id="g4150-05">
+      <ellipse
+         ry="37.880722"
+         rx="36.870567"
+         cy="729.1134"
+         cx="48.467022"
+         id="path3336-5"
+         style="fill:#6441a5;fill-opacity:0.34901961;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <text
+         sodipodi:linespacing="125%"
+         id="text4138-4"
+         y="734.16418"
+         x="24.72843"
+         style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         xml:space="preserve"><tspan
+           style="font-size:16.25px"
+           y="734.16418"
+           x="24.72843"
+           id="tspan4140-97"
+           sodipodi:role="line">Client</tspan></text>
+    </g>
+    <g
+       transform="translate(381.53298,75.391659)"
+       id="g4150-1">
+      <ellipse
+         ry="37.880722"
+         rx="36.870567"
+         cy="729.1134"
+         cx="48.467022"
+         id="path3336-0"
+         style="fill:#6441a5;fill-opacity:0.34901961;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <text
+         sodipodi:linespacing="125%"
+         id="text4138-5"
+         y="734.16418"
+         x="24.72843"
+         style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         xml:space="preserve"><tspan
+           style="font-size:16.25px"
+           y="734.16418"
+           x="24.72843"
+           id="tspan4140-2"
+           sodipodi:role="line">Client</tspan></text>
+    </g>
+    <g
+       transform="translate(469.39012,130.39166)"
+       id="g4150-3">
+      <ellipse
+         ry="37.880722"
+         rx="36.870567"
+         cy="729.1134"
+         cx="48.467022"
+         id="path3336-84"
+         style="fill:#6441a5;fill-opacity:0.34901961;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <text
+         sodipodi:linespacing="125%"
+         id="text4138-2"
+         y="734.16418"
+         x="24.72843"
+         style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         xml:space="preserve"><tspan
+           style="font-size:16.25px"
+           y="734.16418"
+           x="24.72843"
+           id="tspan4140-6"
+           sodipodi:role="line">Client</tspan></text>
+    </g>
+    <g
+       transform="translate(522.96155,25.391663)"
+       id="g4150-2">
+      <ellipse
+         ry="37.880722"
+         rx="36.870567"
+         cy="729.1134"
+         cx="48.467022"
+         id="path3336-99"
+         style="fill:#6441a5;fill-opacity:0.34901961;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <text
+         sodipodi:linespacing="125%"
+         id="text4138-58"
+         y="734.16418"
+         x="24.72843"
+         style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         xml:space="preserve"><tspan
+           style="font-size:16.25px"
+           y="734.16418"
+           x="24.72843"
+           id="tspan4140-5"
+           sodipodi:role="line">Client</tspan></text>
+    </g>
+    <g
+       transform="translate(582.24727,-57.465485)"
+       id="g4150-8">
+      <ellipse
+         ry="37.880722"
+         rx="36.870567"
+         cy="729.1134"
+         cx="48.467022"
+         id="path3336-55"
+         style="fill:#6441a5;fill-opacity:0.34901961;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <text
+         sodipodi:linespacing="125%"
+         id="text4138-1"
+         y="734.16418"
+         x="24.72843"
+         style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         xml:space="preserve"><tspan
+           style="font-size:16.25px"
+           y="734.16418"
+           x="24.72843"
+           id="tspan4140-23"
+           sodipodi:role="line">Client</tspan></text>
+    </g>
+    <g
+       transform="translate(414.39012,-17.465485)"
+       id="g4150-21">
+      <ellipse
+         ry="37.880722"
+         rx="36.870567"
+         cy="729.1134"
+         cx="48.467022"
+         id="path3336-3"
+         style="fill:#6441a5;fill-opacity:0.34901961;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <text
+         sodipodi:linespacing="125%"
+         id="text4138-3"
+         y="734.16418"
+         x="24.72843"
+         style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         xml:space="preserve"><tspan
+           style="font-size:16.25px"
+           y="734.16418"
+           x="24.72843"
+           id="tspan4140-1"
+           sodipodi:role="line">Client</tspan></text>
+    </g>
+    <ellipse
+       style="fill:none;fill-opacity:0.34901961"
+       id="path4279"
+       cx="66.428574"
+       cy="384.86221"
+       rx="45"
+       ry="46.07143" />
+    <g
+       id="g4340">
+      <ellipse
+         ry="61.234596"
+         rx="99.079659"
+         cy="485.92004"
+         cx="124.97432"
+         id="path4283"
+         style="fill:#bc4187;fill-opacity:0.34901961;stroke:#000000;stroke-width:1.3607688;stroke-opacity:1" />
+      <text
+         sodipodi:linespacing="125%"
+         id="text4285"
+         y="475.03372"
+         x="43.153011"
+         style="font-style:normal;font-weight:normal;font-size:22.5px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         xml:space="preserve"><tspan
+           id="tspan4289"
+           y="475.03372"
+           x="43.153011"
+           sodipodi:role="line">Socket Server</tspan></text>
+      <text
+         sodipodi:linespacing="125%"
+         id="text4293"
+         y="496.83524"
+         x="125.13116"
+         style="font-style:normal;font-weight:normal;font-size:12.5px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;text-anchor:middle;text-align:center;"
+         xml:space="preserve"><tspan
+           y="496.83524"
+           x="30.471735"
+           id="tspan4295"
+           sodipodi:role="line">andknuckles.frankerfacez.com</tspan></text>
+    </g>
+    <g
+       id="g4347">
+      <ellipse
+         ry="61.234596"
+         rx="99.079659"
+         cy="460.21936"
+         cx="362.14285"
+         id="path4283-0"
+         style="fill:#bc4187;fill-opacity:0.34901961;stroke:#000000;stroke-width:1.3607688;stroke-opacity:1" />
+      <text
+         sodipodi:linespacing="125%"
+         id="text4285-9"
+         y="449.33301"
+         x="280.32153"
+         style="font-style:normal;font-weight:normal;font-size:22.5px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         xml:space="preserve"><tspan
+           id="tspan4289-7"
+           y="449.33301"
+           x="280.32153"
+           sodipodi:role="line">Socket Server</tspan></text>
+      <text
+         sodipodi:linespacing="125%"
+         id="text4293-0"
+         y="471.13452"
+         x="362.29968"
+         style="font-style:normal;font-weight:normal;font-size:12.5px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         xml:space="preserve"><tspan
+           y="471.13452"
+           x="362.29968"
+           id="tspan4295-6"
+           sodipodi:role="line"><tspan
+   id="tspan4338"
+   style="text-align:center;text-anchor:middle">catbag.franke</tspan>rfacez.com</tspan></text>
+    </g>
+    <g
+       id="g4355">
+      <ellipse
+         ry="61.234596"
+         rx="99.079659"
+         cy="477.36221"
+         cx="602.14288"
+         id="path4283-8"
+         style="fill:#bc4187;fill-opacity:0.34901961;stroke:#000000;stroke-width:1.3607688;stroke-opacity:1" />
+      <text
+         sodipodi:linespacing="125%"
+         id="text4285-2"
+         y="466.47592"
+         x="520.32153"
+         style="font-style:normal;font-weight:normal;font-size:22.5px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         xml:space="preserve"><tspan
+           id="tspan4289-3"
+           y="466.47592"
+           x="520.32153"
+           sodipodi:role="line">Socket Server</tspan></text>
+      <text
+         sodipodi:linespacing="125%"
+         id="text4293-3"
+         y="488.27744"
+         x="602.29968"
+         style="font-style:normal;font-weight:normal;font-size:12.5px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         xml:space="preserve"><tspan
+           y="488.27744"
+           x="602.29968"
+           id="tspan4295-63"
+           sodipodi:role="line">tuturu.frankerfacez.com</tspan></text>
+    </g>
+    <path
+       style="fill:#008000;fill-rule:evenodd;stroke:#008000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend)"
+       d="M 59.599,692.24282 C 91.923882,545.26563 91.923882,545.26563 91.923882,545.26563"
+       id="path4388"
+       inkscape:connector-curvature="0" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;"
+       x="-601.21124"
+       y="210.3251"
+       id="text4682"
+       sodipodi:linespacing="125%"
+       transform="matrix(0.22471085,-0.97442549,0.97442549,0.22471085,0,0)"><tspan
+         sodipodi:role="line"
+         id="tspan4684"
+         x="-601.21124"
+         y="210.3251"
+         style="font-size:11.25px;fill:#008000;"
+         rotate="0 0 0 0"
+         dy="0">TLS</tspan></text>
+    <path
+       style="fill:#008000;fill-rule:evenodd;stroke:#008000;stroke-width:1.01131487px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend-9)"
+       d="M 184.93881,678.07781 C 148.07036,546.28106 148.07036,546.28106 148.07036,546.28106"
+       id="path4388-1"
+       inkscape:connector-curvature="0" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#008000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#marker6763)"
+       d="m 129.8046,761.94335 c -9.26994,-214.42282 0,0 -9.26994,-214.42282"
+       id="path6849"
+       inkscape:connector-curvature="0"
+       sodipodi:nodetypes="cc" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="617.93414"
+       y="-75.716248"
+       id="text4682-9"
+       sodipodi:linespacing="125%"
+       transform="matrix(0.08046399,0.99675752,-0.99675752,0.08046399,0,0)"><tspan
+         sodipodi:role="line"
+         id="tspan4684-8"
+         x="617.93414"
+         y="-75.716248"
+         style="font-size:11.25px;fill:#008000"
+         rotate="0 0 0 0"
+         dy="0">TLS</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="594.69568"
+       y="4.5866504"
+       id="text4682-8"
+       sodipodi:linespacing="125%"
+       transform="matrix(0.27352407,0.96186516,-0.96186516,0.27352407,0,0)"><tspan
+         sodipodi:role="line"
+         id="tspan4684-7"
+         x="594.69568"
+         y="4.5866504"
+         style="font-size:11.25px;fill:#008000"
+         rotate="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0"
+         dy="0">TLS / websocket</tspan></text>
+    <ellipse
+       style="fill:#008080;fill-opacity:0.34901961;stroke:#000000;stroke-width:0.95014036;stroke-linecap:butt;stroke-opacity:1"
+       id="path7045"
+       cx="365.59174"
+       cy="248.32088"
+       rx="127.16779"
+       ry="66.310638" />
+    <path
+       style="fill:#008000;fill-rule:evenodd;stroke:#008000;stroke-width:1.01131487px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend-9-5)"
+       d="M 261.93648,774.11167 192.23807,530.69306"
+       id="path4388-1-8"
+       inkscape:connector-curvature="0"
+       sodipodi:nodetypes="cc" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="672.03656"
+       y="-36.647579"
+       id="text4682-9-9"
+       sodipodi:linespacing="125%"
+       transform="matrix(0.28142998,0.95958177,-0.95958177,0.28142998,0,0)"
+       inkscape:transform-center-x="7.5822135"
+       inkscape:transform-center-y="-21.804086"><tspan
+         sodipodi:role="line"
+         id="tspan4684-8-9"
+         x="672.03656"
+         y="-36.647579"
+         style="font-size:11.25px;fill:#008000"
+         rotate="0 0 0 0"
+         dy="0">TLS</tspan></text>
+    <path
+       style="fill:#000000;fill-rule:evenodd;stroke:#000000;stroke-width:1.01131487px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend-9-5-3)"
+       d="M 345.98646,677.89161 334.8769,521.8512"
+       id="path4388-1-8-5"
+       inkscape:connector-curvature="0"
+       sodipodi:nodetypes="cc" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;"
+       x="583.44501"
+       y="-297.9649"
+       id="text4682-9-9-1"
+       sodipodi:linespacing="125%"
+       transform="matrix(0.07300835,0.99733133,-0.99733133,0.07300835,0,0)"
+       inkscape:transform-center-x="2.8120774"
+       inkscape:transform-center-y="-22.930222"><tspan
+         sodipodi:role="line"
+         id="tspan4684-8-9-6"
+         x="583.44501"
+         y="-297.9649"
+         style="font-size:11.25px;fill:#000000;"
+         rotate="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0">HTTP / websocket</tspan></text>
+    <path
+       style="fill:#008000;fill-rule:evenodd;stroke:#008000;stroke-width:1.01131487px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend-9-5-3-1)"
+       d="M 423.44838,767.12245 383.0444,521.17846"
+       id="path4388-1-8-5-9"
+       inkscape:connector-curvature="0"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:#008000;fill-rule:evenodd;stroke:#008000;stroke-width:1.01131487px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend-9-5-3-1-7)"
+       d="M 461.08578,673.65046 416.64119,514.07451"
+       id="path4388-1-8-5-9-7"
+       inkscape:connector-curvature="0"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:#008000;fill-rule:evenodd;stroke:#008000;stroke-width:1.01131487px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend-9-5-3-1-7-2)"
+       d="m 629.38882,633.44847 6.06304,-98.46171"
+       id="path4388-1-8-5-9-7-7"
+       inkscape:connector-curvature="0"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:#008000;fill-rule:evenodd;stroke:#008000;stroke-width:1.01131487px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend-9-5-3-1-7-4)"
+       d="M 575.85075,715.77591 598.5813,540.54259"
+       id="path4388-1-8-5-9-7-8"
+       inkscape:connector-curvature="0"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:#008000;fill-rule:evenodd;stroke:#008000;stroke-width:1.01131487px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend-9-5-3-1-7-6)"
+       d="M 519.2822,821.33685 534.43661,523.37001"
+       id="path4388-1-8-5-9-7-86"
+       inkscape:connector-curvature="0"
+       sodipodi:nodetypes="cc" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="684.90643"
+       y="-213.68245"
+       id="text4682-9-9-6"
+       sodipodi:linespacing="125%"
+       transform="matrix(0.28142998,0.95958177,-0.95958177,0.28142998,0,0)"
+       inkscape:transform-center-x="7.5822135"
+       inkscape:transform-center-y="-21.804086"><tspan
+         sodipodi:role="line"
+         id="tspan4684-8-9-3"
+         x="684.90643"
+         y="-213.68245"
+         style="font-size:11.25px;fill:#008000"
+         rotate="0 0 0 0"
+         dy="0">TLS</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="687.49579"
+       y="-256.61011"
+       id="text4682-9-9-4"
+       sodipodi:linespacing="125%"
+       transform="matrix(0.28142998,0.95958177,-0.95958177,0.28142998,0,0)"
+       inkscape:transform-center-x="7.5822135"
+       inkscape:transform-center-y="-21.804086"><tspan
+         sodipodi:role="line"
+         id="tspan4684-8-9-5"
+         x="687.49579"
+         y="-256.61011"
+         style="font-size:11.25px;fill:#008000"
+         rotate="0 0 0 0"
+         dy="0">TLS</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="-625.29315"
+       y="560.20447"
+       id="text4682-9-9-3"
+       sodipodi:linespacing="125%"
+       transform="matrix(0.05618991,-0.9984201,0.9984201,0.05618991,0,0)"
+       inkscape:transform-center-x="0.01884435"
+       inkscape:transform-center-y="23.125079"><tspan
+         sodipodi:role="line"
+         id="tspan4684-8-9-37"
+         x="-625.29315"
+         y="560.20447"
+         style="font-size:11.25px;fill:#008000"
+         rotate="0 0 0 0"
+         dy="0">TLS</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="-562.52814"
+       y="663.12689"
+       id="text4682-9-9-3-3"
+       sodipodi:linespacing="125%"
+       transform="matrix(0.13410768,-0.99096677,0.99096677,0.13410768,0,0)"
+       inkscape:transform-center-x="1.8281616"
+       inkscape:transform-center-y="23.040781"><tspan
+         sodipodi:role="line"
+         id="tspan4684-8-9-37-1"
+         x="-562.52814"
+         y="663.12689"
+         style="font-size:11.25px;fill:#008000"
+         rotate="0 0 0 0"
+         dy="0">TLS</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="-557.59875"
+       y="679.40869"
+       id="text4682-9-9-3-5"
+       sodipodi:linespacing="125%"
+       transform="matrix(0.08802633,-0.99611815,0.99611815,0.08802633,0,0)"
+       inkscape:transform-center-x="0.75711632"
+       inkscape:transform-center-y="23.107802"><tspan
+         sodipodi:role="line"
+         id="tspan4684-8-9-37-8"
+         x="-557.59875"
+         y="679.40869"
+         style="font-size:11.25px;fill:#008000"
+         rotate="0 0 0 0"
+         dy="0">TLS</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:22.5px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="278.73459"
+       y="234.37636"
+       id="text15347"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         x="278.73459"
+         y="234.37636"
+         id="tspan15351">Socket Backend</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:12.5px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="361.82379"
+       y="267.38663"
+       id="text4293-0-2"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan4295-6-2"
+         x="361.82379"
+         y="267.38663"><tspan
+   style="text-align:center;text-anchor:middle"
+   id="tspan4338-2">catbag.franke</tspan>rfacez.com</tspan></text>
+    <path
+       style="fill:#000000;fill-rule:evenodd;stroke:#008000;stroke-width:1.01131487px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend-9-5-3-1-7-2-5);marker-start:url(#marker23127)"
+       d="M 125.99509,423.67358 252.01374,279.85602"
+       id="path4388-1-8-5-9-7-7-9"
+       inkscape:connector-curvature="0"
+       sodipodi:nodetypes="cc" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:17.5px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="-204.06422"
+       y="368.93552"
+       id="text17619"
+       sodipodi:linespacing="125%"
+       transform="matrix(0.65402838,-0.75647001,0.75647001,0.65402838,0,0)"><tspan
+         sodipodi:role="line"
+         id="tspan17621"
+         x="-204.06422"
+         y="368.93552">NaCl / HTTP</tspan></text>
+    <path
+       style="fill:#000000;fill-rule:evenodd;stroke:#008000;stroke-width:1.01131487px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend-9-5-3-1-7-2-5-2);marker-start:url(#Arrow1Lstart)"
+       d="M 358.43958,398.54766 357.9365,315.74331"
+       id="path4388-1-8-5-9-7-7-9-3"
+       inkscape:connector-curvature="0"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:#000000;fill-rule:evenodd;stroke:#008000;stroke-width:1.01131487px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend-9-5-3-1-7-2-5-6);marker-start:url(#marker23251)"
+       d="M 598.85589,415.21516 472.53843,284.42857"
+       id="path4388-1-8-5-9-7-7-9-7"
+       inkscape:connector-curvature="0"
+       sodipodi:nodetypes="cc" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:17.5px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="566.948"
+       y="-143.04019"
+       id="text17619-6"
+       sodipodi:linespacing="125%"
+       transform="matrix(0.69625411,0.71779538,-0.71779538,0.69625411,0,0)"><tspan
+         sodipodi:role="line"
+         id="tspan17621-7"
+         x="566.948"
+         y="-143.04019">NaCl / HTTP</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:17.5px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="358.19937"
+       y="367.07816"
+       id="text17619-6-6"
+       sodipodi:linespacing="125%"
+       transform="matrix(0.99999912,-0.001326,0.001326,0.99999912,0,0)"><tspan
+         sodipodi:role="line"
+         id="tspan17621-7-5"
+         x="358.19937"
+         y="367.07816">NaCl / HTTP</tspan></text>
+    <ellipse
+       style="fill:#c7804a;fill-opacity:0.34901962;stroke:#000000;stroke-width:0.89510232;stroke-linecap:butt;stroke-opacity:1"
+       id="path7045-6"
+       cx="366"
+       cy="71.362198"
+       rx="127.1953"
+       ry="58.838161" />
+    <path
+       style="fill:#c87137;fill-rule:evenodd;stroke:#c87137;stroke-width:1.01131487px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-start:url(#marker23417);marker-end:url(#Arrow1Lend-9-5-3-1-7-2-5-2-2)"
+       d="m 361.73807,180.57084 -0.50308,-49.7471"
+       id="path4388-1-8-5-9-7-7-9-3-1"
+       inkscape:connector-curvature="0"
+       sodipodi:nodetypes="cc" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:17.5px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="362.03867"
+       y="158.57924"
+       id="text22558"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan22560"
+         x="362.03867"
+         y="158.57924"
+         style="font-size:10px">(out of scope)</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:12.5px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="362.22787"
+       y="98.034554"
+       id="text4293-0-2-6"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan4295-6-2-6"
+         x="362.22787"
+         y="98.034554"><tspan
+   style="text-align:center;text-anchor:middle"
+   id="tspan4338-2-6">www.franke</tspan>rfacez.com</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:22.5px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="296.80118"
+       y="68.132027"
+       id="text15347-1"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         x="296.80118"
+         y="68.132027"
+         id="tspan15351-4">Web Server</tspan></text>
+  </g>
+</svg>

From faf02a483e3485b6834ceabed60ccf9cb1b5e112 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 2 Dec 2015 17:26:39 -0800
Subject: [PATCH 086/176] Fix centering in svg

---
 socketserver/SocketServerDesign.svg | 835 ++++++++++++++--------------
 1 file changed, 403 insertions(+), 432 deletions(-)

diff --git a/socketserver/SocketServerDesign.svg b/socketserver/SocketServerDesign.svg
index b635d0ba..e2b31d73 100644
--- a/socketserver/SocketServerDesign.svg
+++ b/socketserver/SocketServerDesign.svg
@@ -43,7 +43,7 @@
        inkscape:collect="always">
       <path
          transform="scale(0.8) translate(12.5,0)"
-         style="fill-rule:evenodd;stroke:#c87137;stroke-width:1pt;stroke-opacity:1;fill:#c87137;fill-opacity:1"
+         style="fill-rule:evenodd;stroke:#008000;stroke-width:1pt;stroke-opacity:1;fill:#000000;fill-opacity:1"
          d="M 0.0,0.0 L 5.0,-5.0 L -12.5,0.0 L 5.0,5.0 L 0.0,0.0 z "
          id="path23253" />
     </marker>
@@ -119,21 +119,6 @@
          style="fill-rule:evenodd;stroke:#000000;stroke-width:1pt;stroke-opacity:1;fill:#000000;fill-opacity:1"
          transform="scale(0.8) rotate(180) translate(12.5,0)" />
     </marker>
-    <marker
-       inkscape:stockid="Arrow1Lend"
-       orient="auto"
-       refY="0.0"
-       refX="0.0"
-       id="Arrow1Lend"
-       style="overflow:visible;"
-       inkscape:isstock="true"
-       inkscape:collect="always">
-      <path
-         id="path4397"
-         d="M 0.0,0.0 L 5.0,-5.0 L -12.5,0.0 L 5.0,5.0 L 0.0,0.0 z "
-         style="fill-rule:evenodd;stroke:#008000;stroke-width:1pt;stroke-opacity:1;fill:#008000;fill-opacity:1"
-         transform="scale(0.8) rotate(180) translate(12.5,0)" />
-    </marker>
     <marker
        inkscape:stockid="Arrow1Lend"
        orient="auto"
@@ -329,6 +314,38 @@
          style="fill:#c87137;fill-opacity:1;fill-rule:evenodd;stroke:#c87137;stroke-width:1pt;stroke-opacity:1"
          transform="matrix(-0.8,0,0,-0.8,-10,0)" />
     </marker>
+    <marker
+       inkscape:isstock="true"
+       style="overflow:visible"
+       id="marker6763-4"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Lend"
+       inkscape:collect="always">
+      <path
+         inkscape:connector-curvature="0"
+         transform="matrix(-0.8,0,0,-0.8,-10,0)"
+         style="fill:#008000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:1pt;stroke-opacity:1"
+         d="M 0,0 5,-5 -12.5,0 5,5 0,0 Z"
+         id="path6765-2" />
+    </marker>
+    <marker
+       inkscape:isstock="true"
+       style="overflow:visible"
+       id="marker6763-4-5"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Lend"
+       inkscape:collect="always">
+      <path
+         inkscape:connector-curvature="0"
+         transform="matrix(-0.8,0,0,-0.8,-10,0)"
+         style="fill:#008000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:1pt;stroke-opacity:1"
+         d="M 0,0 5,-5 -12.5,0 5,5 0,0 Z"
+         id="path6765-2-8" />
+    </marker>
   </defs>
   <sodipodi:namedview
      id="base"
@@ -337,9 +354,9 @@
      borderopacity="1.0"
      inkscape:pageopacity="0.0"
      inkscape:pageshadow="2"
-     inkscape:zoom="1"
-     inkscape:cx="361.33859"
-     inkscape:cy="571.17255"
+     inkscape:zoom="2"
+     inkscape:cx="253.89137"
+     inkscape:cy="923.6285"
      inkscape:document-units="px"
      inkscape:current-layer="layer1"
      showgrid="false"
@@ -356,7 +373,7 @@
         <dc:format>image/svg+xml</dc:format>
         <dc:type
            rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
-        <dc:title></dc:title>
+        <dc:title />
       </cc:Work>
     </rdf:RDF>
   </metadata>
@@ -364,235 +381,196 @@
      inkscape:label="Layer 1"
      inkscape:groupmode="layer"
      id="layer1">
-    <g
-       id="g4150">
-      <ellipse
-         ry="37.880722"
-         rx="36.870567"
-         cy="729.1134"
-         cx="48.467022"
-         id="path3336"
-         style="fill:#6441a5;fill-opacity:0.34901962;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
-      <text
-         sodipodi:linespacing="125%"
-         id="text4138"
+    <ellipse
+       style="fill:#6441a5;fill-opacity:0.34901961;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       id="path3336"
+       cx="48.467022"
+       cy="729.1134"
+       rx="36.870567"
+       ry="37.880722" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="47.607765"
+       y="734.16418"
+       id="text4138"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan4140"
+         x="47.607765"
          y="734.16418"
-         x="24.72843"
-         style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-         xml:space="preserve"><tspan
-           style="font-size:16.25px"
-           y="734.16418"
-           x="24.72843"
-           id="tspan4140"
-           sodipodi:role="line">Client</tspan></text>
-    </g>
-    <g
-       transform="translate(82.96155,71.82023)"
-       id="g4150-0">
-      <ellipse
-         ry="37.880722"
-         rx="36.870567"
-         cy="729.1134"
-         cx="48.467022"
-         id="path3336-9"
-         style="fill:#6441a5;fill-opacity:0.34901961;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
-      <text
-         sodipodi:linespacing="125%"
-         id="text4138-6"
-         y="734.16418"
-         x="24.72843"
-         style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-         xml:space="preserve"><tspan
-           style="font-size:16.25px"
-           y="734.16418"
-           x="24.72843"
-           id="tspan4140-3"
-           sodipodi:role="line">Client</tspan></text>
-    </g>
-    <g
-       transform="translate(142.24726,-13.17977)"
-       id="g4150-6">
-      <ellipse
-         ry="37.880722"
-         rx="36.870567"
-         cy="729.1134"
-         cx="48.467022"
-         id="path3336-8"
-         style="fill:#6441a5;fill-opacity:0.34901961;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
-      <text
-         sodipodi:linespacing="125%"
-         id="text4138-0"
-         y="734.16418"
-         x="24.72843"
-         style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-         xml:space="preserve"><tspan
-           style="font-size:16.25px"
-           y="734.16418"
-           x="24.72843"
-           id="tspan4140-7"
-           sodipodi:role="line">Client</tspan></text>
-    </g>
-    <g
-       transform="translate(228.67583,80.391658)"
-       id="g4150-61">
-      <ellipse
-         ry="37.880722"
-         rx="36.870567"
-         cy="729.1134"
-         cx="48.467022"
-         id="path3336-6"
-         style="fill:#6441a5;fill-opacity:0.34901961;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
-      <text
-         sodipodi:linespacing="125%"
-         id="text4138-9"
-         y="734.16418"
-         x="24.72843"
-         style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-         xml:space="preserve"><tspan
-           style="font-size:16.25px"
-           y="734.16418"
-           x="24.72843"
-           id="tspan4140-9"
-           sodipodi:role="line">Client</tspan></text>
-    </g>
-    <g
-       transform="translate(300.1044,-11.751199)"
-       id="g4150-05">
-      <ellipse
-         ry="37.880722"
-         rx="36.870567"
-         cy="729.1134"
-         cx="48.467022"
-         id="path3336-5"
-         style="fill:#6441a5;fill-opacity:0.34901961;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
-      <text
-         sodipodi:linespacing="125%"
-         id="text4138-4"
-         y="734.16418"
-         x="24.72843"
-         style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-         xml:space="preserve"><tspan
-           style="font-size:16.25px"
-           y="734.16418"
-           x="24.72843"
-           id="tspan4140-97"
-           sodipodi:role="line">Client</tspan></text>
-    </g>
-    <g
-       transform="translate(381.53298,75.391659)"
-       id="g4150-1">
-      <ellipse
-         ry="37.880722"
-         rx="36.870567"
-         cy="729.1134"
-         cx="48.467022"
-         id="path3336-0"
-         style="fill:#6441a5;fill-opacity:0.34901961;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
-      <text
-         sodipodi:linespacing="125%"
-         id="text4138-5"
-         y="734.16418"
-         x="24.72843"
-         style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-         xml:space="preserve"><tspan
-           style="font-size:16.25px"
-           y="734.16418"
-           x="24.72843"
-           id="tspan4140-2"
-           sodipodi:role="line">Client</tspan></text>
-    </g>
-    <g
-       transform="translate(469.39012,130.39166)"
-       id="g4150-3">
-      <ellipse
-         ry="37.880722"
-         rx="36.870567"
-         cy="729.1134"
-         cx="48.467022"
-         id="path3336-84"
-         style="fill:#6441a5;fill-opacity:0.34901961;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
-      <text
-         sodipodi:linespacing="125%"
-         id="text4138-2"
-         y="734.16418"
-         x="24.72843"
-         style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-         xml:space="preserve"><tspan
-           style="font-size:16.25px"
-           y="734.16418"
-           x="24.72843"
-           id="tspan4140-6"
-           sodipodi:role="line">Client</tspan></text>
-    </g>
-    <g
-       transform="translate(522.96155,25.391663)"
-       id="g4150-2">
-      <ellipse
-         ry="37.880722"
-         rx="36.870567"
-         cy="729.1134"
-         cx="48.467022"
-         id="path3336-99"
-         style="fill:#6441a5;fill-opacity:0.34901961;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
-      <text
-         sodipodi:linespacing="125%"
-         id="text4138-58"
-         y="734.16418"
-         x="24.72843"
-         style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-         xml:space="preserve"><tspan
-           style="font-size:16.25px"
-           y="734.16418"
-           x="24.72843"
-           id="tspan4140-5"
-           sodipodi:role="line">Client</tspan></text>
-    </g>
-    <g
-       transform="translate(582.24727,-57.465485)"
-       id="g4150-8">
-      <ellipse
-         ry="37.880722"
-         rx="36.870567"
-         cy="729.1134"
-         cx="48.467022"
-         id="path3336-55"
-         style="fill:#6441a5;fill-opacity:0.34901961;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
-      <text
-         sodipodi:linespacing="125%"
-         id="text4138-1"
-         y="734.16418"
-         x="24.72843"
-         style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-         xml:space="preserve"><tspan
-           style="font-size:16.25px"
-           y="734.16418"
-           x="24.72843"
-           id="tspan4140-23"
-           sodipodi:role="line">Client</tspan></text>
-    </g>
-    <g
-       transform="translate(414.39012,-17.465485)"
-       id="g4150-21">
-      <ellipse
-         ry="37.880722"
-         rx="36.870567"
-         cy="729.1134"
-         cx="48.467022"
-         id="path3336-3"
-         style="fill:#6441a5;fill-opacity:0.34901961;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
-      <text
-         sodipodi:linespacing="125%"
-         id="text4138-3"
-         y="734.16418"
-         x="24.72843"
-         style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-         xml:space="preserve"><tspan
-           style="font-size:16.25px"
-           y="734.16418"
-           x="24.72843"
-           id="tspan4140-1"
-           sodipodi:role="line">Client</tspan></text>
-    </g>
+         style="font-size:16.25px;text-align:center;text-anchor:middle">Client</tspan></text>
+    <ellipse
+       style="fill:#6441a5;fill-opacity:0.34901961;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       id="path3336-9"
+       cx="131.42857"
+       cy="800.93365"
+       rx="36.870567"
+       ry="37.880722" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="130.56932"
+       y="805.98444"
+       id="text4138-6"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan4140-3"
+         x="130.56932"
+         y="805.98444"
+         style="font-size:16.25px;text-align:center;text-anchor:middle">Client</tspan></text>
+    <ellipse
+       style="fill:#6441a5;fill-opacity:0.34901961;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       id="path3336-8"
+       cx="190.71428"
+       cy="715.93365"
+       rx="36.870567"
+       ry="37.880722" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="189.85503"
+       y="720.98444"
+       id="text4138-0"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan4140-7"
+         x="189.85503"
+         y="720.98444"
+         style="font-size:16.25px;text-align:center;text-anchor:middle">Client</tspan></text>
+    <ellipse
+       style="fill:#6441a5;fill-opacity:0.34901961;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       id="path3336-6"
+       cx="277.14285"
+       cy="809.50507"
+       rx="36.870567"
+       ry="37.880722" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="276.2836"
+       y="814.55585"
+       id="text4138-9"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan4140-9"
+         x="276.2836"
+         y="814.55585"
+         style="font-size:16.25px;text-align:center;text-anchor:middle">Client</tspan></text>
+    <ellipse
+       style="fill:#6441a5;fill-opacity:0.34901961;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       id="path3336-5"
+       cx="348.57141"
+       cy="717.36218"
+       rx="36.870567"
+       ry="37.880722" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="347.71216"
+       y="722.41296"
+       id="text4138-4"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan4140-97"
+         x="347.71216"
+         y="722.41296"
+         style="font-size:16.25px;text-align:center;text-anchor:middle">Client</tspan></text>
+    <ellipse
+       style="fill:#6441a5;fill-opacity:0.34901961;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       id="path3336-0"
+       cx="430"
+       cy="804.50507"
+       rx="36.870567"
+       ry="37.880722" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="429.14075"
+       y="809.55585"
+       id="text4138-5"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan4140-2"
+         x="429.14075"
+         y="809.55585"
+         style="font-size:16.25px;text-align:center;text-anchor:middle">Client</tspan></text>
+    <ellipse
+       style="fill:#6441a5;fill-opacity:0.34901961;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       id="path3336-84"
+       cx="517.85712"
+       cy="859.50507"
+       rx="36.870567"
+       ry="37.880722" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="516.99786"
+       y="864.55585"
+       id="text4138-2"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan4140-6"
+         x="516.99786"
+         y="864.55585"
+         style="font-size:16.25px;text-align:center;text-anchor:middle">Client</tspan></text>
+    <ellipse
+       style="fill:#6441a5;fill-opacity:0.34901961;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       id="path3336-99"
+       cx="571.42859"
+       cy="754.50507"
+       rx="36.870567"
+       ry="37.880722" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="570.56934"
+       y="759.55585"
+       id="text4138-58"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan4140-5"
+         x="570.56934"
+         y="759.55585"
+         style="font-size:16.25px;text-align:center;text-anchor:middle">Client</tspan></text>
+    <ellipse
+       style="fill:#6441a5;fill-opacity:0.34901961;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       id="path3336-55"
+       cx="630.71429"
+       cy="671.64789"
+       rx="36.870567"
+       ry="37.880722" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="629.85504"
+       y="676.69867"
+       id="text4138-1"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan4140-23"
+         x="629.85504"
+         y="676.69867"
+         style="font-size:16.25px;text-align:center;text-anchor:middle">Client</tspan></text>
+    <ellipse
+       style="fill:#6441a5;fill-opacity:0.34901961;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       id="path3336-3"
+       cx="462.85715"
+       cy="711.64789"
+       rx="36.870567"
+       ry="37.880722" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="461.99789"
+       y="716.69867"
+       id="text4138-3"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan4140-1"
+         x="461.99789"
+         y="716.69867"
+         style="font-size:16.25px;text-align:center;text-anchor:middle">Client</tspan></text>
     <ellipse
        style="fill:none;fill-opacity:0.34901961"
        id="path4279"
@@ -600,113 +578,99 @@
        cy="384.86221"
        rx="45"
        ry="46.07143" />
-    <g
-       id="g4340">
-      <ellipse
-         ry="61.234596"
-         rx="99.079659"
-         cy="485.92004"
-         cx="124.97432"
-         id="path4283"
-         style="fill:#bc4187;fill-opacity:0.34901961;stroke:#000000;stroke-width:1.3607688;stroke-opacity:1" />
-      <text
-         sodipodi:linespacing="125%"
-         id="text4285"
-         y="475.03372"
-         x="43.153011"
-         style="font-style:normal;font-weight:normal;font-size:22.5px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-         xml:space="preserve"><tspan
-           id="tspan4289"
-           y="475.03372"
-           x="43.153011"
-           sodipodi:role="line">Socket Server</tspan></text>
-      <text
-         sodipodi:linespacing="125%"
-         id="text4293"
-         y="496.83524"
-         x="125.13116"
-         style="font-style:normal;font-weight:normal;font-size:12.5px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;text-anchor:middle;text-align:center;"
-         xml:space="preserve"><tspan
-           y="496.83524"
-           x="30.471735"
-           id="tspan4295"
-           sodipodi:role="line">andknuckles.frankerfacez.com</tspan></text>
-    </g>
-    <g
-       id="g4347">
-      <ellipse
-         ry="61.234596"
-         rx="99.079659"
-         cy="460.21936"
-         cx="362.14285"
-         id="path4283-0"
-         style="fill:#bc4187;fill-opacity:0.34901961;stroke:#000000;stroke-width:1.3607688;stroke-opacity:1" />
-      <text
-         sodipodi:linespacing="125%"
-         id="text4285-9"
-         y="449.33301"
-         x="280.32153"
-         style="font-style:normal;font-weight:normal;font-size:22.5px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-         xml:space="preserve"><tspan
-           id="tspan4289-7"
-           y="449.33301"
-           x="280.32153"
-           sodipodi:role="line">Socket Server</tspan></text>
-      <text
-         sodipodi:linespacing="125%"
-         id="text4293-0"
-         y="471.13452"
-         x="362.29968"
-         style="font-style:normal;font-weight:normal;font-size:12.5px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-         xml:space="preserve"><tspan
-           y="471.13452"
-           x="362.29968"
-           id="tspan4295-6"
-           sodipodi:role="line"><tspan
-   id="tspan4338"
-   style="text-align:center;text-anchor:middle">catbag.franke</tspan>rfacez.com</tspan></text>
-    </g>
-    <g
-       id="g4355">
-      <ellipse
-         ry="61.234596"
-         rx="99.079659"
-         cy="477.36221"
-         cx="602.14288"
-         id="path4283-8"
-         style="fill:#bc4187;fill-opacity:0.34901961;stroke:#000000;stroke-width:1.3607688;stroke-opacity:1" />
-      <text
-         sodipodi:linespacing="125%"
-         id="text4285-2"
-         y="466.47592"
-         x="520.32153"
-         style="font-style:normal;font-weight:normal;font-size:22.5px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-         xml:space="preserve"><tspan
-           id="tspan4289-3"
-           y="466.47592"
-           x="520.32153"
-           sodipodi:role="line">Socket Server</tspan></text>
-      <text
-         sodipodi:linespacing="125%"
-         id="text4293-3"
-         y="488.27744"
-         x="602.29968"
-         style="font-style:normal;font-weight:normal;font-size:12.5px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-         xml:space="preserve"><tspan
-           y="488.27744"
-           x="602.29968"
-           id="tspan4295-63"
-           sodipodi:role="line">tuturu.frankerfacez.com</tspan></text>
-    </g>
-    <path
-       style="fill:#008000;fill-rule:evenodd;stroke:#008000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend)"
-       d="M 59.599,692.24282 C 91.923882,545.26563 91.923882,545.26563 91.923882,545.26563"
-       id="path4388"
-       inkscape:connector-curvature="0" />
+    <ellipse
+       style="fill:#bc4187;fill-opacity:0.34901961;stroke:#000000;stroke-width:1.3607688;stroke-opacity:1"
+       id="path4283"
+       cx="124.97432"
+       cy="485.92004"
+       rx="99.079659"
+       ry="61.234596" />
     <text
        xml:space="preserve"
-       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;"
-       x="-601.21124"
+       style="font-style:normal;font-weight:normal;font-size:22.5px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="123.57916"
+       y="475.53372"
+       id="text4285"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         x="123.57916"
+         y="475.53372"
+         id="tspan4289">Socket Server</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:12.5px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;"
+       x="125.13116"
+       y="496.83524"
+       id="text4293"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan4295"
+         x="125.13116"
+         y="496.83524">andknuckles.frankerfacez.com</tspan></text>
+    <ellipse
+       style="fill:#bc4187;fill-opacity:0.34901961;stroke:#000000;stroke-width:1.3607688;stroke-opacity:1"
+       id="path4283-0"
+       cx="362.14285"
+       cy="460.21936"
+       rx="99.079659"
+       ry="61.234596" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:22.5px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="362.24768"
+       y="453.33301"
+       id="text4285-9"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         x="362.24768"
+         y="453.33301"
+         id="tspan4289-7">Socket Server</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:12.5px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="362.75281"
+       y="476.63452"
+       id="text4293-0"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan4295-6"
+         x="362.75281"
+         y="476.63452"><tspan
+   style="text-align:center;text-anchor:middle"
+   id="tspan4338">catbag.franke</tspan>rfacez.com</tspan></text>
+    <ellipse
+       style="fill:#bc4187;fill-opacity:0.34901961;stroke:#000000;stroke-width:1.3607688;stroke-opacity:1"
+       id="path4283-8"
+       cx="602.14288"
+       cy="477.36221"
+       rx="99.079659"
+       ry="61.234596" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:22.5px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="602.24768"
+       y="466.47592"
+       id="text4285-2"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         x="602.24768"
+         y="466.47592"
+         id="tspan4289-3">Socket Server</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:12.5px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="602.29968"
+       y="488.27744"
+       id="text4293-3"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan4295-63"
+         x="602.29968"
+         y="488.27744">tuturu.frankerfacez.com</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;text-anchor:middle;text-align:center;"
+       x="-591.36749"
        y="210.3251"
        id="text4682"
        sodipodi:linespacing="125%"
@@ -715,14 +679,9 @@
          id="tspan4684"
          x="-601.21124"
          y="210.3251"
-         style="font-size:11.25px;fill:#008000;"
+         style="font-size:11.25px;fill:#008000;text-anchor:middle;text-align:center;"
          rotate="0 0 0 0"
          dy="0">TLS</tspan></text>
-    <path
-       style="fill:#008000;fill-rule:evenodd;stroke:#008000;stroke-width:1.01131487px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend-9)"
-       d="M 184.93881,678.07781 C 148.07036,546.28106 148.07036,546.28106 148.07036,546.28106"
-       id="path4388-1"
-       inkscape:connector-curvature="0" />
     <path
        style="fill:none;fill-rule:evenodd;stroke:#008000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#marker6763)"
        d="m 129.8046,761.94335 c -9.26994,-214.42282 0,0 -9.26994,-214.42282"
@@ -731,8 +690,8 @@
        sodipodi:nodetypes="cc" />
     <text
        xml:space="preserve"
-       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       x="617.93414"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;text-anchor:middle;text-align:center;"
+       x="627.77789"
        y="-75.716248"
        id="text4682-9"
        sodipodi:linespacing="125%"
@@ -741,22 +700,22 @@
          id="tspan4684-8"
          x="617.93414"
          y="-75.716248"
-         style="font-size:11.25px;fill:#008000"
+         style="font-size:11.25px;fill:#008000;text-anchor:middle;text-align:center;"
          rotate="0 0 0 0"
          dy="0">TLS</tspan></text>
     <text
        xml:space="preserve"
-       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       x="594.69568"
-       y="4.5866504"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="641.75256"
+       y="4.0350714"
        id="text4682-8"
        sodipodi:linespacing="125%"
        transform="matrix(0.27352407,0.96186516,-0.96186516,0.27352407,0,0)"><tspan
          sodipodi:role="line"
          id="tspan4684-7"
-         x="594.69568"
-         y="4.5866504"
-         style="font-size:11.25px;fill:#008000"
+         x="641.75256"
+         y="4.0350714"
+         style="font-size:11.25px;text-align:center;text-anchor:middle;fill:#008000"
          rotate="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0"
          dy="0">TLS / websocket</tspan></text>
     <ellipse
@@ -774,8 +733,8 @@
        sodipodi:nodetypes="cc" />
     <text
        xml:space="preserve"
-       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       x="672.03656"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;text-anchor:middle;text-align:center;"
+       x="681.88031"
        y="-36.647579"
        id="text4682-9-9"
        sodipodi:linespacing="125%"
@@ -786,7 +745,7 @@
          id="tspan4684-8-9"
          x="672.03656"
          y="-36.647579"
-         style="font-size:11.25px;fill:#008000"
+         style="font-size:11.25px;fill:#008000;text-anchor:middle;text-align:center;"
          rotate="0 0 0 0"
          dy="0">TLS</tspan></text>
     <path
@@ -797,9 +756,9 @@
        sodipodi:nodetypes="cc" />
     <text
        xml:space="preserve"
-       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;"
-       x="583.44501"
-       y="-297.9649"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="633.97571"
+       y="-297.85538"
        id="text4682-9-9-1"
        sodipodi:linespacing="125%"
        transform="matrix(0.07300835,0.99733133,-0.99733133,0.07300835,0,0)"
@@ -807,9 +766,9 @@
        inkscape:transform-center-y="-22.930222"><tspan
          sodipodi:role="line"
          id="tspan4684-8-9-6"
-         x="583.44501"
-         y="-297.9649"
-         style="font-size:11.25px;fill:#000000;"
+         x="633.97571"
+         y="-297.85538"
+         style="font-size:11.25px;text-align:center;text-anchor:middle;fill:#000000"
          rotate="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0">HTTP / websocket</tspan></text>
     <path
        style="fill:#008000;fill-rule:evenodd;stroke:#008000;stroke-width:1.01131487px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend-9-5-3-1)"
@@ -843,8 +802,8 @@
        sodipodi:nodetypes="cc" />
     <text
        xml:space="preserve"
-       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       x="684.90643"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;text-anchor:middle;text-align:center;"
+       x="694.75018"
        y="-213.68245"
        id="text4682-9-9-6"
        sodipodi:linespacing="125%"
@@ -855,13 +814,13 @@
          id="tspan4684-8-9-3"
          x="684.90643"
          y="-213.68245"
-         style="font-size:11.25px;fill:#008000"
+         style="font-size:11.25px;fill:#008000;text-anchor:middle;text-align:center;"
          rotate="0 0 0 0"
          dy="0">TLS</tspan></text>
     <text
        xml:space="preserve"
-       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       x="687.49579"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;text-anchor:middle;text-align:center;"
+       x="697.33954"
        y="-256.61011"
        id="text4682-9-9-4"
        sodipodi:linespacing="125%"
@@ -872,13 +831,13 @@
          id="tspan4684-8-9-5"
          x="687.49579"
          y="-256.61011"
-         style="font-size:11.25px;fill:#008000"
+         style="font-size:11.25px;fill:#008000;text-anchor:middle;text-align:center;"
          rotate="0 0 0 0"
          dy="0">TLS</tspan></text>
     <text
        xml:space="preserve"
-       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       x="-625.29315"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;text-anchor:middle;text-align:center;"
+       x="-615.4494"
        y="560.20447"
        id="text4682-9-9-3"
        sodipodi:linespacing="125%"
@@ -889,13 +848,13 @@
          id="tspan4684-8-9-37"
          x="-625.29315"
          y="560.20447"
-         style="font-size:11.25px;fill:#008000"
+         style="font-size:11.25px;fill:#008000;text-anchor:middle;text-align:center;"
          rotate="0 0 0 0"
          dy="0">TLS</tspan></text>
     <text
        xml:space="preserve"
-       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       x="-562.52814"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;text-anchor:middle;text-align:center;"
+       x="-552.68439"
        y="663.12689"
        id="text4682-9-9-3-3"
        sodipodi:linespacing="125%"
@@ -906,13 +865,13 @@
          id="tspan4684-8-9-37-1"
          x="-562.52814"
          y="663.12689"
-         style="font-size:11.25px;fill:#008000"
+         style="font-size:11.25px;fill:#008000;text-anchor:middle;text-align:center;"
          rotate="0 0 0 0"
          dy="0">TLS</tspan></text>
     <text
        xml:space="preserve"
-       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       x="-557.59875"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;text-anchor:middle;text-align:center;"
+       x="-547.755"
        y="679.40869"
        id="text4682-9-9-3-5"
        sodipodi:linespacing="125%"
@@ -923,31 +882,31 @@
          id="tspan4684-8-9-37-8"
          x="-557.59875"
          y="679.40869"
-         style="font-size:11.25px;fill:#008000"
+         style="font-size:11.25px;fill:#008000;text-anchor:middle;text-align:center;"
          rotate="0 0 0 0"
          dy="0">TLS</tspan></text>
     <text
        xml:space="preserve"
-       style="font-style:normal;font-weight:normal;font-size:22.5px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       x="278.73459"
+       style="font-style:normal;font-weight:normal;font-size:22.5px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="364.87802"
        y="234.37636"
        id="text15347"
        sodipodi:linespacing="125%"><tspan
          sodipodi:role="line"
-         x="278.73459"
+         x="364.87802"
          y="234.37636"
          id="tspan15351">Socket Backend</tspan></text>
     <text
        xml:space="preserve"
-       style="font-style:normal;font-weight:normal;font-size:12.5px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       x="361.82379"
-       y="267.38663"
+       style="font-style:normal;font-weight:normal;font-size:12.5px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="363.27692"
+       y="268.38663"
        id="text4293-0-2"
        sodipodi:linespacing="125%"><tspan
          sodipodi:role="line"
          id="tspan4295-6-2"
-         x="361.82379"
-         y="267.38663"><tspan
+         x="363.27692"
+         y="268.38663"><tspan
    style="text-align:center;text-anchor:middle"
    id="tspan4338-2">catbag.franke</tspan>rfacez.com</tspan></text>
     <path
@@ -958,16 +917,16 @@
        sodipodi:nodetypes="cc" />
     <text
        xml:space="preserve"
-       style="font-style:normal;font-weight:normal;font-size:17.5px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       x="-204.06422"
-       y="368.93552"
+       style="font-style:normal;font-weight:normal;font-size:17.5px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="-142.66475"
+       y="372.49307"
        id="text17619"
        sodipodi:linespacing="125%"
-       transform="matrix(0.65402838,-0.75647001,0.75647001,0.65402838,0,0)"><tspan
+       transform="matrix(0.67329752,-0.73937166,0.73937166,0.67329752,0,0)"><tspan
          sodipodi:role="line"
          id="tspan17621"
-         x="-204.06422"
-         y="368.93552">NaCl / HTTP</tspan></text>
+         x="-142.66475"
+         y="372.49307">NaCl / HTTP</tspan></text>
     <path
        style="fill:#000000;fill-rule:evenodd;stroke:#008000;stroke-width:1.01131487px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend-9-5-3-1-7-2-5-2);marker-start:url(#Arrow1Lstart)"
        d="M 358.43958,398.54766 357.9365,315.74331"
@@ -982,28 +941,28 @@
        sodipodi:nodetypes="cc" />
     <text
        xml:space="preserve"
-       style="font-style:normal;font-weight:normal;font-size:17.5px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       x="566.948"
-       y="-143.04019"
+       style="font-style:normal;font-weight:normal;font-size:17.5px;line-height:125%;font-family:sans-serif;text-align:start;letter-spacing:0px;word-spacing:0px;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="564.79462"
+       y="-145.12895"
        id="text17619-6"
        sodipodi:linespacing="125%"
        transform="matrix(0.69625411,0.71779538,-0.71779538,0.69625411,0,0)"><tspan
          sodipodi:role="line"
          id="tspan17621-7"
-         x="566.948"
-         y="-143.04019">NaCl / HTTP</tspan></text>
+         x="564.79462"
+         y="-145.12895">NaCl / HTTP</tspan></text>
     <text
        xml:space="preserve"
-       style="font-style:normal;font-weight:normal;font-size:17.5px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       x="358.19937"
-       y="367.07816"
+       style="font-style:normal;font-weight:normal;font-size:17.5px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="410.92709"
+       y="366.58148"
        id="text17619-6-6"
        sodipodi:linespacing="125%"
        transform="matrix(0.99999912,-0.001326,0.001326,0.99999912,0,0)"><tspan
          sodipodi:role="line"
          id="tspan17621-7-5"
-         x="358.19937"
-         y="367.07816">NaCl / HTTP</tspan></text>
+         x="410.92709"
+         y="366.58148">NaCl / HTTP</tspan></text>
     <ellipse
        style="fill:#c7804a;fill-opacity:0.34901962;stroke:#000000;stroke-width:0.89510232;stroke-linecap:butt;stroke-opacity:1"
        id="path7045-6"
@@ -1019,39 +978,51 @@
        sodipodi:nodetypes="cc" />
     <text
        xml:space="preserve"
-       style="font-style:normal;font-weight:normal;font-size:17.5px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       x="362.03867"
-       y="158.57924"
+       style="font-style:normal;font-weight:normal;font-size:17.5px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="397.44247"
+       y="159.07924"
        id="text22558"
        sodipodi:linespacing="125%"><tspan
          sodipodi:role="line"
          id="tspan22560"
-         x="362.03867"
-         y="158.57924"
-         style="font-size:10px">(out of scope)</tspan></text>
+         x="397.44247"
+         y="159.07924"
+         style="font-size:10px;text-align:center;text-anchor:middle">(out of scope)</tspan></text>
     <text
        xml:space="preserve"
-       style="font-style:normal;font-weight:normal;font-size:12.5px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       x="362.22787"
-       y="98.034554"
+       style="font-style:normal;font-weight:normal;font-size:12.5px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="366.53732"
+       y="94.034554"
        id="text4293-0-2-6"
        sodipodi:linespacing="125%"><tspan
          sodipodi:role="line"
          id="tspan4295-6-2-6"
-         x="362.22787"
-         y="98.034554"><tspan
+         x="366.53732"
+         y="94.034554"><tspan
    style="text-align:center;text-anchor:middle"
    id="tspan4338-2-6">www.franke</tspan>rfacez.com</tspan></text>
     <text
        xml:space="preserve"
-       style="font-style:normal;font-weight:normal;font-size:22.5px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       x="296.80118"
-       y="68.132027"
+       style="font-style:normal;font-weight:normal;font-size:22.5px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="362.43375"
+       y="67.632027"
        id="text15347-1"
        sodipodi:linespacing="125%"><tspan
          sodipodi:role="line"
-         x="296.80118"
-         y="68.132027"
+         x="362.43375"
+         y="67.632027"
          id="tspan15351-4">Web Server</tspan></text>
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#008000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#marker6763-4)"
+       d="M 59.168385,693.24658 91.617745,544.93825"
+       id="path6849-9"
+       inkscape:connector-curvature="0"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#008000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#marker6763-4-5)"
+       d="m 184.07012,678.33912 -36.14,-132.04489"
+       id="path6849-9-8"
+       inkscape:connector-curvature="0"
+       sodipodi:nodetypes="cc" />
   </g>
 </svg>

From 59fed52797218b8239f311b4d6adfa75dd57d159 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 2 Dec 2015 17:28:15 -0800
Subject: [PATCH 087/176] Split socket writer loop into a function

---
 socketserver/server/handlecore.go | 127 ++++++++++++++----------------
 1 file changed, 58 insertions(+), 69 deletions(-)

diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 2871159f..b612cf16 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -306,76 +306,8 @@ func RunSocketConnection(conn *websocket.Conn) {
 		return nil
 	})
 
-	var errorChan <-chan error = _errorChan
-	var clientChan <-chan ClientMessage = _clientChan
-	var serverMessageChan <-chan ClientMessage = _serverMessageChan
-
 	// All set up, now enter the work loop
-
-	var closeReason websocket.CloseError
-
-RunLoop:
-	for {
-		select {
-		case err := <-errorChan:
-			if err == io.EOF {
-				closeReason = websocket.CloseError{
-					Code: websocket.CloseGoingAway,
-					Text: err.Error(),
-				}
-			} else if closeMsg, isClose := err.(*websocket.CloseError); isClose {
-				closeReason = *closeMsg
-			} else {
-				closeReason = websocket.CloseError{
-					Code: websocket.CloseInternalServerErr,
-					Text: err.Error(),
-				}
-			}
-			break RunLoop
-
-		case msg := <-clientChan:
-			if client.VersionString == "" && msg.Command != HelloCommand {
-				closeReason = CloseFirstMessageNotHello
-				break RunLoop
-			}
-
-			for _, char := range msg.Command {
-				if char == utf8.RuneError {
-					closeReason = CloseNonUTF8Data
-					break RunLoop
-				}
-			}
-
-			DispatchC2SCommand(conn, &client, msg)
-
-		case msg := <-serverMessageChan:
-			if len(serverMessageChan) > sendMessageAbortLength {
-				closeReason = CloseTooManyBufferedMessages
-				break RunLoop
-			}
-			if cls, ok := msg.Arguments.(*websocket.CloseError); ok {
-				closeReason = *cls
-				break RunLoop
-			}
-			SendMessage(conn, msg)
-
-		case <-time.After(1 * time.Minute):
-			client.Mutex.Lock()
-			client.pingCount++
-			tooManyPings := client.pingCount == 5
-			client.Mutex.Unlock()
-			if tooManyPings {
-				closeReason = CloseTimedOut
-				break RunLoop
-			} else {
-				conn.WriteControl(websocket.PingMessage, []byte(strconv.FormatInt(time.Now().Unix(), 10)), getDeadline())
-			}
-
-		case <-StopAcceptingConnectionsCh:
-			closeReason = CloseGoingAway
-			break RunLoop
-		}
-	}
+	closeReason := runSocketWriter(_errorChan, _clientChan, _serverMessageChan, conn, &client)
 
 	// Exit
 	CloseConnection(conn, closeReason)
@@ -407,6 +339,63 @@ RunLoop:
 	}
 }
 
+func runSocketWriter(errorChan <-chan error, clientChan <-chan ClientMessage, serverMessageChan <-chan ClientMessage, conn *websocket.Conn, client *ClientInfo) websocket.CloseError {
+	for {
+		select {
+		case err := <-errorChan:
+			if err == io.EOF {
+				return websocket.CloseError{
+					Code: websocket.CloseGoingAway,
+					Text: err.Error(),
+				}
+			} else if closeMsg, isClose := err.(*websocket.CloseError); isClose {
+				return *closeMsg
+			} else {
+				return websocket.CloseError{
+					Code: websocket.CloseInternalServerErr,
+					Text: err.Error(),
+				}
+			}
+
+		case msg := <-clientChan:
+			if client.VersionString == "" && msg.Command != HelloCommand {
+				return CloseFirstMessageNotHello
+			}
+
+			for _, char := range msg.Command {
+				if char == utf8.RuneError {
+					return CloseNonUTF8Data
+				}
+			}
+
+			DispatchC2SCommand(conn, client, msg)
+
+		case msg := <-serverMessageChan:
+			if len(serverMessageChan) > sendMessageAbortLength {
+				return CloseTooManyBufferedMessages
+			}
+			if cls, ok := msg.Arguments.(*websocket.CloseError); ok {
+				return *cls
+			}
+			SendMessage(conn, msg)
+
+		case <-time.After(1 * time.Minute):
+			client.Mutex.Lock()
+			client.pingCount++
+			tooManyPings := client.pingCount == 5
+			client.Mutex.Unlock()
+			if tooManyPings {
+				return CloseTimedOut
+			} else {
+				conn.WriteControl(websocket.PingMessage, []byte(strconv.FormatInt(time.Now().Unix(), 10)), getDeadline())
+			}
+
+		case <-StopAcceptingConnectionsCh:
+			return CloseGoingAway
+		}
+	}
+}
+
 func getDeadline() time.Time {
 	return time.Now().Add(1 * time.Minute)
 }

From dc09698e3694a0623aaa710a646686364ce08d49 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 2 Dec 2015 19:08:19 -0800
Subject: [PATCH 088/176] Small refactor to bunched command

---
 socketserver/server/backend.go    |  2 +-
 socketserver/server/commands.go   | 48 ++++++++++++++++---------------
 socketserver/server/handlecore.go |  8 +++---
 socketserver/server/types.go      |  4 +--
 4 files changed, 32 insertions(+), 30 deletions(-)

diff --git a/socketserver/server/backend.go b/socketserver/server/backend.go
index 73fa03aa..b4582845 100644
--- a/socketserver/server/backend.go
+++ b/socketserver/server/backend.go
@@ -269,7 +269,7 @@ func GenerateKeys(outputFile, serverID, theirPublicStr string) {
 	var err error
 	output := ConfigFile{
 		ListenAddr:      "0.0.0.0:8001",
-		SSLListenAddr:      "0.0.0.0:443",
+		SSLListenAddr:   "0.0.0.0:443",
 		SocketOrigin:    "localhost:8001",
 		BackendURL:      "http://localhost:8002/ffz",
 		MinMemoryKBytes: defaultMinMemoryKB,
diff --git a/socketserver/server/commands.go b/socketserver/server/commands.go
index b68d4ebf..ee373363 100644
--- a/socketserver/server/commands.go
+++ b/socketserver/server/commands.go
@@ -386,7 +386,7 @@ type bunchSubscriberList struct {
 	Members []bunchSubscriber
 }
 
-type CacheStatus byte
+type cacheStatus byte
 
 const (
 	CacheStatusNotFound = iota
@@ -435,36 +435,38 @@ func bunchCacheJanitor() {
 	}
 }
 
+var emptyCachedBunchedResponse cachedBunchedResponse
+
+func bunchGetCacheStatus(br bunchedRequest, client *ClientInfo) (cacheStatus, cachedBunchedResponse) {
+	bunchCacheLock.RLock()
+	defer bunchCacheLock.RUnlock()
+	cachedResponse, ok := bunchCache[br]
+	if ok && cachedResponse.Timestamp.After(time.Now().Add(-5*time.Minute)) {
+		return CacheStatusFound, cachedResponse
+	} else if ok {
+		return CacheStatusExpired, emptyCachedBunchedResponse
+	}
+	return CacheStatusNotFound, emptyCachedBunchedResponse
+}
+
 // C2SHandleBunchedCommand handles C2S Commands such as `get_link`.
 // It makes a request to the backend server for the data, but any other requests coming in while the first is pending also get the responses from the first one.
 // Additionally, results are cached.
 func C2SHandleBunchedCommand(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
+	// FIXME(riking): Function is too complex
+
 	br := bunchedRequestFromCM(&msg)
 
-	cacheStatus := func() byte {
-		bunchCacheLock.RLock()
-		defer bunchCacheLock.RUnlock()
-		bresp, ok := bunchCache[br]
-		if ok && bresp.Timestamp.After(time.Now().Add(-5*time.Minute)) {
-			client.MsgChannelKeepalive.Add(1)
-			go func() {
-				var rmsg ClientMessage
-				rmsg.Command = SuccessCommand
-				rmsg.MessageID = msg.MessageID
-				rmsg.origArguments = bresp.Response
-				rmsg.parseOrigArguments()
-				client.MessageChannel <- rmsg
-				client.MsgChannelKeepalive.Done()
-			}()
-			return CacheStatusFound
-		} else if ok {
-			return CacheStatusExpired
-		}
-		return CacheStatusNotFound
-	}()
+	cacheStatus, cachedResponse := bunchGetCacheStatus(br, client)
 
 	if cacheStatus == CacheStatusFound {
-		return ClientMessage{Command: AsyncResponseCommand}, nil
+		var response ClientMessage
+		response.Command = SuccessCommand
+		response.MessageID = msg.MessageID
+		response.origArguments = cachedResponse.Response
+		response.parseOrigArguments()
+
+		return response, nil
 	} else if cacheStatus == CacheStatusExpired {
 		// Wake up the lazy janitor
 		bunchCacheCleanupSignal.Signal()
diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index b612cf16..c70880de 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -10,15 +10,15 @@ import (
 	"log"
 	"net/http"
 	"net/url"
+	"os"
+	"os/signal"
 	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
+	"syscall"
 	"time"
 	"unicode/utf8"
-	"os"
-	"os/signal"
-	"syscall"
 )
 
 // SuccessCommand is a Reply Command to indicate success in reply to a C2S Command.
@@ -112,7 +112,7 @@ func shutdownHandler() {
 	StopAcceptingConnections = true
 	close(StopAcceptingConnectionsCh)
 
-	time.Sleep(1*time.Second)
+	time.Sleep(1 * time.Second)
 	os.Exit(0)
 }
 
diff --git a/socketserver/server/types.go b/socketserver/server/types.go
index 2fe42b8a..6fddf4d9 100644
--- a/socketserver/server/types.go
+++ b/socketserver/server/types.go
@@ -13,8 +13,8 @@ const CryptoBoxKeyLength = 32
 
 type ConfigFile struct {
 	// Numeric server id known to the backend
-	ServerID   int
-	ListenAddr string
+	ServerID      int
+	ListenAddr    string
 	SSLListenAddr string
 	// Hostname of the socket server
 	SocketOrigin string

From 3eb7c58e9083eeeb71184387368cda453042f29a Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 2 Dec 2015 19:09:50 -0800
Subject: [PATCH 089/176] Add scaffold for link normalization

---
 socketserver/server/commands.go | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/socketserver/server/commands.go b/socketserver/server/commands.go
index ee373363..72fca25a 100644
--- a/socketserver/server/commands.go
+++ b/socketserver/server/commands.go
@@ -449,6 +449,13 @@ func bunchGetCacheStatus(br bunchedRequest, client *ClientInfo) (cacheStatus, ca
 	return CacheStatusNotFound, emptyCachedBunchedResponse
 }
 
+func normalizeBunchedRequest(br bunchedRequest) bunchedRequest {
+	if br.Command == "get_link" {
+		// TODO
+	}
+	return br
+}
+
 // C2SHandleBunchedCommand handles C2S Commands such as `get_link`.
 // It makes a request to the backend server for the data, but any other requests coming in while the first is pending also get the responses from the first one.
 // Additionally, results are cached.
@@ -456,6 +463,7 @@ func C2SHandleBunchedCommand(conn *websocket.Conn, client *ClientInfo, msg Clien
 	// FIXME(riking): Function is too complex
 
 	br := bunchedRequestFromCM(&msg)
+	br = normalizeBunchedRequest(br)
 
 	cacheStatus, cachedResponse := bunchGetCacheStatus(br, client)
 

From ed71736e07de43cd3296818f36f46d404444c8e7 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sat, 12 Dec 2015 16:00:51 -0800
Subject: [PATCH 090/176] Update tweet tooltip display code

---
 socketserver/server/links.go |  1 +
 src/tokenize.js              | 17 +++++++++++++++--
 2 files changed, 16 insertions(+), 2 deletions(-)
 create mode 100644 socketserver/server/links.go

diff --git a/socketserver/server/links.go b/socketserver/server/links.go
new file mode 100644
index 00000000..abb4e431
--- /dev/null
+++ b/socketserver/server/links.go
@@ -0,0 +1 @@
+package server
diff --git a/src/tokenize.js b/src/tokenize.js
index bd877874..7f7c8610 100644
--- a/src/tokenize.js
+++ b/src/tokenize.js
@@ -95,8 +95,21 @@ var FFZ = window.FrankerFaceZ,
 
 
 		} else if ( link_data.type == "twitter" ) {
-			tooltip = "<b>Tweet By: " + utils.sanitize(link_data.user) + "</b><hr>";
-			tooltip += utils.sanitize(link_data.tweet);
+			/*
+			Display Guidelines - https://about.twitter.com/company/display-requirements
+			[logo] Tweet
+			Ryan Greenberg (@greenberg) 24 Apr 2013
+			"Beware," wrote the prophets of San Francisco, "for what the sun giveth the wind taketh away."
+			 */
+			var ttparts = ["<span class='twitter_logo'></span> Tweet<br>",
+					utils.sanitize(link_data.user_display),
+					" (<b>@" + utils.sanitize(link_data.user) + "</b>) "];
+			var timestamp = utils.parse_date(link_data.timestamp);
+			if ( timestamp )
+				ttparts.push(utils.date_string(timestamp));
+			ttparts.push("<hr>");
+			ttparts.push(link_data.tweet_html || utils.sanitize(link_data.tweet));
+			tooltip = ttparts.join('');
 
 
 		} else if ( link_data.type == "reputation" ) {

From 24693325ae656523268920a535908907cafb2f29 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 16 Dec 2015 09:44:04 -0800
Subject: [PATCH 091/176] Delete unused file

---
 socketserver/server/links.go | 1 -
 1 file changed, 1 deletion(-)
 delete mode 100644 socketserver/server/links.go

diff --git a/socketserver/server/links.go b/socketserver/server/links.go
deleted file mode 100644
index abb4e431..00000000
--- a/socketserver/server/links.go
+++ /dev/null
@@ -1 +0,0 @@
-package server

From 93d93b54a317f418fd941a371b7210f6ee8ec3a1 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 16 Dec 2015 09:44:52 -0800
Subject: [PATCH 092/176] Revert "Update tweet tooltip display code"

This reverts commit 2ecdf8d8d6a183ad7c03da5cca3497fcc01e8e17.
---
 src/tokenize.js | 17 ++---------------
 1 file changed, 2 insertions(+), 15 deletions(-)

diff --git a/src/tokenize.js b/src/tokenize.js
index 7f7c8610..bd877874 100644
--- a/src/tokenize.js
+++ b/src/tokenize.js
@@ -95,21 +95,8 @@ var FFZ = window.FrankerFaceZ,
 
 
 		} else if ( link_data.type == "twitter" ) {
-			/*
-			Display Guidelines - https://about.twitter.com/company/display-requirements
-			[logo] Tweet
-			Ryan Greenberg (@greenberg) 24 Apr 2013
-			"Beware," wrote the prophets of San Francisco, "for what the sun giveth the wind taketh away."
-			 */
-			var ttparts = ["<span class='twitter_logo'></span> Tweet<br>",
-					utils.sanitize(link_data.user_display),
-					" (<b>@" + utils.sanitize(link_data.user) + "</b>) "];
-			var timestamp = utils.parse_date(link_data.timestamp);
-			if ( timestamp )
-				ttparts.push(utils.date_string(timestamp));
-			ttparts.push("<hr>");
-			ttparts.push(link_data.tweet_html || utils.sanitize(link_data.tweet));
-			tooltip = ttparts.join('');
+			tooltip = "<b>Tweet By: " + utils.sanitize(link_data.user) + "</b><hr>";
+			tooltip += utils.sanitize(link_data.tweet);
 
 
 		} else if ( link_data.type == "reputation" ) {

From 03e6e99cb9c922529fed26c9ac585070510574df Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 16 Dec 2015 11:15:17 -0800
Subject: [PATCH 093/176] Add submission of connection reports to logstash

---
 socketserver/cmd/ffzsocketserver/console.go   |   4 +-
 socketserver/server/handlecore.go             | 102 +++++----
 socketserver/server/logstash/elasticsearch.go | 205 ++++++++++++++++++
 socketserver/server/types.go                  |   9 +-
 4 files changed, 264 insertions(+), 56 deletions(-)
 create mode 100644 socketserver/server/logstash/elasticsearch.go

diff --git a/socketserver/cmd/ffzsocketserver/console.go b/socketserver/cmd/ffzsocketserver/console.go
index 21da6e4c..b3a77b0c 100644
--- a/socketserver/cmd/ffzsocketserver/console.go
+++ b/socketserver/cmd/ffzsocketserver/console.go
@@ -6,8 +6,8 @@ import (
 	"github.com/abiosoft/ishell"
 	"github.com/gorilla/websocket"
 	"runtime"
-	"strings"
 	"strconv"
+	"strings"
 )
 
 func commandLineConsole() {
@@ -106,7 +106,7 @@ func commandLineConsole() {
 			server.GlobalSubscriptionLock.RUnlock()
 		}
 
-		msg := server.ClientMessage{ Arguments: &server.CloseRebalance }
+		msg := server.ClientMessage{Arguments: &server.CloseRebalance}
 		server.GlobalSubscriptionLock.RLock()
 		defer server.GlobalSubscriptionLock.RUnlock()
 
diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index c70880de..4d96f704 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -19,6 +19,8 @@ import (
 	"syscall"
 	"time"
 	"unicode/utf8"
+
+	"./logstash"
 )
 
 // SuccessCommand is a Reply Command to indicate success in reply to a C2S Command.
@@ -251,6 +253,10 @@ func RunSocketConnection(conn *websocket.Conn) {
 	// Close the connection when we're done.
 	defer closer()
 
+	var report logstash.ConnectionReport
+	report.ConnectTime = time.Now()
+	report.RemoteAddr = conn.RemoteAddr()
+
 	_clientChan := make(chan ClientMessage)
 	_serverMessageChan := make(chan ClientMessage, sendMessageBufferLength)
 	_errorChan := make(chan error)
@@ -261,44 +267,6 @@ func RunSocketConnection(conn *websocket.Conn) {
 	client.RemoteAddr = conn.RemoteAddr()
 	client.MsgChannelIsDone = stoppedChan
 
-	// Launch receiver goroutine
-	go func(errorChan chan<- error, clientChan chan<- ClientMessage, stoppedChan <-chan struct{}) {
-		var msg ClientMessage
-		var messageType int
-		var packet []byte
-		var err error
-
-		defer close(errorChan)
-		defer close(clientChan)
-
-		for ; err == nil; messageType, packet, err = conn.ReadMessage() {
-			if messageType == websocket.BinaryMessage {
-				err = &CloseGotBinaryMessage
-				break
-			}
-			if messageType == websocket.CloseMessage {
-				err = io.EOF
-				break
-			}
-
-			UnmarshalClientMessage(packet, messageType, &msg)
-			if msg.MessageID == 0 {
-				continue
-			}
-			select {
-			case clientChan <- msg:
-			case <-stoppedChan:
-				return
-			}
-		}
-
-		select {
-		case errorChan <- err:
-		case <-stoppedChan:
-		}
-		// exit goroutine
-	}(_errorChan, _clientChan, stoppedChan)
-
 	conn.SetPongHandler(func(pongBody string) error {
 		client.Mutex.Lock()
 		client.pingCount = 0
@@ -307,10 +275,11 @@ func RunSocketConnection(conn *websocket.Conn) {
 	})
 
 	// All set up, now enter the work loop
-	closeReason := runSocketWriter(_errorChan, _clientChan, _serverMessageChan, conn, &client)
+	go runSocketReader(conn, _errorChan, _clientChan, stoppedChan)
+	closeReason := runSocketWriter(conn, &client, _errorChan, _clientChan, _serverMessageChan)
 
 	// Exit
-	CloseConnection(conn, closeReason)
+	closeConnection(conn, closeReason, &report)
 
 	// Launch message draining goroutine - we aren't out of the pub/sub records
 	go func() {
@@ -334,12 +303,50 @@ func RunSocketConnection(conn *websocket.Conn) {
 
 	if !StopAcceptingConnections {
 		// Don't perform high contention operations when server is closing
-		atomic.AddUint64(&Statistics.ClientDisconnectsTotal, 1)
-		atomic.AddUint64(&Statistics.CurrentClientCount, ^uint64(0))
+		atomic.AddUint64(&Statistics.CurrentClientCount, NegativeOne)
 	}
+
+	logstash.Submit(report)
 }
 
-func runSocketWriter(errorChan <-chan error, clientChan <-chan ClientMessage, serverMessageChan <-chan ClientMessage, conn *websocket.Conn, client *ClientInfo) websocket.CloseError {
+func runSocketReader(conn *websocket.Conn, errorChan chan<- error, clientChan chan<- ClientMessage, stoppedChan <-chan struct{}) {
+	var msg ClientMessage
+	var messageType int
+	var packet []byte
+	var err error
+
+	defer close(errorChan)
+	defer close(clientChan)
+
+	for ; err == nil; messageType, packet, err = conn.ReadMessage() {
+		if messageType == websocket.BinaryMessage {
+			err = &CloseGotBinaryMessage
+			break
+		}
+		if messageType == websocket.CloseMessage {
+			err = io.EOF
+			break
+		}
+
+		UnmarshalClientMessage(packet, messageType, &msg)
+		if msg.MessageID == 0 {
+			continue
+		}
+		select {
+		case clientChan <- msg:
+		case <-stoppedChan:
+			return
+		}
+	}
+
+	select {
+	case errorChan <- err:
+	case <-stoppedChan:
+	}
+	// exit goroutine
+}
+
+func runSocketWriter(conn *websocket.Conn, client *ClientInfo, errorChan <-chan error, clientChan <-chan ClientMessage, serverMessageChan <-chan ClientMessage) websocket.CloseError {
 	for {
 		select {
 		case err := <-errorChan:
@@ -400,7 +407,7 @@ func getDeadline() time.Time {
 	return time.Now().Add(1 * time.Minute)
 }
 
-func CloseConnection(conn *websocket.Conn, closeMsg websocket.CloseError) {
+func closeConnection(conn *websocket.Conn, closeMsg websocket.CloseError, report *esConnectionReport) {
 	closeTxt := closeMsg.Text
 	if strings.Contains(closeTxt, "read: connection reset by peer") {
 		closeTxt = "read: connection reset by peer"
@@ -409,9 +416,10 @@ func CloseConnection(conn *websocket.Conn, closeMsg websocket.CloseError) {
 	} else if closeMsg.Code == 1001 {
 		closeTxt = "clean shutdown"
 	}
-	// todo kibana cannot analyze these
-	Statistics.DisconnectCodes[strconv.Itoa(closeMsg.Code)]++
-	Statistics.DisconnectReasons[closeTxt]++
+
+	report.DisconnectCode = closeMsg.Code
+	report.DisconnectReason = closeTxt
+	report.DisconnectTime = time.Now()
 
 	conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(closeMsg.Code, closeMsg.Text), getDeadline())
 	conn.Close()
diff --git a/socketserver/server/logstash/elasticsearch.go b/socketserver/server/logstash/elasticsearch.go
new file mode 100644
index 00000000..b8c88299
--- /dev/null
+++ b/socketserver/server/logstash/elasticsearch.go
@@ -0,0 +1,205 @@
+package server
+
+import (
+	"bytes"
+	"crypto/rand"
+	"encoding/base64"
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/url"
+	"sync"
+	"time"
+)
+
+// ID is a 128-bit ID for an elasticsearch document.
+// Textually, it is base64-encoded.
+// The Next() method increments the ID.
+type ID struct {
+	High uint64
+	Low  uint64
+}
+
+// Text converts the ID into a base64 string.
+func (id ID) String() string {
+	var buf bytes.Buffer
+	buf.Grow(21)
+	enc := base64.NewEncoder(base64.StdEncoding, &buf)
+	var bytes [16]byte
+	binary.LittleEndian.PutUint64(bytes[0:8], id.High)
+	binary.LittleEndian.PutUint64(bytes[8:16], id.Low)
+	enc.Write(bytes[:])
+	enc.Close()
+	return buf.String()
+}
+
+// Next increments the ID and returns the prior state.
+// Overflow is not checked because it's a uint64, do you really expect me to overflow that
+func (id *ID) Next() ID {
+	ret := ID{
+		High: id.High,
+		Low:  id.Low,
+	}
+	id.Low++
+	return ret
+}
+
+var idPool = sync.Pool{New: func() interface{} {
+	var bytes [16]byte
+	n, err := rand.Reader.Read(bytes[:])
+	if n != 16 || err != nil {
+		panic(fmt.Errorf("Short read from crypto/rand: %v", err))
+	}
+
+	return &ID{
+		High: binary.LittleEndian.Uint64(bytes[0:8]),
+		Low:  binary.LittleEndian.Uint64(bytes[8:16]),
+	}
+}}
+
+func ExampleID_Next() {
+	id := idPool.Get().(*ID).Next()
+	fmt.Println(id)
+	idPool.Put(id)
+}
+
+// Report is the interface presented to the Submit() function.
+// FillReport() is satisfied by ReportBasic, but ReportType must always be specified.
+type Report interface {
+	FillReport() error
+	ReportType() string
+
+	GetID() string
+	GetTimestamp() time.Time
+}
+
+// ReportBasic is the essential fields of any report.
+type ReportBasic struct {
+	ID        string
+	Timestamp time.Time
+	Host      string
+}
+
+// FillReport sets the Host and Timestamp fields.
+func (report *ReportBasic) FillReport() error {
+	report.Host = hostMarker
+	report.Timestamp = time.Now()
+	id := idPool.Get().(*ID).Next()
+	report.ID = id.String()
+	idPool.Put(id)
+	return nil
+}
+
+func (report *ReportBasic) GetID() string {
+	return report.ID
+}
+
+func (report *ReportBasic) GetTimestamp() time.Time {
+	return report.Timestamp
+}
+
+type ConnectionReport struct {
+	ReportBasic
+
+	ConnectTime    time.Time
+	DisconnectTime time.Time
+	// calculated
+	ConnectionDuration time.Duration
+
+	DisconnectCode   int
+	DisconnectReason string
+
+	RemoteAddr net.Addr
+}
+
+// FillReport sets all the calculated fields, and calls esReportBasic.FillReport().
+func (report *ConnectionReport) FillReport() error {
+	report.ReportBasic.FillReport()
+	report.ConnectionDuration = report.DisconnectTime.Sub(report.ConnectTime)
+	return nil
+}
+
+func (report *ConnectionReport) ReportType() string {
+	return "conn"
+}
+
+var serverPresent bool
+var esClient http.Client
+var submitChan chan Report
+var serverBase, indexPrefix, hostMarker string
+
+func checkServerPresent() {
+	if serverBase == "" {
+		serverBase = "http://localhost:9200"
+	}
+	if indexPrefix == "" {
+		indexPrefix = "sockreport"
+	}
+
+	urlHealth := fmt.Sprintf("%s/_cluster/health", serverBase)
+	resp, err := esClient.Get(urlHealth)
+	if err == nil {
+		resp.Body.Close()
+		serverPresent = true
+		submitChan = make(chan Report, 8)
+		go submissionWorker()
+	} else {
+		serverPresent = false
+	}
+}
+
+// Setup sets up the global variables for the package.
+func Setup(ESServer, ESIndexPrefix, ESHostname string) {
+	serverBase = ESServer
+	indexPrefix = ESIndexPrefix
+	hostMarker = ESHostname
+	checkServerPresent()
+}
+
+// Submit inserts a report into elasticsearch (this is basically a manual logstash).
+func Submit(report Report) {
+	if !serverPresent {
+		return
+	}
+
+	report.FillReport()
+	submitChan <- report
+}
+
+func submissionWorker() {
+	for report := range submitChan {
+		time := report.GetTimestamp()
+		rType := report.ReportType()
+
+		// prefix-type-date
+		indexName := fmt.Sprintf("%s-%s-%d-%d-%d", indexPrefix, rType, time.Year(), time.Month(), time.Day())
+		// base/index/type/id
+		putUrl, err := url.Parse(fmt.Sprintf("%s/%s/%s/%s", serverBase, indexName, rType, report.GetID()))
+		if err != nil {
+			panic(fmt.Errorf("logstash: cannot parse url: %v", err))
+		}
+		body, err := json.Marshal(report)
+		if err != nil {
+			panic(fmt.Errorf("logstash: cannot marshal json: %v", err))
+		}
+
+		req := &http.Request{
+			Method: "PUT",
+			URL:    putUrl,
+			Body:   ioutil.NopCloser(bytes.NewReader(body)),
+		}
+
+		resp, err := esClient.Do(req)
+
+		if err != nil {
+			// ignore, the show must go on
+		} else {
+			io.Copy(ioutil.Discard, resp.Body)
+			resp.Body.Close()
+		}
+	}
+}
diff --git a/socketserver/server/types.go b/socketserver/server/types.go
index 6fddf4d9..fe6193ba 100644
--- a/socketserver/server/types.go
+++ b/socketserver/server/types.go
@@ -11,6 +11,8 @@ import (
 
 const CryptoBoxKeyLength = 32
 
+const NegativeOne = ^uint64(0)
+
 type ConfigFile struct {
 	// Numeric server id known to the backend
 	ServerID      int
@@ -121,13 +123,6 @@ type ClientInfo struct {
 	pingCount int
 }
 
-type esReportBasic struct {
-	Timestamp time.Time
-	Host      string
-}
-type esDisconnectReport struct {
-}
-
 func VersionFromString(v string) ClientVersion {
 	var cv ClientVersion
 	fmt.Sscanf(v, "ffz_%d.%d.%d", &cv.Major, &cv.Minor, &cv.Revision)

From 75cf4fbb4687e12dfd249f7acd9e4511d8f6e052 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 16 Dec 2015 11:47:51 -0800
Subject: [PATCH 094/176] rename package to 'logstasher'

---
 socketserver/server/handlecore.go                      | 10 ++++++----
 .../server/{logstash => logstasher}/elasticsearch.go   |  3 ++-
 2 files changed, 8 insertions(+), 5 deletions(-)
 rename socketserver/server/{logstash => logstasher}/elasticsearch.go (98%)

diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 4d96f704..d502d071 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -20,7 +20,7 @@ import (
 	"time"
 	"unicode/utf8"
 
-	"./logstash"
+	"./logstasher"
 )
 
 // SuccessCommand is a Reply Command to indicate success in reply to a C2S Command.
@@ -96,6 +96,8 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 		resp.Body.Close()
 	}
 
+	logstasher.Setup(Configuration.ESServer, Configuration.ESIndexPrefix, Configuration.ESHostName)
+
 	go authorizationJanitor()
 	go bunchCacheJanitor()
 	go pubsubJanitor()
@@ -253,7 +255,7 @@ func RunSocketConnection(conn *websocket.Conn) {
 	// Close the connection when we're done.
 	defer closer()
 
-	var report logstash.ConnectionReport
+	var report logstasher.ConnectionReport
 	report.ConnectTime = time.Now()
 	report.RemoteAddr = conn.RemoteAddr()
 
@@ -306,7 +308,7 @@ func RunSocketConnection(conn *websocket.Conn) {
 		atomic.AddUint64(&Statistics.CurrentClientCount, NegativeOne)
 	}
 
-	logstash.Submit(report)
+	logstasher.Submit(&report)
 }
 
 func runSocketReader(conn *websocket.Conn, errorChan chan<- error, clientChan chan<- ClientMessage, stoppedChan <-chan struct{}) {
@@ -407,7 +409,7 @@ func getDeadline() time.Time {
 	return time.Now().Add(1 * time.Minute)
 }
 
-func closeConnection(conn *websocket.Conn, closeMsg websocket.CloseError, report *esConnectionReport) {
+func closeConnection(conn *websocket.Conn, closeMsg websocket.CloseError, report *logstasher.ConnectionReport) {
 	closeTxt := closeMsg.Text
 	if strings.Contains(closeTxt, "read: connection reset by peer") {
 		closeTxt = "read: connection reset by peer"
diff --git a/socketserver/server/logstash/elasticsearch.go b/socketserver/server/logstasher/elasticsearch.go
similarity index 98%
rename from socketserver/server/logstash/elasticsearch.go
rename to socketserver/server/logstasher/elasticsearch.go
index b8c88299..9eece14c 100644
--- a/socketserver/server/logstash/elasticsearch.go
+++ b/socketserver/server/logstasher/elasticsearch.go
@@ -1,4 +1,4 @@
-package server
+package logstasher
 
 import (
 	"bytes"
@@ -146,6 +146,7 @@ func checkServerPresent() {
 		resp.Body.Close()
 		serverPresent = true
 		submitChan = make(chan Report, 8)
+		fmt.Println("elasticsearch reports enabled")
 		go submissionWorker()
 	} else {
 		serverPresent = false

From 717acb3e40221b5b590af4932188304c9dc45b97 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 16 Dec 2015 11:48:37 -0800
Subject: [PATCH 095/176] Oh yeah, the config option

---
 socketserver/server/handlecore.go | 4 +++-
 socketserver/server/types.go      | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index d502d071..9fbb1af7 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -96,7 +96,9 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 		resp.Body.Close()
 	}
 
-	logstasher.Setup(Configuration.ESServer, Configuration.ESIndexPrefix, Configuration.ESHostName)
+	if Configuration.UseESLogStashing {
+		logstasher.Setup(Configuration.ESServer, Configuration.ESIndexPrefix, Configuration.ESHostName)
+	}
 
 	go authorizationJanitor()
 	go bunchCacheJanitor()
diff --git a/socketserver/server/types.go b/socketserver/server/types.go
index fe6193ba..6d7c68a9 100644
--- a/socketserver/server/types.go
+++ b/socketserver/server/types.go
@@ -31,7 +31,7 @@ type ConfigFile struct {
 	SSLCertificateFile string
 	SSLKeyFile         string
 
-	UseElasticSearch bool
+	UseESLogStashing bool
 	ESServer         string
 	ESIndexPrefix    string
 	ESHostName       string

From 405ad22372003397b86ea9c8285451a523ab8f22 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 16 Dec 2015 11:58:48 -0800
Subject: [PATCH 096/176] Fix a data race

---
 socketserver/server/commands.go   | 3 +--
 socketserver/server/handlecore.go | 1 +
 socketserver/server/stats.go      | 9 +++++++++
 3 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/socketserver/server/commands.go b/socketserver/server/commands.go
index 72fca25a..fe219ccf 100644
--- a/socketserver/server/commands.go
+++ b/socketserver/server/commands.go
@@ -50,8 +50,7 @@ func DispatchC2SCommand(conn *websocket.Conn, client *ClientInfo, msg ClientMess
 		handler = C2SHandleRemoteCommand
 	}
 
-	Statistics.CommandsIssuedTotal++
-	Statistics.CommandsIssuedMap[msg.Command]++
+	CommandCounter <- msg.Command
 
 	response, err := callHandler(handler, conn, client, msg)
 
diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 9fbb1af7..8a6ea4b6 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -104,6 +104,7 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 	go bunchCacheJanitor()
 	go pubsubJanitor()
 	go aggregateDataSender()
+	go commandCounter()
 
 	go ircConnection()
 	go shutdownHandler()
diff --git a/socketserver/server/stats.go b/socketserver/server/stats.go
index a64a315b..eecfceab 100644
--- a/socketserver/server/stats.go
+++ b/socketserver/server/stats.go
@@ -59,6 +59,15 @@ type StatsData struct {
 // I don't really care.
 var Statistics = newStatsData()
 
+var CommandCounter = make(chan Command, 10)
+
+func commandCounter() {
+	for cmd := range CommandCounter {
+		Statistics.CommandsIssuedTotal++
+		Statistics.CommandsIssuedMap[cmd]++
+	}
+}
+
 const StatsDataVersion = 5
 const pageSize = 4096
 

From 6b88b40538e768d68dffbca8c724ff357c4f06d7 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 16 Dec 2015 12:00:37 -0800
Subject: [PATCH 097/176] Fix data race, add stat

---
 socketserver/server/handlecore.go | 3 ++-
 socketserver/server/stats.go      | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 8a6ea4b6..92268b5f 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -160,6 +160,7 @@ func HTTPHandleRootURL(w http.ResponseWriter, r *http.Request) {
 		updateSysMem()
 
 		if Statistics.SysMemTotalKB-Statistics.SysMemFreeKB < Configuration.MinMemoryKBytes {
+			atomic.AddUint64(&Statistics.LowMemDroppedConnections, 1)
 			w.WriteHeader(503)
 			return
 		}
@@ -439,7 +440,7 @@ func SendMessage(conn *websocket.Conn, msg ClientMessage) {
 	}
 	conn.SetWriteDeadline(getDeadline())
 	conn.WriteMessage(messageType, packet)
-	Statistics.MessagesSent++
+	atomic.AddUint64(&Statistics.MessagesSent, 1)
 }
 
 // UnmarshalClientMessage unpacks websocket TextMessage into a ClientMessage provided in the `v` parameter.
diff --git a/socketserver/server/stats.go b/socketserver/server/stats.go
index eecfceab..7ba62f1b 100644
--- a/socketserver/server/stats.go
+++ b/socketserver/server/stats.go
@@ -30,6 +30,8 @@ type StatsData struct {
 	MemoryInUseKB uint64
 	MemoryRSSKB   uint64
 
+	LowMemDroppedConnections uint64
+
 	MemPerClientBytes uint64
 
 	CpuUsagePct float64

From 4b2449aecfc6267b324610057419269b34841f23 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 16 Dec 2015 12:47:19 -0800
Subject: [PATCH 098/176] Remove SocketOrigin config, edit url building

---
 socketserver/server/backend.go | 12 ++++++++----
 socketserver/server/types.go   |  2 --
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/socketserver/server/backend.go b/socketserver/server/backend.go
index b4582845..8601d2d0 100644
--- a/socketserver/server/backend.go
+++ b/socketserver/server/backend.go
@@ -18,6 +18,11 @@ import (
 	"time"
 )
 
+const bPathAnnounceStartup = "/startup"
+const bPathAddTopic = "/topics"
+const bPathAggStats = "/stats"
+const bPathOtherCommand = "/cmd/"
+
 var backendHTTPClient http.Client
 var backendURL string
 var responseCache *cache.Cache
@@ -39,9 +44,9 @@ func setupBackend(config *ConfigFile) {
 	}
 	responseCache = cache.New(60*time.Second, 120*time.Second)
 
-	postStatisticsURL = fmt.Sprintf("%s/stats", backendURL)
-	addTopicURL = fmt.Sprintf("%s/topics", backendURL)
-	announceStartupURL = fmt.Sprintf("%s/startup", backendURL)
+	announceStartupURL = fmt.Sprintf("%s%s", backendURL, bPathAnnounceStartup)
+	addTopicURL = fmt.Sprintf("%s%s", backendURL, bPathAddTopic)
+	postStatisticsURL = fmt.Sprintf("%s%s", backendURL, bPathAggStats)
 
 	messageBufferPool.New = New4KByteBuffer
 
@@ -270,7 +275,6 @@ func GenerateKeys(outputFile, serverID, theirPublicStr string) {
 	output := ConfigFile{
 		ListenAddr:      "0.0.0.0:8001",
 		SSLListenAddr:   "0.0.0.0:443",
-		SocketOrigin:    "localhost:8001",
 		BackendURL:      "http://localhost:8002/ffz",
 		MinMemoryKBytes: defaultMinMemoryKB,
 	}
diff --git a/socketserver/server/types.go b/socketserver/server/types.go
index 6d7c68a9..d827912c 100644
--- a/socketserver/server/types.go
+++ b/socketserver/server/types.go
@@ -18,8 +18,6 @@ type ConfigFile struct {
 	ServerID      int
 	ListenAddr    string
 	SSLListenAddr string
-	// Hostname of the socket server
-	SocketOrigin string
 	// URL to the backend server
 	BackendURL string
 

From 26747a1104ca4ed094d2282d95fc0364f853217e Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 16 Dec 2015 13:59:51 -0800
Subject: [PATCH 099/176] Fix - non-logged-in needs a reply if auth needed

Refactor for testing - all janitors have sleep in a different function
from the work, so tests can trigger them
---
 socketserver/server/commands.go      |  20 +++--
 socketserver/server/handlecore.go    |   6 ++
 socketserver/server/irc.go           | 125 ++++++++++++++-------------
 socketserver/server/subscriptions.go |  32 ++++---
 4 files changed, 104 insertions(+), 79 deletions(-)

diff --git a/socketserver/server/commands.go b/socketserver/server/commands.go
index fe219ccf..83dddb90 100644
--- a/socketserver/server/commands.go
+++ b/socketserver/server/commands.go
@@ -21,10 +21,10 @@ type Command string
 type CommandHandler func(*websocket.Conn, *ClientInfo, ClientMessage) (ClientMessage, error)
 
 var commandHandlers = map[Command]CommandHandler{
-	HelloCommand: C2SHello,
-	"ping":       C2SPing,
-	"setuser":    C2SSetUser,
-	"ready":      C2SReady,
+	HelloCommand:   C2SHello,
+	"ping":         C2SPing,
+	SetUserCommand: C2SSetUser,
+	ReadyCommand:   C2SReady,
 
 	"sub":   C2SSubscribe,
 	"unsub": C2SUnsubscribe,
@@ -143,7 +143,6 @@ func C2SSetUser(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rm
 		go client.StartAuthorization(func(_ *ClientInfo, _ bool) {
 			client.MsgChannelKeepalive.Done()
 		})
-
 	}
 
 	return ResponseSuccess, nil
@@ -316,11 +315,11 @@ func C2SEmoticonUses(conn *websocket.Conn, client *ClientInfo, msg ClientMessage
 func aggregateDataSender() {
 	for {
 		time.Sleep(5 * time.Minute)
-		doSendAggregateData()
+		aggregateDataSender_do()
 	}
 }
 
-func doSendAggregateData() {
+func aggregateDataSender_do() {
 	followEventsLock.Lock()
 	follows := followEvents
 	followEvents = nil
@@ -538,11 +537,18 @@ func C2SHandleRemoteCommand(conn *websocket.Conn, client *ClientInfo, msg Client
 }
 
 const AuthorizationFailedErrorString = "Failed to verify your Twitch username."
+const AuthorizationNeededError = "You must be signed in to use that command."
 
 func doRemoteCommand(conn *websocket.Conn, msg ClientMessage, client *ClientInfo) {
 	resp, err := SendRemoteCommandCached(string(msg.Command), msg.origArguments, client.AuthInfo)
 
 	if err == ErrAuthorizationNeeded {
+		if client.TwitchUsername == "" {
+			// Not logged in
+			client.MessageChannel <- ClientMessage{MessageID: msg.MessageID, Command: ErrorCommand, Arguments: AuthorizationNeededError}
+			client.MsgChannelKeepalive.Done()
+			return
+		}
 		client.StartAuthorization(func(_ *ClientInfo, success bool) {
 			if success {
 				doRemoteCommand(conn, msg, client)
diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 92268b5f..380d7549 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -34,6 +34,12 @@ const ErrorCommand Command = "error"
 // Sending any other command will result in a CloseFirstMessageNotHello.
 const HelloCommand Command = "hello"
 
+// ReadyCommand is a C2S Command.
+// It indicates that the client is finished sending the initial 'sub' commands and the server should send the backlog.
+const ReadyCommand Command = "ready"
+
+const SetUserCommand Command = "set_user"
+
 // AuthorizeCommand is a S2C Command sent as part of Twitch username validation.
 const AuthorizeCommand Command = "do_authorize"
 
diff --git a/socketserver/server/irc.go b/socketserver/server/irc.go
index a9a95edf..b92717d4 100644
--- a/socketserver/server/irc.go
+++ b/socketserver/server/irc.go
@@ -10,6 +10,7 @@ import (
 	"strings"
 	"sync"
 	"time"
+	"errors"
 )
 
 type AuthCallback func(client *ClientInfo, successful bool)
@@ -39,28 +40,31 @@ func AddPendingAuthorization(client *ClientInfo, challenge string, callback Auth
 func authorizationJanitor() {
 	for {
 		time.Sleep(5 * time.Minute)
-
-		func() {
-			cullTime := time.Now().Add(-30 * time.Minute)
-
-			PendingAuthLock.Lock()
-			defer PendingAuthLock.Unlock()
-
-			newPendingAuths := make([]PendingAuthorization, 0, len(PendingAuths))
-
-			for _, v := range PendingAuths {
-				if !cullTime.After(v.EnteredAt) {
-					newPendingAuths = append(newPendingAuths, v)
-				}
-			}
-
-			PendingAuths = newPendingAuths
-		}()
+		authorizationJanitor_do()
 	}
 }
 
+func authorizationJanitor_do() {
+	cullTime := time.Now().Add(-30 * time.Minute)
+
+	PendingAuthLock.Lock()
+	defer PendingAuthLock.Unlock()
+
+	newPendingAuths := make([]PendingAuthorization, 0, len(PendingAuths))
+
+	for _, v := range PendingAuths {
+		if !cullTime.After(v.EnteredAt) {
+			newPendingAuths = append(newPendingAuths, v)
+		} else {
+			v.Callback(v.Client, false)
+		}
+	}
+
+	PendingAuths = newPendingAuths
+}
+
 func (client *ClientInfo) StartAuthorization(callback AuthCallback) {
-	fmt.Println(DEBUG, "startig auth for user", client.TwitchUsername, client.RemoteAddr)
+	fmt.Println(DEBUG, "starting auth for user", client.TwitchUsername, client.RemoteAddr)
 	var nonce [32]byte
 	_, err := rand.Read(nonce[:])
 	if err != nil {
@@ -88,6 +92,8 @@ const AuthCommand = "AUTH"
 
 const DEBUG = "DEBUG"
 
+var errChallengeNotFound = errors.New("did not find a challenge solved by that message")
+
 func ircConnection() {
 
 	c := irc.SimpleClient("justinfan123")
@@ -113,46 +119,7 @@ func ircConnection() {
 		submittedUser := line.Nick
 		submittedChallenge := msgArray[1]
 
-		var auth PendingAuthorization
-		var idx int = -1
-
-		PendingAuthLock.Lock()
-		for i, v := range PendingAuths {
-			if v.Client.TwitchUsername == submittedUser && v.Challenge == submittedChallenge {
-				auth = v
-				idx = i
-				break
-			}
-		}
-		if idx != -1 {
-			PendingAuths = append(PendingAuths[:idx], PendingAuths[idx+1:]...)
-		}
-		PendingAuthLock.Unlock()
-
-		if idx == -1 {
-			fmt.Println(DEBUG, "discarded msg - challenge not found", line.Raw)
-			return
-		}
-
-		// auth is valid, and removed from pending list
-
-		fmt.Println(DEBUG, "authorization success for user", auth.Client.TwitchUsername)
-		var usernameChanged bool
-		auth.Client.Mutex.Lock()
-		if auth.Client.TwitchUsername == submittedUser { // recheck condition
-			auth.Client.UsernameValidated = true
-		} else {
-			usernameChanged = true
-		}
-		auth.Client.Mutex.Unlock()
-
-		if auth.Callback != nil {
-			if !usernameChanged {
-				auth.Callback(auth.Client, true)
-			} else {
-				auth.Callback(auth.Client, false)
-			}
-		}
+		submitAuth(submittedUser, submittedChallenge)
 	})
 
 	err := c.ConnectTo("irc.twitch.tv")
@@ -161,3 +128,45 @@ func ircConnection() {
 	}
 
 }
+
+func submitAuth(user, challenge string) error {
+	var auth PendingAuthorization
+	var idx int = -1
+
+	PendingAuthLock.Lock()
+	for i, v := range PendingAuths {
+		if v.Client.TwitchUsername == user && v.Challenge == challenge {
+			auth = v
+			idx = i
+			break
+		}
+	}
+	if idx != -1 {
+		PendingAuths = append(PendingAuths[:idx], PendingAuths[idx+1:]...)
+	}
+	PendingAuthLock.Unlock()
+
+	if idx == -1 {
+		return errChallengeNotFound // perhaps it was for another socket server
+	}
+
+	// auth is valid, and removed from pending list
+
+	fmt.Println(DEBUG, "authorization success for user", auth.Client.TwitchUsername)
+	var usernameChanged bool
+	auth.Client.Mutex.Lock()
+	if auth.Client.TwitchUsername == user { // recheck condition
+		auth.Client.UsernameValidated = true
+	} else {
+		usernameChanged = true
+	}
+	auth.Client.Mutex.Unlock()
+
+	if auth.Callback != nil {
+		if !usernameChanged {
+			auth.Callback(auth.Client, true)
+		} else {
+			auth.Callback(auth.Client, false)
+		}
+	}
+}
\ No newline at end of file
diff --git a/socketserver/server/subscriptions.go b/socketserver/server/subscriptions.go
index 3a139165..8609e7c6 100644
--- a/socketserver/server/subscriptions.go
+++ b/socketserver/server/subscriptions.go
@@ -149,21 +149,25 @@ const ReapingDelay = 1 * time.Minute
 func pubsubJanitor() {
 	for {
 		time.Sleep(ReapingDelay)
-		var cleanedUp = make([]string, 0, 6)
-		ChatSubscriptionLock.Lock()
-		for key, val := range ChatSubscriptionInfo {
-			if val == nil || len(val.Members) == 0 {
-				delete(ChatSubscriptionInfo, key)
-				cleanedUp = append(cleanedUp, key)
-			}
-		}
-		ChatSubscriptionLock.Unlock()
+		pubsubJanitor_do()
+	}
+}
 
-		if len(cleanedUp) != 0 {
-			err := SendCleanupTopicsNotice(cleanedUp)
-			if err != nil {
-				log.Println("error reporting cleaned subs:", err)
-			}
+func pubsubJanitor_do() {
+	var cleanedUp = make([]string, 0, 6)
+	ChatSubscriptionLock.Lock()
+	for key, val := range ChatSubscriptionInfo {
+		if val == nil || len(val.Members) == 0 {
+			delete(ChatSubscriptionInfo, key)
+			cleanedUp = append(cleanedUp, key)
+		}
+	}
+	ChatSubscriptionLock.Unlock()
+
+	if len(cleanedUp) != 0 {
+		err := SendCleanupTopicsNotice(cleanedUp)
+		if err != nil {
+			log.Println("error reporting cleaned subs:", err)
 		}
 	}
 }

From c0500457a65f028f906cffbb158d9e80f83695c7 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 16 Dec 2015 14:14:19 -0800
Subject: [PATCH 100/176] [BREAKING] Add tests for username auth, fix form key
 names

Also, testing infrastructure
---
 socketserver/server/backend.go            |   4 +-
 socketserver/server/backend_test.go       | 110 ++++++++++++++++
 socketserver/server/irc.go                |  11 +-
 socketserver/server/subscriptions_test.go | 149 ++++++++++++++++++++--
 4 files changed, 252 insertions(+), 22 deletions(-)

diff --git a/socketserver/server/backend.go b/socketserver/server/backend.go
index 8601d2d0..ea986371 100644
--- a/socketserver/server/backend.go
+++ b/socketserver/server/backend.go
@@ -142,9 +142,9 @@ func SendRemoteCommand(remoteCommand, data string, auth AuthInfo) (responseStr s
 	destURL := fmt.Sprintf("%s/cmd/%s", backendURL, remoteCommand)
 	var authKey string
 	if auth.UsernameValidated {
-		authKey = "usernameClaimed"
+		authKey = "usernameVerified"
 	} else {
-		authKey = "username"
+		authKey = "usernameClaimed"
 	}
 
 	formData := url.Values{
diff --git a/socketserver/server/backend_test.go b/socketserver/server/backend_test.go
index 7043d9f3..7a252f83 100644
--- a/socketserver/server/backend_test.go
+++ b/socketserver/server/backend_test.go
@@ -2,7 +2,9 @@ package server
 
 import (
 	"crypto/rand"
+	"fmt"
 	"golang.org/x/crypto/nacl/box"
+	"net/http"
 	"net/url"
 	"testing"
 )
@@ -44,3 +46,111 @@ func TestSealRequest(t *testing.T) {
 		t.Errorf("Failed to round-trip, got back %v", unsealedValues)
 	}
 }
+
+const MethodIsPost = "POST"
+
+type ExpectedBackendRequest struct {
+	ResponseCode int
+	Path         string
+	// Method       string // always POST
+	PostForm *url.Values
+	Response string
+}
+
+func (er *ExpectedBackendRequest) String() string {
+	if MethodIsPost == "" {
+		return er.Path
+	}
+	return fmt.Sprint("%s %s: %s", MethodIsPost, er.Path, er.PostForm.Encode())
+}
+
+type BackendRequestChecker struct {
+	ExpectedRequests []ExpectedBackendRequest
+
+	currentRequest int
+	tb             testing.TB
+}
+
+func NewBackendRequestChecker(tb testing.TB, urls ...ExpectedBackendRequest) *BackendRequestChecker {
+	return &BackendRequestChecker{ExpectedRequests: urls, tb: tb, currentRequest: 0}
+}
+
+func (backend *BackendRequestChecker) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.Method != MethodIsPost {
+		backend.tb.Errorf("Bad backend request: was not a POST. %v", r)
+		return
+	}
+
+	r.ParseForm()
+
+	unsealedForm, err := UnsealRequest(r.PostForm)
+	if err != nil {
+		backend.tb.Errorf("Failed to unseal backend request: %v", err)
+	}
+
+	if backend.currentRequest >= len(backend.ExpectedRequests) {
+		backend.tb.Errorf("Unexpected backend request: %s %s: %s", r.Method, r.URL, unsealedForm)
+		return
+	}
+
+	cur := backend.ExpectedRequests[backend.currentRequest]
+	backend.currentRequest++
+
+	defer func() {
+		w.WriteHeader(cur.ResponseCode)
+		if cur.Response != "" {
+			w.Write([]byte(cur.Response))
+		}
+	}()
+
+	if cur.Path != "" {
+		if r.URL.Path != cur.Path {
+			backend.tb.Errorf("Bad backend request. Expected %v, got %s %s", cur, r.Method, r.URL)
+			return
+		}
+	}
+
+	if cur.PostForm != nil {
+		anyErr := compareForms(backend.tb, "Different form contents", *cur.PostForm, unsealedForm)
+		if anyErr {
+			backend.tb.Errorf("...in %s %s: %s", r.Method, r.URL, unsealedForm.Encode())
+		}
+	}
+}
+
+func (backend *BackendRequestChecker) Close() error {
+	if backend.currentRequest < len(backend.ExpectedRequests) {
+		backend.tb.Errorf("Not all requests sent, got %d out of %d", backend.currentRequest, len(backend.ExpectedRequests))
+	}
+	return nil
+}
+
+func compareForms(tb testing.TB, ctx string, expectedForm, gotForm url.Values) (anyErrors bool) {
+	for k, expVal := range expectedForm {
+		gotVal, ok := gotForm[k]
+		if !ok {
+			tb.Errorf("%s: Form[%s]: Expected %v, (got nothing)", ctx, k, expVal)
+			anyErrors = true
+			continue
+		}
+		if len(expVal) != len(gotVal) {
+			tb.Errorf("%s: Form[%s]: Expected %d%v, Got %d%v", ctx, k, len(expVal), expVal, len(gotVal), gotVal)
+			anyErrors = true
+			continue
+		}
+		for i, el := range expVal {
+			if gotVal[i] != el {
+				tb.Errorf("%s: Form[%s][%d]: Expected %s, Got %s", ctx, k, i, el, gotVal[i])
+				anyErrors = true
+			}
+		}
+	}
+	for k, gotVal := range gotForm {
+		_, ok := expectedForm[k]
+		if !ok {
+			tb.Errorf("%s: Form[%s]: (expected nothing), Got %v", ctx, k, gotVal)
+			anyErrors = true
+		}
+	}
+	return anyErrors
+}
diff --git a/socketserver/server/irc.go b/socketserver/server/irc.go
index b92717d4..6834652e 100644
--- a/socketserver/server/irc.go
+++ b/socketserver/server/irc.go
@@ -4,13 +4,13 @@ import (
 	"bytes"
 	"crypto/rand"
 	"encoding/base64"
+	"errors"
 	"fmt"
 	irc "github.com/fluffle/goirc/client"
 	"log"
 	"strings"
 	"sync"
 	"time"
-	"errors"
 )
 
 type AuthCallback func(client *ClientInfo, successful bool)
@@ -106,13 +106,11 @@ func ircConnection() {
 		channel := line.Args[0]
 		msg := line.Args[1]
 		if channel != AuthChannel || !strings.HasPrefix(msg, AuthCommand) || !line.Public() {
-			fmt.Println(DEBUG, "discarded msg", line.Raw)
 			return
 		}
 
 		msgArray := strings.Split(msg, " ")
 		if len(msgArray) != 2 {
-			fmt.Println(DEBUG, "discarded msg - not 2 strings", line.Raw)
 			return
 		}
 
@@ -129,7 +127,7 @@ func ircConnection() {
 
 }
 
-func submitAuth(user, challenge string) error {
+func submitAuth(user, challenge string) {
 	var auth PendingAuthorization
 	var idx int = -1
 
@@ -147,12 +145,11 @@ func submitAuth(user, challenge string) error {
 	PendingAuthLock.Unlock()
 
 	if idx == -1 {
-		return errChallengeNotFound // perhaps it was for another socket server
+		return // perhaps it was for another socket server
 	}
 
 	// auth is valid, and removed from pending list
 
-	fmt.Println(DEBUG, "authorization success for user", auth.Client.TwitchUsername)
 	var usernameChanged bool
 	auth.Client.Mutex.Lock()
 	if auth.Client.TwitchUsername == user { // recheck condition
@@ -169,4 +166,4 @@ func submitAuth(user, challenge string) error {
 			auth.Callback(auth.Client, false)
 		}
 	}
-}
\ No newline at end of file
+}
diff --git a/socketserver/server/subscriptions_test.go b/socketserver/server/subscriptions_test.go
index a9a09ac0..7a50b31d 100644
--- a/socketserver/server/subscriptions_test.go
+++ b/socketserver/server/subscriptions_test.go
@@ -147,8 +147,8 @@ type TURLs struct {
 	SavePubMsg     string // cached_pub
 }
 
-func TGetUrls(testserver *httptest.Server) TURLs {
-	addr := testserver.Listener.Addr().String()
+func TGetUrls(socketserver *httptest.Server, backend *httptest.Server) TURLs {
+	addr := socketserver.Listener.Addr().String()
 	return TURLs{
 		Websocket:      fmt.Sprintf("ws://%s/", addr),
 		Origin:         fmt.Sprintf("http://%s", addr),
@@ -157,7 +157,13 @@ func TGetUrls(testserver *httptest.Server) TURLs {
 	}
 }
 
-func TSetup(testserver **httptest.Server, urls *TURLs) {
+const (
+	SetupWantSocketServer = 1 << iota
+	SetupWantBackendServer
+	SetupWantURLs
+)
+
+func TSetup(flags int, backendChecker *BackendRequestChecker) (socketserver *httptest.Server, backend *httptest.Server, urls TURLs) {
 	DumpBacklogData()
 
 	ioutil.WriteFile("index.html", []byte(`
@@ -174,28 +180,34 @@ func TSetup(testserver **httptest.Server, urls *TURLs) {
 	&mdash; CatBag by <a href="http://www.twitch.tv/wolsk">Wolsk</a>
 </div>
 </div>`), 0600)
+
 	conf := &ConfigFile{
 		ServerID:         20,
 		UseSSL:           false,
-		SocketOrigin:     "localhost:2002",
 		OurPublicKey:     []byte{176, 149, 72, 209, 35, 42, 110, 220, 22, 236, 212, 129, 213, 199, 1, 227, 185, 167, 150, 159, 117, 202, 164, 100, 9, 107, 45, 141, 122, 221, 155, 73},
 		OurPrivateKey:    []byte{247, 133, 147, 194, 70, 240, 211, 216, 223, 16, 241, 253, 120, 14, 198, 74, 237, 180, 89, 33, 146, 146, 140, 58, 88, 160, 2, 246, 112, 35, 239, 87},
 		BackendPublicKey: []byte{19, 163, 37, 157, 50, 139, 193, 85, 229, 47, 166, 21, 153, 231, 31, 133, 41, 158, 8, 53, 73, 0, 113, 91, 13, 181, 131, 248, 176, 18, 1, 107},
 	}
+
+	if flags&SetupWantBackendServer != 0 {
+		backend = httptest.NewServer(backendChecker)
+		conf.BackendURL = fmt.Sprintf("http://%s", backend.Listener.Addr().String())
+	}
+
 	Configuration = conf
 	setupBackend(conf)
 
-	if testserver != nil {
+	if flags&SetupWantSocketServer != 0 {
 		serveMux := http.NewServeMux()
 		SetupServerAndHandle(conf, serveMux)
 
-		tserv := httptest.NewUnstartedServer(serveMux)
-		*testserver = tserv
-		tserv.Start()
-		if urls != nil {
-			*urls = TGetUrls(tserv)
-		}
+		socketserver = httptest.NewServer(serveMux)
 	}
+
+	if flags&SetupWantURLs != 0 {
+		urls = TGetUrls(socketserver, backend)
+	}
+	return
 }
 
 func TestSubscriptionAndPublish(t *testing.T) {
@@ -220,9 +232,18 @@ func TestSubscriptionAndPublish(t *testing.T) {
 
 	var server *httptest.Server
 	var urls TURLs
-	TSetup(&server, &urls)
+
+	var backendExpected = NewBackendRequestChecker(t,
+		ExpectedBackendRequest{200, bPathAnnounceStartup, &url.Values{"startup": []string{"1"}}, ""},
+		ExpectedBackendRequest{200, bPathAddTopic, &url.Values{"channels": []string{TestChannelName1}, "added": []string{"t"}}, "ok"},
+		ExpectedBackendRequest{200, bPathAddTopic, &url.Values{"channels": []string{TestChannelName2}, "added": []string{"t"}}, "ok"},
+		ExpectedBackendRequest{200, bPathAddTopic, &url.Values{"channels": []string{TestChannelName3}, "added": []string{"t"}}, "ok"},
+	)
+	server, _, urls = TSetup(SetupWantSocketServer|SetupWantBackendServer|SetupWantURLs, backendExpected)
+
 	defer server.CloseClientConnections()
 	defer unsubscribeAllClients()
+	defer backendExpected.Close()
 
 	var conn *websocket.Conn
 	var resp *http.Response
@@ -277,6 +298,7 @@ func TestSubscriptionAndPublish(t *testing.T) {
 	}
 
 	doneWg.Add(1)
+	readyWg.Wait() // enforce ordering
 	readyWg.Add(1)
 	go func(conn *websocket.Conn) {
 		TSendMessage(t, conn, 1, HelloCommand, []interface{}{"ffz_0.0-test", uuid.NewV4().String()})
@@ -306,6 +328,7 @@ func TestSubscriptionAndPublish(t *testing.T) {
 	}
 
 	doneWg.Add(1)
+	readyWg.Wait() // enforce ordering
 	readyWg.Add(1)
 	go func(conn *websocket.Conn) {
 		TSendMessage(t, conn, 1, HelloCommand, []interface{}{"ffz_0.0-test", uuid.NewV4().String()})
@@ -402,6 +425,106 @@ func TestSubscriptionAndPublish(t *testing.T) {
 	server.Close()
 }
 
+func TestRestrictedCommands(t *testing.T) {
+	var doneWg sync.WaitGroup
+	var readyWg sync.WaitGroup
+
+	const TestCommandNeedsAuth = "needsauth"
+	const TestRequestData = "123456"
+	const TestRequestDataJSON = "\"" + TestRequestData + "\""
+	const TestReplyData = "success"
+	const TestUsername = "sirstendec"
+
+	var server *httptest.Server
+	var urls TURLs
+
+	var backendExpected = NewBackendRequestChecker(t,
+		ExpectedBackendRequest{200, bPathAnnounceStartup, &url.Values{"startup": []string{"1"}}, ""},
+		ExpectedBackendRequest{401, fmt.Sprintf("%s%s", bPathOtherCommand, TestCommandNeedsAuth), &url.Values{"usernameClaimed": []string{""}, "clientData": []string{TestRequestDataJSON}}, ""},
+		ExpectedBackendRequest{401, fmt.Sprintf("%s%s", bPathOtherCommand, TestCommandNeedsAuth), &url.Values{"usernameClaimed": []string{TestUsername}, "clientData": []string{TestRequestDataJSON}}, ""},
+		ExpectedBackendRequest{200, fmt.Sprintf("%s%s", bPathOtherCommand, TestCommandNeedsAuth), &url.Values{"usernameVerified": []string{TestUsername}, "clientData": []string{TestRequestDataJSON}}, fmt.Sprintf("\"%s\"", TestReplyData)},
+	)
+	server, _, urls = TSetup(SetupWantSocketServer|SetupWantBackendServer|SetupWantURLs, backendExpected)
+
+	defer server.CloseClientConnections()
+	defer unsubscribeAllClients()
+	defer backendExpected.Close()
+
+	var conn *websocket.Conn
+	var err error
+	var challengeChan = make(chan string)
+
+	var headers http.Header = make(http.Header)
+	headers.Set("Origin", TwitchDotTv)
+
+	// Client 1
+	conn, _, err = websocket.DefaultDialer.Dial(urls.Websocket, headers)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	doneWg.Add(1)
+	readyWg.Add(1)
+	go func(conn *websocket.Conn) {
+		defer doneWg.Done()
+		defer conn.Close()
+		TSendMessage(t, conn, 1, HelloCommand, []interface{}{"ffz_0.0-test", uuid.NewV4().String()})
+		TReceiveExpectedMessage(t, conn, 1, SuccessCommand, IgnoreReceivedArguments)
+		TSendMessage(t, conn, 2, ReadyCommand, 0)
+		TReceiveExpectedMessage(t, conn, 2, SuccessCommand, nil)
+
+		// Should get immediate refusal because no username set
+		TSendMessage(t, conn, 3, TestCommandNeedsAuth, TestRequestData)
+		TReceiveExpectedMessage(t, conn, 3, ErrorCommand, AuthorizationNeededError)
+
+		// Set a username
+		TSendMessage(t, conn, 4, SetUserCommand, TestUsername)
+		TReceiveExpectedMessage(t, conn, 4, SuccessCommand, nil)
+
+		// Should get authorization prompt
+		TSendMessage(t, conn, 5, TestCommandNeedsAuth, TestRequestData)
+		readyWg.Done()
+		msg, success := TReceiveExpectedMessage(t, conn, -1, AuthorizeCommand, IgnoreReceivedArguments)
+		if !success {
+			t.Error("recieve authorize command failed, cannot continue")
+			return
+		}
+		challenge, err := msg.ArgumentsAsString()
+		if err != nil {
+			t.Error(err)
+			return
+		}
+		challengeChan <- challenge // mocked: sending challenge to IRC server, IRC server sends challenge to socket server
+
+		TReceiveExpectedMessage(t, conn, 5, SuccessCommand, TestReplyData)
+	}(conn)
+
+	readyWg.Wait()
+
+	challenge := <-challengeChan
+	PendingAuthLock.Lock()
+	found := false
+	for _, v := range PendingAuths {
+		if conn.LocalAddr().String() == v.Client.RemoteAddr.String() {
+			found = true
+			if v.Challenge != challenge {
+				t.Error("Challenge in array was not what client got")
+			}
+			break
+		}
+	}
+	PendingAuthLock.Unlock()
+	if !found {
+		t.Fatal("Did not find authorization challenge in the pending auths array")
+	}
+
+	submitAuth(TestUsername, challenge)
+
+	doneWg.Wait()
+	server.Close()
+}
+
 func BenchmarkUserSubscriptionSinglePublish(b *testing.B) {
 	var doneWg sync.WaitGroup
 	var readyWg sync.WaitGroup
@@ -429,7 +552,7 @@ func BenchmarkUserSubscriptionSinglePublish(b *testing.B) {
 
 	var server *httptest.Server
 	var urls TURLs
-	TSetup(&server, &urls)
+	server, _, urls = TSetup(SetupWantSocketServer|SetupWantURLs, nil)
 	defer unsubscribeAllClients()
 
 	var headers http.Header = make(http.Header)

From b236c34c945e430665f83ce40d04f6988efdfeae Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 16 Dec 2015 14:19:23 -0800
Subject: [PATCH 101/176] Don't send IPs and usernames until I know how to
 delete them

---
 socketserver/server/handlecore.go               | 10 ++++++----
 socketserver/server/logstasher/elasticsearch.go |  5 ++++-
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 380d7549..46c0e1ce 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -265,10 +265,6 @@ func RunSocketConnection(conn *websocket.Conn) {
 	// Close the connection when we're done.
 	defer closer()
 
-	var report logstasher.ConnectionReport
-	report.ConnectTime = time.Now()
-	report.RemoteAddr = conn.RemoteAddr()
-
 	_clientChan := make(chan ClientMessage)
 	_serverMessageChan := make(chan ClientMessage, sendMessageBufferLength)
 	_errorChan := make(chan error)
@@ -279,6 +275,10 @@ func RunSocketConnection(conn *websocket.Conn) {
 	client.RemoteAddr = conn.RemoteAddr()
 	client.MsgChannelIsDone = stoppedChan
 
+	var report logstasher.ConnectionReport
+	report.ConnectTime = time.Now()
+	report.RemoteAddr = client.RemoteAddr
+
 	conn.SetPongHandler(func(pongBody string) error {
 		client.Mutex.Lock()
 		client.pingCount = 0
@@ -318,6 +318,8 @@ func RunSocketConnection(conn *websocket.Conn) {
 		atomic.AddUint64(&Statistics.CurrentClientCount, NegativeOne)
 	}
 
+	report.UsernameWasValidated = client.UsernameValidated
+	report.TwitchUsername = client.TwitchUsername
 	logstasher.Submit(&report)
 }
 
diff --git a/socketserver/server/logstasher/elasticsearch.go b/socketserver/server/logstasher/elasticsearch.go
index 9eece14c..c505e00e 100644
--- a/socketserver/server/logstasher/elasticsearch.go
+++ b/socketserver/server/logstasher/elasticsearch.go
@@ -113,7 +113,10 @@ type ConnectionReport struct {
 	DisconnectCode   int
 	DisconnectReason string
 
-	RemoteAddr net.Addr
+	UsernameWasValidated bool
+
+	RemoteAddr     net.Addr `json:"-"` // not transmitted until I can figure out data minimization
+	TwitchUsername string   `json:"-"` // also not transmitted
 }
 
 // FillReport sets all the calculated fields, and calls esReportBasic.FillReport().

From 9f5eb5818ae0c852b87bcbb6770cf3709448e8f9 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 16 Dec 2015 14:20:50 -0800
Subject: [PATCH 102/176] Remove cruft code that was double-closing connections

---
 socketserver/server/handlecore.go | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 46c0e1ce..e28afaa6 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -255,16 +255,6 @@ func RunSocketConnection(conn *websocket.Conn) {
 	atomic.AddUint64(&Statistics.ClientConnectsTotal, 1)
 	atomic.AddUint64(&Statistics.CurrentClientCount, 1)
 
-	var _closer sync.Once
-	closer := func() {
-		_closer.Do(func() {
-			conn.Close()
-		})
-	}
-
-	// Close the connection when we're done.
-	defer closer()
-
 	_clientChan := make(chan ClientMessage)
 	_serverMessageChan := make(chan ClientMessage, sendMessageBufferLength)
 	_errorChan := make(chan error)

From 6f612f881eaf7fe07841a4aaef91a77fd34d8231 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 16 Dec 2015 14:23:28 -0800
Subject: [PATCH 103/176] Fix race condition during testing

---
 socketserver/server/handlecore.go | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index e28afaa6..6e5ced54 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -59,6 +59,8 @@ var ResponseSuccess = ClientMessage{Command: SuccessCommand}
 // Configuration is the active ConfigFile.
 var Configuration *ConfigFile
 
+var janitorsOnce sync.Once
+
 // SetupServerAndHandle starts all background goroutines and registers HTTP listeners on the given ServeMux.
 // Essentially, this function completely preps the server for a http.ListenAndServe call.
 // (Uses http.DefaultServeMux if `serveMux` is nil.)
@@ -106,14 +108,16 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 		logstasher.Setup(Configuration.ESServer, Configuration.ESIndexPrefix, Configuration.ESHostName)
 	}
 
-	go authorizationJanitor()
-	go bunchCacheJanitor()
-	go pubsubJanitor()
-	go aggregateDataSender()
-	go commandCounter()
+	janitorsOnce.Do(func() {
+		go authorizationJanitor()
+		go bunchCacheJanitor()
+		go pubsubJanitor()
+		go aggregateDataSender()
+		go commandCounter()
 
-	go ircConnection()
-	go shutdownHandler()
+		go ircConnection()
+		go shutdownHandler()
+	})
 }
 
 func shutdownHandler() {

From c6891425d5c66d8f354beca72086387fa159c7dc Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 16 Dec 2015 14:24:10 -0800
Subject: [PATCH 104/176] kill debug prints

---
 socketserver/server/irc.go | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/socketserver/server/irc.go b/socketserver/server/irc.go
index 6834652e..e0c2fae5 100644
--- a/socketserver/server/irc.go
+++ b/socketserver/server/irc.go
@@ -64,7 +64,6 @@ func authorizationJanitor_do() {
 }
 
 func (client *ClientInfo) StartAuthorization(callback AuthCallback) {
-	fmt.Println(DEBUG, "starting auth for user", client.TwitchUsername, client.RemoteAddr)
 	var nonce [32]byte
 	_, err := rand.Read(nonce[:])
 	if err != nil {
@@ -79,10 +78,8 @@ func (client *ClientInfo) StartAuthorization(callback AuthCallback) {
 	enc.Close()
 	challenge := buf.String()
 
-	fmt.Println(DEBUG, "adding to auth array")
 	AddPendingAuthorization(client, challenge, callback)
 
-	fmt.Println(DEBUG, "sending auth message")
 	client.MessageChannel <- ClientMessage{MessageID: -1, Command: AuthorizeCommand, Arguments: challenge}
 }
 

From 7552a91b3d92e03dd6f3f04030339a4d3d5f53f1 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 16 Dec 2015 14:25:23 -0800
Subject: [PATCH 105/176] derp, unused import..

---
 socketserver/server/irc.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/socketserver/server/irc.go b/socketserver/server/irc.go
index e0c2fae5..6e4df282 100644
--- a/socketserver/server/irc.go
+++ b/socketserver/server/irc.go
@@ -5,7 +5,6 @@ import (
 	"crypto/rand"
 	"encoding/base64"
 	"errors"
-	"fmt"
 	irc "github.com/fluffle/goirc/client"
 	"log"
 	"strings"

From 4509a4245f11da5d793c44981cf16e67865e64ed Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 16 Dec 2015 14:31:29 -0800
Subject: [PATCH 106/176] Move all testing infra to the same file

---
 socketserver/server/backend_test.go       | 126 --------
 socketserver/server/subscriptions_test.go | 217 +-------------
 socketserver/server/testinfra_test.go     | 331 ++++++++++++++++++++++
 3 files changed, 341 insertions(+), 333 deletions(-)
 create mode 100644 socketserver/server/testinfra_test.go

diff --git a/socketserver/server/backend_test.go b/socketserver/server/backend_test.go
index 7a252f83..266aea8e 100644
--- a/socketserver/server/backend_test.go
+++ b/socketserver/server/backend_test.go
@@ -1,28 +1,10 @@
 package server
 
 import (
-	"crypto/rand"
-	"fmt"
-	"golang.org/x/crypto/nacl/box"
-	"net/http"
 	"net/url"
 	"testing"
 )
 
-func SetupRandomKeys(t testing.TB) {
-	_, senderPrivate, err := box.GenerateKey(rand.Reader)
-	if err != nil {
-		t.Fatal(err)
-	}
-	receiverPublic, _, err := box.GenerateKey(rand.Reader)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	box.Precompute(&backendSharedKey, receiverPublic, senderPrivate)
-	messageBufferPool.New = New4KByteBuffer
-}
-
 func TestSealRequest(t *testing.T) {
 	SetupRandomKeys(t)
 
@@ -46,111 +28,3 @@ func TestSealRequest(t *testing.T) {
 		t.Errorf("Failed to round-trip, got back %v", unsealedValues)
 	}
 }
-
-const MethodIsPost = "POST"
-
-type ExpectedBackendRequest struct {
-	ResponseCode int
-	Path         string
-	// Method       string // always POST
-	PostForm *url.Values
-	Response string
-}
-
-func (er *ExpectedBackendRequest) String() string {
-	if MethodIsPost == "" {
-		return er.Path
-	}
-	return fmt.Sprint("%s %s: %s", MethodIsPost, er.Path, er.PostForm.Encode())
-}
-
-type BackendRequestChecker struct {
-	ExpectedRequests []ExpectedBackendRequest
-
-	currentRequest int
-	tb             testing.TB
-}
-
-func NewBackendRequestChecker(tb testing.TB, urls ...ExpectedBackendRequest) *BackendRequestChecker {
-	return &BackendRequestChecker{ExpectedRequests: urls, tb: tb, currentRequest: 0}
-}
-
-func (backend *BackendRequestChecker) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	if r.Method != MethodIsPost {
-		backend.tb.Errorf("Bad backend request: was not a POST. %v", r)
-		return
-	}
-
-	r.ParseForm()
-
-	unsealedForm, err := UnsealRequest(r.PostForm)
-	if err != nil {
-		backend.tb.Errorf("Failed to unseal backend request: %v", err)
-	}
-
-	if backend.currentRequest >= len(backend.ExpectedRequests) {
-		backend.tb.Errorf("Unexpected backend request: %s %s: %s", r.Method, r.URL, unsealedForm)
-		return
-	}
-
-	cur := backend.ExpectedRequests[backend.currentRequest]
-	backend.currentRequest++
-
-	defer func() {
-		w.WriteHeader(cur.ResponseCode)
-		if cur.Response != "" {
-			w.Write([]byte(cur.Response))
-		}
-	}()
-
-	if cur.Path != "" {
-		if r.URL.Path != cur.Path {
-			backend.tb.Errorf("Bad backend request. Expected %v, got %s %s", cur, r.Method, r.URL)
-			return
-		}
-	}
-
-	if cur.PostForm != nil {
-		anyErr := compareForms(backend.tb, "Different form contents", *cur.PostForm, unsealedForm)
-		if anyErr {
-			backend.tb.Errorf("...in %s %s: %s", r.Method, r.URL, unsealedForm.Encode())
-		}
-	}
-}
-
-func (backend *BackendRequestChecker) Close() error {
-	if backend.currentRequest < len(backend.ExpectedRequests) {
-		backend.tb.Errorf("Not all requests sent, got %d out of %d", backend.currentRequest, len(backend.ExpectedRequests))
-	}
-	return nil
-}
-
-func compareForms(tb testing.TB, ctx string, expectedForm, gotForm url.Values) (anyErrors bool) {
-	for k, expVal := range expectedForm {
-		gotVal, ok := gotForm[k]
-		if !ok {
-			tb.Errorf("%s: Form[%s]: Expected %v, (got nothing)", ctx, k, expVal)
-			anyErrors = true
-			continue
-		}
-		if len(expVal) != len(gotVal) {
-			tb.Errorf("%s: Form[%s]: Expected %d%v, Got %d%v", ctx, k, len(expVal), expVal, len(gotVal), gotVal)
-			anyErrors = true
-			continue
-		}
-		for i, el := range expVal {
-			if gotVal[i] != el {
-				tb.Errorf("%s: Form[%s][%d]: Expected %s, Got %s", ctx, k, i, el, gotVal[i])
-				anyErrors = true
-			}
-		}
-	}
-	for k, gotVal := range gotForm {
-		_, ok := expectedForm[k]
-		if !ok {
-			tb.Errorf("%s: Form[%s]: (expected nothing), Got %v", ctx, k, gotVal)
-			anyErrors = true
-		}
-	}
-	return anyErrors
-}
diff --git a/socketserver/server/subscriptions_test.go b/socketserver/server/subscriptions_test.go
index 7a50b31d..a12bddc3 100644
--- a/socketserver/server/subscriptions_test.go
+++ b/socketserver/server/subscriptions_test.go
@@ -1,215 +1,18 @@
 package server
 
 import (
-	"encoding/json"
 	"fmt"
 	"github.com/gorilla/websocket"
 	"github.com/satori/go.uuid"
-	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
-	"os"
 	"strconv"
 	"sync"
 	"syscall"
 	"testing"
-	"time"
 )
 
-func TCountOpenFDs() uint64 {
-	ary, _ := ioutil.ReadDir(fmt.Sprintf("/proc/%d/fd", os.Getpid()))
-	return uint64(len(ary))
-}
-
-const IgnoreReceivedArguments = 1 + 2i
-
-func TReceiveExpectedMessage(tb testing.TB, conn *websocket.Conn, messageID int, command Command, arguments interface{}) (ClientMessage, bool) {
-	var msg ClientMessage
-	var fail bool
-	messageType, packet, err := conn.ReadMessage()
-	if err != nil {
-		tb.Error(err)
-		return msg, false
-	}
-	if messageType != websocket.TextMessage {
-		tb.Error("got non-text message", packet)
-		return msg, false
-	}
-
-	err = UnmarshalClientMessage(packet, messageType, &msg)
-	if err != nil {
-		tb.Error(err)
-		return msg, false
-	}
-	if msg.MessageID != messageID {
-		tb.Error("Message ID was wrong. Expected", messageID, ", got", msg.MessageID, ":", msg)
-		fail = true
-	}
-	if msg.Command != command {
-		tb.Error("Command was wrong. Expected", command, ", got", msg.Command, ":", msg)
-		fail = true
-	}
-	if arguments != IgnoreReceivedArguments {
-		if arguments == nil {
-			if msg.origArguments != "" {
-				tb.Error("Arguments are wrong. Expected", arguments, ", got", msg.Arguments, ":", msg)
-			}
-		} else {
-			argBytes, _ := json.Marshal(arguments)
-			if msg.origArguments != string(argBytes) {
-				tb.Error("Arguments are wrong. Expected", arguments, ", got", msg.Arguments, ":", msg)
-			}
-		}
-	}
-	return msg, !fail
-}
-
-func TSendMessage(tb testing.TB, conn *websocket.Conn, messageID int, command Command, arguments interface{}) bool {
-	SendMessage(conn, ClientMessage{MessageID: messageID, Command: command, Arguments: arguments})
-	return true
-}
-
-func TSealForSavePubMsg(tb testing.TB, cmd Command, channel string, arguments interface{}, deleteMode bool) (url.Values, error) {
-	form := url.Values{}
-	form.Set("cmd", string(cmd))
-	argsBytes, err := json.Marshal(arguments)
-	if err != nil {
-		tb.Error(err)
-		return nil, err
-	}
-	form.Set("args", string(argsBytes))
-	form.Set("channel", channel)
-	if deleteMode {
-		form.Set("delete", "1")
-	}
-	form.Set("time", time.Now().Format(time.UnixDate))
-
-	sealed, err := SealRequest(form)
-	if err != nil {
-		tb.Error(err)
-		return nil, err
-	}
-	return sealed, nil
-}
-
-func TSealForUncachedPubMsg(tb testing.TB, cmd Command, channel string, arguments interface{}, scope MessageTargetType, deleteMode bool) (url.Values, error) {
-	form := url.Values{}
-	form.Set("cmd", string(cmd))
-	argsBytes, err := json.Marshal(arguments)
-	if err != nil {
-		tb.Error(err)
-		return nil, err
-	}
-	form.Set("args", string(argsBytes))
-	form.Set("channel", channel)
-	if deleteMode {
-		form.Set("delete", "1")
-	}
-	form.Set("time", time.Now().Format(time.UnixDate))
-	form.Set("scope", scope.String())
-
-	sealed, err := SealRequest(form)
-	if err != nil {
-		tb.Error(err)
-		return nil, err
-	}
-	return sealed, nil
-}
-
-func TCheckResponse(tb testing.TB, resp *http.Response, expected string, desc string) bool {
-	var failed bool
-	respBytes, err := ioutil.ReadAll(resp.Body)
-	resp.Body.Close()
-	respStr := string(respBytes)
-
-	if err != nil {
-		tb.Error(err)
-		failed = true
-	}
-
-	if resp.StatusCode != 200 {
-		tb.Error("Publish failed: ", resp.StatusCode, respStr)
-		failed = true
-	}
-
-	if respStr != expected {
-		tb.Errorf("Got wrong response from server. %s Expected: '%s' Got: '%s'", desc, expected, respStr)
-		failed = true
-	}
-	return !failed
-}
-
-type TURLs struct {
-	Websocket      string
-	Origin         string
-	UncachedPubMsg string // uncached_pub
-	SavePubMsg     string // cached_pub
-}
-
-func TGetUrls(socketserver *httptest.Server, backend *httptest.Server) TURLs {
-	addr := socketserver.Listener.Addr().String()
-	return TURLs{
-		Websocket:      fmt.Sprintf("ws://%s/", addr),
-		Origin:         fmt.Sprintf("http://%s", addr),
-		UncachedPubMsg: fmt.Sprintf("http://%s/uncached_pub", addr),
-		SavePubMsg:     fmt.Sprintf("http://%s/cached_pub", addr),
-	}
-}
-
-const (
-	SetupWantSocketServer = 1 << iota
-	SetupWantBackendServer
-	SetupWantURLs
-)
-
-func TSetup(flags int, backendChecker *BackendRequestChecker) (socketserver *httptest.Server, backend *httptest.Server, urls TURLs) {
-	DumpBacklogData()
-
-	ioutil.WriteFile("index.html", []byte(`
-<!DOCTYPE html>
-<title>CatBag</title>
-<link rel="stylesheet" href="//cdn.frankerfacez.com/script/catbag.css">
-<div id="container">
-<div id="zf0"></div><div id="zf1"></div><div id="zf2"></div>
-<div id="zf3"></div><div id="zf4"></div><div id="zf5"></div>
-<div id="zf6"></div><div id="zf7"></div><div id="zf8"></div>
-<div id="zf9"></div><div id="catbag"></div>
-<div id="bottom">
-	A <a href="http://www.frankerfacez.com/">FrankerFaceZ</a> Service
-	&mdash; CatBag by <a href="http://www.twitch.tv/wolsk">Wolsk</a>
-</div>
-</div>`), 0600)
-
-	conf := &ConfigFile{
-		ServerID:         20,
-		UseSSL:           false,
-		OurPublicKey:     []byte{176, 149, 72, 209, 35, 42, 110, 220, 22, 236, 212, 129, 213, 199, 1, 227, 185, 167, 150, 159, 117, 202, 164, 100, 9, 107, 45, 141, 122, 221, 155, 73},
-		OurPrivateKey:    []byte{247, 133, 147, 194, 70, 240, 211, 216, 223, 16, 241, 253, 120, 14, 198, 74, 237, 180, 89, 33, 146, 146, 140, 58, 88, 160, 2, 246, 112, 35, 239, 87},
-		BackendPublicKey: []byte{19, 163, 37, 157, 50, 139, 193, 85, 229, 47, 166, 21, 153, 231, 31, 133, 41, 158, 8, 53, 73, 0, 113, 91, 13, 181, 131, 248, 176, 18, 1, 107},
-	}
-
-	if flags&SetupWantBackendServer != 0 {
-		backend = httptest.NewServer(backendChecker)
-		conf.BackendURL = fmt.Sprintf("http://%s", backend.Listener.Addr().String())
-	}
-
-	Configuration = conf
-	setupBackend(conf)
-
-	if flags&SetupWantSocketServer != 0 {
-		serveMux := http.NewServeMux()
-		SetupServerAndHandle(conf, serveMux)
-
-		socketserver = httptest.NewServer(serveMux)
-	}
-
-	if flags&SetupWantURLs != 0 {
-		urls = TGetUrls(socketserver, backend)
-	}
-	return
-}
-
 func TestSubscriptionAndPublish(t *testing.T) {
 	var doneWg sync.WaitGroup
 	var readyWg sync.WaitGroup
@@ -233,11 +36,11 @@ func TestSubscriptionAndPublish(t *testing.T) {
 	var server *httptest.Server
 	var urls TURLs
 
-	var backendExpected = NewBackendRequestChecker(t,
-		ExpectedBackendRequest{200, bPathAnnounceStartup, &url.Values{"startup": []string{"1"}}, ""},
-		ExpectedBackendRequest{200, bPathAddTopic, &url.Values{"channels": []string{TestChannelName1}, "added": []string{"t"}}, "ok"},
-		ExpectedBackendRequest{200, bPathAddTopic, &url.Values{"channels": []string{TestChannelName2}, "added": []string{"t"}}, "ok"},
-		ExpectedBackendRequest{200, bPathAddTopic, &url.Values{"channels": []string{TestChannelName3}, "added": []string{"t"}}, "ok"},
+	var backendExpected = NewTBackendRequestChecker(t,
+		TExpectedBackendRequest{200, bPathAnnounceStartup, &url.Values{"startup": []string{"1"}}, ""},
+		TExpectedBackendRequest{200, bPathAddTopic, &url.Values{"channels": []string{TestChannelName1}, "added": []string{"t"}}, "ok"},
+		TExpectedBackendRequest{200, bPathAddTopic, &url.Values{"channels": []string{TestChannelName2}, "added": []string{"t"}}, "ok"},
+		TExpectedBackendRequest{200, bPathAddTopic, &url.Values{"channels": []string{TestChannelName3}, "added": []string{"t"}}, "ok"},
 	)
 	server, _, urls = TSetup(SetupWantSocketServer|SetupWantBackendServer|SetupWantURLs, backendExpected)
 
@@ -438,11 +241,11 @@ func TestRestrictedCommands(t *testing.T) {
 	var server *httptest.Server
 	var urls TURLs
 
-	var backendExpected = NewBackendRequestChecker(t,
-		ExpectedBackendRequest{200, bPathAnnounceStartup, &url.Values{"startup": []string{"1"}}, ""},
-		ExpectedBackendRequest{401, fmt.Sprintf("%s%s", bPathOtherCommand, TestCommandNeedsAuth), &url.Values{"usernameClaimed": []string{""}, "clientData": []string{TestRequestDataJSON}}, ""},
-		ExpectedBackendRequest{401, fmt.Sprintf("%s%s", bPathOtherCommand, TestCommandNeedsAuth), &url.Values{"usernameClaimed": []string{TestUsername}, "clientData": []string{TestRequestDataJSON}}, ""},
-		ExpectedBackendRequest{200, fmt.Sprintf("%s%s", bPathOtherCommand, TestCommandNeedsAuth), &url.Values{"usernameVerified": []string{TestUsername}, "clientData": []string{TestRequestDataJSON}}, fmt.Sprintf("\"%s\"", TestReplyData)},
+	var backendExpected = NewTBackendRequestChecker(t,
+		TExpectedBackendRequest{200, bPathAnnounceStartup, &url.Values{"startup": []string{"1"}}, ""},
+		TExpectedBackendRequest{401, fmt.Sprintf("%s%s", bPathOtherCommand, TestCommandNeedsAuth), &url.Values{"usernameClaimed": []string{""}, "clientData": []string{TestRequestDataJSON}}, ""},
+		TExpectedBackendRequest{401, fmt.Sprintf("%s%s", bPathOtherCommand, TestCommandNeedsAuth), &url.Values{"usernameClaimed": []string{TestUsername}, "clientData": []string{TestRequestDataJSON}}, ""},
+		TExpectedBackendRequest{200, fmt.Sprintf("%s%s", bPathOtherCommand, TestCommandNeedsAuth), &url.Values{"usernameVerified": []string{TestUsername}, "clientData": []string{TestRequestDataJSON}}, fmt.Sprintf("\"%s\"", TestReplyData)},
 	)
 	server, _, urls = TSetup(SetupWantSocketServer|SetupWantBackendServer|SetupWantURLs, backendExpected)
 
diff --git a/socketserver/server/testinfra_test.go b/socketserver/server/testinfra_test.go
new file mode 100644
index 00000000..d0326847
--- /dev/null
+++ b/socketserver/server/testinfra_test.go
@@ -0,0 +1,331 @@
+package server
+
+import (
+	"crypto/rand"
+	"encoding/json"
+	"fmt"
+	"github.com/gorilla/websocket"
+	"golang.org/x/crypto/nacl/box"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"testing"
+	"time"
+)
+
+func SetupRandomKeys(t testing.TB) {
+	_, senderPrivate, err := box.GenerateKey(rand.Reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+	receiverPublic, _, err := box.GenerateKey(rand.Reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	box.Precompute(&backendSharedKey, receiverPublic, senderPrivate)
+	messageBufferPool.New = New4KByteBuffer
+}
+
+const MethodIsPost = "POST"
+
+type TExpectedBackendRequest struct {
+	ResponseCode int
+	Path         string
+	// Method       string // always POST
+	PostForm *url.Values
+	Response string
+}
+
+func (er *TExpectedBackendRequest) String() string {
+	if MethodIsPost == "" {
+		return er.Path
+	}
+	return fmt.Sprint("%s %s: %s", MethodIsPost, er.Path, er.PostForm.Encode())
+}
+
+type TBackendRequestChecker struct {
+	ExpectedRequests []TExpectedBackendRequest
+
+	currentRequest int
+	tb             testing.TB
+}
+
+func NewTBackendRequestChecker(tb testing.TB, urls ...TExpectedBackendRequest) *TBackendRequestChecker {
+	return &TBackendRequestChecker{ExpectedRequests: urls, tb: tb, currentRequest: 0}
+}
+
+func (backend *TBackendRequestChecker) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.Method != MethodIsPost {
+		backend.tb.Errorf("Bad backend request: was not a POST. %v", r)
+		return
+	}
+
+	r.ParseForm()
+
+	unsealedForm, err := UnsealRequest(r.PostForm)
+	if err != nil {
+		backend.tb.Errorf("Failed to unseal backend request: %v", err)
+	}
+
+	if backend.currentRequest >= len(backend.ExpectedRequests) {
+		backend.tb.Errorf("Unexpected backend request: %s %s: %s", r.Method, r.URL, unsealedForm)
+		return
+	}
+
+	cur := backend.ExpectedRequests[backend.currentRequest]
+	backend.currentRequest++
+
+	defer func() {
+		w.WriteHeader(cur.ResponseCode)
+		if cur.Response != "" {
+			w.Write([]byte(cur.Response))
+		}
+	}()
+
+	if cur.Path != "" {
+		if r.URL.Path != cur.Path {
+			backend.tb.Errorf("Bad backend request. Expected %v, got %s %s", cur, r.Method, r.URL)
+			return
+		}
+	}
+
+	if cur.PostForm != nil {
+		anyErr := TcompareForms(backend.tb, "Different form contents", *cur.PostForm, unsealedForm)
+		if anyErr {
+			backend.tb.Errorf("...in %s %s: %s", r.Method, r.URL, unsealedForm.Encode())
+		}
+	}
+}
+
+func (backend *TBackendRequestChecker) Close() error {
+	if backend.currentRequest < len(backend.ExpectedRequests) {
+		backend.tb.Errorf("Not all requests sent, got %d out of %d", backend.currentRequest, len(backend.ExpectedRequests))
+	}
+	return nil
+}
+
+func TcompareForms(tb testing.TB, ctx string, expectedForm, gotForm url.Values) (anyErrors bool) {
+	for k, expVal := range expectedForm {
+		gotVal, ok := gotForm[k]
+		if !ok {
+			tb.Errorf("%s: Form[%s]: Expected %v, (got nothing)", ctx, k, expVal)
+			anyErrors = true
+			continue
+		}
+		if len(expVal) != len(gotVal) {
+			tb.Errorf("%s: Form[%s]: Expected %d%v, Got %d%v", ctx, k, len(expVal), expVal, len(gotVal), gotVal)
+			anyErrors = true
+			continue
+		}
+		for i, el := range expVal {
+			if gotVal[i] != el {
+				tb.Errorf("%s: Form[%s][%d]: Expected %s, Got %s", ctx, k, i, el, gotVal[i])
+				anyErrors = true
+			}
+		}
+	}
+	for k, gotVal := range gotForm {
+		_, ok := expectedForm[k]
+		if !ok {
+			tb.Errorf("%s: Form[%s]: (expected nothing), Got %v", ctx, k, gotVal)
+			anyErrors = true
+		}
+	}
+	return anyErrors
+}
+
+func TCountOpenFDs() uint64 {
+	ary, _ := ioutil.ReadDir(fmt.Sprintf("/proc/%d/fd", os.Getpid()))
+	return uint64(len(ary))
+}
+
+const IgnoreReceivedArguments = 1 + 2i
+
+func TReceiveExpectedMessage(tb testing.TB, conn *websocket.Conn, messageID int, command Command, arguments interface{}) (ClientMessage, bool) {
+	var msg ClientMessage
+	var fail bool
+	messageType, packet, err := conn.ReadMessage()
+	if err != nil {
+		tb.Error(err)
+		return msg, false
+	}
+	if messageType != websocket.TextMessage {
+		tb.Error("got non-text message", packet)
+		return msg, false
+	}
+
+	err = UnmarshalClientMessage(packet, messageType, &msg)
+	if err != nil {
+		tb.Error(err)
+		return msg, false
+	}
+	if msg.MessageID != messageID {
+		tb.Error("Message ID was wrong. Expected", messageID, ", got", msg.MessageID, ":", msg)
+		fail = true
+	}
+	if msg.Command != command {
+		tb.Error("Command was wrong. Expected", command, ", got", msg.Command, ":", msg)
+		fail = true
+	}
+	if arguments != IgnoreReceivedArguments {
+		if arguments == nil {
+			if msg.origArguments != "" {
+				tb.Error("Arguments are wrong. Expected", arguments, ", got", msg.Arguments, ":", msg)
+			}
+		} else {
+			argBytes, _ := json.Marshal(arguments)
+			if msg.origArguments != string(argBytes) {
+				tb.Error("Arguments are wrong. Expected", arguments, ", got", msg.Arguments, ":", msg)
+			}
+		}
+	}
+	return msg, !fail
+}
+
+func TSendMessage(tb testing.TB, conn *websocket.Conn, messageID int, command Command, arguments interface{}) bool {
+	SendMessage(conn, ClientMessage{MessageID: messageID, Command: command, Arguments: arguments})
+	return true
+}
+
+func TSealForSavePubMsg(tb testing.TB, cmd Command, channel string, arguments interface{}, deleteMode bool) (url.Values, error) {
+	form := url.Values{}
+	form.Set("cmd", string(cmd))
+	argsBytes, err := json.Marshal(arguments)
+	if err != nil {
+		tb.Error(err)
+		return nil, err
+	}
+	form.Set("args", string(argsBytes))
+	form.Set("channel", channel)
+	if deleteMode {
+		form.Set("delete", "1")
+	}
+	form.Set("time", time.Now().Format(time.UnixDate))
+
+	sealed, err := SealRequest(form)
+	if err != nil {
+		tb.Error(err)
+		return nil, err
+	}
+	return sealed, nil
+}
+
+func TSealForUncachedPubMsg(tb testing.TB, cmd Command, channel string, arguments interface{}, scope MessageTargetType, deleteMode bool) (url.Values, error) {
+	form := url.Values{}
+	form.Set("cmd", string(cmd))
+	argsBytes, err := json.Marshal(arguments)
+	if err != nil {
+		tb.Error(err)
+		return nil, err
+	}
+	form.Set("args", string(argsBytes))
+	form.Set("channel", channel)
+	if deleteMode {
+		form.Set("delete", "1")
+	}
+	form.Set("time", time.Now().Format(time.UnixDate))
+	form.Set("scope", scope.String())
+
+	sealed, err := SealRequest(form)
+	if err != nil {
+		tb.Error(err)
+		return nil, err
+	}
+	return sealed, nil
+}
+
+func TCheckResponse(tb testing.TB, resp *http.Response, expected string, desc string) bool {
+	var failed bool
+	respBytes, err := ioutil.ReadAll(resp.Body)
+	resp.Body.Close()
+	respStr := string(respBytes)
+
+	if err != nil {
+		tb.Error(err)
+		failed = true
+	}
+
+	if resp.StatusCode != 200 {
+		tb.Error("Publish failed: ", resp.StatusCode, respStr)
+		failed = true
+	}
+
+	if respStr != expected {
+		tb.Errorf("Got wrong response from server. %s Expected: '%s' Got: '%s'", desc, expected, respStr)
+		failed = true
+	}
+	return !failed
+}
+
+type TURLs struct {
+	Websocket      string
+	Origin         string
+	UncachedPubMsg string // uncached_pub
+	SavePubMsg     string // cached_pub
+}
+
+func TGetUrls(socketserver *httptest.Server, backend *httptest.Server) TURLs {
+	addr := socketserver.Listener.Addr().String()
+	return TURLs{
+		Websocket:      fmt.Sprintf("ws://%s/", addr),
+		Origin:         fmt.Sprintf("http://%s", addr),
+		UncachedPubMsg: fmt.Sprintf("http://%s/uncached_pub", addr),
+		SavePubMsg:     fmt.Sprintf("http://%s/cached_pub", addr),
+	}
+}
+
+const (
+	SetupWantSocketServer = 1 << iota
+	SetupWantBackendServer
+	SetupWantURLs
+)
+
+func TSetup(flags int, backendChecker *TBackendRequestChecker) (socketserver *httptest.Server, backend *httptest.Server, urls TURLs) {
+	DumpBacklogData()
+
+	ioutil.WriteFile("index.html", []byte(`
+<!DOCTYPE html>
+<title>CatBag</title>
+<link rel="stylesheet" href="//cdn.frankerfacez.com/script/catbag.css">
+<div id="container">
+<div id="zf0"></div><div id="zf1"></div><div id="zf2"></div>
+<div id="zf3"></div><div id="zf4"></div><div id="zf5"></div>
+<div id="zf6"></div><div id="zf7"></div><div id="zf8"></div>
+<div id="zf9"></div><div id="catbag"></div>
+<div id="bottom">
+	A <a href="http://www.frankerfacez.com/">FrankerFaceZ</a> Service
+	&mdash; CatBag by <a href="http://www.twitch.tv/wolsk">Wolsk</a>
+</div>
+</div>`), 0600)
+
+	conf := &ConfigFile{
+		ServerID:         20,
+		UseSSL:           false,
+		OurPublicKey:     []byte{176, 149, 72, 209, 35, 42, 110, 220, 22, 236, 212, 129, 213, 199, 1, 227, 185, 167, 150, 159, 117, 202, 164, 100, 9, 107, 45, 141, 122, 221, 155, 73},
+		OurPrivateKey:    []byte{247, 133, 147, 194, 70, 240, 211, 216, 223, 16, 241, 253, 120, 14, 198, 74, 237, 180, 89, 33, 146, 146, 140, 58, 88, 160, 2, 246, 112, 35, 239, 87},
+		BackendPublicKey: []byte{19, 163, 37, 157, 50, 139, 193, 85, 229, 47, 166, 21, 153, 231, 31, 133, 41, 158, 8, 53, 73, 0, 113, 91, 13, 181, 131, 248, 176, 18, 1, 107},
+	}
+
+	if flags&SetupWantBackendServer != 0 {
+		backend = httptest.NewServer(backendChecker)
+		conf.BackendURL = fmt.Sprintf("http://%s", backend.Listener.Addr().String())
+	}
+
+	Configuration = conf
+	setupBackend(conf)
+
+	if flags&SetupWantSocketServer != 0 {
+		serveMux := http.NewServeMux()
+		SetupServerAndHandle(conf, serveMux)
+
+		socketserver = httptest.NewServer(serveMux)
+	}
+
+	if flags&SetupWantURLs != 0 {
+		urls = TGetUrls(socketserver, backend)
+	}
+	return
+}

From ddc5e02cd72501e4857350ae7ae2130368acbf01 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 16 Dec 2015 14:34:05 -0800
Subject: [PATCH 107/176] Move TSetup to top of file, use in TestSealRequest

---
 socketserver/server/backend_test.go   |   6 +-
 socketserver/server/testinfra_test.go | 114 +++++++++++---------------
 2 files changed, 54 insertions(+), 66 deletions(-)

diff --git a/socketserver/server/backend_test.go b/socketserver/server/backend_test.go
index 266aea8e..89cdbf39 100644
--- a/socketserver/server/backend_test.go
+++ b/socketserver/server/backend_test.go
@@ -6,7 +6,7 @@ import (
 )
 
 func TestSealRequest(t *testing.T) {
-	SetupRandomKeys(t)
+	TSetup(0, nil)
 
 	values := url.Values{
 		"QuickBrownFox": []string{"LazyDog"},
@@ -28,3 +28,7 @@ func TestSealRequest(t *testing.T) {
 		t.Errorf("Failed to round-trip, got back %v", unsealedValues)
 	}
 }
+
+func TestSendRemoteCommand(t *testing.T) {
+
+}
diff --git a/socketserver/server/testinfra_test.go b/socketserver/server/testinfra_test.go
index d0326847..4639d9f8 100644
--- a/socketserver/server/testinfra_test.go
+++ b/socketserver/server/testinfra_test.go
@@ -1,11 +1,9 @@
 package server
 
 import (
-	"crypto/rand"
 	"encoding/json"
 	"fmt"
 	"github.com/gorilla/websocket"
-	"golang.org/x/crypto/nacl/box"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
@@ -15,18 +13,57 @@ import (
 	"time"
 )
 
-func SetupRandomKeys(t testing.TB) {
-	_, senderPrivate, err := box.GenerateKey(rand.Reader)
-	if err != nil {
-		t.Fatal(err)
-	}
-	receiverPublic, _, err := box.GenerateKey(rand.Reader)
-	if err != nil {
-		t.Fatal(err)
+const (
+	SetupWantSocketServer = 1 << iota
+	SetupWantBackendServer
+	SetupWantURLs
+)
+
+func TSetup(flags int, backendChecker *TBackendRequestChecker) (socketserver *httptest.Server, backend *httptest.Server, urls TURLs) {
+	DumpBacklogData()
+
+	ioutil.WriteFile("index.html", []byte(`
+<!DOCTYPE html>
+<title>CatBag</title>
+<link rel="stylesheet" href="//cdn.frankerfacez.com/script/catbag.css">
+<div id="container">
+<div id="zf0"></div><div id="zf1"></div><div id="zf2"></div>
+<div id="zf3"></div><div id="zf4"></div><div id="zf5"></div>
+<div id="zf6"></div><div id="zf7"></div><div id="zf8"></div>
+<div id="zf9"></div><div id="catbag"></div>
+<div id="bottom">
+	A <a href="http://www.frankerfacez.com/">FrankerFaceZ</a> Service
+	&mdash; CatBag by <a href="http://www.twitch.tv/wolsk">Wolsk</a>
+</div>
+</div>`), 0600)
+
+	conf := &ConfigFile{
+		ServerID:         20,
+		UseSSL:           false,
+		OurPublicKey:     []byte{176, 149, 72, 209, 35, 42, 110, 220, 22, 236, 212, 129, 213, 199, 1, 227, 185, 167, 150, 159, 117, 202, 164, 100, 9, 107, 45, 141, 122, 221, 155, 73},
+		OurPrivateKey:    []byte{247, 133, 147, 194, 70, 240, 211, 216, 223, 16, 241, 253, 120, 14, 198, 74, 237, 180, 89, 33, 146, 146, 140, 58, 88, 160, 2, 246, 112, 35, 239, 87},
+		BackendPublicKey: []byte{19, 163, 37, 157, 50, 139, 193, 85, 229, 47, 166, 21, 153, 231, 31, 133, 41, 158, 8, 53, 73, 0, 113, 91, 13, 181, 131, 248, 176, 18, 1, 107},
 	}
 
-	box.Precompute(&backendSharedKey, receiverPublic, senderPrivate)
-	messageBufferPool.New = New4KByteBuffer
+	if flags&SetupWantBackendServer != 0 {
+		backend = httptest.NewServer(backendChecker)
+		conf.BackendURL = fmt.Sprintf("http://%s", backend.Listener.Addr().String())
+	}
+
+	Configuration = conf
+	setupBackend(conf)
+
+	if flags&SetupWantSocketServer != 0 {
+		serveMux := http.NewServeMux()
+		SetupServerAndHandle(conf, serveMux)
+
+		socketserver = httptest.NewServer(serveMux)
+	}
+
+	if flags&SetupWantURLs != 0 {
+		urls = TGetUrls(socketserver, backend)
+	}
+	return
 }
 
 const MethodIsPost = "POST"
@@ -276,56 +313,3 @@ func TGetUrls(socketserver *httptest.Server, backend *httptest.Server) TURLs {
 		SavePubMsg:     fmt.Sprintf("http://%s/cached_pub", addr),
 	}
 }
-
-const (
-	SetupWantSocketServer = 1 << iota
-	SetupWantBackendServer
-	SetupWantURLs
-)
-
-func TSetup(flags int, backendChecker *TBackendRequestChecker) (socketserver *httptest.Server, backend *httptest.Server, urls TURLs) {
-	DumpBacklogData()
-
-	ioutil.WriteFile("index.html", []byte(`
-<!DOCTYPE html>
-<title>CatBag</title>
-<link rel="stylesheet" href="//cdn.frankerfacez.com/script/catbag.css">
-<div id="container">
-<div id="zf0"></div><div id="zf1"></div><div id="zf2"></div>
-<div id="zf3"></div><div id="zf4"></div><div id="zf5"></div>
-<div id="zf6"></div><div id="zf7"></div><div id="zf8"></div>
-<div id="zf9"></div><div id="catbag"></div>
-<div id="bottom">
-	A <a href="http://www.frankerfacez.com/">FrankerFaceZ</a> Service
-	&mdash; CatBag by <a href="http://www.twitch.tv/wolsk">Wolsk</a>
-</div>
-</div>`), 0600)
-
-	conf := &ConfigFile{
-		ServerID:         20,
-		UseSSL:           false,
-		OurPublicKey:     []byte{176, 149, 72, 209, 35, 42, 110, 220, 22, 236, 212, 129, 213, 199, 1, 227, 185, 167, 150, 159, 117, 202, 164, 100, 9, 107, 45, 141, 122, 221, 155, 73},
-		OurPrivateKey:    []byte{247, 133, 147, 194, 70, 240, 211, 216, 223, 16, 241, 253, 120, 14, 198, 74, 237, 180, 89, 33, 146, 146, 140, 58, 88, 160, 2, 246, 112, 35, 239, 87},
-		BackendPublicKey: []byte{19, 163, 37, 157, 50, 139, 193, 85, 229, 47, 166, 21, 153, 231, 31, 133, 41, 158, 8, 53, 73, 0, 113, 91, 13, 181, 131, 248, 176, 18, 1, 107},
-	}
-
-	if flags&SetupWantBackendServer != 0 {
-		backend = httptest.NewServer(backendChecker)
-		conf.BackendURL = fmt.Sprintf("http://%s", backend.Listener.Addr().String())
-	}
-
-	Configuration = conf
-	setupBackend(conf)
-
-	if flags&SetupWantSocketServer != 0 {
-		serveMux := http.NewServeMux()
-		SetupServerAndHandle(conf, serveMux)
-
-		socketserver = httptest.NewServer(serveMux)
-	}
-
-	if flags&SetupWantURLs != 0 {
-		urls = TGetUrls(socketserver, backend)
-	}
-	return
-}

From 286488891e858135d9a2a840e671281071b1988f Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 16 Dec 2015 15:35:35 -0800
Subject: [PATCH 108/176] Tests for SendRemoteCommand[Cached]

---
 socketserver/server/backend_test.go       | 97 ++++++++++++++++++++++-
 socketserver/server/subscriptions_test.go | 16 ++--
 socketserver/server/testinfra_test.go     | 28 +++++--
 3 files changed, 126 insertions(+), 15 deletions(-)

diff --git a/socketserver/server/backend_test.go b/socketserver/server/backend_test.go
index 89cdbf39..f6b30831 100644
--- a/socketserver/server/backend_test.go
+++ b/socketserver/server/backend_test.go
@@ -3,10 +3,14 @@ package server
 import (
 	"net/url"
 	"testing"
+	"net/http"
+	. "gopkg.in/check.v1"
 )
 
+func Test(t *testing.T) { TestingT(t) }
+
 func TestSealRequest(t *testing.T) {
-	TSetup(0, nil)
+	TSetup(SetupNoServers, nil)
 
 	values := url.Values{
 		"QuickBrownFox": []string{"LazyDog"},
@@ -29,6 +33,95 @@ func TestSealRequest(t *testing.T) {
 	}
 }
 
-func TestSendRemoteCommand(t *testing.T) {
+type BackendSuite struct{}
+var _ = Suite(&BackendSuite{})
 
+func (s *BackendSuite) TestSendRemoteCommand(c *C) {
+	const TestCommand1 = "somecommand"
+	const TestCommand2 = "other"
+	const PathTestCommand1 = "/cmd/" + TestCommand1
+	const PathTestCommand2 = "/cmd/" + TestCommand2
+	const TestData1 = "623478.32"
+	const TestData2 = "\"Hello, there\""
+	const TestData3 = "3"
+	const TestUsername = "sirstendec"
+	const TestResponse1 = "asfdg"
+	const TestResponse2 = "yuiop"
+	const TestErrorText = "{\"err\":\"some kind of special error\"}"
+
+	var AnonAuthInfo = AuthInfo{}
+	var NonValidatedAuthInfo = AuthInfo{TwitchUsername: TestUsername}
+	var ValidatedAuthInfo = AuthInfo{TwitchUsername: TestUsername, UsernameValidated: true}
+
+	headersCacheTwoSeconds := http.Header{"FFZ-Cache": []string{"2"}}
+	headersCacheInvalid := http.Header{"FFZ-Cache": []string{"NotANumber"}}
+	headersApplicationJson := http.Header{"Content-Type": []string{"application/json"}}
+
+	backend := NewTBackendRequestChecker(c,
+		TExpectedBackendRequest{200, PathTestCommand1, &url.Values{"clientData": []string{TestData1}, "usernameClaimed": []string{""}}, TestResponse1, nil},
+		TExpectedBackendRequest{200, PathTestCommand1, &url.Values{"clientData": []string{TestData1}, "usernameClaimed": []string{""}}, TestResponse2, nil},
+		TExpectedBackendRequest{200, PathTestCommand1, &url.Values{"clientData": []string{TestData1}, "usernameClaimed": []string{TestUsername}}, TestResponse1, nil},
+		TExpectedBackendRequest{200, PathTestCommand1, &url.Values{"clientData": []string{TestData1}, "usernameVerified": []string{TestUsername}}, TestResponse1, nil},
+		TExpectedBackendRequest{200, PathTestCommand2, &url.Values{"clientData": []string{TestData2}, "usernameClaimed": []string{TestUsername}}, TestResponse1, headersCacheTwoSeconds},
+		// cached
+		// cached
+		TExpectedBackendRequest{200, PathTestCommand2, &url.Values{"clientData": []string{TestData1}, "usernameClaimed": []string{TestUsername}}, TestResponse2, headersCacheTwoSeconds},
+		TExpectedBackendRequest{401, PathTestCommand1, &url.Values{"clientData": []string{TestData1}, "usernameClaimed": []string{TestUsername}}, "", nil},
+		TExpectedBackendRequest{503, PathTestCommand1, &url.Values{"clientData": []string{TestData1}, "usernameClaimed": []string{TestUsername}}, "", nil},
+		TExpectedBackendRequest{418, PathTestCommand1, &url.Values{"clientData": []string{TestData1}, "usernameClaimed": []string{TestUsername}}, TestErrorText, headersApplicationJson},
+		TExpectedBackendRequest{200, PathTestCommand2, &url.Values{"clientData": []string{TestData3}, "usernameClaimed": []string{TestUsername}}, TestResponse1, headersCacheInvalid},
+	)
+	_, _, _ = TSetup(SetupWantBackendServer, backend)
+	defer backend.Close()
+
+	var resp string
+	var err error
+
+	resp, err = SendRemoteCommand(TestCommand1, TestData1, AnonAuthInfo)
+	c.Check(resp, Equals, TestResponse1)
+	c.Check(err, IsNil)
+
+	resp, err = SendRemoteCommand(TestCommand1, TestData1, AnonAuthInfo)
+	c.Check(resp, Equals, TestResponse2)
+	c.Check(err, IsNil)
+
+	resp, err = SendRemoteCommand(TestCommand1, TestData1, NonValidatedAuthInfo)
+	c.Check(resp, Equals, TestResponse1)
+	c.Check(err, IsNil)
+
+	resp, err = SendRemoteCommand(TestCommand1, TestData1, ValidatedAuthInfo)
+	c.Check(resp, Equals, TestResponse1)
+	c.Check(err, IsNil)
+	// cache save
+	resp, err = SendRemoteCommandCached(TestCommand2, TestData2, NonValidatedAuthInfo)
+	c.Check(resp, Equals, TestResponse1)
+	c.Check(err, IsNil)
+
+	resp, err = SendRemoteCommandCached(TestCommand2, TestData2, NonValidatedAuthInfo) // cache hit
+	c.Check(resp, Equals, TestResponse1)
+	c.Check(err, IsNil)
+
+	resp, err = SendRemoteCommandCached(TestCommand2, TestData2, AnonAuthInfo) // cache hit
+	c.Check(resp, Equals, TestResponse1)
+	c.Check(err, IsNil)
+	// cache miss - data is different
+	resp, err = SendRemoteCommandCached(TestCommand2, TestData1, NonValidatedAuthInfo)
+	c.Check(resp, Equals, TestResponse2)
+	c.Check(err, IsNil)
+
+	resp, err = SendRemoteCommand(TestCommand1, TestData1, NonValidatedAuthInfo)
+	c.Check(resp, Equals, "")
+	c.Check(err, Equals, ErrAuthorizationNeeded)
+
+	resp, err = SendRemoteCommand(TestCommand1, TestData1, NonValidatedAuthInfo)
+	c.Check(resp, Equals, "")
+	c.Check(err, ErrorMatches, "backend http error: 503")
+
+	resp, err = SendRemoteCommand(TestCommand1, TestData1, NonValidatedAuthInfo)
+	c.Check(resp, Equals, "")
+	c.Check(err, ErrorMatches, TestErrorText)
+
+	resp, err = SendRemoteCommand(TestCommand2, TestData3, NonValidatedAuthInfo)
+	c.Check(resp, Equals, "")
+	c.Check(err, ErrorMatches, "The RPC server returned a non-integer cache duration: .*")
 }
diff --git a/socketserver/server/subscriptions_test.go b/socketserver/server/subscriptions_test.go
index a12bddc3..4fabc66e 100644
--- a/socketserver/server/subscriptions_test.go
+++ b/socketserver/server/subscriptions_test.go
@@ -37,10 +37,10 @@ func TestSubscriptionAndPublish(t *testing.T) {
 	var urls TURLs
 
 	var backendExpected = NewTBackendRequestChecker(t,
-		TExpectedBackendRequest{200, bPathAnnounceStartup, &url.Values{"startup": []string{"1"}}, ""},
-		TExpectedBackendRequest{200, bPathAddTopic, &url.Values{"channels": []string{TestChannelName1}, "added": []string{"t"}}, "ok"},
-		TExpectedBackendRequest{200, bPathAddTopic, &url.Values{"channels": []string{TestChannelName2}, "added": []string{"t"}}, "ok"},
-		TExpectedBackendRequest{200, bPathAddTopic, &url.Values{"channels": []string{TestChannelName3}, "added": []string{"t"}}, "ok"},
+		TExpectedBackendRequest{200, bPathAnnounceStartup, &url.Values{"startup": []string{"1"}}, "", nil},
+		TExpectedBackendRequest{200, bPathAddTopic, &url.Values{"channels": []string{TestChannelName1}, "added": []string{"t"}}, "ok", nil},
+		TExpectedBackendRequest{200, bPathAddTopic, &url.Values{"channels": []string{TestChannelName2}, "added": []string{"t"}}, "ok", nil},
+		TExpectedBackendRequest{200, bPathAddTopic, &url.Values{"channels": []string{TestChannelName3}, "added": []string{"t"}}, "ok", nil},
 	)
 	server, _, urls = TSetup(SetupWantSocketServer|SetupWantBackendServer|SetupWantURLs, backendExpected)
 
@@ -242,10 +242,10 @@ func TestRestrictedCommands(t *testing.T) {
 	var urls TURLs
 
 	var backendExpected = NewTBackendRequestChecker(t,
-		TExpectedBackendRequest{200, bPathAnnounceStartup, &url.Values{"startup": []string{"1"}}, ""},
-		TExpectedBackendRequest{401, fmt.Sprintf("%s%s", bPathOtherCommand, TestCommandNeedsAuth), &url.Values{"usernameClaimed": []string{""}, "clientData": []string{TestRequestDataJSON}}, ""},
-		TExpectedBackendRequest{401, fmt.Sprintf("%s%s", bPathOtherCommand, TestCommandNeedsAuth), &url.Values{"usernameClaimed": []string{TestUsername}, "clientData": []string{TestRequestDataJSON}}, ""},
-		TExpectedBackendRequest{200, fmt.Sprintf("%s%s", bPathOtherCommand, TestCommandNeedsAuth), &url.Values{"usernameVerified": []string{TestUsername}, "clientData": []string{TestRequestDataJSON}}, fmt.Sprintf("\"%s\"", TestReplyData)},
+		TExpectedBackendRequest{200, bPathAnnounceStartup, &url.Values{"startup": []string{"1"}}, "", nil},
+		TExpectedBackendRequest{401, fmt.Sprintf("%s%s", bPathOtherCommand, TestCommandNeedsAuth), &url.Values{"usernameClaimed": []string{""}, "clientData": []string{TestRequestDataJSON}}, "", nil},
+		TExpectedBackendRequest{401, fmt.Sprintf("%s%s", bPathOtherCommand, TestCommandNeedsAuth), &url.Values{"usernameClaimed": []string{TestUsername}, "clientData": []string{TestRequestDataJSON}}, "", nil},
+		TExpectedBackendRequest{200, fmt.Sprintf("%s%s", bPathOtherCommand, TestCommandNeedsAuth), &url.Values{"usernameVerified": []string{TestUsername}, "clientData": []string{TestRequestDataJSON}}, fmt.Sprintf("\"%s\"", TestReplyData), nil},
 	)
 	server, _, urls = TSetup(SetupWantSocketServer|SetupWantBackendServer|SetupWantURLs, backendExpected)
 
diff --git a/socketserver/server/testinfra_test.go b/socketserver/server/testinfra_test.go
index 4639d9f8..0c1e6a03 100644
--- a/socketserver/server/testinfra_test.go
+++ b/socketserver/server/testinfra_test.go
@@ -18,6 +18,7 @@ const (
 	SetupWantBackendServer
 	SetupWantURLs
 )
+const SetupNoServers = 0
 
 func TSetup(flags int, backendChecker *TBackendRequestChecker) (socketserver *httptest.Server, backend *httptest.Server, urls TURLs) {
 	DumpBacklogData()
@@ -66,14 +67,20 @@ func TSetup(flags int, backendChecker *TBackendRequestChecker) (socketserver *ht
 	return
 }
 
+type TBC interface {
+	Error(args ...interface{})
+	Errorf(format string, args ...interface{})
+}
+
 const MethodIsPost = "POST"
 
 type TExpectedBackendRequest struct {
 	ResponseCode int
 	Path         string
 	// Method       string // always POST
-	PostForm *url.Values
-	Response string
+	PostForm        *url.Values
+	Response        string
+	ResponseHeaders http.Header
 }
 
 func (er *TExpectedBackendRequest) String() string {
@@ -87,10 +94,10 @@ type TBackendRequestChecker struct {
 	ExpectedRequests []TExpectedBackendRequest
 
 	currentRequest int
-	tb             testing.TB
+	tb             TBC
 }
 
-func NewTBackendRequestChecker(tb testing.TB, urls ...TExpectedBackendRequest) *TBackendRequestChecker {
+func NewTBackendRequestChecker(tb TBC, urls ...TExpectedBackendRequest) *TBackendRequestChecker {
 	return &TBackendRequestChecker{ExpectedRequests: urls, tb: tb, currentRequest: 0}
 }
 
@@ -115,6 +122,17 @@ func (backend *TBackendRequestChecker) ServeHTTP(w http.ResponseWriter, r *http.
 	cur := backend.ExpectedRequests[backend.currentRequest]
 	backend.currentRequest++
 
+	headers := w.Header()
+	for k, v := range cur.ResponseHeaders {
+		if len(v) == 1 {
+			headers.Set(k, v[0])
+		} else if len(v) == 0 {
+			headers.Del(k)
+		} else {
+			for _, hv := range v { headers.Add(k, hv) }
+		}
+	}
+
 	defer func() {
 		w.WriteHeader(cur.ResponseCode)
 		if cur.Response != "" {
@@ -144,7 +162,7 @@ func (backend *TBackendRequestChecker) Close() error {
 	return nil
 }
 
-func TcompareForms(tb testing.TB, ctx string, expectedForm, gotForm url.Values) (anyErrors bool) {
+func TcompareForms(tb TBC, ctx string, expectedForm, gotForm url.Values) (anyErrors bool) {
 	for k, expVal := range expectedForm {
 		gotVal, ok := gotForm[k]
 		if !ok {

From bbe8b41fed6978db76fd38e8f89f1928a0d424a9 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 16 Dec 2015 15:51:12 -0800
Subject: [PATCH 109/176] Use username= and authenticated=

---
 socketserver/server/backend.go            | 14 +++++++-------
 socketserver/server/backend_test.go       | 20 ++++++++++----------
 socketserver/server/subscriptions_test.go |  6 +++---
 3 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/socketserver/server/backend.go b/socketserver/server/backend.go
index ea986371..eea73ead 100644
--- a/socketserver/server/backend.go
+++ b/socketserver/server/backend.go
@@ -140,16 +140,16 @@ func SendRemoteCommandCached(remoteCommand, data string, auth AuthInfo) (string,
 // `usernameClaimed` depending on whether AuthInfo.UsernameValidates is true is AuthInfo.TwitchUsername.
 func SendRemoteCommand(remoteCommand, data string, auth AuthInfo) (responseStr string, err error) {
 	destURL := fmt.Sprintf("%s/cmd/%s", backendURL, remoteCommand)
-	var authKey string
-	if auth.UsernameValidated {
-		authKey = "usernameVerified"
-	} else {
-		authKey = "usernameClaimed"
-	}
 
 	formData := url.Values{
 		"clientData": []string{data},
-		authKey:      []string{auth.TwitchUsername},
+		"username":   []string{auth.TwitchUsername},
+	}
+
+	if auth.UsernameValidated {
+		formData.Set("authenticated", "1")
+	} else {
+		formData.Set("authenticated", "0")
 	}
 
 	sealedForm, err := SealRequest(formData)
diff --git a/socketserver/server/backend_test.go b/socketserver/server/backend_test.go
index f6b30831..b16782e4 100644
--- a/socketserver/server/backend_test.go
+++ b/socketserver/server/backend_test.go
@@ -58,18 +58,18 @@ func (s *BackendSuite) TestSendRemoteCommand(c *C) {
 	headersApplicationJson := http.Header{"Content-Type": []string{"application/json"}}
 
 	backend := NewTBackendRequestChecker(c,
-		TExpectedBackendRequest{200, PathTestCommand1, &url.Values{"clientData": []string{TestData1}, "usernameClaimed": []string{""}}, TestResponse1, nil},
-		TExpectedBackendRequest{200, PathTestCommand1, &url.Values{"clientData": []string{TestData1}, "usernameClaimed": []string{""}}, TestResponse2, nil},
-		TExpectedBackendRequest{200, PathTestCommand1, &url.Values{"clientData": []string{TestData1}, "usernameClaimed": []string{TestUsername}}, TestResponse1, nil},
-		TExpectedBackendRequest{200, PathTestCommand1, &url.Values{"clientData": []string{TestData1}, "usernameVerified": []string{TestUsername}}, TestResponse1, nil},
-		TExpectedBackendRequest{200, PathTestCommand2, &url.Values{"clientData": []string{TestData2}, "usernameClaimed": []string{TestUsername}}, TestResponse1, headersCacheTwoSeconds},
+		TExpectedBackendRequest{200, PathTestCommand1, &url.Values{"clientData": []string{TestData1}, "authenticated": []string{"0"}, "username": []string{""}}, TestResponse1, nil},
+		TExpectedBackendRequest{200, PathTestCommand1, &url.Values{"clientData": []string{TestData1}, "authenticated": []string{"0"}, "username": []string{""}}, TestResponse2, nil},
+		TExpectedBackendRequest{200, PathTestCommand1, &url.Values{"clientData": []string{TestData1}, "authenticated": []string{"0"}, "username": []string{TestUsername}}, TestResponse1, nil},
+		TExpectedBackendRequest{200, PathTestCommand1, &url.Values{"clientData": []string{TestData1}, "authenticated": []string{"1"}, "username": []string{TestUsername}}, TestResponse1, nil},
+		TExpectedBackendRequest{200, PathTestCommand2, &url.Values{"clientData": []string{TestData2}, "authenticated": []string{"0"}, "username": []string{TestUsername}}, TestResponse1, headersCacheTwoSeconds},
 		// cached
 		// cached
-		TExpectedBackendRequest{200, PathTestCommand2, &url.Values{"clientData": []string{TestData1}, "usernameClaimed": []string{TestUsername}}, TestResponse2, headersCacheTwoSeconds},
-		TExpectedBackendRequest{401, PathTestCommand1, &url.Values{"clientData": []string{TestData1}, "usernameClaimed": []string{TestUsername}}, "", nil},
-		TExpectedBackendRequest{503, PathTestCommand1, &url.Values{"clientData": []string{TestData1}, "usernameClaimed": []string{TestUsername}}, "", nil},
-		TExpectedBackendRequest{418, PathTestCommand1, &url.Values{"clientData": []string{TestData1}, "usernameClaimed": []string{TestUsername}}, TestErrorText, headersApplicationJson},
-		TExpectedBackendRequest{200, PathTestCommand2, &url.Values{"clientData": []string{TestData3}, "usernameClaimed": []string{TestUsername}}, TestResponse1, headersCacheInvalid},
+		TExpectedBackendRequest{200, PathTestCommand2, &url.Values{"clientData": []string{TestData1}, "authenticated": []string{"0"}, "username": []string{TestUsername}}, TestResponse2, headersCacheTwoSeconds},
+		TExpectedBackendRequest{401, PathTestCommand1, &url.Values{"clientData": []string{TestData1}, "authenticated": []string{"0"}, "username": []string{TestUsername}}, "", nil},
+		TExpectedBackendRequest{503, PathTestCommand1, &url.Values{"clientData": []string{TestData1}, "authenticated": []string{"0"}, "username": []string{TestUsername}}, "", nil},
+		TExpectedBackendRequest{418, PathTestCommand1, &url.Values{"clientData": []string{TestData1}, "authenticated": []string{"0"}, "username": []string{TestUsername}}, TestErrorText, headersApplicationJson},
+		TExpectedBackendRequest{200, PathTestCommand2, &url.Values{"clientData": []string{TestData3}, "authenticated": []string{"0"}, "username": []string{TestUsername}}, TestResponse1, headersCacheInvalid},
 	)
 	_, _, _ = TSetup(SetupWantBackendServer, backend)
 	defer backend.Close()
diff --git a/socketserver/server/subscriptions_test.go b/socketserver/server/subscriptions_test.go
index 4fabc66e..d53202c1 100644
--- a/socketserver/server/subscriptions_test.go
+++ b/socketserver/server/subscriptions_test.go
@@ -243,9 +243,9 @@ func TestRestrictedCommands(t *testing.T) {
 
 	var backendExpected = NewTBackendRequestChecker(t,
 		TExpectedBackendRequest{200, bPathAnnounceStartup, &url.Values{"startup": []string{"1"}}, "", nil},
-		TExpectedBackendRequest{401, fmt.Sprintf("%s%s", bPathOtherCommand, TestCommandNeedsAuth), &url.Values{"usernameClaimed": []string{""}, "clientData": []string{TestRequestDataJSON}}, "", nil},
-		TExpectedBackendRequest{401, fmt.Sprintf("%s%s", bPathOtherCommand, TestCommandNeedsAuth), &url.Values{"usernameClaimed": []string{TestUsername}, "clientData": []string{TestRequestDataJSON}}, "", nil},
-		TExpectedBackendRequest{200, fmt.Sprintf("%s%s", bPathOtherCommand, TestCommandNeedsAuth), &url.Values{"usernameVerified": []string{TestUsername}, "clientData": []string{TestRequestDataJSON}}, fmt.Sprintf("\"%s\"", TestReplyData), nil},
+		TExpectedBackendRequest{401, fmt.Sprintf("%s%s", bPathOtherCommand, TestCommandNeedsAuth), &url.Values{"authenticated": []string{"0"}, "username": []string{""}, "clientData": []string{TestRequestDataJSON}}, "", nil},
+		TExpectedBackendRequest{401, fmt.Sprintf("%s%s", bPathOtherCommand, TestCommandNeedsAuth), &url.Values{"authenticated": []string{"0"}, "username": []string{TestUsername}, "clientData": []string{TestRequestDataJSON}}, "", nil},
+		TExpectedBackendRequest{200, fmt.Sprintf("%s%s", bPathOtherCommand, TestCommandNeedsAuth), &url.Values{"authenticated": []string{"1"}, "username": []string{TestUsername}, "clientData": []string{TestRequestDataJSON}}, fmt.Sprintf("\"%s\"", TestReplyData), nil},
 	)
 	server, _, urls = TSetup(SetupWantSocketServer|SetupWantBackendServer|SetupWantURLs, backendExpected)
 

From 18c1abd3e3144420b4a516118082c14ac7b4279e Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 23 Dec 2015 21:55:15 -0800
Subject: [PATCH 110/176] Bugfixes: div0, wrong low-mem condition, race

---
 socketserver/server/handlecore.go     | 26 ++++++++++++++++----------
 socketserver/server/stats.go          |  2 +-
 socketserver/server/testinfra_test.go |  8 +++++++-
 3 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 6e5ced54..1be2ef9c 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -12,6 +12,7 @@ import (
 	"net/url"
 	"os"
 	"os/signal"
+	"runtime"
 	"strconv"
 	"strings"
 	"sync"
@@ -108,16 +109,21 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 		logstasher.Setup(Configuration.ESServer, Configuration.ESIndexPrefix, Configuration.ESHostName)
 	}
 
-	janitorsOnce.Do(func() {
-		go authorizationJanitor()
-		go bunchCacheJanitor()
-		go pubsubJanitor()
-		go aggregateDataSender()
-		go commandCounter()
+	janitorsOnce.Do(startJanitors)
+}
 
-		go ircConnection()
-		go shutdownHandler()
-	})
+// startJanitors starts the 'is_init_func' goroutines
+func startJanitors() {
+	loadUniqueUsers()
+
+	go authorizationJanitor()
+	go bunchCacheJanitor()
+	go pubsubJanitor()
+	go aggregateDataSender()
+	go commandCounter()
+
+	go ircConnection()
+	go shutdownHandler()
 }
 
 func shutdownHandler() {
@@ -169,7 +175,7 @@ func HTTPHandleRootURL(w http.ResponseWriter, r *http.Request) {
 	if r.Header.Get("Connection") == "Upgrade" {
 		updateSysMem()
 
-		if Statistics.SysMemTotalKB-Statistics.SysMemFreeKB < Configuration.MinMemoryKBytes {
+		if Statistics.SysMemFreeKB > 0 && Statistics.SysMemFreeKB < Configuration.MinMemoryKBytes {
 			atomic.AddUint64(&Statistics.LowMemDroppedConnections, 1)
 			w.WriteHeader(503)
 			return
diff --git a/socketserver/server/stats.go b/socketserver/server/stats.go
index 7ba62f1b..d9fb11f3 100644
--- a/socketserver/server/stats.go
+++ b/socketserver/server/stats.go
@@ -122,7 +122,7 @@ func updatePeriodicStats() {
 
 			Statistics.CpuUsagePct = 100 * float64(userTicks+sysTicks) / (timeDiff.Seconds() * float64(ticksPerSecond))
 			Statistics.MemoryRSSKB = uint64(pstat.Rss * pageSize / 1024)
-			Statistics.MemPerClientBytes = (Statistics.MemoryRSSKB * 1024) / Statistics.CurrentClientCount
+			Statistics.MemPerClientBytes = (Statistics.MemoryRSSKB * 1024) / (Statistics.CurrentClientCount + 1)
 		}
 		updateSysMem()
 	}
diff --git a/socketserver/server/testinfra_test.go b/socketserver/server/testinfra_test.go
index 0c1e6a03..e2c1abef 100644
--- a/socketserver/server/testinfra_test.go
+++ b/socketserver/server/testinfra_test.go
@@ -95,6 +95,7 @@ type TBackendRequestChecker struct {
 
 	currentRequest int
 	tb             TBC
+	mutex          sync.Mutex
 }
 
 func NewTBackendRequestChecker(tb TBC, urls ...TExpectedBackendRequest) *TBackendRequestChecker {
@@ -102,6 +103,9 @@ func NewTBackendRequestChecker(tb TBC, urls ...TExpectedBackendRequest) *TBacken
 }
 
 func (backend *TBackendRequestChecker) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	backend.mutex.Lock()
+	defer backend.mutex.Unlock()
+
 	if r.Method != MethodIsPost {
 		backend.tb.Errorf("Bad backend request: was not a POST. %v", r)
 		return
@@ -129,7 +133,9 @@ func (backend *TBackendRequestChecker) ServeHTTP(w http.ResponseWriter, r *http.
 		} else if len(v) == 0 {
 			headers.Del(k)
 		} else {
-			for _, hv := range v { headers.Add(k, hv) }
+			for _, hv := range v {
+				headers.Add(k, hv)
+			}
 		}
 	}
 

From 89ff64f52c2e0c9f34e3f8f6d0ce8fcae7d51bb3 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 23 Dec 2015 21:56:56 -0800
Subject: [PATCH 111/176] Add comments, add dumpStackOnCtrlZ() for tests

---
 socketserver/server/backend_test.go       |  5 +++--
 socketserver/server/commands.go           |  2 ++
 socketserver/server/handlecore.go         | 14 ++++++++++++++
 socketserver/server/irc.go                |  2 ++
 socketserver/server/stats.go              |  8 +++++++-
 socketserver/server/subscriptions.go      |  2 +-
 socketserver/server/subscriptions_test.go |  2 ++
 socketserver/server/testinfra_test.go     |  9 ++++++++-
 socketserver/server/tickspersecond.go     |  3 +++
 9 files changed, 42 insertions(+), 5 deletions(-)

diff --git a/socketserver/server/backend_test.go b/socketserver/server/backend_test.go
index b16782e4..540571f6 100644
--- a/socketserver/server/backend_test.go
+++ b/socketserver/server/backend_test.go
@@ -1,10 +1,10 @@
 package server
 
 import (
+	. "gopkg.in/check.v1"
+	"net/http"
 	"net/url"
 	"testing"
-	"net/http"
-	. "gopkg.in/check.v1"
 )
 
 func Test(t *testing.T) { TestingT(t) }
@@ -34,6 +34,7 @@ func TestSealRequest(t *testing.T) {
 }
 
 type BackendSuite struct{}
+
 var _ = Suite(&BackendSuite{})
 
 func (s *BackendSuite) TestSendRemoteCommand(c *C) {
diff --git a/socketserver/server/commands.go b/socketserver/server/commands.go
index 83dddb90..665256c9 100644
--- a/socketserver/server/commands.go
+++ b/socketserver/server/commands.go
@@ -312,6 +312,7 @@ func C2SEmoticonUses(conn *websocket.Conn, client *ClientInfo, msg ClientMessage
 	return ResponseSuccess, nil
 }
 
+// is_init_func
 func aggregateDataSender() {
 	for {
 		time.Sleep(5 * time.Minute)
@@ -403,6 +404,7 @@ func bunchedRequestFromCM(msg *ClientMessage) bunchedRequest {
 	return bunchedRequest{Command: msg.Command, Param: msg.origArguments}
 }
 
+// is_init_func
 func bunchCacheJanitor() {
 	go func() {
 		for {
diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 1be2ef9c..c61dfa32 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -126,6 +126,7 @@ func startJanitors() {
 	go shutdownHandler()
 }
 
+// is_init_func
 func shutdownHandler() {
 	ch := make(chan os.Signal)
 	signal.Notify(ch, syscall.SIGUSR1)
@@ -139,6 +140,19 @@ func shutdownHandler() {
 	os.Exit(0)
 }
 
+// is_init_func +test
+func dumpStackOnCtrlZ() {
+	ch := make(chan os.Signal)
+	signal.Notify(ch, syscall.SIGTSTP)
+	for _ = range ch {
+		fmt.Println("Got ^Z")
+
+		buf := make([]byte, 10000)
+		byteCnt := runtime.Stack(buf, true)
+		fmt.Println(string(buf[:byteCnt]))
+	}
+}
+
 // SocketUpgrader is the websocket.Upgrader currently in use.
 var SocketUpgrader = websocket.Upgrader{
 	ReadBufferSize:  1024,
diff --git a/socketserver/server/irc.go b/socketserver/server/irc.go
index 6e4df282..568cb534 100644
--- a/socketserver/server/irc.go
+++ b/socketserver/server/irc.go
@@ -36,6 +36,7 @@ func AddPendingAuthorization(client *ClientInfo, challenge string, callback Auth
 	})
 }
 
+// is_init_func
 func authorizationJanitor() {
 	for {
 		time.Sleep(5 * time.Minute)
@@ -90,6 +91,7 @@ const DEBUG = "DEBUG"
 
 var errChallengeNotFound = errors.New("did not find a challenge solved by that message")
 
+// is_init_func
 func ircConnection() {
 
 	c := irc.SimpleClient("justinfan123")
diff --git a/socketserver/server/stats.go b/socketserver/server/stats.go
index d9fb11f3..115cfefb 100644
--- a/socketserver/server/stats.go
+++ b/socketserver/server/stats.go
@@ -58,11 +58,14 @@ type StatsData struct {
 // Its structure should be versioned as it is exposed via JSON.
 //
 // Note as to threaded access - this is soft/fun data and not critical to data integrity.
-// I don't really care.
+// Fix anything that -race turns up, but otherwise it's not too much of a problem.
 var Statistics = newStatsData()
 
+// CommandCounter is a channel for race-free counting of command usage.
 var CommandCounter = make(chan Command, 10)
 
+// commandCounter receives from the CommandCounter channel and uses the value to increment the values in Statistics.
+// is_init_func
 func commandCounter() {
 	for cmd := range CommandCounter {
 		Statistics.CommandsIssuedTotal++
@@ -70,6 +73,7 @@ func commandCounter() {
 	}
 }
 
+// StatsDataVersion
 const StatsDataVersion = 5
 const pageSize = 4096
 
@@ -145,6 +149,7 @@ func updatePeriodicStats() {
 var sysMemLastUpdate time.Time
 var sysMemUpdateLock sync.Mutex
 
+// updateSysMem reads the system's available RAM.
 func updateSysMem() {
 	if time.Now().Add(-2 * time.Second).After(sysMemLastUpdate) {
 		sysMemUpdateLock.Lock()
@@ -163,6 +168,7 @@ func updateSysMem() {
 	}
 }
 
+// HTTPShowStatistics handles the /stats endpoint. It writes out the Statistics object as indented JSON.
 func HTTPShowStatistics(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json")
 
diff --git a/socketserver/server/subscriptions.go b/socketserver/server/subscriptions.go
index 8609e7c6..ecd73a0a 100644
--- a/socketserver/server/subscriptions.go
+++ b/socketserver/server/subscriptions.go
@@ -145,7 +145,7 @@ func unsubscribeAllClients() {
 const ReapingDelay = 1 * time.Minute
 
 // Checks ChatSubscriptionInfo for entries with no subscribers every ReapingDelay.
-// Started from SetupServer().
+// is_init_func
 func pubsubJanitor() {
 	for {
 		time.Sleep(ReapingDelay)
diff --git a/socketserver/server/subscriptions_test.go b/socketserver/server/subscriptions_test.go
index d53202c1..27ed6fe3 100644
--- a/socketserver/server/subscriptions_test.go
+++ b/socketserver/server/subscriptions_test.go
@@ -11,6 +11,7 @@ import (
 	"sync"
 	"syscall"
 	"testing"
+	"time"
 )
 
 func TestSubscriptionAndPublish(t *testing.T) {
@@ -132,6 +133,7 @@ func TestSubscriptionAndPublish(t *testing.T) {
 
 	doneWg.Add(1)
 	readyWg.Wait() // enforce ordering
+	time.Sleep(2 * time.Millisecond)
 	readyWg.Add(1)
 	go func(conn *websocket.Conn) {
 		TSendMessage(t, conn, 1, HelloCommand, []interface{}{"ffz_0.0-test", uuid.NewV4().String()})
diff --git a/socketserver/server/testinfra_test.go b/socketserver/server/testinfra_test.go
index e2c1abef..07fb063c 100644
--- a/socketserver/server/testinfra_test.go
+++ b/socketserver/server/testinfra_test.go
@@ -9,6 +9,7 @@ import (
 	"net/http/httptest"
 	"net/url"
 	"os"
+	"sync"
 	"testing"
 	"time"
 )
@@ -20,7 +21,13 @@ const (
 )
 const SetupNoServers = 0
 
+var signalCatch sync.Once
+
 func TSetup(flags int, backendChecker *TBackendRequestChecker) (socketserver *httptest.Server, backend *httptest.Server, urls TURLs) {
+	signalCatch.Do(func() {
+		go dumpStackOnCtrlZ()
+	})
+
 	DumpBacklogData()
 
 	ioutil.WriteFile("index.html", []byte(`
@@ -36,7 +43,7 @@ func TSetup(flags int, backendChecker *TBackendRequestChecker) (socketserver *ht
 	A <a href="http://www.frankerfacez.com/">FrankerFaceZ</a> Service
 	&mdash; CatBag by <a href="http://www.twitch.tv/wolsk">Wolsk</a>
 </div>
-</div>`), 0600)
+</div>`), 0644)
 
 	conf := &ConfigFile{
 		ServerID:         20,
diff --git a/socketserver/server/tickspersecond.go b/socketserver/server/tickspersecond.go
index b545740d..36e99516 100644
--- a/socketserver/server/tickspersecond.go
+++ b/socketserver/server/tickspersecond.go
@@ -6,4 +6,7 @@ package server
 // }
 import "C"
 
+// note: this seems to add 0.1s to compile time on my machine
 var ticksPerSecond = int(C.get_ticks_per_second())
+
+//var ticksPerSecond = 100

From 81c477cd6b2a70c4bb325451ac5ceea3d0fd558d Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 23 Dec 2015 21:57:33 -0800
Subject: [PATCH 112/176] Add unique client counting

---
 socketserver/server/commands.go           |   2 +
 socketserver/server/handlecore.go         |  12 +-
 socketserver/server/stats.go              |   4 +
 socketserver/server/subscriptions_test.go |   7 +
 socketserver/server/testinfra_test.go     |   9 +
 socketserver/server/usercount.go          | 271 ++++++++++++++++++++++
 socketserver/server/usercount_test.go     |  70 ++++++
 7 files changed, 374 insertions(+), 1 deletion(-)
 create mode 100644 socketserver/server/usercount.go
 create mode 100644 socketserver/server/usercount_test.go

diff --git a/socketserver/server/commands.go b/socketserver/server/commands.go
index 665256c9..0e8004c9 100644
--- a/socketserver/server/commands.go
+++ b/socketserver/server/commands.go
@@ -103,6 +103,8 @@ func C2SHello(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg
 		client.ClientID = uuid.NewV4()
 	}
 
+	uniqueUserChannel <- client.ClientID
+
 	SubscribeGlobal(client)
 	SubscribeDefaults(client)
 
diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index c61dfa32..451b9223 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -85,8 +85,9 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 	BannerHTML = bannerBytes
 
 	serveMux.HandleFunc("/", HTTPHandleRootURL)
-	serveMux.Handle("/.well-known/", http.FileServer(http.FileSystem(http.Dir("/tmp/letsencrypt/"))))
+	serveMux.Handle("/.well-known/", http.FileServer(http.Dir("/tmp/letsencrypt/")))
 	serveMux.HandleFunc("/stats", HTTPShowStatistics)
+	serveMux.HandleFunc("/hll/", HTTPShowHLL)
 
 	serveMux.HandleFunc("/drop_backlog", HTTPBackendDropBacklog)
 	serveMux.HandleFunc("/uncached_pub", HTTPBackendUncachedPublish)
@@ -130,13 +131,22 @@ func startJanitors() {
 func shutdownHandler() {
 	ch := make(chan os.Signal)
 	signal.Notify(ch, syscall.SIGUSR1)
+	signal.Notify(ch, syscall.SIGTERM)
 	<-ch
 	log.Println("Shutting down...")
 
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		writeAllHLLs()
+		wg.Done()
+	}()
+
 	StopAcceptingConnections = true
 	close(StopAcceptingConnectionsCh)
 
 	time.Sleep(1 * time.Second)
+	wg.Wait()
 	os.Exit(0)
 }
 
diff --git a/socketserver/server/stats.go b/socketserver/server/stats.go
index 115cfefb..0275d4d7 100644
--- a/socketserver/server/stats.go
+++ b/socketserver/server/stats.go
@@ -165,6 +165,10 @@ func updateSysMem() {
 	if err == nil {
 		Statistics.SysMemTotalKB = memInfo.MemTotal
 		Statistics.SysMemFreeKB = memInfo.MemAvailable
+
+		if memInfo.MemAvailable > 0 && memInfo.MemAvailable < Configuration.MinMemoryKBytes {
+			writeAllHLLs()
+		}
 	}
 }
 
diff --git a/socketserver/server/subscriptions_test.go b/socketserver/server/subscriptions_test.go
index 27ed6fe3..d2e561d2 100644
--- a/socketserver/server/subscriptions_test.go
+++ b/socketserver/server/subscriptions_test.go
@@ -228,6 +228,13 @@ func TestSubscriptionAndPublish(t *testing.T) {
 
 	doneWg.Wait()
 	server.Close()
+
+	for _, period := range periods {
+		clientCount := readHLL(period)
+		if clientCount < 3 || clientCount > 5 {
+			t.Error("clientCount outside acceptable range: expected 4, got ", clientCount)
+		}
+	}
 }
 
 func TestRestrictedCommands(t *testing.T) {
diff --git a/socketserver/server/testinfra_test.go b/socketserver/server/testinfra_test.go
index 07fb063c..9c03fa78 100644
--- a/socketserver/server/testinfra_test.go
+++ b/socketserver/server/testinfra_test.go
@@ -64,6 +64,7 @@ func TSetup(flags int, backendChecker *TBackendRequestChecker) (socketserver *ht
 	if flags&SetupWantSocketServer != 0 {
 		serveMux := http.NewServeMux()
 		SetupServerAndHandle(conf, serveMux)
+		dumpUniqueUsers()
 
 		socketserver = httptest.NewServer(serveMux)
 	}
@@ -344,3 +345,11 @@ func TGetUrls(socketserver *httptest.Server, backend *httptest.Server) TURLs {
 		SavePubMsg:     fmt.Sprintf("http://%s/cached_pub", addr),
 	}
 }
+
+func TCheckHLLValue(tb testing.TB, expected uint64, actual uint64) {
+	high := uint64(float64(expected) * 1.05)
+	low := uint64(float64(expected) * 0.95)
+	if actual < low || actual > high {
+		tb.Errorf("Count outside expected range. Expected %d, Got %d", expected, actual)
+	}
+}
diff --git a/socketserver/server/usercount.go b/socketserver/server/usercount.go
new file mode 100644
index 00000000..9ac614fa
--- /dev/null
+++ b/socketserver/server/usercount.go
@@ -0,0 +1,271 @@
+package server
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/binary"
+	"encoding/gob"
+	"fmt"
+	"net/http"
+	"io/ioutil"
+	"log"
+	"os"
+	"time"
+
+	"github.com/clarkduvall/hyperloglog"
+	"github.com/satori/go.uuid"
+)
+
+// uuidHash implements a hash for uuid.UUID by XORing the random bits.
+type uuidHash uuid.UUID
+
+func (u uuidHash) Sum64() uint64 {
+	var valLow, valHigh uint64
+	valLow = binary.LittleEndian.Uint64(u[0:8])
+	valHigh = binary.LittleEndian.Uint64(u[8:16])
+	return valLow ^ valHigh
+}
+
+type PeriodUniqueUsers struct {
+	Start   time.Time
+	End     time.Time
+	Counter *hyperloglog.HyperLogLogPlus
+}
+
+type usageToken struct{}
+
+const (
+	periodDaily = iota
+	periodWeekly
+	periodMonthly
+)
+
+var periods [3]int = [3]int{periodDaily, periodWeekly, periodMonthly}
+
+const uniqCountDir = "./uniques"
+const usersDailyFmt = "daily-%d-%d-%d.gob"  // d-m-y
+const usersWeeklyFmt = "weekly-%d-%d.gob"   // w-y
+const usersMonthlyFmt = "monthly-%d-%d.gob" // m-y
+const counterPrecision uint8 = 12
+
+var uniqueCounters [3]PeriodUniqueUsers
+var uniqueUserChannel chan uuid.UUID
+var uniqueCtrWritingToken chan usageToken
+
+var counterLocation *time.Location = time.FixedZone("UTC-5", int((time.Hour*-5)/time.Second))
+
+// getCounterPeriod calculates the start and end timestamps for the HLL measurement period that includes the 'at' timestamp.
+func getCounterPeriod(which int, at time.Time) (start time.Time, end time.Time) {
+	year, month, day := at.Date()
+
+	switch which {
+	case periodDaily:
+		start = time.Date(year, month, day, 0, 0, 0, 0, counterLocation)
+		end = time.Date(year, month, day+1, 0, 0, 0, 0, counterLocation)
+	case periodWeekly:
+		dayOffset := at.Weekday() - time.Sunday
+		start = time.Date(year, month, day-int(dayOffset), 0, 0, 0, 0, counterLocation)
+		end = time.Date(year, month, day-int(dayOffset)+7, 0, 0, 0, 0, counterLocation)
+	case periodMonthly:
+		start = time.Date(year, month, 1, 0, 0, 0, 0, counterLocation)
+		end = time.Date(year, month+1, 1, 0, 0, 0, 0, counterLocation)
+	}
+	return start, end
+}
+
+// getHLLFilename returns the filename for the saved HLL whose measurement period covers the given time.
+func getHLLFilename(which int, at time.Time) string {
+	var filename string
+	switch which {
+	case periodDaily:
+		year, month, day := at.Date()
+		filename = fmt.Sprintf(usersDailyFmt, day, month, year)
+	case periodWeekly:
+		year, week := at.ISOWeek()
+		filename = fmt.Sprintf(usersWeeklyFmt, week, year)
+	case periodMonthly:
+		year, month, _ := at.Date()
+		filename = fmt.Sprintf(usersMonthlyFmt, month, year)
+	}
+	return fmt.Sprintf("%s/%s", uniqCountDir, filename)
+}
+
+// loadHLL loads a HLL from disk and stores the result in dest.Counter.
+// If dest.Counter is nil, it will be initialized. (This is a useful side-effect.)
+// If dest is one of the uniqueCounters, the usageToken must be held.
+func loadHLL(which int, at time.Time, dest *PeriodUniqueUsers) error {
+	fileBytes, err := ioutil.ReadFile(getHLLFilename(which, at))
+	if err != nil {
+		return err
+	}
+
+	if dest.Counter == nil {
+		dest.Counter, _ = hyperloglog.NewPlus(counterPrecision)
+	}
+
+	dec := gob.NewDecoder(bytes.NewReader(fileBytes))
+	err = dec.Decode(dest.Counter)
+	if err != nil {
+		log.Panicln(err)
+		return err
+	}
+	return nil
+}
+
+// writeHLL writes the indicated HLL to disk.
+// The function takes the usageToken.
+func writeHLL(which int) error {
+	token := <-uniqueCtrWritingToken
+	result := writeHLL_do(which)
+	uniqueCtrWritingToken <- token
+	return result
+}
+
+// writeHLL_do writes out the HLL indicated by `which` to disk.
+// The usageToken must be held when calling this function.
+func writeHLL_do(which int) error {
+	counter := uniqueCounters[which]
+	filename := getHLLFilename(which, counter.Start)
+	file, err := os.Create(filename)
+	if err != nil {
+		return err
+	}
+	enc := gob.NewEncoder(file)
+	enc.Encode(counter.Counter)
+	return file.Close()
+}
+
+// readHLL reads the current value of the indicated HLL counter.
+// The function takes the usageToken.
+func readHLL(which int) uint64 {
+	token := <-uniqueCtrWritingToken
+	result := uniqueCounters[which].Counter.Count()
+	uniqueCtrWritingToken <- token
+	return result
+}
+
+// writeAllHLLs writes out all in-memory HLLs to disk.
+// The function takes the usageToken.
+func writeAllHLLs() error {
+	var err, err2 error
+	token := <-uniqueCtrWritingToken
+	for _, period := range periods {
+		err2 = writeHLL_do(period)
+		if err == nil {
+			err = err2
+		}
+	}
+	uniqueCtrWritingToken <- token
+	return err
+}
+
+var hllFileServer = http.StripPrefix("/hll", http.FileServer(http.Dir(uniqCountDir)))
+func HTTPShowHLL(w http.ResponseWriter, r *http.Request) {
+	hllFileServer.ServeHTTP(w, r)
+}
+
+// loadUniqueUsers loads the previous HLLs into memory.
+// is_init_func
+func loadUniqueUsers() {
+	err := os.MkdirAll(uniqCountDir, 0755)
+	if err != nil {
+		log.Panicln("could not make unique users data dir:", err)
+	}
+
+	now := time.Now().In(counterLocation)
+	for _, period := range periods {
+		uniqueCounters[period].Start, uniqueCounters[period].End = getCounterPeriod(period, now)
+		err := loadHLL(period, now, &uniqueCounters[period])
+		if err != nil && os.IsNotExist(err) {
+			// errors are bad precisions
+			uniqueCounters[period].Counter, _ = hyperloglog.NewPlus(counterPrecision)
+		} else if err != nil && !os.IsNotExist(err) {
+			log.Panicln("failed to load unique users data:", err)
+		}
+	}
+
+	uniqueUserChannel = make(chan uuid.UUID)
+	uniqueCtrWritingToken = make(chan usageToken)
+	go processNewUsers()
+	go rolloverCounters()
+	uniqueCtrWritingToken <- usageToken{}
+}
+
+// dumpUniqueUsers dumps all the data in uniqueCounters.
+func dumpUniqueUsers() {
+	token := <-uniqueCtrWritingToken
+
+	for _, period := range periods {
+		uniqueCounters[period].Counter.Clear()
+	}
+
+	uniqueCtrWritingToken <- token
+}
+
+// processNewUsers reads uniqueUserChannel, and also dispatches the writing token.
+// This function is the primary writer of uniqueCounters, so it makes sense for it to hold the token.
+// is_init_func
+func processNewUsers() {
+	token := <-uniqueCtrWritingToken
+
+	for {
+		select {
+		case u := <-uniqueUserChannel:
+			hashed := uuidHash(u)
+			for _, period := range periods {
+				uniqueCounters[period].Counter.Add(hashed)
+			}
+		case uniqueCtrWritingToken <- token:
+			// relinquish token. important that there is only one of this going on
+			// otherwise we thrash
+			token = <-uniqueCtrWritingToken
+		}
+	}
+}
+
+func getNextMidnight() time.Time {
+	now := time.Now().In(counterLocation)
+	year, month, day := now.Date()
+	return time.Date(year, month, day+1, 0, 0, 1, 0, counterLocation)
+}
+
+// is_init_func
+func rolloverCounters() {
+	for {
+		time.Sleep(getNextMidnight().Sub(time.Now()))
+		rolloverCounters_do()
+	}
+}
+
+func rolloverCounters_do() {
+	var token usageToken
+	var now time.Time
+
+	token = <-uniqueCtrWritingToken
+	now = time.Now().In(counterLocation)
+	for _, period := range periods {
+		if now.After(uniqueCounters[period].End) {
+			// Cycle for period
+			err := writeHLL_do(period)
+			if err != nil {
+				log.Println("could not cycle unique user counter:", err)
+
+				// Attempt to rescue the data into the log
+				var buf bytes.Buffer
+				bytes, err := uniqueCounters[period].Counter.GobEncode()
+				if err == nil {
+					enc := base64.NewEncoder(base64.StdEncoding, &buf)
+					enc.Write(bytes)
+					enc.Close()
+					log.Print("data for ", getHLLFilename(period, now), ":", buf.String())
+				}
+			}
+
+			uniqueCounters[period].Start, uniqueCounters[period].End = getCounterPeriod(period, now)
+			// errors are bad precisions, so we can ignore
+			uniqueCounters[period].Counter, _ = hyperloglog.NewPlus(counterPrecision)
+		}
+	}
+
+	uniqueCtrWritingToken <- token
+}
diff --git a/socketserver/server/usercount_test.go b/socketserver/server/usercount_test.go
new file mode 100644
index 00000000..4b436d49
--- /dev/null
+++ b/socketserver/server/usercount_test.go
@@ -0,0 +1,70 @@
+package server
+
+import (
+	"github.com/satori/go.uuid"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"testing"
+	"time"
+)
+
+func TestUniqueConnections(t *testing.T) {
+	const TestExpectedCount = 1000
+
+	testStart := time.Now().In(counterLocation)
+
+	var server *httptest.Server
+	var backendExpected = NewTBackendRequestChecker(t,
+		TExpectedBackendRequest{200, bPathAnnounceStartup, &url.Values{"startup": []string{"1"}}, "", nil},
+	)
+	server, _, _ = TSetup(SetupWantSocketServer|SetupWantBackendServer, backendExpected)
+
+	defer server.CloseClientConnections()
+	defer unsubscribeAllClients()
+	defer backendExpected.Close()
+
+	dumpUniqueUsers()
+
+	for i := 0; i < TestExpectedCount; i++ {
+		uuid := uuid.NewV4()
+		uniqueUserChannel <- uuid
+		uniqueUserChannel <- uuid
+	}
+
+	TCheckHLLValue(t, TestExpectedCount, readHLL(periodDaily))
+	TCheckHLLValue(t, TestExpectedCount, readHLL(periodWeekly))
+	TCheckHLLValue(t, TestExpectedCount, readHLL(periodMonthly))
+
+	token := <-uniqueCtrWritingToken
+	uniqueCounters[periodDaily].End = time.Now().In(counterLocation).Add(-1 * time.Second)
+	uniqueCtrWritingToken <- token
+
+	rolloverCounters_do()
+
+	for i := 0; i < TestExpectedCount; i++ {
+		uuid := uuid.NewV4()
+		uniqueUserChannel <- uuid
+		uniqueUserChannel <- uuid
+	}
+
+	TCheckHLLValue(t, TestExpectedCount, readHLL(periodDaily))
+	TCheckHLLValue(t, TestExpectedCount*2, readHLL(periodWeekly))
+	TCheckHLLValue(t, TestExpectedCount*2, readHLL(periodMonthly))
+
+	// Check: Merging the two days results in 2000
+	// note: rolloverCounters_do() wrote out a file, and loadHLL() is reading it back
+	var loadDest PeriodUniqueUsers
+	loadHLL(periodDaily, testStart, &loadDest)
+
+	token = <-uniqueCtrWritingToken
+	loadDest.Counter.Merge(uniqueCounters[periodDaily].Counter)
+	uniqueCtrWritingToken <- token
+
+	TCheckHLLValue(t, TestExpectedCount*2, loadDest.Counter.Count())
+}
+
+func TestUniqueUsersCleanup(t *testing.T) {
+	// Not a test. Removes old files.
+	os.RemoveAll(uniqCountDir)
+}

From 2ddfa2b02cdf6a8e6f91bc45b633604e1d6ffd9b Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 23 Dec 2015 22:18:11 -0800
Subject: [PATCH 113/176] Add /hll_force_write

---
 socketserver/server/handlecore.go | 1 +
 socketserver/server/usercount.go  | 6 ++++++
 2 files changed, 7 insertions(+)

diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 451b9223..49d73c53 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -88,6 +88,7 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 	serveMux.Handle("/.well-known/", http.FileServer(http.Dir("/tmp/letsencrypt/")))
 	serveMux.HandleFunc("/stats", HTTPShowStatistics)
 	serveMux.HandleFunc("/hll/", HTTPShowHLL)
+	serveMux.HandleFunc("/hll_force_write", HTTPWriteHLL)
 
 	serveMux.HandleFunc("/drop_backlog", HTTPBackendDropBacklog)
 	serveMux.HandleFunc("/uncached_pub", HTTPBackendUncachedPublish)
diff --git a/socketserver/server/usercount.go b/socketserver/server/usercount.go
index 9ac614fa..44ad68e3 100644
--- a/socketserver/server/usercount.go
+++ b/socketserver/server/usercount.go
@@ -164,6 +164,12 @@ func HTTPShowHLL(w http.ResponseWriter, r *http.Request) {
 	hllFileServer.ServeHTTP(w, r)
 }
 
+func HTTPWriteHLL(w http.ResponseWriter, r *http.Request) {
+	writeAllHLLs()
+	w.WriteHeader(200)
+	w.Write([]byte("ok"))
+}
+
 // loadUniqueUsers loads the previous HLLs into memory.
 // is_init_func
 func loadUniqueUsers() {

From 987286a6074ca40addca6c05443ab32fd2d8240d Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 23 Dec 2015 22:35:52 -0800
Subject: [PATCH 114/176] Commit script to merge HLL counts

---
 socketserver/cmd/mergecounts/mergecounts.go | 78 +++++++++++++++++++++
 socketserver/server/usercount.go            |  8 +--
 2 files changed, 82 insertions(+), 4 deletions(-)
 create mode 100644 socketserver/cmd/mergecounts/mergecounts.go

diff --git a/socketserver/cmd/mergecounts/mergecounts.go b/socketserver/cmd/mergecounts/mergecounts.go
new file mode 100644
index 00000000..cb503c08
--- /dev/null
+++ b/socketserver/cmd/mergecounts/mergecounts.go
@@ -0,0 +1,78 @@
+package main
+
+import (
+	"github.com/clarkduvall/hyperloglog"
+	"flag"
+	"fmt"
+	"../../server"
+	"net/http"
+	"encoding/gob"
+	"os"
+)
+
+var SERVERS = []string{
+	"https://catbag.frankerfacez.com",
+	"https://andknuckles.frankerfacez.com",
+	"https://tuturu.frankerfacez.com",
+}
+
+const folderPrefix = "/hll/"
+
+const HELP = `
+Usage: mergecounts [filename]
+
+Downloads the file /hll/filename from the 3 FFZ socket servers, merges the contents, and prints the total cardinality.
+
+Filename should be in one of the following formats:
+
+  daily-25-12-2015.gob
+  weekly-51-2015.gob
+  monthly-12-2015.gob
+`
+
+func main() {
+	flag.Parse()
+	if flag.NArg() < 1 {
+		fmt.Print(HELP)
+		os.Exit(2)
+		return
+	}
+
+	filename := flag.Arg(1)
+	hll, err := DownloadAll(filename)
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+		return
+	}
+
+	fmt.Println(hll.Count())
+}
+
+func DownloadAll(filename string) (*hyperloglog.HyperLogLogPlus, error) {
+	result, _ := hyperloglog.NewPlus(server.CounterPrecision)
+
+	for _, server := range SERVERS {
+		singleHLL, err := DownloadHLL(fmt.Sprintf("%s%s%s", server, folderPrefix, filename))
+		if err != nil {
+			return nil, err
+		}
+		result.Merge(singleHLL)
+	}
+
+	return result, nil
+}
+
+func DownloadHLL(url string) (*hyperloglog.HyperLogLogPlus, error) {
+	result, _ := hyperloglog.NewPlus(server.CounterPrecision)
+
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	dec := gob.NewDecoder(resp.Body)
+	dec.Decode(result)
+	resp.Body.Close()
+
+	return result, nil
+}
diff --git a/socketserver/server/usercount.go b/socketserver/server/usercount.go
index 44ad68e3..ef9522c9 100644
--- a/socketserver/server/usercount.go
+++ b/socketserver/server/usercount.go
@@ -46,7 +46,7 @@ const uniqCountDir = "./uniques"
 const usersDailyFmt = "daily-%d-%d-%d.gob"  // d-m-y
 const usersWeeklyFmt = "weekly-%d-%d.gob"   // w-y
 const usersMonthlyFmt = "monthly-%d-%d.gob" // m-y
-const counterPrecision uint8 = 12
+const CounterPrecision uint8 = 12
 
 var uniqueCounters [3]PeriodUniqueUsers
 var uniqueUserChannel chan uuid.UUID
@@ -100,7 +100,7 @@ func loadHLL(which int, at time.Time, dest *PeriodUniqueUsers) error {
 	}
 
 	if dest.Counter == nil {
-		dest.Counter, _ = hyperloglog.NewPlus(counterPrecision)
+		dest.Counter, _ = hyperloglog.NewPlus(CounterPrecision)
 	}
 
 	dec := gob.NewDecoder(bytes.NewReader(fileBytes))
@@ -184,7 +184,7 @@ func loadUniqueUsers() {
 		err := loadHLL(period, now, &uniqueCounters[period])
 		if err != nil && os.IsNotExist(err) {
 			// errors are bad precisions
-			uniqueCounters[period].Counter, _ = hyperloglog.NewPlus(counterPrecision)
+			uniqueCounters[period].Counter, _ = hyperloglog.NewPlus(CounterPrecision)
 		} else if err != nil && !os.IsNotExist(err) {
 			log.Panicln("failed to load unique users data:", err)
 		}
@@ -269,7 +269,7 @@ func rolloverCounters_do() {
 
 			uniqueCounters[period].Start, uniqueCounters[period].End = getCounterPeriod(period, now)
 			// errors are bad precisions, so we can ignore
-			uniqueCounters[period].Counter, _ = hyperloglog.NewPlus(counterPrecision)
+			uniqueCounters[period].Counter, _ = hyperloglog.NewPlus(CounterPrecision)
 		}
 	}
 

From daeb9d1a0610c105b025018da0da52720bdc0c20 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 23 Dec 2015 23:36:10 -0800
Subject: [PATCH 115/176] try gob.registername??

---
 socketserver/cmd/mergecounts/mergecounts.go | 10 ++++++++--
 socketserver/server/usercount.go            |  7 ++++---
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/socketserver/cmd/mergecounts/mergecounts.go b/socketserver/cmd/mergecounts/mergecounts.go
index cb503c08..2a63a893 100644
--- a/socketserver/cmd/mergecounts/mergecounts.go
+++ b/socketserver/cmd/mergecounts/mergecounts.go
@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"encoding/gob"
 	"os"
+	"github.com/satori/go.uuid"
 )
 
 var SERVERS = []string{
@@ -37,6 +38,7 @@ func main() {
 		os.Exit(2)
 		return
 	}
+	gob.RegisterName("hyperloglog", hyperloglog.HyperLogLogPlus{})
 
 	filename := flag.Arg(1)
 	hll, err := DownloadAll(filename)
@@ -70,9 +72,13 @@ func DownloadHLL(url string) (*hyperloglog.HyperLogLogPlus, error) {
 	if err != nil {
 		return nil, err
 	}
+	defer resp.Body.Close()
 	dec := gob.NewDecoder(resp.Body)
-	dec.Decode(result)
-	resp.Body.Close()
+	err = dec.Decode(result)
+	if err != nil {
+		return nil, err
+	}
+	fmt.Println(url, result.Count())
 
 	return result, nil
 }
diff --git a/socketserver/server/usercount.go b/socketserver/server/usercount.go
index ef9522c9..fbacab95 100644
--- a/socketserver/server/usercount.go
+++ b/socketserver/server/usercount.go
@@ -17,9 +17,9 @@ import (
 )
 
 // uuidHash implements a hash for uuid.UUID by XORing the random bits.
-type uuidHash uuid.UUID
+type UuidHash uuid.UUID
 
-func (u uuidHash) Sum64() uint64 {
+func (u UuidHash) Sum64() uint64 {
 	var valLow, valHigh uint64
 	valLow = binary.LittleEndian.Uint64(u[0:8])
 	valHigh = binary.LittleEndian.Uint64(u[8:16])
@@ -173,6 +173,7 @@ func HTTPWriteHLL(w http.ResponseWriter, r *http.Request) {
 // loadUniqueUsers loads the previous HLLs into memory.
 // is_init_func
 func loadUniqueUsers() {
+	gob.RegisterName("hyperloglog", hyperloglog.HyperLogLogPlus{})
 	err := os.MkdirAll(uniqCountDir, 0755)
 	if err != nil {
 		log.Panicln("could not make unique users data dir:", err)
@@ -217,7 +218,7 @@ func processNewUsers() {
 	for {
 		select {
 		case u := <-uniqueUserChannel:
-			hashed := uuidHash(u)
+			hashed := UuidHash(u)
 			for _, period := range periods {
 				uniqueCounters[period].Counter.Add(hashed)
 			}

From b22efabfed2e191d516e15b5b5f3752c107a5c1a Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 23 Dec 2015 23:54:03 -0800
Subject: [PATCH 116/176] flag.Arg(0) not flag.Arg(1) you dummy

---
 socketserver/cmd/mergecounts/mergecounts.go | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/socketserver/cmd/mergecounts/mergecounts.go b/socketserver/cmd/mergecounts/mergecounts.go
index 2a63a893..a127bde6 100644
--- a/socketserver/cmd/mergecounts/mergecounts.go
+++ b/socketserver/cmd/mergecounts/mergecounts.go
@@ -8,7 +8,6 @@ import (
 	"net/http"
 	"encoding/gob"
 	"os"
-	"github.com/satori/go.uuid"
 )
 
 var SERVERS = []string{
@@ -31,6 +30,8 @@ Filename should be in one of the following formats:
   monthly-12-2015.gob
 `
 
+var forceWrite = flag.Bool("f", false, "force servers to write out their current")
+
 func main() {
 	flag.Parse()
 	if flag.NArg() < 1 {
@@ -38,9 +39,12 @@ func main() {
 		os.Exit(2)
 		return
 	}
-	gob.RegisterName("hyperloglog", hyperloglog.HyperLogLogPlus{})
 
-	filename := flag.Arg(1)
+	if *forceWrite {
+		ForceWrite()
+	}
+
+	filename := flag.Arg(0)
 	hll, err := DownloadAll(filename)
 	if err != nil {
 		fmt.Println(err)
@@ -51,6 +55,17 @@ func main() {
 	fmt.Println(hll.Count())
 }
 
+func ForceWrite() {
+	for _, server := range SERVERS {
+		resp, err := http.Get(fmt.Sprintf("%s/hll_force_write", server))
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+		resp.Body.Close()
+	}
+}
+
 func DownloadAll(filename string) (*hyperloglog.HyperLogLogPlus, error) {
 	result, _ := hyperloglog.NewPlus(server.CounterPrecision)
 

From 1a709d6e51bec65654f44d8ce612bad615c70e29 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 3 Jan 2016 08:57:37 -0800
Subject: [PATCH 117/176] lmao time.UnixDate sounds like a great idea doesn't
 it

---
 socketserver/cmd/mergecounts/mergecounts.go | 16 +++++++++-------
 socketserver/server/publisher.go            |  4 +++-
 socketserver/server/usercount.go            |  7 +++++--
 3 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/socketserver/cmd/mergecounts/mergecounts.go b/socketserver/cmd/mergecounts/mergecounts.go
index a127bde6..4eff882f 100644
--- a/socketserver/cmd/mergecounts/mergecounts.go
+++ b/socketserver/cmd/mergecounts/mergecounts.go
@@ -1,12 +1,12 @@
 package main
 
 import (
-	"github.com/clarkduvall/hyperloglog"
+	"../../server"
+	"encoding/gob"
 	"flag"
 	"fmt"
-	"../../server"
+	"github.com/clarkduvall/hyperloglog"
 	"net/http"
-	"encoding/gob"
 	"os"
 )
 
@@ -40,10 +40,6 @@ func main() {
 		return
 	}
 
-	if *forceWrite {
-		ForceWrite()
-	}
-
 	filename := flag.Arg(0)
 	hll, err := DownloadAll(filename)
 	if err != nil {
@@ -70,6 +66,12 @@ func DownloadAll(filename string) (*hyperloglog.HyperLogLogPlus, error) {
 	result, _ := hyperloglog.NewPlus(server.CounterPrecision)
 
 	for _, server := range SERVERS {
+		if *forceWrite {
+			resp, err := http.Get(fmt.Sprintf("%s/hll_force_write", server))
+			if err == nil {
+				resp.Body.Close()
+			}
+		}
 		singleHLL, err := DownloadHLL(fmt.Sprintf("%s%s%s", server, folderPrefix, filename))
 		if err != nil {
 			return nil, err
diff --git a/socketserver/server/publisher.go b/socketserver/server/publisher.go
index cad5b62a..8e722682 100644
--- a/socketserver/server/publisher.go
+++ b/socketserver/server/publisher.go
@@ -241,11 +241,13 @@ func HTTPBackendCachedPublish(w http.ResponseWriter, r *http.Request) {
 	channel := formData.Get("channel")
 	deleteMode := formData.Get("delete") != ""
 	timeStr := formData.Get("time")
-	timestamp, err := time.Parse(time.UnixDate, timeStr)
+	timeNum, err := strconv.ParseInt(timeStr, 10, 64)
 	if err != nil {
 		w.WriteHeader(422)
 		fmt.Fprintf(w, "error parsing time: %v", err)
+		return
 	}
+	timestamp := time.Unix(timeNum, 0)
 
 	cacheinfo, ok := S2CCommandsCacheInfo[cmd]
 	if !ok {
diff --git a/socketserver/server/usercount.go b/socketserver/server/usercount.go
index fbacab95..7cdbed67 100644
--- a/socketserver/server/usercount.go
+++ b/socketserver/server/usercount.go
@@ -6,9 +6,9 @@ import (
 	"encoding/binary"
 	"encoding/gob"
 	"fmt"
-	"net/http"
 	"io/ioutil"
 	"log"
+	"net/http"
 	"os"
 	"time"
 
@@ -160,6 +160,7 @@ func writeAllHLLs() error {
 }
 
 var hllFileServer = http.StripPrefix("/hll", http.FileServer(http.Dir(uniqCountDir)))
+
 func HTTPShowHLL(w http.ResponseWriter, r *http.Request) {
 	hllFileServer.ServeHTTP(w, r)
 }
@@ -239,7 +240,9 @@ func getNextMidnight() time.Time {
 // is_init_func
 func rolloverCounters() {
 	for {
-		time.Sleep(getNextMidnight().Sub(time.Now()))
+		duration := getNextMidnight().Sub(time.Now())
+		//	fmt.Println(duration)
+		time.Sleep(duration)
 		rolloverCounters_do()
 	}
 }

From c634e50b5d92fb9cc11f6ba6597a78bf48163246 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 3 Jan 2016 09:31:22 -0800
Subject: [PATCH 118/176] CahcedLastMessages was not initialized

---
 socketserver/server/publisher.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/socketserver/server/publisher.go b/socketserver/server/publisher.go
index 8e722682..02843f4c 100644
--- a/socketserver/server/publisher.go
+++ b/socketserver/server/publisher.go
@@ -73,11 +73,11 @@ type LastSavedMessage struct {
 // map is command -> channel -> data
 
 // CacheTypeLastOnly. Cleaned up by reaper goroutine every ~hour.
-var CachedLastMessages map[Command]map[string]LastSavedMessage
+var CachedLastMessages = make(map[Command]map[string]LastSavedMessage)
 var CachedLSMLock sync.RWMutex
 
 // CacheTypePersistent. Never cleaned.
-var PersistentLastMessages map[Command]map[string]LastSavedMessage
+var PersistentLastMessages = make(map[Command]map[string]LastSavedMessage)
 var PersistentLSMLock sync.RWMutex
 
 // DumpBacklogData drops all /cached_pub data.

From 41a40d360dd23a3bbd91029a3ba9141de4384dba Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 3 Jan 2016 13:50:33 -0800
Subject: [PATCH 119/176] Fix backend error handling, maybe

---
 socketserver/server/backend.go | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/socketserver/server/backend.go b/socketserver/server/backend.go
index eea73ead..5cad2f58 100644
--- a/socketserver/server/backend.go
+++ b/socketserver/server/backend.go
@@ -115,10 +115,13 @@ func HTTPBackendUncachedPublish(w http.ResponseWriter, r *http.Request) {
 }
 
 // ErrForwardedFromBackend is an error returned by the backend server.
-type ErrForwardedFromBackend string
+type ErrForwardedFromBackend struct {
+	JSONError interface{}
+}
 
 func (bfe ErrForwardedFromBackend) Error() string {
-	return string(bfe)
+	bytes, _ := json.Marshal(bfe)
+	return string(bytes)
 }
 
 // ErrAuthorizationNeeded is emitted when the backend replies with HTTP 401.
@@ -174,7 +177,12 @@ func SendRemoteCommand(remoteCommand, data string, auth AuthInfo) (responseStr s
 		return "", ErrAuthorizationNeeded
 	} else if resp.StatusCode != 200 {
 		if resp.Header.Get("Content-Type") == "application/json" {
-			return "", ErrForwardedFromBackend(responseStr)
+			var err2 ErrForwardedFromBackend
+			err := json.Unmarshal(respBytes, &err2.JSONError)
+			if err != nil {
+				return "", fmt.Errorf("error decoding json error from backend: %v | %s", err, responseStr)
+			}
+			return "", err2
 		}
 		return "", httpError(resp.StatusCode)
 	}

From 9ca9b6b213d53297d048c3a2f4d82ba6439c1a72 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 3 Jan 2016 13:54:26 -0800
Subject: [PATCH 120/176] Actually give ErrForwardedFromBackend special
 handling >.>

---
 socketserver/server/commands.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/socketserver/server/commands.go b/socketserver/server/commands.go
index 0e8004c9..3f424fa5 100644
--- a/socketserver/server/commands.go
+++ b/socketserver/server/commands.go
@@ -562,6 +562,8 @@ func doRemoteCommand(conn *websocket.Conn, msg ClientMessage, client *ClientInfo
 			}
 		})
 		return // without keepalive.Done()
+	} else if bfe, ok := err.(ErrForwardedFromBackend); ok {
+		client.MessageChannel <- ClientMessage{MessageID: msg.MessageID, Command: ErrorCommand, Arguments: bfe.JSONError}
 	} else if err != nil {
 		client.MessageChannel <- ClientMessage{MessageID: msg.MessageID, Command: ErrorCommand, Arguments: err.Error()}
 	} else {

From e117766eb654c1aa2bc8003b4783ccbad4bab502 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Fri, 15 Jan 2016 20:54:30 -0800
Subject: [PATCH 121/176] Rip out the weekly/monthly HLLs - they're redundant

The weekly data can be constructed much more flexibly by just using the
daily data.
---
 socketserver/cmd/mergecounts/mergecounts.go |   2 +-
 socketserver/server/backend.go              |   2 +-
 socketserver/server/handlecore.go           |   2 +-
 socketserver/server/stats.go                |   6 +-
 socketserver/server/subscriptions_test.go   |   8 +-
 socketserver/server/testinfra_test.go       |   3 +-
 socketserver/server/usercount.go            | 160 ++++++++------------
 socketserver/server/usercount_test.go       |  15 +-
 8 files changed, 76 insertions(+), 122 deletions(-)

diff --git a/socketserver/cmd/mergecounts/mergecounts.go b/socketserver/cmd/mergecounts/mergecounts.go
index 4eff882f..af3ec83b 100644
--- a/socketserver/cmd/mergecounts/mergecounts.go
+++ b/socketserver/cmd/mergecounts/mergecounts.go
@@ -11,7 +11,7 @@ import (
 )
 
 var SERVERS = []string{
-	"https://catbag.frankerfacez.com",
+//	"https://catbag.frankerfacez.com",
 	"https://andknuckles.frankerfacez.com",
 	"https://tuturu.frankerfacez.com",
 }
diff --git a/socketserver/server/backend.go b/socketserver/server/backend.go
index 5cad2f58..1ec4751c 100644
--- a/socketserver/server/backend.go
+++ b/socketserver/server/backend.go
@@ -120,7 +120,7 @@ type ErrForwardedFromBackend struct {
 }
 
 func (bfe ErrForwardedFromBackend) Error() string {
-	bytes, _ := json.Marshal(bfe)
+	bytes, _ := json.Marshal(bfe.JSONError)
 	return string(bytes)
 }
 
diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 49d73c53..f068e142 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -139,7 +139,7 @@ func shutdownHandler() {
 	var wg sync.WaitGroup
 	wg.Add(1)
 	go func() {
-		writeAllHLLs()
+		writeHLL()
 		wg.Done()
 	}()
 
diff --git a/socketserver/server/stats.go b/socketserver/server/stats.go
index 0275d4d7..24fd52dc 100644
--- a/socketserver/server/stats.go
+++ b/socketserver/server/stats.go
@@ -165,10 +165,10 @@ func updateSysMem() {
 	if err == nil {
 		Statistics.SysMemTotalKB = memInfo.MemTotal
 		Statistics.SysMemFreeKB = memInfo.MemAvailable
+	}
 
-		if memInfo.MemAvailable > 0 && memInfo.MemAvailable < Configuration.MinMemoryKBytes {
-			writeAllHLLs()
-		}
+	{
+		writeHLL()
 	}
 }
 
diff --git a/socketserver/server/subscriptions_test.go b/socketserver/server/subscriptions_test.go
index d2e561d2..c0c04720 100644
--- a/socketserver/server/subscriptions_test.go
+++ b/socketserver/server/subscriptions_test.go
@@ -229,11 +229,9 @@ func TestSubscriptionAndPublish(t *testing.T) {
 	doneWg.Wait()
 	server.Close()
 
-	for _, period := range periods {
-		clientCount := readHLL(period)
-		if clientCount < 3 || clientCount > 5 {
-			t.Error("clientCount outside acceptable range: expected 4, got ", clientCount)
-		}
+	clientCount := readCurrentHLL()
+	if clientCount < 3 || clientCount > 5 {
+		t.Error("clientCount outside acceptable range: expected 4, got ", clientCount)
 	}
 }
 
diff --git a/socketserver/server/testinfra_test.go b/socketserver/server/testinfra_test.go
index 9c03fa78..bc7c5e0b 100644
--- a/socketserver/server/testinfra_test.go
+++ b/socketserver/server/testinfra_test.go
@@ -12,6 +12,7 @@ import (
 	"sync"
 	"testing"
 	"time"
+	"strconv"
 )
 
 const (
@@ -272,7 +273,7 @@ func TSealForSavePubMsg(tb testing.TB, cmd Command, channel string, arguments in
 	if deleteMode {
 		form.Set("delete", "1")
 	}
-	form.Set("time", time.Now().Format(time.UnixDate))
+	form.Set("time", strconv.FormatInt(time.Now().Unix(), 10))
 
 	sealed, err := SealRequest(form)
 	if err != nil {
diff --git a/socketserver/server/usercount.go b/socketserver/server/usercount.go
index 7cdbed67..74360172 100644
--- a/socketserver/server/usercount.go
+++ b/socketserver/server/usercount.go
@@ -14,6 +14,7 @@ import (
 
 	"github.com/clarkduvall/hyperloglog"
 	"github.com/satori/go.uuid"
+	"io"
 )
 
 // uuidHash implements a hash for uuid.UUID by XORing the random bits.
@@ -34,67 +35,37 @@ type PeriodUniqueUsers struct {
 
 type usageToken struct{}
 
-const (
-	periodDaily = iota
-	periodWeekly
-	periodMonthly
-)
-
-var periods [3]int = [3]int{periodDaily, periodWeekly, periodMonthly}
-
 const uniqCountDir = "./uniques"
-const usersDailyFmt = "daily-%d-%d-%d.gob"  // d-m-y
-const usersWeeklyFmt = "weekly-%d-%d.gob"   // w-y
-const usersMonthlyFmt = "monthly-%d-%d.gob" // m-y
+const usersDailyFmt = "daily-%d-%d-%d.gob" // d-m-y
 const CounterPrecision uint8 = 12
 
-var uniqueCounters [3]PeriodUniqueUsers
+var uniqueCounter PeriodUniqueUsers
 var uniqueUserChannel chan uuid.UUID
 var uniqueCtrWritingToken chan usageToken
 
 var counterLocation *time.Location = time.FixedZone("UTC-5", int((time.Hour*-5)/time.Second))
 
 // getCounterPeriod calculates the start and end timestamps for the HLL measurement period that includes the 'at' timestamp.
-func getCounterPeriod(which int, at time.Time) (start time.Time, end time.Time) {
+func getCounterPeriod(at time.Time) (start time.Time, end time.Time) {
 	year, month, day := at.Date()
-
-	switch which {
-	case periodDaily:
-		start = time.Date(year, month, day, 0, 0, 0, 0, counterLocation)
-		end = time.Date(year, month, day+1, 0, 0, 0, 0, counterLocation)
-	case periodWeekly:
-		dayOffset := at.Weekday() - time.Sunday
-		start = time.Date(year, month, day-int(dayOffset), 0, 0, 0, 0, counterLocation)
-		end = time.Date(year, month, day-int(dayOffset)+7, 0, 0, 0, 0, counterLocation)
-	case periodMonthly:
-		start = time.Date(year, month, 1, 0, 0, 0, 0, counterLocation)
-		end = time.Date(year, month+1, 1, 0, 0, 0, 0, counterLocation)
-	}
+	start = time.Date(year, month, day, 0, 0, 0, 0, counterLocation)
+	end = time.Date(year, month, day+1, 0, 0, 0, 0, counterLocation)
 	return start, end
 }
 
 // getHLLFilename returns the filename for the saved HLL whose measurement period covers the given time.
-func getHLLFilename(which int, at time.Time) string {
+func getHLLFilename(at time.Time) string {
 	var filename string
-	switch which {
-	case periodDaily:
-		year, month, day := at.Date()
-		filename = fmt.Sprintf(usersDailyFmt, day, month, year)
-	case periodWeekly:
-		year, week := at.ISOWeek()
-		filename = fmt.Sprintf(usersWeeklyFmt, week, year)
-	case periodMonthly:
-		year, month, _ := at.Date()
-		filename = fmt.Sprintf(usersMonthlyFmt, month, year)
-	}
+	year, month, day := at.Date()
+	filename = fmt.Sprintf(usersDailyFmt, day, month, year)
 	return fmt.Sprintf("%s/%s", uniqCountDir, filename)
 }
 
 // loadHLL loads a HLL from disk and stores the result in dest.Counter.
 // If dest.Counter is nil, it will be initialized. (This is a useful side-effect.)
 // If dest is one of the uniqueCounters, the usageToken must be held.
-func loadHLL(which int, at time.Time, dest *PeriodUniqueUsers) error {
-	fileBytes, err := ioutil.ReadFile(getHLLFilename(which, at))
+func loadHLL(at time.Time, dest *PeriodUniqueUsers) error {
+	fileBytes, err := ioutil.ReadFile(getHLLFilename(at))
 	if err != nil {
 		return err
 	}
@@ -107,6 +78,7 @@ func loadHLL(which int, at time.Time, dest *PeriodUniqueUsers) error {
 	err = dec.Decode(dest.Counter)
 	if err != nil {
 		log.Panicln(err)
+
 		return err
 	}
 	return nil
@@ -114,51 +86,42 @@ func loadHLL(which int, at time.Time, dest *PeriodUniqueUsers) error {
 
 // writeHLL writes the indicated HLL to disk.
 // The function takes the usageToken.
-func writeHLL(which int) error {
+func writeHLL() error {
 	token := <-uniqueCtrWritingToken
-	result := writeHLL_do(which)
+	result := writeHLL_do(&uniqueCounter)
 	uniqueCtrWritingToken <- token
 	return result
 }
 
 // writeHLL_do writes out the HLL indicated by `which` to disk.
 // The usageToken must be held when calling this function.
-func writeHLL_do(which int) error {
-	counter := uniqueCounters[which]
-	filename := getHLLFilename(which, counter.Start)
+func writeHLL_do(hll *PeriodUniqueUsers) (err error) {
+	filename := getHLLFilename(hll.Start)
 	file, err := os.Create(filename)
 	if err != nil {
 		return err
 	}
+
+	defer func(file io.Closer) {
+		fileErr := file.Close()
+		if err == nil {
+			err = fileErr
+		}
+	}(file)
+
 	enc := gob.NewEncoder(file)
-	enc.Encode(counter.Counter)
-	return file.Close()
+	return enc.Encode(hll.Counter)
 }
 
-// readHLL reads the current value of the indicated HLL counter.
+// readCurrentHLL reads the current value of the active HLL counter.
 // The function takes the usageToken.
-func readHLL(which int) uint64 {
+func readCurrentHLL() uint64 {
 	token := <-uniqueCtrWritingToken
-	result := uniqueCounters[which].Counter.Count()
+	result := uniqueCounter.Counter.Count()
 	uniqueCtrWritingToken <- token
 	return result
 }
 
-// writeAllHLLs writes out all in-memory HLLs to disk.
-// The function takes the usageToken.
-func writeAllHLLs() error {
-	var err, err2 error
-	token := <-uniqueCtrWritingToken
-	for _, period := range periods {
-		err2 = writeHLL_do(period)
-		if err == nil {
-			err = err2
-		}
-	}
-	uniqueCtrWritingToken <- token
-	return err
-}
-
 var hllFileServer = http.StripPrefix("/hll", http.FileServer(http.Dir(uniqCountDir)))
 
 func HTTPShowHLL(w http.ResponseWriter, r *http.Request) {
@@ -166,7 +129,7 @@ func HTTPShowHLL(w http.ResponseWriter, r *http.Request) {
 }
 
 func HTTPWriteHLL(w http.ResponseWriter, r *http.Request) {
-	writeAllHLLs()
+	writeHLL()
 	w.WriteHeader(200)
 	w.Write([]byte("ok"))
 }
@@ -181,15 +144,18 @@ func loadUniqueUsers() {
 	}
 
 	now := time.Now().In(counterLocation)
-	for _, period := range periods {
-		uniqueCounters[period].Start, uniqueCounters[period].End = getCounterPeriod(period, now)
-		err := loadHLL(period, now, &uniqueCounters[period])
-		if err != nil && os.IsNotExist(err) {
-			// errors are bad precisions
-			uniqueCounters[period].Counter, _ = hyperloglog.NewPlus(CounterPrecision)
-		} else if err != nil && !os.IsNotExist(err) {
-			log.Panicln("failed to load unique users data:", err)
-		}
+	uniqueCounter.Start, uniqueCounter.End = getCounterPeriod(now)
+	err = loadHLL(now, &uniqueCounter)
+	isIgnorableError := err != nil && (false ||
+		(os.IsNotExist(err)) ||
+		(err == io.EOF))
+
+	if isIgnorableError {
+		// file didn't finish writing
+		// errors in NewPlus are bad precisions
+		uniqueCounter.Counter, _ = hyperloglog.NewPlus(CounterPrecision)
+	} else if err != nil {
+		log.Panicln("failed to load unique users data:", err)
 	}
 
 	uniqueUserChannel = make(chan uuid.UUID)
@@ -203,9 +169,7 @@ func loadUniqueUsers() {
 func dumpUniqueUsers() {
 	token := <-uniqueCtrWritingToken
 
-	for _, period := range periods {
-		uniqueCounters[period].Counter.Clear()
-	}
+	uniqueCounter.Counter.Clear()
 
 	uniqueCtrWritingToken <- token
 }
@@ -220,9 +184,7 @@ func processNewUsers() {
 		select {
 		case u := <-uniqueUserChannel:
 			hashed := UuidHash(u)
-			for _, period := range periods {
-				uniqueCounters[period].Counter.Add(hashed)
-			}
+			uniqueCounter.Counter.Add(hashed)
 		case uniqueCtrWritingToken <- token:
 			// relinquish token. important that there is only one of this going on
 			// otherwise we thrash
@@ -253,29 +215,25 @@ func rolloverCounters_do() {
 
 	token = <-uniqueCtrWritingToken
 	now = time.Now().In(counterLocation)
-	for _, period := range periods {
-		if now.After(uniqueCounters[period].End) {
-			// Cycle for period
-			err := writeHLL_do(period)
-			if err != nil {
-				log.Println("could not cycle unique user counter:", err)
+	// Cycle for period
+	err := writeHLL_do(&uniqueCounter)
+	if err != nil {
+		log.Println("could not cycle unique user counter:", err)
 
-				// Attempt to rescue the data into the log
-				var buf bytes.Buffer
-				bytes, err := uniqueCounters[period].Counter.GobEncode()
-				if err == nil {
-					enc := base64.NewEncoder(base64.StdEncoding, &buf)
-					enc.Write(bytes)
-					enc.Close()
-					log.Print("data for ", getHLLFilename(period, now), ":", buf.String())
-				}
-			}
-
-			uniqueCounters[period].Start, uniqueCounters[period].End = getCounterPeriod(period, now)
-			// errors are bad precisions, so we can ignore
-			uniqueCounters[period].Counter, _ = hyperloglog.NewPlus(CounterPrecision)
+		// Attempt to rescue the data into the log
+		var buf bytes.Buffer
+		bytes, err := uniqueCounter.Counter.GobEncode()
+		if err == nil {
+			enc := base64.NewEncoder(base64.StdEncoding, &buf)
+			enc.Write(bytes)
+			enc.Close()
+			log.Print("data for ", getHLLFilename(uniqueCounter.Start), ":", buf.String())
 		}
 	}
 
+	uniqueCounter.Start, uniqueCounter.End = getCounterPeriod(now)
+	// errors are bad precisions, so we can ignore
+	uniqueCounter.Counter, _ = hyperloglog.NewPlus(CounterPrecision)
+
 	uniqueCtrWritingToken <- token
 }
diff --git a/socketserver/server/usercount_test.go b/socketserver/server/usercount_test.go
index 4b436d49..c6ed2050 100644
--- a/socketserver/server/usercount_test.go
+++ b/socketserver/server/usercount_test.go
@@ -32,12 +32,10 @@ func TestUniqueConnections(t *testing.T) {
 		uniqueUserChannel <- uuid
 	}
 
-	TCheckHLLValue(t, TestExpectedCount, readHLL(periodDaily))
-	TCheckHLLValue(t, TestExpectedCount, readHLL(periodWeekly))
-	TCheckHLLValue(t, TestExpectedCount, readHLL(periodMonthly))
+	TCheckHLLValue(t, TestExpectedCount, readCurrentHLL())
 
 	token := <-uniqueCtrWritingToken
-	uniqueCounters[periodDaily].End = time.Now().In(counterLocation).Add(-1 * time.Second)
+	uniqueCounter.End = time.Now().In(counterLocation).Add(-1 * time.Second)
 	uniqueCtrWritingToken <- token
 
 	rolloverCounters_do()
@@ -48,17 +46,16 @@ func TestUniqueConnections(t *testing.T) {
 		uniqueUserChannel <- uuid
 	}
 
-	TCheckHLLValue(t, TestExpectedCount, readHLL(periodDaily))
-	TCheckHLLValue(t, TestExpectedCount*2, readHLL(periodWeekly))
-	TCheckHLLValue(t, TestExpectedCount*2, readHLL(periodMonthly))
+	TCheckHLLValue(t, TestExpectedCount, readCurrentHLL())
 
 	// Check: Merging the two days results in 2000
 	// note: rolloverCounters_do() wrote out a file, and loadHLL() is reading it back
+	// TODO need to rewrite some of the test to make this work
 	var loadDest PeriodUniqueUsers
-	loadHLL(periodDaily, testStart, &loadDest)
+	loadHLL(testStart, &loadDest)
 
 	token = <-uniqueCtrWritingToken
-	loadDest.Counter.Merge(uniqueCounters[periodDaily].Counter)
+	loadDest.Counter.Merge(uniqueCounter.Counter)
 	uniqueCtrWritingToken <- token
 
 	TCheckHLLValue(t, TestExpectedCount*2, loadDest.Counter.Count())

From ba73186a99601fb570201d20c9e90e7f9514c7c6 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Fri, 15 Jan 2016 20:59:33 -0800
Subject: [PATCH 122/176] Add MaxClientCount to configuration

---
 socketserver/server/handlecore.go | 10 ++++++++++
 socketserver/server/types.go      |  8 ++++++++
 2 files changed, 18 insertions(+)

diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index f068e142..900a74d5 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -203,9 +203,19 @@ func HTTPHandleRootURL(w http.ResponseWriter, r *http.Request) {
 		if Statistics.SysMemFreeKB > 0 && Statistics.SysMemFreeKB < Configuration.MinMemoryKBytes {
 			atomic.AddUint64(&Statistics.LowMemDroppedConnections, 1)
 			w.WriteHeader(503)
+			fmt.Fprint(w, "error: low memory")
 			return
 		}
 
+		if Configuration.MaxClientCount != 0 {
+			curClients := atomic.LoadUint64(&Statistics.CurrentClientCount)
+			if curClients >= Configuration.MaxClientCount {
+				w.WriteHeader(503)
+				fmt.Fprint(w, "error: client limit reached")
+				return
+			}
+		}
+
 		conn, err := SocketUpgrader.Upgrade(w, r, nil)
 		if err != nil {
 			fmt.Fprintf(w, "error: %v", err)
diff --git a/socketserver/server/types.go b/socketserver/server/types.go
index d827912c..04614b2a 100644
--- a/socketserver/server/types.go
+++ b/socketserver/server/types.go
@@ -16,17 +16,24 @@ const NegativeOne = ^uint64(0)
 type ConfigFile struct {
 	// Numeric server id known to the backend
 	ServerID      int
+	// Address to bind the HTTP server to on startup.
 	ListenAddr    string
+	// Address to bind the TLS server to on startup.
 	SSLListenAddr string
 	// URL to the backend server
 	BackendURL string
 
 	// Minimum memory to accept a new connection
 	MinMemoryKBytes uint64
+	// Maximum # of clients that can be connected. 0 to disable.
+	MaxClientCount  uint64
 
 	// SSL/TLS
+	// Enable the use of SSL.
 	UseSSL             bool
+	// Path to certificate file.
 	SSLCertificateFile string
+	// Path to key file.
 	SSLKeyFile         string
 
 	UseESLogStashing bool
@@ -39,6 +46,7 @@ type ConfigFile struct {
 	OurPublicKey     []byte
 	BackendPublicKey []byte
 
+	// Request username validation from all new clients.
 	SendAuthToNewClients bool
 }
 

From 133544bddd5feff1e4ffce2046409ede7d2e5a14 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Fri, 15 Jan 2016 21:22:36 -0800
Subject: [PATCH 123/176] Oops, forgot to count disconnects...

---
 socketserver/server/handlecore.go | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 900a74d5..2d0940ca 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -351,11 +351,12 @@ func RunSocketConnection(conn *websocket.Conn) {
 	if !StopAcceptingConnections {
 		// Don't perform high contention operations when server is closing
 		atomic.AddUint64(&Statistics.CurrentClientCount, NegativeOne)
-	}
+		atomic.AddUint64(&Statistics.ClientDisconnectsTotal, 1)
 
-	report.UsernameWasValidated = client.UsernameValidated
-	report.TwitchUsername = client.TwitchUsername
-	logstasher.Submit(&report)
+		report.UsernameWasValidated = client.UsernameValidated
+		report.TwitchUsername = client.TwitchUsername
+		logstasher.Submit(&report)
+	}
 }
 
 func runSocketReader(conn *websocket.Conn, errorChan chan<- error, clientChan chan<- ClientMessage, stoppedChan <-chan struct{}) {

From 15658e58c3acde545c3a2332da395285d45a1cd2 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sat, 16 Jan 2016 17:49:39 -0800
Subject: [PATCH 124/176] Oops

---
 socketserver/server/handlecore.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 2d0940ca..301d8abc 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -39,7 +39,7 @@ const HelloCommand Command = "hello"
 // It indicates that the client is finished sending the initial 'sub' commands and the server should send the backlog.
 const ReadyCommand Command = "ready"
 
-const SetUserCommand Command = "set_user"
+const SetUserCommand Command = "setuser"
 
 // AuthorizeCommand is a S2C Command sent as part of Twitch username validation.
 const AuthorizeCommand Command = "do_authorize"

From 56b86ad4e35c2689e2951212b88f985edcdb5f73 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 17 Jan 2016 10:31:39 -0800
Subject: [PATCH 125/176] Expose some functions

---
 socketserver/cmd/statsweb/statsweb.go |  3 +++
 socketserver/server/usercount.go      | 22 +++++++++++-----------
 2 files changed, 14 insertions(+), 11 deletions(-)
 create mode 100644 socketserver/cmd/statsweb/statsweb.go

diff --git a/socketserver/cmd/statsweb/statsweb.go b/socketserver/cmd/statsweb/statsweb.go
new file mode 100644
index 00000000..1110061a
--- /dev/null
+++ b/socketserver/cmd/statsweb/statsweb.go
@@ -0,0 +1,3 @@
+package main
+
+
diff --git a/socketserver/server/usercount.go b/socketserver/server/usercount.go
index 74360172..fb6e5ad9 100644
--- a/socketserver/server/usercount.go
+++ b/socketserver/server/usercount.go
@@ -36,7 +36,7 @@ type PeriodUniqueUsers struct {
 type usageToken struct{}
 
 const uniqCountDir = "./uniques"
-const usersDailyFmt = "daily-%d-%d-%d.gob" // d-m-y
+const UsersDailyFmt = "daily-%d-%d-%d.gob" // d-m-y
 const CounterPrecision uint8 = 12
 
 var uniqueCounter PeriodUniqueUsers
@@ -45,19 +45,19 @@ var uniqueCtrWritingToken chan usageToken
 
 var counterLocation *time.Location = time.FixedZone("UTC-5", int((time.Hour*-5)/time.Second))
 
-// getCounterPeriod calculates the start and end timestamps for the HLL measurement period that includes the 'at' timestamp.
-func getCounterPeriod(at time.Time) (start time.Time, end time.Time) {
+// GetCounterPeriod calculates the start and end timestamps for the HLL measurement period that includes the 'at' timestamp.
+func GetCounterPeriod(at time.Time) (start time.Time, end time.Time) {
 	year, month, day := at.Date()
 	start = time.Date(year, month, day, 0, 0, 0, 0, counterLocation)
 	end = time.Date(year, month, day+1, 0, 0, 0, 0, counterLocation)
 	return start, end
 }
 
-// getHLLFilename returns the filename for the saved HLL whose measurement period covers the given time.
-func getHLLFilename(at time.Time) string {
+// GetHLLFilename returns the filename for the saved HLL whose measurement period covers the given time.
+func GetHLLFilename(at time.Time) string {
 	var filename string
 	year, month, day := at.Date()
-	filename = fmt.Sprintf(usersDailyFmt, day, month, year)
+	filename = fmt.Sprintf(UsersDailyFmt, day, month, year)
 	return fmt.Sprintf("%s/%s", uniqCountDir, filename)
 }
 
@@ -65,7 +65,7 @@ func getHLLFilename(at time.Time) string {
 // If dest.Counter is nil, it will be initialized. (This is a useful side-effect.)
 // If dest is one of the uniqueCounters, the usageToken must be held.
 func loadHLL(at time.Time, dest *PeriodUniqueUsers) error {
-	fileBytes, err := ioutil.ReadFile(getHLLFilename(at))
+	fileBytes, err := ioutil.ReadFile(GetHLLFilename(at))
 	if err != nil {
 		return err
 	}
@@ -96,7 +96,7 @@ func writeHLL() error {
 // writeHLL_do writes out the HLL indicated by `which` to disk.
 // The usageToken must be held when calling this function.
 func writeHLL_do(hll *PeriodUniqueUsers) (err error) {
-	filename := getHLLFilename(hll.Start)
+	filename := GetHLLFilename(hll.Start)
 	file, err := os.Create(filename)
 	if err != nil {
 		return err
@@ -144,7 +144,7 @@ func loadUniqueUsers() {
 	}
 
 	now := time.Now().In(counterLocation)
-	uniqueCounter.Start, uniqueCounter.End = getCounterPeriod(now)
+	uniqueCounter.Start, uniqueCounter.End = GetCounterPeriod(now)
 	err = loadHLL(now, &uniqueCounter)
 	isIgnorableError := err != nil && (false ||
 		(os.IsNotExist(err)) ||
@@ -227,11 +227,11 @@ func rolloverCounters_do() {
 			enc := base64.NewEncoder(base64.StdEncoding, &buf)
 			enc.Write(bytes)
 			enc.Close()
-			log.Print("data for ", getHLLFilename(uniqueCounter.Start), ":", buf.String())
+			log.Print("data for ", GetHLLFilename(uniqueCounter.Start), ":", buf.String())
 		}
 	}
 
-	uniqueCounter.Start, uniqueCounter.End = getCounterPeriod(now)
+	uniqueCounter.Start, uniqueCounter.End = GetCounterPeriod(now)
 	// errors are bad precisions, so we can ignore
 	uniqueCounter.Counter, _ = hyperloglog.NewPlus(CounterPrecision)
 

From 9fc946e373233ea7b1ab9eaa63cd2cafa0fdb096 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 17 Jan 2016 11:11:21 -0800
Subject: [PATCH 126/176] Start of statsweb program

---
 socketserver/cmd/statsweb/.gitignore  |  3 ++
 socketserver/cmd/statsweb/config.go   | 74 +++++++++++++++++++++++++++
 socketserver/cmd/statsweb/config.json |  1 +
 socketserver/cmd/statsweb/statsweb.go | 37 ++++++++++++++
 socketserver/server/handlecore.go     |  2 +-
 socketserver/server/usercount.go      | 19 ++++---
 socketserver/server/usercount_test.go |  4 +-
 7 files changed, 130 insertions(+), 10 deletions(-)
 create mode 100644 socketserver/cmd/statsweb/.gitignore
 create mode 100644 socketserver/cmd/statsweb/config.go
 create mode 100644 socketserver/cmd/statsweb/config.json

diff --git a/socketserver/cmd/statsweb/.gitignore b/socketserver/cmd/statsweb/.gitignore
new file mode 100644
index 00000000..3dddabd8
--- /dev/null
+++ b/socketserver/cmd/statsweb/.gitignore
@@ -0,0 +1,3 @@
+database.sqlite
+gobcache/
+statsweb
diff --git a/socketserver/cmd/statsweb/config.go b/socketserver/cmd/statsweb/config.go
new file mode 100644
index 00000000..ae882d38
--- /dev/null
+++ b/socketserver/cmd/statsweb/config.go
@@ -0,0 +1,74 @@
+package main
+
+import (
+	"os"
+	"fmt"
+	"encoding/json"
+)
+
+type ConfigFile struct {
+	ListenAddr       string
+	DatabaseLocation string
+	GobFilesLocation string
+}
+
+func makeConfig() {
+	config.ListenAddr = "localhost:3000"
+	home, ok := os.LookupEnv("HOME")
+	if ok {
+		config.DatabaseLocation = fmt.Sprintf("%s/.ffzstatsweb/database.sqlite", home)
+		config.GobFilesLocation = fmt.Sprintf("%s/.ffzstatsweb/gobcache", home)
+		os.MkdirAll(config.GobFilesLocation, 0644)
+	} else {
+		config.DatabaseLocation = "./database.sqlite"
+		config.GobFilesLocation = "./gobcache"
+		os.MkdirAll(config.GobFilesLocation, 0644)
+	}
+	file, err := os.Create(*configLocation)
+	if err != nil {
+		fmt.Printf("Error: could not create config file: %v\n", err)
+		os.Exit(ExitCodeBadConfig)
+		return
+	}
+	enc := json.NewEncoder(file)
+	err = enc.Encode(config)
+	if err != nil {
+		fmt.Printf("Error: could not write config file: %v\n", err)
+		os.Exit(ExitCodeBadConfig)
+		return
+	}
+	err = file.Close()
+	if err != nil {
+		fmt.Printf("Error: could not write config file: %v\n", err)
+		os.Exit(ExitCodeBadConfig)
+		return
+	}
+	return
+}
+
+func loadConfig() {
+	file, err := os.Open(*configLocation)
+	if err != nil {
+		if os.IsNotExist(err) {
+			fmt.Println("You must create a config file with -genconf")
+		} else {
+			fmt.Printf("Error: could not load config file: %v", err)
+		}
+		os.Exit(ExitCodeBadConfig)
+		return
+	}
+	dec := json.NewDecoder(file)
+	err = dec.Decode(&config)
+	if err != nil {
+		fmt.Printf("Error: could not load config file: %v\n", err)
+		os.Exit(ExitCodeBadConfig)
+		return
+	}
+	err = file.Close()
+	if err != nil {
+		fmt.Printf("Error: could not load config file: %v\n", err)
+		os.Exit(ExitCodeBadConfig)
+		return
+	}
+	return
+}
\ No newline at end of file
diff --git a/socketserver/cmd/statsweb/config.json b/socketserver/cmd/statsweb/config.json
new file mode 100644
index 00000000..07cfa8e3
--- /dev/null
+++ b/socketserver/cmd/statsweb/config.json
@@ -0,0 +1 @@
+{"ListenAddr":"localhost:3000","DatabaseLocation":"./database.sqlite","GobFilesLocation":"./gobcache"}
diff --git a/socketserver/cmd/statsweb/statsweb.go b/socketserver/cmd/statsweb/statsweb.go
index 1110061a..683754a4 100644
--- a/socketserver/cmd/statsweb/statsweb.go
+++ b/socketserver/cmd/statsweb/statsweb.go
@@ -1,3 +1,40 @@
 package main
 
+import (
+	"net/http"
+	"flag"
+	"github.com/clarkduvall/hyperloglog"
+	"time"
+	"bitbucket.org/stendec/frankerfacez/socketserver/server"
+)
 
+var configLocation = flag.String("config", "./config.json", "Location of the configuration file. Defaults to ./config.json")
+var genConfig = flag.Bool("genconf", false, "Generate a new configuration file.")
+
+var config ConfigFile
+
+const ExitCodeBadConfig = 2
+
+func main() {
+	flag.Parse()
+
+	if *genConfig {
+		makeConfig()
+		return
+	}
+
+	loadConfig()
+
+	http.ListenAndServe(config.ListenAddr, http.DefaultServeMux)
+}
+
+func combineDateRange(from time.Time, to time.Time, dest *hyperloglog.HyperLogLogPlus) error {
+	from = server.TruncateToMidnight(from)
+	to = server.TruncateToMidnight(to)
+	year, month, day := from.Date()
+	for current := from; current.Before(to); day = day + 1 {
+		current = time.Date(year, month, day, 0, 0, 0, 0, server.CounterLocation)
+
+	}
+	return nil
+}
\ No newline at end of file
diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 301d8abc..ed0ae1f5 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -21,7 +21,7 @@ import (
 	"time"
 	"unicode/utf8"
 
-	"./logstasher"
+	"bitbucket.org/stendec/frankerfacez/socketserver/server/logstasher"
 )
 
 // SuccessCommand is a Reply Command to indicate success in reply to a C2S Command.
diff --git a/socketserver/server/usercount.go b/socketserver/server/usercount.go
index fb6e5ad9..d46316d0 100644
--- a/socketserver/server/usercount.go
+++ b/socketserver/server/usercount.go
@@ -43,13 +43,18 @@ var uniqueCounter PeriodUniqueUsers
 var uniqueUserChannel chan uuid.UUID
 var uniqueCtrWritingToken chan usageToken
 
-var counterLocation *time.Location = time.FixedZone("UTC-5", int((time.Hour*-5)/time.Second))
+var CounterLocation *time.Location = time.FixedZone("UTC-5", int((time.Hour*-5)/time.Second))
+
+func TruncateToMidnight(at time.Time) time.Time {
+	year, month, day := at.Date()
+	return time.Date(year, month, day, 0, 0, 0, 0, CounterLocation)
+}
 
 // GetCounterPeriod calculates the start and end timestamps for the HLL measurement period that includes the 'at' timestamp.
 func GetCounterPeriod(at time.Time) (start time.Time, end time.Time) {
 	year, month, day := at.Date()
-	start = time.Date(year, month, day, 0, 0, 0, 0, counterLocation)
-	end = time.Date(year, month, day+1, 0, 0, 0, 0, counterLocation)
+	start = time.Date(year, month, day, 0, 0, 0, 0, CounterLocation)
+	end = time.Date(year, month, day+1, 0, 0, 0, 0, CounterLocation)
 	return start, end
 }
 
@@ -143,7 +148,7 @@ func loadUniqueUsers() {
 		log.Panicln("could not make unique users data dir:", err)
 	}
 
-	now := time.Now().In(counterLocation)
+	now := time.Now().In(CounterLocation)
 	uniqueCounter.Start, uniqueCounter.End = GetCounterPeriod(now)
 	err = loadHLL(now, &uniqueCounter)
 	isIgnorableError := err != nil && (false ||
@@ -194,9 +199,9 @@ func processNewUsers() {
 }
 
 func getNextMidnight() time.Time {
-	now := time.Now().In(counterLocation)
+	now := time.Now().In(CounterLocation)
 	year, month, day := now.Date()
-	return time.Date(year, month, day+1, 0, 0, 1, 0, counterLocation)
+	return time.Date(year, month, day+1, 0, 0, 1, 0, CounterLocation)
 }
 
 // is_init_func
@@ -214,7 +219,7 @@ func rolloverCounters_do() {
 	var now time.Time
 
 	token = <-uniqueCtrWritingToken
-	now = time.Now().In(counterLocation)
+	now = time.Now().In(CounterLocation)
 	// Cycle for period
 	err := writeHLL_do(&uniqueCounter)
 	if err != nil {
diff --git a/socketserver/server/usercount_test.go b/socketserver/server/usercount_test.go
index c6ed2050..6c5a8d2f 100644
--- a/socketserver/server/usercount_test.go
+++ b/socketserver/server/usercount_test.go
@@ -12,7 +12,7 @@ import (
 func TestUniqueConnections(t *testing.T) {
 	const TestExpectedCount = 1000
 
-	testStart := time.Now().In(counterLocation)
+	testStart := time.Now().In(CounterLocation)
 
 	var server *httptest.Server
 	var backendExpected = NewTBackendRequestChecker(t,
@@ -35,7 +35,7 @@ func TestUniqueConnections(t *testing.T) {
 	TCheckHLLValue(t, TestExpectedCount, readCurrentHLL())
 
 	token := <-uniqueCtrWritingToken
-	uniqueCounter.End = time.Now().In(counterLocation).Add(-1 * time.Second)
+	uniqueCounter.End = time.Now().In(CounterLocation).Add(-1 * time.Second)
 	uniqueCtrWritingToken <- token
 
 	rolloverCounters_do()

From b0ae3c27c6ed2c136b42956b2362fa888e9e4f30 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 17 Jan 2016 11:12:19 -0800
Subject: [PATCH 127/176] Add more .gitignore files

---
 socketserver/cmd/ffzsocketserver/.gitignore | 3 +++
 socketserver/cmd/mergecounts/.gitignore     | 1 +
 2 files changed, 4 insertions(+)
 create mode 100644 socketserver/cmd/ffzsocketserver/.gitignore
 create mode 100644 socketserver/cmd/mergecounts/.gitignore

diff --git a/socketserver/cmd/ffzsocketserver/.gitignore b/socketserver/cmd/ffzsocketserver/.gitignore
new file mode 100644
index 00000000..43852e2e
--- /dev/null
+++ b/socketserver/cmd/ffzsocketserver/.gitignore
@@ -0,0 +1,3 @@
+config.json
+ffzsocketserver
+uniques/
diff --git a/socketserver/cmd/mergecounts/.gitignore b/socketserver/cmd/mergecounts/.gitignore
new file mode 100644
index 00000000..5b97e97f
--- /dev/null
+++ b/socketserver/cmd/mergecounts/.gitignore
@@ -0,0 +1 @@
+mergecounts

From a327f6cf57b1f17ae70abf5dd45ee3840c1c511d Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 17 Jan 2016 12:53:15 -0800
Subject: [PATCH 128/176] That's a good stopping point

---
 socketserver/cmd/statsweb/html.go             |  59 +++++++
 socketserver/cmd/statsweb/statsweb.go         | 156 ++++++++++++++++++
 .../cmd/statsweb/webroot/cal_entry.hbs        |   6 +
 .../cmd/statsweb/webroot/calendar.hbs         |  18 ++
 .../cmd/statsweb/webroot/layout.template.html |  15 ++
 5 files changed, 254 insertions(+)
 create mode 100644 socketserver/cmd/statsweb/html.go
 create mode 100644 socketserver/cmd/statsweb/webroot/cal_entry.hbs
 create mode 100644 socketserver/cmd/statsweb/webroot/calendar.hbs
 create mode 100644 socketserver/cmd/statsweb/webroot/layout.template.html

diff --git a/socketserver/cmd/statsweb/html.go b/socketserver/cmd/statsweb/html.go
new file mode 100644
index 00000000..5634c1f1
--- /dev/null
+++ b/socketserver/cmd/statsweb/html.go
@@ -0,0 +1,59 @@
+package main
+
+import (
+	"html/template"
+	"net/http"
+	"time"
+	"bitbucket.org/stendec/frankerfacez/socketserver/server"
+)
+
+type CalendarData struct {
+	Weeks []CalWeekData
+}
+type CalWeekData struct {
+	Days []CalDayData
+}
+type CalDayData struct {
+	NoData bool
+	Date int
+	UniqUsers int
+}
+
+type CalendarMonthInfo struct {
+	Year int
+	Month time.Month
+	// Ranges from -5 to +1.
+	// A value of +1 means the 1st of the month is a Sunday.
+	// A value of 0 means the 1st of the month is a Monday.
+	// A value of -5 means the 1st of the month is a Saturday.
+	FirstSundayOffset int
+	// True if the calendar for this month needs six sundays.
+	NeedSixSundays bool
+}
+
+func GetMonthInfo(at time.Time) CalendarMonthInfo {
+	year, month, _ := at.Date()
+	// 1 (start of month) - weekday of start of month = day offset of start of week at start of month
+	monthWeekStartDay := 1 - time.Date(year, month, 1, 0, 0, 0, 0, server.CounterLocation).Weekday()
+	// first day on calendar + 6 weeks < end of month?
+	sixthSundayDay := monthWeekStartDay + 5*7
+	sixthSundayDate := time.Date(year, month, sixthSundayDay, 0, 0, 0, 0, server.CounterLocation)
+	var needSixSundays bool = false
+	if sixthSundayDate.Month() == month {
+		needSixSundays = true
+	}
+
+	return CalendarMonthInfo{
+		Year: year,
+		Month: month,
+		FirstSundayOffset: monthWeekStartDay,
+		NeedSixSundays: needSixSundays,
+	}
+}
+
+func renderCalendar(w http.ResponseWriter, at time.Time) {
+	layout, err := template.ParseFiles("./webroot/layout.template.html", "./webroot/cal_entry.hbs", "./webroot/calendar.hbs")
+	data := CalendarData{}
+	data.Weeks = make([]CalWeekData, 6)
+
+}
diff --git a/socketserver/cmd/statsweb/statsweb.go b/socketserver/cmd/statsweb/statsweb.go
index 683754a4..6944be6f 100644
--- a/socketserver/cmd/statsweb/statsweb.go
+++ b/socketserver/cmd/statsweb/statsweb.go
@@ -6,6 +6,11 @@ import (
 	"github.com/clarkduvall/hyperloglog"
 	"time"
 	"bitbucket.org/stendec/frankerfacez/socketserver/server"
+	"net/url"
+	"fmt"
+	"strings"
+	"errors"
+	"github.com/dustin/gojson"
 )
 
 var configLocation = flag.String("config", "./config.json", "Location of the configuration file. Defaults to ./config.json")
@@ -25,9 +30,160 @@ func main() {
 
 	loadConfig()
 
+	http.HandleFunc("/api", ServeAPI)
 	http.ListenAndServe(config.ListenAddr, http.DefaultServeMux)
 }
 
+const RequestURIName = "q"
+const separatorRange = "~"
+const separatorAdd = " "
+const jsonErrMalformedRequest = `{"status":"error","error":"malformed request uri"}`
+const jsonErrBlankRequest = `{"status":"error","error":"no queries given"}`
+const statusError = "error"
+const statusPartial = "partial"
+const statusOk = "ok"
+type apiResponse struct {
+	Status string `json:"status"`
+	Responses []requestResponse `json:"resp"`
+}
+type requestResponse struct {
+	Status string `json:"status"`
+	Request string `json:"req"`
+	Error string `json:"error,omitempty"`
+	Count uint64 `json:"count,omitempty"`
+}
+
+type serverFilter struct {
+	// TODO
+}
+func (sf *serverFilter) IsServerAllowed(server string) {
+	return true
+}
+const serverFilterAll serverFilter
+
+func ServeAPI(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+
+	u, err := url.ParseRequestURI(r.RequestURI)
+	if err != nil {
+		w.WriteHeader(400)
+		fmt.Fprint(w, jsonErrMalformedRequest)
+		return
+	}
+
+	query := u.Query()
+	reqCount := len(query[RequestURIName])
+	if reqCount == 0 {
+		w.WriteHeader(400)
+		fmt.Fprint(w, jsonErrBlankRequest)
+		return
+	}
+
+	resp := apiResponse{Status: statusOk}
+	resp.Responses = make([]requestResponse, reqCount)
+	for i, v := range query[RequestURIName] {
+		resp.Responses[i] = processSingleRequest(v)
+	}
+	for _, v := range resp.Responses {
+		if v.Status == statusError {
+			resp.Status = statusPartial
+			break
+		}
+	}
+
+	w.WriteHeader(200)
+	enc := json.NewEncoder(w)
+	enc.Encode(resp)
+}
+
+const errRangeFormatIncorrect = "incorrect range format, must be yyyy-mm-dd~yyyy-mm-dd"
+
+func processSingleRequest(req string) (result requestResponse) {
+	// Forms:
+	// Single: 2016-01-02
+	// Range: 2016-01-03~2016-01-09
+	// Add disparate: 2016-01-02 2016-01-03 2016-01-09 2016-01-10
+	// NOTE: Spaces are uri-encoded as +
+	// Add ranges: 2016-01-04~2016-01-08 2016-01-11~2016-01-15
+	var hll hyperloglog.HyperLogLogPlus, _ = hyperloglog.NewPlus(server.CounterPrecision)
+	addSplit := strings.Split(req, separatorAdd)
+
+	result.Request = req
+	result.Status = statusOk
+
+	outerLoop:
+	for _, split1 := range addSplit {
+		if len(split1) == 0 {
+			continue
+		}
+
+		rangeSplit := strings.Split(split1, separatorRange)
+		if len(rangeSplit) == 1 {
+			at, err := parseDate(rangeSplit[0])
+			if err != nil {
+				result.Status = statusError
+				result.Error = err.Error()
+				break outerLoop
+			}
+			err = addSingleDate(at, serverFilterAll, &hll)
+			if err != nil {
+				result.Status = statusError
+				result.Error = err.Error()
+				break outerLoop
+			}
+		} else if len(rangeSplit) == 2 {
+			from, err := parseDate(rangeSplit[0])
+			if err != nil {
+				result.Status = statusError
+				result.Error = err.Error()
+				break outerLoop
+			}
+			to, err := parseDate(rangeSplit[1])
+			if err != nil {
+				result.Status = statusError
+				result.Error = err.Error()
+				break outerLoop
+			}
+			err = addRange(from, to, serverFilterAll, &hll)
+			if err != nil {
+				result.Status = statusError
+				result.Error = err.Error()
+				break outerLoop
+			}
+		} else {
+			result.Status = statusError
+			result.Error = errRangeFormatIncorrect
+			break outerLoop
+		}
+	}
+
+	if result.Status == statusOk {
+		result.Count = hll.Count()
+	}
+	return result
+}
+
+var errBadDate = errors.New("bad date format, must be yyyy-mm-dd")
+var zeroTime = time.Unix(0, 0)
+
+func parseDate(dateStr string) (time.Time, error) {
+	var year, month, day int
+	n, err := fmt.Sscanf(dateStr, "%d-%d-%d", &year, &month, &day)
+	if err != nil || n != 3 {
+		return zeroTime, errBadDate
+	}
+	return time.Date(year, month, day, 0, 0, 0, 0, server.CounterLocation)
+}
+
+func addSingleDate(at time.Time, filter serverFilter, dest *hyperloglog.HyperLogLogPlus) error {
+	// TODO
+	return nil
+}
+
+func addRange(start time.Time, end time.Time, filter serverFilter, dest *hyperloglog.HyperLogLogPlus) error {
+
+}
+
 func combineDateRange(from time.Time, to time.Time, dest *hyperloglog.HyperLogLogPlus) error {
 	from = server.TruncateToMidnight(from)
 	to = server.TruncateToMidnight(to)
diff --git a/socketserver/cmd/statsweb/webroot/cal_entry.hbs b/socketserver/cmd/statsweb/webroot/cal_entry.hbs
new file mode 100644
index 00000000..cf178cf3
--- /dev/null
+++ b/socketserver/cmd/statsweb/webroot/cal_entry.hbs
@@ -0,0 +1,6 @@
+<td class="calentry {{if .NoData}}no_data{{end}}">
+    <span class="date">{{.Date}}</span>
+    {{if not .NoData}}
+        <span class="uniqusers">{{.UniqUsers}}</span>
+    {{end}}
+</td>
diff --git a/socketserver/cmd/statsweb/webroot/calendar.hbs b/socketserver/cmd/statsweb/webroot/calendar.hbs
new file mode 100644
index 00000000..1a8070ce
--- /dev/null
+++ b/socketserver/cmd/statsweb/webroot/calendar.hbs
@@ -0,0 +1,18 @@
+<table class="calendar">
+    <thead>
+    <th>Sunday</th>
+    <th>Monday</th>
+    <th>Tuesday</th>
+    <th>Wednesday</th>
+    <th>Thursday</th>
+    <th>Friday</th>
+    <th>Saturday</th>
+    </thead>
+    <tbody>
+    {{range .Weeks}}
+        {{range .Days}}
+            {{template "cal_entry"}}
+        {{end}}
+    {{end}}
+    </tbody>
+</table>
\ No newline at end of file
diff --git a/socketserver/cmd/statsweb/webroot/layout.template.html b/socketserver/cmd/statsweb/webroot/layout.template.html
new file mode 100644
index 00000000..09c24acf
--- /dev/null
+++ b/socketserver/cmd/statsweb/webroot/layout.template.html
@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Socket Server Stats Dashboard</title>
+</head>
+<body>
+<div id="header">
+    {{template "header"}}
+</div>
+<div id="main">
+    {{template "content"}}
+</div>
+</body>
+</html>

From aca50d9de5eed0a0d9c984f8cdda6062596a91da Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 17 Jan 2016 14:09:09 -0800
Subject: [PATCH 129/176] Add documentation to ProcessSingleGetRequest

Make a collectError() function to reduce repetitiveness
---
 socketserver/cmd/statsweb/config.go   |   6 +-
 socketserver/cmd/statsweb/html.go     |  14 ++--
 socketserver/cmd/statsweb/servers.go  |  70 ++++++++++++++++
 socketserver/cmd/statsweb/statsweb.go | 115 ++++++++++++++++----------
 4 files changed, 152 insertions(+), 53 deletions(-)
 create mode 100644 socketserver/cmd/statsweb/servers.go

diff --git a/socketserver/cmd/statsweb/config.go b/socketserver/cmd/statsweb/config.go
index ae882d38..6ddca585 100644
--- a/socketserver/cmd/statsweb/config.go
+++ b/socketserver/cmd/statsweb/config.go
@@ -1,9 +1,9 @@
 package main
 
 import (
-	"os"
-	"fmt"
 	"encoding/json"
+	"fmt"
+	"os"
 )
 
 type ConfigFile struct {
@@ -71,4 +71,4 @@ func loadConfig() {
 		return
 	}
 	return
-}
\ No newline at end of file
+}
diff --git a/socketserver/cmd/statsweb/html.go b/socketserver/cmd/statsweb/html.go
index 5634c1f1..303c4ddf 100644
--- a/socketserver/cmd/statsweb/html.go
+++ b/socketserver/cmd/statsweb/html.go
@@ -1,10 +1,10 @@
 package main
 
 import (
+	"bitbucket.org/stendec/frankerfacez/socketserver/server"
 	"html/template"
 	"net/http"
 	"time"
-	"bitbucket.org/stendec/frankerfacez/socketserver/server"
 )
 
 type CalendarData struct {
@@ -14,13 +14,13 @@ type CalWeekData struct {
 	Days []CalDayData
 }
 type CalDayData struct {
-	NoData bool
-	Date int
+	NoData    bool
+	Date      int
 	UniqUsers int
 }
 
 type CalendarMonthInfo struct {
-	Year int
+	Year  int
 	Month time.Month
 	// Ranges from -5 to +1.
 	// A value of +1 means the 1st of the month is a Sunday.
@@ -44,10 +44,10 @@ func GetMonthInfo(at time.Time) CalendarMonthInfo {
 	}
 
 	return CalendarMonthInfo{
-		Year: year,
-		Month: month,
+		Year:              year,
+		Month:             month,
 		FirstSundayOffset: monthWeekStartDay,
-		NeedSixSundays: needSixSundays,
+		NeedSixSundays:    needSixSundays,
 	}
 }
 
diff --git a/socketserver/cmd/statsweb/servers.go b/socketserver/cmd/statsweb/servers.go
new file mode 100644
index 00000000..83f9bffc
--- /dev/null
+++ b/socketserver/cmd/statsweb/servers.go
@@ -0,0 +1,70 @@
+package main
+
+type serverFilter struct {
+	// Mode is false for blacklist, true for whitelist
+	Mode bool
+	Special string[]
+}
+
+const serverFilterModeBlacklist = false
+const serverFilterModeWhitelist = true
+
+func (sf *serverFilter) IsServerAllowed(server string) {
+	for _, v := range sf.Special {
+		if server == v {
+			return sf.Mode
+		}
+	}
+	return !sf.Mode
+}
+
+func (sf *serverFilter) Remove(server string) {
+	if sf.Mode == serverFilterModeWhitelist {
+		var idx int = -1
+		for i, v := range sf.Special {
+			if server == v {
+				idx = i
+				break
+			}
+		}
+		if idx != -1 {
+			var lenMinusOne = len(sf.Special)-1
+			sf.Special[idx] = sf.Special[lenMinusOne]
+			sf.Special = sf.Special[:lenMinusOne]
+		}
+	} else {
+		for _, v := range sf.Special {
+			if server == v {
+				return
+			}
+		}
+		sf.Special = append(sf.Special, server)
+	}
+}
+
+func (sf *serverFilter) Add(server string) {
+	if sf.Mode == serverFilterModeBlacklist {
+		var idx int = -1
+		for i, v := range sf.Special {
+			if server == v {
+				idx = i
+				break
+			}
+		}
+		if idx != -1 {
+			var lenMinusOne = len(sf.Special)-1
+			sf.Special[idx] = sf.Special[lenMinusOne]
+			sf.Special = sf.Special[:lenMinusOne]
+		}
+	} else {
+		for _, v := range sf.Special {
+			if server == v {
+				return
+			}
+		}
+		sf.Special = append(sf.Special, server)
+	}
+}
+
+const serverFilterAll serverFilter = serverFilter{Mode: serverFilterModeBlacklist}
+const serverFilterNone serverFilter = serverFilter{Mode: serverFilterModeWhitelist}
diff --git a/socketserver/cmd/statsweb/statsweb.go b/socketserver/cmd/statsweb/statsweb.go
index 6944be6f..284eb1d5 100644
--- a/socketserver/cmd/statsweb/statsweb.go
+++ b/socketserver/cmd/statsweb/statsweb.go
@@ -30,13 +30,14 @@ func main() {
 
 	loadConfig()
 
-	http.HandleFunc("/api", ServeAPI)
+	http.HandleFunc("/api/get", ServeAPIGet)
 	http.ListenAndServe(config.ListenAddr, http.DefaultServeMux)
 }
 
 const RequestURIName = "q"
 const separatorRange = "~"
 const separatorAdd = " "
+const separatorServer = "@"
 const jsonErrMalformedRequest = `{"status":"error","error":"malformed request uri"}`
 const jsonErrBlankRequest = `{"status":"error","error":"no queries given"}`
 const statusError = "error"
@@ -53,15 +54,7 @@ type requestResponse struct {
 	Count uint64 `json:"count,omitempty"`
 }
 
-type serverFilter struct {
-	// TODO
-}
-func (sf *serverFilter) IsServerAllowed(server string) {
-	return true
-}
-const serverFilterAll serverFilter
-
-func ServeAPI(w http.ResponseWriter, r *http.Request) {
+func ServeAPIGet(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json")
 
 	u, err := url.ParseRequestURI(r.RequestURI)
@@ -82,7 +75,10 @@ func ServeAPI(w http.ResponseWriter, r *http.Request) {
 	resp := apiResponse{Status: statusOk}
 	resp.Responses = make([]requestResponse, reqCount)
 	for i, v := range query[RequestURIName] {
-		resp.Responses[i] = processSingleRequest(v)
+		if len(v) == 0 {
+			continue
+		}
+		resp.Responses[i] = ProcessSingleGetRequest(v)
 	}
 	for _, v := range resp.Responses {
 		if v.Status == statusError {
@@ -96,20 +92,61 @@ func ServeAPI(w http.ResponseWriter, r *http.Request) {
 	enc.Encode(resp)
 }
 
-const errRangeFormatIncorrect = "incorrect range format, must be yyyy-mm-dd~yyyy-mm-dd"
+const errRangeFormatIncorrect = errors.New("incorrect range format, must be yyyy-mm-dd~yyyy-mm-dd")
 
-func processSingleRequest(req string) (result requestResponse) {
-	// Forms:
-	// Single: 2016-01-02
-	// Range: 2016-01-03~2016-01-09
-	// Add disparate: 2016-01-02 2016-01-03 2016-01-09 2016-01-10
-	// NOTE: Spaces are uri-encoded as +
-	// Add ranges: 2016-01-04~2016-01-08 2016-01-11~2016-01-15
+// ProcessSingleGetRequest takes a request string and pulls the unique user data for the given dates and filters.
+//
+// The request string is in the following format:
+//
+//   Request = AddDateRanges [ "@" ServerFilter ] .
+//   ServerFilter = [ "!" ] ServerName { " " ServerName } .
+//   ServerName = { "a" … "z" } .
+//   AddDateRanges = DateMaybeRange { " " DateMaybeRange } .
+//   DateMaybeRange = DateRange | Date .
+//   DateRange = Date "~" Date .
+//   Date = Year "-" Month "-" Day .
+//   Year = number number number number .
+//   Month = number number .
+//   Day = number number .
+//   number = "0" … "9" .
+//
+// Example of a well-formed request:
+//
+//   2016-01-04~2016-01-08 2016-01-11~2016-01-15@andknuckles tuturu
+//
+// Remember that spaces are urlencoded as "+", so the HTTP request to send to retrieve that data would be this:
+//
+//   /api/get?q=2016-01-04~2016-01-08+2016-01-11~2016-01-15%40andknuckles+tuturu
+//
+// If a ServerFilter is specified, only users connecting to the specified servers will be included in the count.
+//
+// It does not matter if a date is specified multiple times, due to the data format used.
+func ProcessSingleGetRequest(req string) (result requestResponse) {
 	var hll hyperloglog.HyperLogLogPlus, _ = hyperloglog.NewPlus(server.CounterPrecision)
-	addSplit := strings.Split(req, separatorAdd)
 
 	result.Request = req
 	result.Status = statusOk
+	filter := serverFilterAll
+
+	collectError := func(err error) bool {
+		if err != nil {
+			result.Status = statusError
+			result.Error = err.Error()
+			return true
+		}
+		return false
+	}
+
+	serverSplit := strings.Split(req, separatorServer)
+	if len(serverSplit) == 2 {
+		filter = serverFilterNone
+		serversOnly := strings.Split(serverSplit[1], separatorAdd)
+		for _, v := range serversOnly {
+			filter.Add(v)
+		}
+	}
+
+	addSplit := strings.Split(serverSplit[0], separatorAdd)
 
 	outerLoop:
 	for _, split1 := range addSplit {
@@ -119,40 +156,31 @@ func processSingleRequest(req string) (result requestResponse) {
 
 		rangeSplit := strings.Split(split1, separatorRange)
 		if len(rangeSplit) == 1 {
-			at, err := parseDate(rangeSplit[0])
-			if err != nil {
-				result.Status = statusError
-				result.Error = err.Error()
+			at, err := parseDateFromRequest(rangeSplit[0])
+			if collectError(err) {
 				break outerLoop
 			}
-			err = addSingleDate(at, serverFilterAll, &hll)
-			if err != nil {
-				result.Status = statusError
-				result.Error = err.Error()
+
+			err = addSingleDate(at, filter, &hll)
+			if collectError(err) {
 				break outerLoop
 			}
 		} else if len(rangeSplit) == 2 {
-			from, err := parseDate(rangeSplit[0])
-			if err != nil {
-				result.Status = statusError
-				result.Error = err.Error()
+			from, err := parseDateFromRequest(rangeSplit[0])
+			if collectError(err) {
 				break outerLoop
 			}
-			to, err := parseDate(rangeSplit[1])
-			if err != nil {
-				result.Status = statusError
-				result.Error = err.Error()
+			to, err := parseDateFromRequest(rangeSplit[1])
+			if collectError(err) {
 				break outerLoop
 			}
-			err = addRange(from, to, serverFilterAll, &hll)
-			if err != nil {
-				result.Status = statusError
-				result.Error = err.Error()
+
+			err = addRange(from, to, filter, &hll)
+			if collectError(err) {
 				break outerLoop
 			}
 		} else {
-			result.Status = statusError
-			result.Error = errRangeFormatIncorrect
+			collectError(errRangeFormatIncorrect)
 			break outerLoop
 		}
 	}
@@ -166,7 +194,7 @@ func processSingleRequest(req string) (result requestResponse) {
 var errBadDate = errors.New("bad date format, must be yyyy-mm-dd")
 var zeroTime = time.Unix(0, 0)
 
-func parseDate(dateStr string) (time.Time, error) {
+func parseDateFromRequest(dateStr string) (time.Time, error) {
 	var year, month, day int
 	n, err := fmt.Sscanf(dateStr, "%d-%d-%d", &year, &month, &day)
 	if err != nil || n != 3 {
@@ -182,6 +210,7 @@ func addSingleDate(at time.Time, filter serverFilter, dest *hyperloglog.HyperLog
 
 func addRange(start time.Time, end time.Time, filter serverFilter, dest *hyperloglog.HyperLogLogPlus) error {
 
+	return nil
 }
 
 func combineDateRange(from time.Time, to time.Time, dest *hyperloglog.HyperLogLogPlus) error {

From 1023b6ed11da492c5414fe3626e15d62b5f445d3 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 17 Jan 2016 16:50:17 -0800
Subject: [PATCH 130/176] implement addRange, add pprof to server

---
 .../cmd/ffzsocketserver/socketserver.go       |   2 +
 socketserver/cmd/statsweb/config.go           |   4 +-
 socketserver/cmd/statsweb/html.go             |   8 +-
 socketserver/cmd/statsweb/servers.go          | 276 +++++++++++++++++-
 socketserver/cmd/statsweb/statsweb.go         | 102 +++++--
 socketserver/server/handlecore.go             |   6 +-
 6 files changed, 362 insertions(+), 36 deletions(-)

diff --git a/socketserver/cmd/ffzsocketserver/socketserver.go b/socketserver/cmd/ffzsocketserver/socketserver.go
index fedcdf80..8b56d707 100644
--- a/socketserver/cmd/ffzsocketserver/socketserver.go
+++ b/socketserver/cmd/ffzsocketserver/socketserver.go
@@ -11,6 +11,8 @@ import (
 	"os"
 )
 
+import _ "net/http/pprof"
+
 var configFilename = flag.String("config", "config.json", "Configuration file, including the keypairs for the NaCl crypto library, for communicating with the backend.")
 var flagGenerateKeys = flag.Bool("genkeys", false, "Generate NaCl keys instead of serving requests.\nArguments: [int serverId] [base64 backendPublic]\nThe backend public key can either be specified in base64 on the command line, or put in the json file later.")
 
diff --git a/socketserver/cmd/statsweb/config.go b/socketserver/cmd/statsweb/config.go
index 6ddca585..04591f44 100644
--- a/socketserver/cmd/statsweb/config.go
+++ b/socketserver/cmd/statsweb/config.go
@@ -18,11 +18,11 @@ func makeConfig() {
 	if ok {
 		config.DatabaseLocation = fmt.Sprintf("%s/.ffzstatsweb/database.sqlite", home)
 		config.GobFilesLocation = fmt.Sprintf("%s/.ffzstatsweb/gobcache", home)
-		os.MkdirAll(config.GobFilesLocation, 0644)
+		os.MkdirAll(config.GobFilesLocation, 0755)
 	} else {
 		config.DatabaseLocation = "./database.sqlite"
 		config.GobFilesLocation = "./gobcache"
-		os.MkdirAll(config.GobFilesLocation, 0644)
+		os.MkdirAll(config.GobFilesLocation, 0755)
 	}
 	file, err := os.Create(*configLocation)
 	if err != nil {
diff --git a/socketserver/cmd/statsweb/html.go b/socketserver/cmd/statsweb/html.go
index 303c4ddf..a2bca5d4 100644
--- a/socketserver/cmd/statsweb/html.go
+++ b/socketserver/cmd/statsweb/html.go
@@ -33,8 +33,9 @@ type CalendarMonthInfo struct {
 
 func GetMonthInfo(at time.Time) CalendarMonthInfo {
 	year, month, _ := at.Date()
-	// 1 (start of month) - weekday of start of month = day offset of start of week at start of month
-	monthWeekStartDay := 1 - time.Date(year, month, 1, 0, 0, 0, 0, server.CounterLocation).Weekday()
+	monthStartWeekday := time.Date(year, month, 1, 0, 0, 0, 0, server.CounterLocation).Weekday()
+	// 1 (start of month) - weekday of start of month = day offset of start of week at start of mont
+	monthWeekStartDay := 1 - int(monthStartWeekday)
 	// first day on calendar + 6 weeks < end of month?
 	sixthSundayDay := monthWeekStartDay + 5*7
 	sixthSundayDate := time.Date(year, month, sixthSundayDay, 0, 0, 0, 0, server.CounterLocation)
@@ -55,5 +56,6 @@ func renderCalendar(w http.ResponseWriter, at time.Time) {
 	layout, err := template.ParseFiles("./webroot/layout.template.html", "./webroot/cal_entry.hbs", "./webroot/calendar.hbs")
 	data := CalendarData{}
 	data.Weeks = make([]CalWeekData, 6)
-
+	_ = layout
+	_ = err
 }
diff --git a/socketserver/cmd/statsweb/servers.go b/socketserver/cmd/statsweb/servers.go
index 83f9bffc..659b430b 100644
--- a/socketserver/cmd/statsweb/servers.go
+++ b/socketserver/cmd/statsweb/servers.go
@@ -1,17 +1,32 @@
 package main
 
+import (
+	"time"
+	"io"
+	"fmt"
+	"os"
+	"net/http"
+	"errors"
+	"sync"
+	"github.com/hashicorp/golang-lru"
+	"github.com/clarkduvall/hyperloglog"
+	"bitbucket.org/stendec/frankerfacez/socketserver/server"
+	"encoding/gob"
+)
+
 type serverFilter struct {
 	// Mode is false for blacklist, true for whitelist
-	Mode bool
-	Special string[]
+	Mode    bool
+	Special []string
 }
 
 const serverFilterModeBlacklist = false
 const serverFilterModeWhitelist = true
 
-func (sf *serverFilter) IsServerAllowed(server string) {
+func (sf *serverFilter) IsServerAllowed(server *serverInfo) bool {
+	name := server.subdomain
 	for _, v := range sf.Special {
-		if server == v {
+		if name == v {
 			return sf.Mode
 		}
 	}
@@ -28,7 +43,7 @@ func (sf *serverFilter) Remove(server string) {
 			}
 		}
 		if idx != -1 {
-			var lenMinusOne = len(sf.Special)-1
+			var lenMinusOne = len(sf.Special) - 1
 			sf.Special[idx] = sf.Special[lenMinusOne]
 			sf.Special = sf.Special[:lenMinusOne]
 		}
@@ -52,7 +67,7 @@ func (sf *serverFilter) Add(server string) {
 			}
 		}
 		if idx != -1 {
-			var lenMinusOne = len(sf.Special)-1
+			var lenMinusOne = len(sf.Special) - 1
 			sf.Special[idx] = sf.Special[lenMinusOne]
 			sf.Special = sf.Special[:lenMinusOne]
 		}
@@ -66,5 +81,250 @@ func (sf *serverFilter) Add(server string) {
 	}
 }
 
-const serverFilterAll serverFilter = serverFilter{Mode: serverFilterModeBlacklist}
-const serverFilterNone serverFilter = serverFilter{Mode: serverFilterModeWhitelist}
+var serverFilterAll serverFilter = serverFilter{Mode: serverFilterModeBlacklist}
+var serverFilterNone serverFilter = serverFilter{Mode: serverFilterModeWhitelist}
+
+func cannotCacheHLL(at time.Time) bool {
+	now := time.Now()
+	now.Add(-25 * time.Hour)
+	return now.Before(at)
+}
+
+var ServerNames = []string{
+	"catbag",
+	"andknuckles",
+	"tuturu",
+}
+
+var httpClient http.Client
+
+const serverNameSuffix = ".frankerfacez.com"
+
+const failedStateThreshold = 4
+
+var ErrServerInFailedState = errors.New("server has been down recently and not recovered")
+var ErrServerHasNoData = errors.New("no data for specified date")
+
+type errServerNot200 struct {
+	StatusCode int
+	StatusText string
+}
+
+func (e *errServerNot200) Error() string {
+	return fmt.Sprintf("The server responded with %d %s", e.StatusCode, e.StatusText)
+}
+func Not200Error(resp *http.Response) *errServerNot200 {
+	return &errServerNot200{
+		StatusCode: resp.StatusCode,
+		StatusText: resp.Status,
+	}
+}
+
+func getHLLCacheKey(at time.Time) string {
+	year, month, day := at.Date()
+	return fmt.Sprintf("%d-%d-%d", year, month, day)
+}
+
+type serverInfo struct {
+	subdomain    string
+
+	memcache     *lru.TwoQueueCache
+
+	FailedState  bool
+	FailureErr   error
+	failureCount int
+
+	lock         sync.Mutex
+}
+
+func (si *serverInfo) Setup(subdomain string) {
+	si.subdomain = subdomain
+	tq, err := lru.New2Q(60)
+	if err != nil {
+		panic(err)
+	}
+	si.memcache = tq
+}
+
+// GetHLL gets the HLL from
+func (si *serverInfo) GetHLL(at time.Time) (*hyperloglog.HyperLogLogPlus, error) {
+	if cannotCacheHLL(at) {
+		err := si.ForceWrite()
+		if err != nil {
+			return nil, err
+		}
+		reader, err := si.DownloadHLL(at)
+		if err != nil {
+			return nil, err
+		}
+		fmt.Printf("downloaded hll %s:%s\n", si.subdomain, getHLLCacheKey(at))
+		return loadHLLFromStream(reader)
+	}
+
+	hll, ok := si.PeekHLL(at)
+	if ok {
+		fmt.Printf("got cached hll %s:%s\n", si.subdomain, getHLLCacheKey(at))
+		return hll, nil
+	}
+
+	reader, err := si.OpenHLL(at)
+	if err != nil {
+		// continue to download
+	} else {
+		fmt.Printf("opened hll %s:%s\n", si.subdomain, getHLLCacheKey(at))
+		return loadHLLFromStream(reader)
+	}
+
+	reader, err = si.DownloadHLL(at)
+	if err != nil {
+		if err == ErrServerHasNoData {
+			return hyperloglog.NewPlus(server.CounterPrecision)
+		}
+		return nil, err
+	}
+	fmt.Printf("downloaded hll %s:%s\n", si.subdomain, getHLLCacheKey(at))
+	return loadHLLFromStream(reader)
+}
+
+func loadHLLFromStream(reader io.ReadCloser) (*hyperloglog.HyperLogLogPlus, error) {
+	defer reader.Close()
+	hll, _ := hyperloglog.NewPlus(server.CounterPrecision)
+	dec := gob.NewDecoder(reader)
+	err := dec.Decode(hll)
+	if err != nil {
+		return nil, err
+	}
+	return hll, nil
+}
+
+// PeekHLL tries to grab a HLL from the memcache without downloading it or hitting the disk.
+func (si *serverInfo) PeekHLL(at time.Time) (*hyperloglog.HyperLogLogPlus, bool) {
+	if cannotCacheHLL(at) {
+		return nil, false
+	}
+
+	key := getHLLCacheKey(at)
+	hll, ok := si.memcache.Get(key)
+	if ok {
+		return hll.(*hyperloglog.HyperLogLogPlus), true
+	}
+
+	return nil, false
+}
+
+func (si *serverInfo) OpenHLL(at time.Time) (io.ReadCloser, error) {
+	year, month, day := at.Date()
+	filename := fmt.Sprintf("%s/%s/%d-%d-%d.gob", config.GobFilesLocation, si.subdomain, year, month, day)
+
+	file, err := os.Open(filename)
+	if err == nil {
+		return file, nil
+	}
+	// file is nil
+	if !os.IsNotExist(err) {
+		return nil, err
+	}
+
+	return nil, os.ErrNotExist
+}
+
+func (si *serverInfo) DownloadHLL(at time.Time) (io.ReadCloser, error) {
+	if si.FailedState {
+		return nil, ErrServerInFailedState
+	}
+	si.lock.Lock()
+	defer si.lock.Unlock()
+
+	year, month, day := at.Date()
+	url := fmt.Sprintf("https://%s/hll/daily-%d-%d-%d.gob", si.Domain(), day, month, year)
+	resp, err := httpClient.Get(url)
+	if err != nil {
+		si.ServerFailed(err)
+		return nil, err
+	}
+	if resp.StatusCode == 404 {
+		return nil, ErrServerHasNoData
+	}
+	if resp.StatusCode != 200 {
+		err = Not200Error(resp)
+		si.ServerFailed(err)
+		return nil, err
+	}
+
+	filename := fmt.Sprintf("%s/%s/%d-%d-%d.gob", config.GobFilesLocation, si.subdomain, year, month, day)
+	file, err := os.OpenFile(filename, os.O_CREATE | os.O_EXCL | os.O_RDWR, 0644)
+	if os.IsNotExist(err) {
+		os.MkdirAll(fmt.Sprintf("%s/%s", config.GobFilesLocation, si.subdomain), 0755)
+		file, err = os.OpenFile(filename, os.O_CREATE | os.O_EXCL | os.O_RDWR, 0644)
+	}
+	if err != nil {
+		resp.Body.Close()
+		return nil, fmt.Errorf("downloadhll: error opening file for writing: %v", err)
+	}
+
+	return &teeReadCloser{r: resp.Body, w: file}, nil
+}
+
+func (si *serverInfo) ForceWrite() error {
+	if si.FailedState {
+		return ErrServerInFailedState
+	}
+
+	url := fmt.Sprintf("https://%s/hll_force_write", si.Domain())
+	resp, err := httpClient.Get(url)
+	if err != nil {
+		si.ServerFailed(err)
+		return err
+	}
+	if resp.StatusCode != 200 {
+		err = Not200Error(resp)
+		si.ServerFailed(err)
+		return err
+	}
+	resp.Body.Close()
+	return nil
+}
+
+func (si *serverInfo) Domain() string {
+	return fmt.Sprintf("%s%s", si.subdomain, serverNameSuffix)
+}
+
+func (si *serverInfo) ServerFailed(err error) {
+	si.lock.Lock()
+	defer si.lock.Unlock()
+	si.failureCount++
+	if si.failureCount > failedStateThreshold {
+		fmt.Printf("Server %s entering failed state\n", si.subdomain)
+		si.FailedState = true
+		si.FailureErr = err
+		go recoveryCheck(si)
+	}
+}
+
+func recoveryCheck(si *serverInfo) {
+	// TODO check for server recovery
+}
+
+type teeReadCloser struct {
+	r io.ReadCloser
+	w io.WriteCloser
+}
+
+func (t *teeReadCloser) Read(p []byte) (n int, err error) {
+	n, err = t.r.Read(p)
+	if n > 0 {
+		if n, err := t.w.Write(p[:n]); err != nil {
+			return n, err
+		}
+	}
+	return
+}
+
+func (t *teeReadCloser) Close() error {
+	err1 := t.r.Close()
+	err2 := t.w.Close()
+	if err1 != nil {
+		return err1
+	}
+	return err2
+}
diff --git a/socketserver/cmd/statsweb/statsweb.go b/socketserver/cmd/statsweb/statsweb.go
index 284eb1d5..9fce5f6e 100644
--- a/socketserver/cmd/statsweb/statsweb.go
+++ b/socketserver/cmd/statsweb/statsweb.go
@@ -10,7 +10,8 @@ import (
 	"fmt"
 	"strings"
 	"errors"
-	"github.com/dustin/gojson"
+	"encoding/json"
+	"sync"
 )
 
 var configLocation = flag.String("config", "./config.json", "Location of the configuration file. Defaults to ./config.json")
@@ -20,6 +21,8 @@ var config ConfigFile
 
 const ExitCodeBadConfig = 2
 
+var allServers []*serverInfo
+
 func main() {
 	flag.Parse()
 
@@ -30,6 +33,12 @@ func main() {
 
 	loadConfig()
 
+	allServers = make([]*serverInfo, len(ServerNames))
+	for i, v := range ServerNames {
+		allServers[i] = &serverInfo{}
+		allServers[i].Setup(v)
+	}
+
 	http.HandleFunc("/api/get", ServeAPIGet)
 	http.ListenAndServe(config.ListenAddr, http.DefaultServeMux)
 }
@@ -43,10 +52,12 @@ const jsonErrBlankRequest = `{"status":"error","error":"no queries given"}`
 const statusError = "error"
 const statusPartial = "partial"
 const statusOk = "ok"
+
 type apiResponse struct {
 	Status string `json:"status"`
 	Responses []requestResponse `json:"resp"`
 }
+
 type requestResponse struct {
 	Status string `json:"status"`
 	Request string `json:"req"`
@@ -81,7 +92,7 @@ func ServeAPIGet(w http.ResponseWriter, r *http.Request) {
 		resp.Responses[i] = ProcessSingleGetRequest(v)
 	}
 	for _, v := range resp.Responses {
-		if v.Status == statusError {
+		if v.Status != statusOk {
 			resp.Status = statusPartial
 			break
 		}
@@ -92,7 +103,7 @@ func ServeAPIGet(w http.ResponseWriter, r *http.Request) {
 	enc.Encode(resp)
 }
 
-const errRangeFormatIncorrect = errors.New("incorrect range format, must be yyyy-mm-dd~yyyy-mm-dd")
+var errRangeFormatIncorrect = errors.New("incorrect range format, must be yyyy-mm-dd~yyyy-mm-dd")
 
 // ProcessSingleGetRequest takes a request string and pulls the unique user data for the given dates and filters.
 //
@@ -122,14 +133,18 @@ const errRangeFormatIncorrect = errors.New("incorrect range format, must be yyyy
 //
 // It does not matter if a date is specified multiple times, due to the data format used.
 func ProcessSingleGetRequest(req string) (result requestResponse) {
-	var hll hyperloglog.HyperLogLogPlus, _ = hyperloglog.NewPlus(server.CounterPrecision)
+	fmt.Println("processing request:", req)
+	hll, _ := hyperloglog.NewPlus(server.CounterPrecision)
 
 	result.Request = req
 	result.Status = statusOk
 	filter := serverFilterAll
 
 	collectError := func(err error) bool {
-		if err != nil {
+		if err == ErrServerInFailedState {
+			result.Status = statusPartial
+			return false
+		} else if err != nil {
 			result.Status = statusError
 			result.Error = err.Error()
 			return true
@@ -161,7 +176,7 @@ func ProcessSingleGetRequest(req string) (result requestResponse) {
 				break outerLoop
 			}
 
-			err = addSingleDate(at, filter, &hll)
+			err = addSingleDate(at, filter, hll)
 			if collectError(err) {
 				break outerLoop
 			}
@@ -175,7 +190,7 @@ func ProcessSingleGetRequest(req string) (result requestResponse) {
 				break outerLoop
 			}
 
-			err = addRange(from, to, filter, &hll)
+			err = addRange(from, to, filter, hll)
 			if collectError(err) {
 				break outerLoop
 			}
@@ -200,26 +215,73 @@ func parseDateFromRequest(dateStr string) (time.Time, error) {
 	if err != nil || n != 3 {
 		return zeroTime, errBadDate
 	}
-	return time.Date(year, month, day, 0, 0, 0, 0, server.CounterLocation)
+	return time.Date(year, time.Month(month), day, 0, 0, 0, 0, server.CounterLocation), nil
+}
+
+type hllAndError struct {
+	hll *hyperloglog.HyperLogLogPlus
+	err error
 }
 
 func addSingleDate(at time.Time, filter serverFilter, dest *hyperloglog.HyperLogLogPlus) error {
-	// TODO
-	return nil
+	var partialErr error
+	for _, si := range allServers {
+		if filter.IsServerAllowed(si) {
+			hll, err2 := si.GetHLL(at)
+			if err2 == ErrServerInFailedState {
+				partialErr = err2
+			} else if err2 != nil {
+				return err2
+			} else {
+				dest.Merge(hll)
+			}
+		}
+	}
+	return partialErr
 }
 
 func addRange(start time.Time, end time.Time, filter serverFilter, dest *hyperloglog.HyperLogLogPlus) error {
+	end = server.TruncateToMidnight(end)
+	year, month, day := start.Date()
+	var partialErr error
+	var myAllServers = make([]*serverInfo, 0, len(allServers))
+	for _, si := range allServers {
+		if filter.IsServerAllowed(si) {
+			myAllServers = append(myAllServers, si)
+		}
+	}
 
-	return nil
+	var ch = make(chan hllAndError)
+	var wg sync.WaitGroup
+	for current := start; current.Before(end); day = day + 1 {
+		current = time.Date(year, month, day, 0, 0, 0, 0, server.CounterLocation)
+		for _, si := range myAllServers {
+			wg.Add(1)
+			go getHLL(ch, si, current)
+		}
+	}
+
+	go func() {
+		wg.Wait()
+		close(ch)
+	}()
+
+	for pair := range ch {
+		wg.Done()
+		hll, err := pair.hll, pair.err
+		if err != nil {
+			if partialErr == nil || partialErr == ErrServerInFailedState {
+				partialErr = err
+			}
+		} else {
+			dest.Merge(hll)
+		}
+	}
+
+	return partialErr
 }
 
-func combineDateRange(from time.Time, to time.Time, dest *hyperloglog.HyperLogLogPlus) error {
-	from = server.TruncateToMidnight(from)
-	to = server.TruncateToMidnight(to)
-	year, month, day := from.Date()
-	for current := from; current.Before(to); day = day + 1 {
-		current = time.Date(year, month, day, 0, 0, 0, 0, server.CounterLocation)
-
-	}
-	return nil
+func getHLL(ch chan hllAndError, si *serverInfo, at time.Time) {
+	hll, err := si.GetHLL(at)
+	ch <- hllAndError{hll: hll, err: err}
 }
\ No newline at end of file
diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index ed0ae1f5..ec705987 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -177,7 +177,7 @@ var SocketUpgrader = websocket.Upgrader{
 // Memes go here.
 var BannerHTML []byte
 
-// StopAcceptingConnections is closed while the server is shutting down.
+// StopAcceptingConnectionsCh is closed while the server is shutting down.
 var StopAcceptingConnectionsCh = make(chan struct{})
 var StopAcceptingConnections = false
 
@@ -283,12 +283,12 @@ const sendMessageBufferLength = 125
 const sendMessageAbortLength = 50
 
 // RunSocketConnection contains the main run loop of a websocket connection.
-
+//
 // First, it sets up the channels, the ClientInfo object, and the pong frame handler.
 // It starts the reader goroutine pointing at the newly created channels.
 // The function then enters the run loop (a `for{select{}}`).
 // The run loop is broken when an object is received on errorChan, or if `hello` is not the first C2S Command.
-
+//
 // After the run loop stops, the function launches a goroutine to drain
 // client.MessageChannel, signals the reader goroutine to stop, unsubscribes
 // from all pub/sub channels, waits on MsgChannelKeepalive (remember, the

From b3cbec4560cb582fe00b39ee5b75a4f3ffb9ac2e Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 17 Jan 2016 17:05:52 -0800
Subject: [PATCH 131/176] this is why i wasn't using relative imports

---
 socketserver/server/handlecore.go | 27 +++++++++++++--------------
 1 file changed, 13 insertions(+), 14 deletions(-)

diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index ec705987..314385a2 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -20,8 +20,6 @@ import (
 	"syscall"
 	"time"
 	"unicode/utf8"
-
-	"bitbucket.org/stendec/frankerfacez/socketserver/server/logstasher"
 )
 
 // SuccessCommand is a Reply Command to indicate success in reply to a C2S Command.
@@ -108,7 +106,7 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 	}
 
 	if Configuration.UseESLogStashing {
-		logstasher.Setup(Configuration.ESServer, Configuration.ESIndexPrefix, Configuration.ESHostName)
+		// logstasher.Setup(Configuration.ESServer, Configuration.ESIndexPrefix, Configuration.ESHostName)
 	}
 
 	janitorsOnce.Do(startJanitors)
@@ -310,9 +308,9 @@ func RunSocketConnection(conn *websocket.Conn) {
 	client.RemoteAddr = conn.RemoteAddr()
 	client.MsgChannelIsDone = stoppedChan
 
-	var report logstasher.ConnectionReport
-	report.ConnectTime = time.Now()
-	report.RemoteAddr = client.RemoteAddr
+	// var report logstasher.ConnectionReport
+	// report.ConnectTime = time.Now()
+	// report.RemoteAddr = client.RemoteAddr
 
 	conn.SetPongHandler(func(pongBody string) error {
 		client.Mutex.Lock()
@@ -326,7 +324,8 @@ func RunSocketConnection(conn *websocket.Conn) {
 	closeReason := runSocketWriter(conn, &client, _errorChan, _clientChan, _serverMessageChan)
 
 	// Exit
-	closeConnection(conn, closeReason, &report)
+	closeConnection(conn, closeReason)
+	// closeConnection(conn, closeReason, &report)
 
 	// Launch message draining goroutine - we aren't out of the pub/sub records
 	go func() {
@@ -353,9 +352,9 @@ func RunSocketConnection(conn *websocket.Conn) {
 		atomic.AddUint64(&Statistics.CurrentClientCount, NegativeOne)
 		atomic.AddUint64(&Statistics.ClientDisconnectsTotal, 1)
 
-		report.UsernameWasValidated = client.UsernameValidated
-		report.TwitchUsername = client.TwitchUsername
-		logstasher.Submit(&report)
+		// report.UsernameWasValidated = client.UsernameValidated
+		// report.TwitchUsername = client.TwitchUsername
+		// logstasher.Submit(&report)
 	}
 }
 
@@ -457,7 +456,7 @@ func getDeadline() time.Time {
 	return time.Now().Add(1 * time.Minute)
 }
 
-func closeConnection(conn *websocket.Conn, closeMsg websocket.CloseError, report *logstasher.ConnectionReport) {
+func closeConnection(conn *websocket.Conn, closeMsg websocket.CloseError) {
 	closeTxt := closeMsg.Text
 	if strings.Contains(closeTxt, "read: connection reset by peer") {
 		closeTxt = "read: connection reset by peer"
@@ -467,9 +466,9 @@ func closeConnection(conn *websocket.Conn, closeMsg websocket.CloseError, report
 		closeTxt = "clean shutdown"
 	}
 
-	report.DisconnectCode = closeMsg.Code
-	report.DisconnectReason = closeTxt
-	report.DisconnectTime = time.Now()
+	// report.DisconnectCode = closeMsg.Code
+	// report.DisconnectReason = closeTxt
+	// report.DisconnectTime = time.Now()
 
 	conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(closeMsg.Code, closeMsg.Text), getDeadline())
 	conn.Close()

From c85e8b10c3e4b06254ea591bcfa8d0c6fb47d51c Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 17 Jan 2016 17:28:33 -0800
Subject: [PATCH 132/176] Memory: Reduce send buffer length, entirely too large

---
 socketserver/server/handlecore.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 314385a2..4bbf852b 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -277,8 +277,8 @@ var CloseNonUTF8Data = websocket.CloseError{
 	Text: "Non UTF8 data recieved. Network corruption likely.",
 }
 
-const sendMessageBufferLength = 125
-const sendMessageAbortLength = 50
+const sendMessageBufferLength = 30
+const sendMessageAbortLength = 20
 
 // RunSocketConnection contains the main run loop of a websocket connection.
 //

From abb032f0c1f44b2d7d7cbe45f49dcc87edb42a14 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 17 Jan 2016 17:45:37 -0800
Subject: [PATCH 133/176] memory: Use a string pool for Commands

---
 socketserver/server/backend.go    |  6 ++---
 socketserver/server/commands.go   | 19 ++++++++++++++++
 socketserver/server/handlecore.go |  7 ++++--
 socketserver/server/intern.go     | 37 +++++++++++++++++++++++++++++++
 socketserver/server/publisher.go  |  2 +-
 5 files changed, 65 insertions(+), 6 deletions(-)
 create mode 100644 socketserver/server/intern.go

diff --git a/socketserver/server/backend.go b/socketserver/server/backend.go
index 1ec4751c..e93afeab 100644
--- a/socketserver/server/backend.go
+++ b/socketserver/server/backend.go
@@ -62,7 +62,7 @@ func getCacheKey(remoteCommand, data string) string {
 	return fmt.Sprintf("%s/%s", remoteCommand, data)
 }
 
-// HBackendPublishRequest handles the /uncached_pub route.
+// HTTPBackendUncachedPublish handles the /uncached_pub route.
 // The backend can POST here to publish a message to clients with no caching.
 // The POST arguments are `cmd`, `args`, `channel`, and `scope`.
 // The `scope` argument is required because no attempt is made to infer the scope from the command, unlike /cached_pub.
@@ -93,7 +93,7 @@ func HTTPBackendUncachedPublish(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	cm := ClientMessage{MessageID: -1, Command: Command(cmd), origArguments: json}
+	cm := ClientMessage{MessageID: -1, Command: CommandPool.Intern(cmd), origArguments: json}
 	cm.parseOrigArguments()
 	var count int
 
@@ -219,7 +219,7 @@ type ErrBackendNotOK struct {
 	Code     int
 }
 
-// Implements the error interface.
+// Error Implements the error interface.
 func (noe ErrBackendNotOK) Error() string {
 	return fmt.Sprintf("backend returned %d: %s", noe.Code, noe.Response)
 }
diff --git a/socketserver/server/commands.go b/socketserver/server/commands.go
index 3f424fa5..6865d2e2 100644
--- a/socketserver/server/commands.go
+++ b/socketserver/server/commands.go
@@ -41,6 +41,25 @@ var commandHandlers = map[Command]CommandHandler{
 	"user_history":          C2SHandleRemoteCommand,
 }
 
+func internCommands() {
+	CommandPool = NewStringPool()
+	CommandPool._Intern_Setup(HelloCommand)
+	CommandPool._Intern_Setup("ping")
+	CommandPool._Intern_Setup(SetUserCommand)
+	CommandPool._Intern_Setup(ReadyCommand)
+	CommandPool._Intern_Setup("sub")
+	CommandPool._Intern_Setup("unsub")
+	CommandPool._Intern_Setup("track_follow")
+	CommandPool._Intern_Setup("emoticon_uses")
+	CommandPool._Intern_Setup("twitch_emote")
+	CommandPool._Intern_Setup("get_link")
+	CommandPool._Intern_Setup("get_display_name")
+	CommandPool._Intern_Setup("update_follow_buttons")
+	CommandPool._Intern_Setup("chat_history")
+	CommandPool._Intern_Setup("user_history")
+	CommandPool._Intern_Setup("adjacent_history")
+}
+
 // DispatchC2SCommand handles a C2S Command in the provided ClientMessage.
 // It calls the correct CommandHandler function, catching panics.
 // It sends either the returned Reply ClientMessage, setting the correct messageID, or sends an ErrorCommand
diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 4bbf852b..057cd456 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -60,6 +60,8 @@ var Configuration *ConfigFile
 
 var janitorsOnce sync.Once
 
+var CommandPool StringPool
+
 // SetupServerAndHandle starts all background goroutines and registers HTTP listeners on the given ServeMux.
 // Essentially, this function completely preps the server for a http.ListenAndServe call.
 // (Uses http.DefaultServeMux if `serveMux` is nil.)
@@ -115,6 +117,7 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 // startJanitors starts the 'is_init_func' goroutines
 func startJanitors() {
 	loadUniqueUsers()
+	internCommands()
 
 	go authorizationJanitor()
 	go bunchCacheJanitor()
@@ -508,11 +511,11 @@ func UnmarshalClientMessage(data []byte, payloadType int, v interface{}) (err er
 
 	spaceIdx = strings.IndexRune(dataStr, ' ')
 	if spaceIdx == -1 {
-		out.Command = Command(dataStr)
+		out.Command = CommandPool.Intern(dataStr)
 		out.Arguments = nil
 		return nil
 	} else {
-		out.Command = Command(dataStr[:spaceIdx])
+		out.Command = CommandPool.Intern(dataStr[:spaceIdx])
 	}
 	dataStr = dataStr[spaceIdx+1:]
 	argumentsJSON := dataStr
diff --git a/socketserver/server/intern.go b/socketserver/server/intern.go
new file mode 100644
index 00000000..e2eb8e1e
--- /dev/null
+++ b/socketserver/server/intern.go
@@ -0,0 +1,37 @@
+package server
+
+import (
+	"sync"
+)
+
+type StringPool struct {
+	sync.RWMutex
+	lookup map[string]Command
+}
+
+func NewStringPool() *StringPool {
+	return &StringPool{lookup: make(map[string]Command)}
+}
+
+// doesn't lock, doesn't check for dupes.
+func (p *StringPool) _Intern_Setup(s string) {
+	p.lookup[s] = Command(s)
+}
+
+func (p *StringPool) Intern(s string) Command {
+	p.RLock()
+	ss, exists := p.lookup[s]
+	p.RUnlock()
+	if exists {
+		return ss
+	}
+
+	p.Lock()
+	defer p.Unlock()
+	ss, exists = p.lookup[s]
+	if exists {
+		return ss
+	}
+	p.lookup[s] = Command(string([]byte(s)))
+	return s
+}
diff --git a/socketserver/server/publisher.go b/socketserver/server/publisher.go
index 02843f4c..7c454a35 100644
--- a/socketserver/server/publisher.go
+++ b/socketserver/server/publisher.go
@@ -236,7 +236,7 @@ func HTTPBackendCachedPublish(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	cmd := Command(formData.Get("cmd"))
+	cmd := CommandPool.Intern(formData.Get("cmd"))
 	json := formData.Get("args")
 	channel := formData.Get("channel")
 	deleteMode := formData.Get("delete") != ""

From 387ada9c9cb96ce195653457a9d65ea9dd5ee70a Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 17 Jan 2016 17:46:34 -0800
Subject: [PATCH 134/176] memory: Make a copy of origArguments

---
 socketserver/server/handlecore.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 057cd456..677f54dc 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -518,7 +518,7 @@ func UnmarshalClientMessage(data []byte, payloadType int, v interface{}) (err er
 		out.Command = CommandPool.Intern(dataStr[:spaceIdx])
 	}
 	dataStr = dataStr[spaceIdx+1:]
-	argumentsJSON := dataStr
+	argumentsJSON := string([]byte(dataStr))
 	out.origArguments = argumentsJSON
 	err = out.parseOrigArguments()
 	if err != nil {

From 7654dcdf8ddc4c04b71af8c92ab586155a7ed250 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 17 Jan 2016 18:01:21 -0800
Subject: [PATCH 135/176] Source cleanup, and some string copies

---
 socketserver/server/commands.go           | 12 ++--
 socketserver/server/handlecore.go         | 12 ++--
 socketserver/server/intern.go             |  5 +-
 socketserver/server/irc.go                |  5 --
 socketserver/server/publisher.go          | 72 ++++++-----------------
 socketserver/server/stats.go              |  2 +-
 socketserver/server/subscriptions.go      |  4 +-
 socketserver/server/subscriptions_test.go |  6 +-
 socketserver/server/types.go              | 16 +++--
 socketserver/server/usercount.go          |  2 +-
 10 files changed, 47 insertions(+), 89 deletions(-)

diff --git a/socketserver/server/commands.go b/socketserver/server/commands.go
index 6865d2e2..8f77b2e4 100644
--- a/socketserver/server/commands.go
+++ b/socketserver/server/commands.go
@@ -17,7 +17,7 @@ import (
 // The Commands sent from Client -> Server and Server -> Client are disjoint sets.
 type Command string
 
-// CommandHandler is a RPC handler assosciated with a Command.
+// CommandHandler is a RPC handler associated with a Command.
 type CommandHandler func(*websocket.Conn, *ClientInfo, ClientMessage) (ClientMessage, error)
 
 var commandHandlers = map[Command]CommandHandler{
@@ -43,10 +43,10 @@ var commandHandlers = map[Command]CommandHandler{
 
 func internCommands() {
 	CommandPool = NewStringPool()
-	CommandPool._Intern_Setup(HelloCommand)
+	CommandPool._Intern_Setup(string(HelloCommand))
 	CommandPool._Intern_Setup("ping")
-	CommandPool._Intern_Setup(SetUserCommand)
-	CommandPool._Intern_Setup(ReadyCommand)
+	CommandPool._Intern_Setup(string(SetUserCommand))
+	CommandPool._Intern_Setup(string(ReadyCommand))
 	CommandPool._Intern_Setup("sub")
 	CommandPool._Intern_Setup("unsub")
 	CommandPool._Intern_Setup("track_follow")
@@ -155,8 +155,8 @@ func C2SSetUser(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rm
 	}
 
 	client.Mutex.Lock()
-	client.TwitchUsername = username
 	client.UsernameValidated = false
+	client.TwitchUsername = username
 	client.Mutex.Unlock()
 
 	if Configuration.SendAuthToNewClients {
@@ -262,7 +262,7 @@ func C2STrackFollow(conn *websocket.Conn, client *ClientInfo, msg ClientMessage)
 	now := time.Now()
 
 	followEventsLock.Lock()
-	followEvents = append(followEvents, followEvent{client.TwitchUsername, channel, following, now})
+	followEvents = append(followEvents, followEvent{User: client.TwitchUsername, Channel: channel, NowFollowing: following, Timestamp: now})
 	followEventsLock.Unlock()
 
 	return ResponseSuccess, nil
diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 677f54dc..1486d502 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -60,7 +60,7 @@ var Configuration *ConfigFile
 
 var janitorsOnce sync.Once
 
-var CommandPool StringPool
+var CommandPool *StringPool
 
 // SetupServerAndHandle starts all background goroutines and registers HTTP listeners on the given ServeMux.
 // Essentially, this function completely preps the server for a http.ListenAndServe call.
@@ -573,7 +573,7 @@ func MarshalClientMessage(clientMessage interface{}) (payloadType int, data []by
 	return websocket.TextMessage, []byte(dataStr), nil
 }
 
-// Convenience method: Parse the arguments of the ClientMessage as a single string.
+// ArgumentsAsString parses the arguments of the ClientMessage as a single string.
 func (cm *ClientMessage) ArgumentsAsString() (string1 string, err error) {
 	var ok bool
 	string1, ok = cm.Arguments.(string)
@@ -585,7 +585,7 @@ func (cm *ClientMessage) ArgumentsAsString() (string1 string, err error) {
 	}
 }
 
-// Convenience method: Parse the arguments of the ClientMessage as a single int.
+// ArgumentsAsInt parses the arguments of the ClientMessage as a single int.
 func (cm *ClientMessage) ArgumentsAsInt() (int1 int64, err error) {
 	var ok bool
 	var num float64
@@ -599,7 +599,7 @@ func (cm *ClientMessage) ArgumentsAsInt() (int1 int64, err error) {
 	}
 }
 
-// Convenience method: Parse the arguments of the ClientMessage as an array of two strings.
+// ArgumentsAsTwoStrings parses the arguments of the ClientMessage as an array of two strings.
 func (cm *ClientMessage) ArgumentsAsTwoStrings() (string1, string2 string, err error) {
 	var ok bool
 	var ary []interface{}
@@ -630,7 +630,7 @@ func (cm *ClientMessage) ArgumentsAsTwoStrings() (string1, string2 string, err e
 	}
 }
 
-// Convenience method: Parse the arguments of the ClientMessage as an array of a string and an int.
+// ArgumentsAsStringAndInt parses the arguments of the ClientMessage as an array of a string and an int.
 func (cm *ClientMessage) ArgumentsAsStringAndInt() (string1 string, int int64, err error) {
 	var ok bool
 	var ary []interface{}
@@ -663,7 +663,7 @@ func (cm *ClientMessage) ArgumentsAsStringAndInt() (string1 string, int int64, e
 	}
 }
 
-// Convenience method: Parse the arguments of the ClientMessage as an array of a string and an int.
+// ArgumentsAsStringAndBool parses the arguments of the ClientMessage as an array of a string and an int.
 func (cm *ClientMessage) ArgumentsAsStringAndBool() (str string, flag bool, err error) {
 	var ok bool
 	var ary []interface{}
diff --git a/socketserver/server/intern.go b/socketserver/server/intern.go
index e2eb8e1e..fb319d5a 100644
--- a/socketserver/server/intern.go
+++ b/socketserver/server/intern.go
@@ -32,6 +32,7 @@ func (p *StringPool) Intern(s string) Command {
 	if exists {
 		return ss
 	}
-	p.lookup[s] = Command(string([]byte(s)))
-	return s
+	ss = Command(string([]byte(s))) // make a copy
+	p.lookup[s] = ss
+	return ss
 }
diff --git a/socketserver/server/irc.go b/socketserver/server/irc.go
index 568cb534..98234829 100644
--- a/socketserver/server/irc.go
+++ b/socketserver/server/irc.go
@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"crypto/rand"
 	"encoding/base64"
-	"errors"
 	irc "github.com/fluffle/goirc/client"
 	"log"
 	"strings"
@@ -87,10 +86,6 @@ const AuthChannelName = "frankerfacezauthorizer"
 const AuthChannel = "#" + AuthChannelName
 const AuthCommand = "AUTH"
 
-const DEBUG = "DEBUG"
-
-var errChallengeNotFound = errors.New("did not find a challenge solved by that message")
-
 // is_init_func
 func ircConnection() {
 
diff --git a/socketserver/server/publisher.go b/socketserver/server/publisher.go
index 7c454a35..a11c471e 100644
--- a/socketserver/server/publisher.go
+++ b/socketserver/server/publisher.go
@@ -4,7 +4,6 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
-	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -33,36 +32,37 @@ var S2CCommandsCacheInfo = map[Command]PushCommandCacheInfo{
 type BacklogCacheType int
 
 const (
-	// This is not a cache type.
+	// CacheTypeInvalid is the sentinel value.
 	CacheTypeInvalid BacklogCacheType = iota
-	// This message cannot be cached.
+	// CacheTypeNever is a message that cannot be cached.
 	CacheTypeNever
-	// Save only the last copy of this message, and always send it when the backlog is requested.
+	// CacheTypeLastOnly means to save only the last copy of this message,
+	// and always send it when the backlog is requested.
 	CacheTypeLastOnly
-	// Save this backlog data to disk with its timestamp.
-	// Send it when the backlog is requested, or after a reconnect if it was updated.
+	// CacheTypePersistent means to save the last copy of this message,
+	// and always send it when the backlog is requested, but do not clean it periodically.
 	CacheTypePersistent
 )
 
 type MessageTargetType int
 
 const (
-	// This is not a message target.
+	// MsgTargetTypeInvalid is the sentinel value.
 	MsgTargetTypeInvalid MessageTargetType = iota
-	// This message is targeted to all users in a chat
+	// MsgTargetTypeChat is a message is targeted to all users in a particular chat.
 	MsgTargetTypeChat
-	// This message is targeted to all users in multiple chats
+	// MsgTargetTypeMultichat is a message is targeted to all users in multiple chats.
 	MsgTargetTypeMultichat
-	// This message is sent to all FFZ users.
+	// MsgTargetTypeGlobal is a message sent to all FFZ users.
 	MsgTargetTypeGlobal
 )
 
 // note: see types.go for methods on these
 
-// Returned by BacklogCacheType.UnmarshalJSON()
+// ErrorUnrecognizedCacheType is returned by BacklogCacheType.UnmarshalJSON()
 var ErrorUnrecognizedCacheType = errors.New("Invalid value for cachetype")
 
-// Returned by MessageTargetType.UnmarshalJSON()
+// ErrorUnrecognizedTargetType is returned by MessageTargetType.UnmarshalJSON()
 var ErrorUnrecognizedTargetType = errors.New("Invalid value for message target")
 
 type LastSavedMessage struct {
@@ -72,11 +72,11 @@ type LastSavedMessage struct {
 
 // map is command -> channel -> data
 
-// CacheTypeLastOnly. Cleaned up by reaper goroutine every ~hour.
+// CachedLastMessages is of CacheTypeLastOnly. Cleaned up by reaper goroutine every ~hour.
 var CachedLastMessages = make(map[Command]map[string]LastSavedMessage)
 var CachedLSMLock sync.RWMutex
 
-// CacheTypePersistent. Never cleaned.
+// PersistentLastMessages is of CacheTypePersistent. Never cleaned.
 var PersistentLastMessages = make(map[Command]map[string]LastSavedMessage)
 var PersistentLSMLock sync.RWMutex
 
@@ -135,50 +135,11 @@ func SendBacklogForNewClient(client *ClientInfo) {
 	CachedLSMLock.RUnlock()
 }
 
-// insertionSort implements insertion sort.
-// CacheTypeTimestamps should use insertion sort for O(N) average performance.
-// (The average case is the array is still sorted after insertion of the new item.)
-func insertionSort(ary sort.Interface) {
-	for i := 1; i < ary.Len(); i++ {
-		for j := i; j > 0 && ary.Less(j, j-1); j-- {
-			ary.Swap(j, j-1)
-		}
-	}
-}
-
 type timestampArray interface {
 	Len() int
 	GetTime(int) time.Time
 }
 
-func findFirstNewMessage(ary timestampArray, disconnectTime time.Time) (idx int) {
-	len := ary.Len()
-	i := len
-
-	// Walk backwards until we find GetTime() before disconnectTime
-	step := 1
-	for i > 0 {
-		i -= step
-		if i < 0 {
-			i = 0
-		}
-		if !ary.GetTime(i).After(disconnectTime) {
-			break
-		}
-		step = int(float64(step)*1.5) + 1
-	}
-
-	// Walk forwards until we find GetTime() after disconnectTime
-	for i < len && !ary.GetTime(i).After(disconnectTime) {
-		i++
-	}
-
-	if i == len {
-		return -1
-	}
-	return i
-}
-
 func SaveLastMessage(which map[Command]map[string]LastSavedMessage, locker sync.Locker, cmd Command, channel string, timestamp time.Time, data string, deleting bool) {
 	locker.Lock()
 	defer locker.Unlock()
@@ -195,7 +156,7 @@ func SaveLastMessage(which map[Command]map[string]LastSavedMessage, locker sync.
 	if deleting {
 		delete(chanMap, channel)
 	} else {
-		chanMap[channel] = LastSavedMessage{timestamp, data}
+		chanMap[channel] = LastSavedMessage{Timestamp: timestamp, Data: data}
 	}
 }
 
@@ -224,7 +185,8 @@ func HTTPBackendDropBacklog(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// Publish a message to clients, and update the in-server cache for the message.
+// HTTPBackendCachedPublish handles the /cached_pub route.
+// It publishes a message to clients, and then updates the in-server cache for the message.
 // notes:
 // `scope` is implicit in the command
 func HTTPBackendCachedPublish(w http.ResponseWriter, r *http.Request) {
diff --git a/socketserver/server/stats.go b/socketserver/server/stats.go
index 24fd52dc..8b4d8d9c 100644
--- a/socketserver/server/stats.go
+++ b/socketserver/server/stats.go
@@ -73,7 +73,7 @@ func commandCounter() {
 	}
 }
 
-// StatsDataVersion
+// StatsDataVersion is the version of the StatsData struct.
 const StatsDataVersion = 5
 const pageSize = 4096
 
diff --git a/socketserver/server/subscriptions.go b/socketserver/server/subscriptions.go
index ecd73a0a..890e2e19 100644
--- a/socketserver/server/subscriptions.go
+++ b/socketserver/server/subscriptions.go
@@ -99,7 +99,9 @@ func UnsubscribeSingleChat(client *ClientInfo, channelName string) {
 	ChatSubscriptionLock.RUnlock()
 }
 
-// Unsubscribe the client from all channels, AND clear the CurrentChannels / WatchingChannels fields.
+// UnsubscribeAll will unsubscribe the client from all channels,
+// AND clear the CurrentChannels / WatchingChannels fields.
+//
 // Locks:
 //   - read lock to top-level maps
 //   - write lock to SubscriptionInfos
diff --git a/socketserver/server/subscriptions_test.go b/socketserver/server/subscriptions_test.go
index c0c04720..c24f88e8 100644
--- a/socketserver/server/subscriptions_test.go
+++ b/socketserver/server/subscriptions_test.go
@@ -30,9 +30,9 @@ func TestSubscriptionAndPublish(t *testing.T) {
 	const TestData3 = false
 	var TestData4 = []interface{}{"str1", "str2", "str3"}
 
-	S2CCommandsCacheInfo[TestCommandChan] = PushCommandCacheInfo{CacheTypeLastOnly, MsgTargetTypeChat}
-	S2CCommandsCacheInfo[TestCommandMulti] = PushCommandCacheInfo{CacheTypeLastOnly, MsgTargetTypeMultichat}
-	S2CCommandsCacheInfo[TestCommandGlobal] = PushCommandCacheInfo{CacheTypeLastOnly, MsgTargetTypeGlobal}
+	S2CCommandsCacheInfo[TestCommandChan] = PushCommandCacheInfo{Caching: CacheTypeLastOnly, Target: MsgTargetTypeChat}
+	S2CCommandsCacheInfo[TestCommandMulti] = PushCommandCacheInfo{Caching: CacheTypeLastOnly, Target: MsgTargetTypeMultichat}
+	S2CCommandsCacheInfo[TestCommandGlobal] = PushCommandCacheInfo{Caching: CacheTypeLastOnly, Target: MsgTargetTypeGlobal}
 
 	var server *httptest.Server
 	var urls TURLs
diff --git a/socketserver/server/types.go b/socketserver/server/types.go
index 04614b2a..3bbca724 100644
--- a/socketserver/server/types.go
+++ b/socketserver/server/types.go
@@ -9,8 +9,6 @@ import (
 	"time"
 )
 
-const CryptoBoxKeyLength = 32
-
 const NegativeOne = ^uint64(0)
 
 type ConfigFile struct {
@@ -94,7 +92,7 @@ type ClientInfo struct {
 	// If it seems to be a performance problem, we can split this.
 	Mutex sync.Mutex
 
-	// TODO(riking) - does this need to be protected cross-thread?
+	// Info about the client's username and whether or not we have verified it.
 	AuthInfo
 
 	RemoteAddr net.Addr
@@ -187,15 +185,15 @@ func BacklogCacheTypeByName(name string) (bct BacklogCacheType) {
 	return
 }
 
-// Implements Stringer
+// String implements Stringer
 func (bct BacklogCacheType) String() string { return bct.Name() }
 
-// Implements json.Marshaler
+// MarshalJSON implements json.Marshaler
 func (bct BacklogCacheType) MarshalJSON() ([]byte, error) {
 	return json.Marshal(bct.Name())
 }
 
-// Implements json.Unmarshaler
+// UnmarshalJSON implements json.Unmarshaler
 func (bct *BacklogCacheType) UnmarshalJSON(data []byte) error {
 	var str string
 	err := json.Unmarshal(data, &str)
@@ -240,15 +238,15 @@ func MessageTargetTypeByName(name string) (mtt MessageTargetType) {
 	return
 }
 
-// Implements Stringer
+// String implements Stringer
 func (mtt MessageTargetType) String() string { return mtt.Name() }
 
-// Implements json.Marshaler
+// MarshalJSON implements json.Marshaler
 func (mtt MessageTargetType) MarshalJSON() ([]byte, error) {
 	return json.Marshal(mtt.Name())
 }
 
-// Implements json.Unmarshaler
+// UnmarshalJSON implements json.Unmarshaler
 func (mtt *MessageTargetType) UnmarshalJSON(data []byte) error {
 	var str string
 	err := json.Unmarshal(data, &str)
diff --git a/socketserver/server/usercount.go b/socketserver/server/usercount.go
index d46316d0..5390a3e8 100644
--- a/socketserver/server/usercount.go
+++ b/socketserver/server/usercount.go
@@ -17,7 +17,7 @@ import (
 	"io"
 )
 
-// uuidHash implements a hash for uuid.UUID by XORing the random bits.
+// UuidHash implements a hash for uuid.UUID by XORing the random bits.
 type UuidHash uuid.UUID
 
 func (u UuidHash) Sum64() uint64 {

From fdbcfe98ddad39503b44ca59d946b9d7f784e6c3 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 17 Jan 2016 18:40:08 -0800
Subject: [PATCH 136/176] move internCommands() to func init()

---
 socketserver/server/handlecore.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 1486d502..bdda501c 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -114,10 +114,13 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 	janitorsOnce.Do(startJanitors)
 }
 
+func init() {
+	internCommands()
+}
+
 // startJanitors starts the 'is_init_func' goroutines
 func startJanitors() {
 	loadUniqueUsers()
-	internCommands()
 
 	go authorizationJanitor()
 	go bunchCacheJanitor()

From 43ecbff65641c2f266b0121208669850ad58ad6f Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 17 Jan 2016 19:46:01 -0800
Subject: [PATCH 137/176] Treat all strings from read buffers as gc-tainted

Rule: Copies must be made before retaining the string.
---
 socketserver/server/backend.go    |  2 +-
 socketserver/server/commands.go   | 22 ++++++++++++++++------
 socketserver/server/handlecore.go |  8 +++++---
 socketserver/server/intern.go     | 16 ++++++++++------
 socketserver/server/publisher.go  |  2 +-
 socketserver/server/utils.go      |  4 ++++
 6 files changed, 37 insertions(+), 17 deletions(-)

diff --git a/socketserver/server/backend.go b/socketserver/server/backend.go
index e93afeab..55ccf9f1 100644
--- a/socketserver/server/backend.go
+++ b/socketserver/server/backend.go
@@ -93,7 +93,7 @@ func HTTPBackendUncachedPublish(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	cm := ClientMessage{MessageID: -1, Command: CommandPool.Intern(cmd), origArguments: json}
+	cm := ClientMessage{MessageID: -1, Command: CommandPool.InternCommand(cmd), origArguments: json}
 	cm.parseOrigArguments()
 	var count int
 
diff --git a/socketserver/server/commands.go b/socketserver/server/commands.go
index 8f77b2e4..6d868512 100644
--- a/socketserver/server/commands.go
+++ b/socketserver/server/commands.go
@@ -41,7 +41,10 @@ var commandHandlers = map[Command]CommandHandler{
 	"user_history":          C2SHandleRemoteCommand,
 }
 
-func internCommands() {
+func setupInterning() {
+	PubSubChannelPool = NewStringPool()
+	TwitchChannelPool = NewStringPool()
+
 	CommandPool = NewStringPool()
 	CommandPool._Intern_Setup(string(HelloCommand))
 	CommandPool._Intern_Setup("ping")
@@ -114,7 +117,7 @@ func C2SHello(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg
 		return
 	}
 
-	client.VersionString = version
+	client.VersionString = copyString(version)
 	client.Version = VersionFromString(version)
 
 	client.ClientID = uuid.FromStringOrNil(clientID)
@@ -154,6 +157,8 @@ func C2SSetUser(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rm
 		return
 	}
 
+	username = copyString(username)
+
 	client.Mutex.Lock()
 	client.UsernameValidated = false
 	client.TwitchUsername = username
@@ -198,11 +203,12 @@ func C2SReady(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg
 
 func C2SSubscribe(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
 	channel, err := msg.ArgumentsAsString()
-
 	if err != nil {
 		return
 	}
 
+	channel = PubSubChannelPool.Intern(channel)
+
 	client.Mutex.Lock()
 	AddToSliceS(&client.CurrentChannels, channel)
 	if usePendingSubscrptionsBacklog {
@@ -219,11 +225,12 @@ func C2SSubscribe(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (
 // It removes the channel from ClientInfo.CurrentChannels and calls UnsubscribeSingleChat.
 func C2SUnsubscribe(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
 	channel, err := msg.ArgumentsAsString()
-
 	if err != nil {
 		return
 	}
 
+	channel = PubSubChannelPool.Intern(channel)
+
 	client.Mutex.Lock()
 	RemoveFromSliceS(&client.CurrentChannels, channel)
 	client.Mutex.Unlock()
@@ -261,6 +268,8 @@ func C2STrackFollow(conn *websocket.Conn, client *ClientInfo, msg ClientMessage)
 	}
 	now := time.Now()
 
+	channel = TwitchChannelPool.Intern(channel)
+
 	followEventsLock.Lock()
 	followEvents = append(followEvents, followEvent{User: client.TwitchUsername, Channel: channel, NowFollowing: following, Timestamp: now})
 	followEventsLock.Unlock()
@@ -323,6 +332,7 @@ func C2SEmoticonUses(conn *websocket.Conn, client *ClientInfo, msg ClientMessage
 			if count > 200 {
 				count = 200
 			}
+			roomName = TwitchChannelPool.Intern(roomName)
 			destMapInner[roomName] += count
 			total += count
 		}
@@ -422,7 +432,7 @@ var bunchCacheCleanupSignal = sync.NewCond(&bunchCacheLock)
 var bunchCacheLastCleanup time.Time
 
 func bunchedRequestFromCM(msg *ClientMessage) bunchedRequest {
-	return bunchedRequest{Command: msg.Command, Param: msg.origArguments}
+	return bunchedRequest{Command: msg.Command, Param: copyString(msg.origArguments)}
 }
 
 // is_init_func
@@ -563,7 +573,7 @@ const AuthorizationFailedErrorString = "Failed to verify your Twitch username."
 const AuthorizationNeededError = "You must be signed in to use that command."
 
 func doRemoteCommand(conn *websocket.Conn, msg ClientMessage, client *ClientInfo) {
-	resp, err := SendRemoteCommandCached(string(msg.Command), msg.origArguments, client.AuthInfo)
+	resp, err := SendRemoteCommandCached(string(msg.Command), copyString(msg.origArguments), client.AuthInfo)
 
 	if err == ErrAuthorizationNeeded {
 		if client.TwitchUsername == "" {
diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index bdda501c..058c628e 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -61,6 +61,8 @@ var Configuration *ConfigFile
 var janitorsOnce sync.Once
 
 var CommandPool *StringPool
+var PubSubChannelPool *StringPool
+var TwitchChannelPool *StringPool
 
 // SetupServerAndHandle starts all background goroutines and registers HTTP listeners on the given ServeMux.
 // Essentially, this function completely preps the server for a http.ListenAndServe call.
@@ -115,7 +117,7 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 }
 
 func init() {
-	internCommands()
+	setupInterning()
 }
 
 // startJanitors starts the 'is_init_func' goroutines
@@ -514,11 +516,11 @@ func UnmarshalClientMessage(data []byte, payloadType int, v interface{}) (err er
 
 	spaceIdx = strings.IndexRune(dataStr, ' ')
 	if spaceIdx == -1 {
-		out.Command = CommandPool.Intern(dataStr)
+		out.Command = CommandPool.InternCommand(dataStr)
 		out.Arguments = nil
 		return nil
 	} else {
-		out.Command = CommandPool.Intern(dataStr[:spaceIdx])
+		out.Command = CommandPool.InternCommand(dataStr[:spaceIdx])
 	}
 	dataStr = dataStr[spaceIdx+1:]
 	argumentsJSON := string([]byte(dataStr))
diff --git a/socketserver/server/intern.go b/socketserver/server/intern.go
index fb319d5a..2f9cf416 100644
--- a/socketserver/server/intern.go
+++ b/socketserver/server/intern.go
@@ -6,19 +6,23 @@ import (
 
 type StringPool struct {
 	sync.RWMutex
-	lookup map[string]Command
+	lookup map[string]string
 }
 
 func NewStringPool() *StringPool {
-	return &StringPool{lookup: make(map[string]Command)}
+	return &StringPool{lookup: make(map[string]string)}
 }
 
 // doesn't lock, doesn't check for dupes.
 func (p *StringPool) _Intern_Setup(s string) {
-	p.lookup[s] = Command(s)
+	p.lookup[s] = s
 }
 
-func (p *StringPool) Intern(s string) Command {
+func (p *StringPool) InternCommand(s string) Command {
+	return Command(p.Intern(s))
+}
+
+func (p *StringPool) Intern(s string) string {
 	p.RLock()
 	ss, exists := p.lookup[s]
 	p.RUnlock()
@@ -32,7 +36,7 @@ func (p *StringPool) Intern(s string) Command {
 	if exists {
 		return ss
 	}
-	ss = Command(string([]byte(s))) // make a copy
-	p.lookup[s] = ss
+	ss = copyString(s)
+	p.lookup[ss] = ss
 	return ss
 }
diff --git a/socketserver/server/publisher.go b/socketserver/server/publisher.go
index a11c471e..2cfd4f4d 100644
--- a/socketserver/server/publisher.go
+++ b/socketserver/server/publisher.go
@@ -198,7 +198,7 @@ func HTTPBackendCachedPublish(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	cmd := CommandPool.Intern(formData.Get("cmd"))
+	cmd := CommandPool.InternCommand(formData.Get("cmd"))
 	json := formData.Get("args")
 	channel := formData.Get("channel")
 	deleteMode := formData.Get("delete") != ""
diff --git a/socketserver/server/utils.go b/socketserver/server/utils.go
index 2cfa6a37..d30ec2a0 100644
--- a/socketserver/server/utils.go
+++ b/socketserver/server/utils.go
@@ -27,6 +27,10 @@ func New4KByteBuffer() interface{} {
 	return make([]byte, 0, 4096)
 }
 
+func copyString(s string) string {
+	return string([]byte(s))
+}
+
 func SealRequest(form url.Values) (url.Values, error) {
 	var nonce [24]byte
 	var err error

From b870eac3f361c599a15f563cab697be1649d7322 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 17 Jan 2016 19:51:10 -0800
Subject: [PATCH 138/176] okay yeah i'm tired of that buildtime penalty

---
 socketserver/server/tickspersecond.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/socketserver/server/tickspersecond.go b/socketserver/server/tickspersecond.go
index 36e99516..160eec7c 100644
--- a/socketserver/server/tickspersecond.go
+++ b/socketserver/server/tickspersecond.go
@@ -4,9 +4,9 @@ package server
 // long get_ticks_per_second() {
 //   return sysconf(_SC_CLK_TCK);
 // }
-import "C"
+//import "C"
 
 // note: this seems to add 0.1s to compile time on my machine
-var ticksPerSecond = int(C.get_ticks_per_second())
+//var ticksPerSecond = int(C.get_ticks_per_second())
 
-//var ticksPerSecond = 100
+var ticksPerSecond = 100

From af19d5f30a814f7074fda456106585ccc003077d Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 27 Jan 2016 18:04:28 -0800
Subject: [PATCH 139/176] one line diff boys

---
 socketserver/server/publisher.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/socketserver/server/publisher.go b/socketserver/server/publisher.go
index 2cfd4f4d..366f1065 100644
--- a/socketserver/server/publisher.go
+++ b/socketserver/server/publisher.go
@@ -22,7 +22,7 @@ var S2CCommandsCacheInfo = map[Command]PushCommandCacheInfo{
 	// follow_buttons: extra follow buttons below the stream
 	"follow_sets":    {CacheTypePersistent, MsgTargetTypeChat},
 	"follow_buttons": {CacheTypePersistent, MsgTargetTypeChat},
-	"srl_race":       {CacheTypeLastOnly, MsgTargetTypeChat},
+	"srl_race":       {CacheTypeLastOnly, MsgTargetTypeMultichat},
 
 	/// Chatter/viewer counts
 	"chatters": {CacheTypeLastOnly, MsgTargetTypeChat},

From 1113c3a766e27ba96b3ab7de9b0cf2325a05d4bf Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 27 Jan 2016 19:40:17 -0800
Subject: [PATCH 140/176] lock once during multichan publish

---
 socketserver/server/publisher.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/socketserver/server/publisher.go b/socketserver/server/publisher.go
index 366f1065..45bfd9f3 100644
--- a/socketserver/server/publisher.go
+++ b/socketserver/server/publisher.go
@@ -230,9 +230,12 @@ func HTTPBackendCachedPublish(w http.ResponseWriter, r *http.Request) {
 		count = PublishToChannel(channel, msg)
 	} else if cacheinfo.Caching == CacheTypeLastOnly && cacheinfo.Target == MsgTargetTypeMultichat {
 		channels := strings.Split(channel, ",")
+		var dummyLock sync.Mutex
+		CachedLSMLock.Lock()
 		for _, channel := range channels {
-			SaveLastMessage(CachedLastMessages, &CachedLSMLock, cmd, channel, timestamp, json, deleteMode)
+			SaveLastMessage(CachedLastMessages, &dummyLock, cmd, channel, timestamp, json, deleteMode)
 		}
+		CachedLSMLock.Unlock()
 		count = PublishToMultiple(channels, msg)
 	}
 

From cddd13ba16dcf34152d85d36d8f33ef4ff5ff081 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 3 Feb 2016 22:01:47 -0800
Subject: [PATCH 141/176] Allow HTTPS twitch pages access to socket server

---
 socketserver/server/handlecore.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 058c628e..c3b52842 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -51,6 +51,7 @@ const defaultMinMemoryKB = 1024 * 24
 
 // TwitchDotTv is the http origin for twitch.tv.
 const TwitchDotTv = "http://www.twitch.tv"
+const TwitchDotTvHTTPS = "https://www.twitch.tv"
 
 // ResponseSuccess is a Reply ClientMessage with the MessageID not yet filled out.
 var ResponseSuccess = ClientMessage{Command: SuccessCommand}
@@ -175,7 +176,7 @@ var SocketUpgrader = websocket.Upgrader{
 	ReadBufferSize:  1024,
 	WriteBufferSize: 1024,
 	CheckOrigin: func(r *http.Request) bool {
-		return r.Header.Get("Origin") == TwitchDotTv
+		return r.Header.Get("Origin") == TwitchDotTv || r.Header.Get("Origin") == TwitchDotTvHTTPS
 	},
 }
 

From bba5d8f3447f94d0585004a00f15293822c2a247 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 3 Feb 2016 22:06:46 -0800
Subject: [PATCH 142/176] Use a regexp instead to match origin

---
 socketserver/server/handlecore.go         |  9 +++++----
 socketserver/server/subscriptions_test.go | 11 ++++++++---
 2 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index c3b52842..ed0f41f4 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -20,6 +20,7 @@ import (
 	"syscall"
 	"time"
 	"unicode/utf8"
+	"regexp"
 )
 
 // SuccessCommand is a Reply Command to indicate success in reply to a C2S Command.
@@ -49,9 +50,9 @@ const AsyncResponseCommand Command = "_async"
 
 const defaultMinMemoryKB = 1024 * 24
 
-// TwitchDotTv is the http origin for twitch.tv.
-const TwitchDotTv = "http://www.twitch.tv"
-const TwitchDotTvHTTPS = "https://www.twitch.tv"
+// DotTwitchDotTv is the .twitch.tv suffix.
+const DotTwitchDotTv = ".twitch.tv"
+var OriginRegexp = regexp.MustCompile(DotTwitchDotTv + "$")
 
 // ResponseSuccess is a Reply ClientMessage with the MessageID not yet filled out.
 var ResponseSuccess = ClientMessage{Command: SuccessCommand}
@@ -176,7 +177,7 @@ var SocketUpgrader = websocket.Upgrader{
 	ReadBufferSize:  1024,
 	WriteBufferSize: 1024,
 	CheckOrigin: func(r *http.Request) bool {
-		return r.Header.Get("Origin") == TwitchDotTv || r.Header.Get("Origin") == TwitchDotTvHTTPS
+		return OriginRegexp.MatchString(r.Header.Get("Origin"))
 	},
 }
 
diff --git a/socketserver/server/subscriptions_test.go b/socketserver/server/subscriptions_test.go
index c24f88e8..132dda74 100644
--- a/socketserver/server/subscriptions_test.go
+++ b/socketserver/server/subscriptions_test.go
@@ -14,6 +14,8 @@ import (
 	"time"
 )
 
+const TestOrigin = "http://www.twitch.tv"
+
 func TestSubscriptionAndPublish(t *testing.T) {
 	var doneWg sync.WaitGroup
 	var readyWg sync.WaitGroup
@@ -54,7 +56,7 @@ func TestSubscriptionAndPublish(t *testing.T) {
 	var err error
 
 	var headers http.Header = make(http.Header)
-	headers.Set("Origin", TwitchDotTv)
+	headers.Set("Origin", TestOrigin)
 
 	// client 1: sub ch1, ch2
 	// client 2: sub ch1, ch3
@@ -72,6 +74,9 @@ func TestSubscriptionAndPublish(t *testing.T) {
 		return
 	}
 
+	// both origins need testing
+	headers.Set("Origin", "https://www.twitch.tv")
+
 	doneWg.Add(1)
 	readyWg.Add(1)
 	go func(conn *websocket.Conn) {
@@ -265,7 +270,7 @@ func TestRestrictedCommands(t *testing.T) {
 	var challengeChan = make(chan string)
 
 	var headers http.Header = make(http.Header)
-	headers.Set("Origin", TwitchDotTv)
+	headers.Set("Origin", TestOrigin)
 
 	// Client 1
 	conn, _, err = websocket.DefaultDialer.Dial(urls.Websocket, headers)
@@ -366,7 +371,7 @@ func BenchmarkUserSubscriptionSinglePublish(b *testing.B) {
 	defer unsubscribeAllClients()
 
 	var headers http.Header = make(http.Header)
-	headers.Set("Origin", TwitchDotTv)
+	headers.Set("Origin", TestOrigin)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {

From 4874d2577f752f3ef8972ea496c688af2e819afa Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 3 Feb 2016 22:06:57 -0800
Subject: [PATCH 143/176] Update the archetecture drawing

---
 socketserver/SocketServerDesign.svg | 289 +++++++++++++++++++---------
 1 file changed, 194 insertions(+), 95 deletions(-)

diff --git a/socketserver/SocketServerDesign.svg b/socketserver/SocketServerDesign.svg
index e2b31d73..6a141697 100644
--- a/socketserver/SocketServerDesign.svg
+++ b/socketserver/SocketServerDesign.svg
@@ -9,9 +9,9 @@
    xmlns="http://www.w3.org/2000/svg"
    xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
    xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
-   width="210mm"
-   height="297mm"
-   viewBox="0 0 744.09448819 1052.3622047"
+   width="198.96094mm"
+   height="254.84174mm"
+   viewBox="0 0 704.9797 902.98252"
    id="svg2"
    version="1.1"
    inkscape:version="0.91 r13725"
@@ -21,103 +21,110 @@
     <marker
        inkscape:stockid="Arrow1Lstart"
        orient="auto"
-       refY="0.0"
-       refX="0.0"
+       refY="0"
+       refX="0"
        id="marker23417"
        style="overflow:visible"
        inkscape:isstock="true">
       <path
          id="path23419"
-         d="M 0.0,0.0 L 5.0,-5.0 L -12.5,0.0 L 5.0,5.0 L 0.0,0.0 z "
-         style="fill-rule:evenodd;stroke:#c87137;stroke-width:1pt;stroke-opacity:1;fill:#c87137;fill-opacity:1"
-         transform="scale(0.8) translate(12.5,0)" />
+         d="M 0,0 5,-5 -12.5,0 5,5 0,0 Z"
+         style="fill:#c87137;fill-opacity:1;fill-rule:evenodd;stroke:#c87137;stroke-width:1pt;stroke-opacity:1"
+         transform="matrix(0.8,0,0,0.8,10,0)"
+         inkscape:connector-curvature="0" />
     </marker>
     <marker
        inkscape:isstock="true"
        style="overflow:visible"
        id="marker23251"
-       refX="0.0"
-       refY="0.0"
+       refX="0"
+       refY="0"
        orient="auto"
        inkscape:stockid="Arrow1Lstart"
        inkscape:collect="always">
       <path
-         transform="scale(0.8) translate(12.5,0)"
-         style="fill-rule:evenodd;stroke:#008000;stroke-width:1pt;stroke-opacity:1;fill:#000000;fill-opacity:1"
-         d="M 0.0,0.0 L 5.0,-5.0 L -12.5,0.0 L 5.0,5.0 L 0.0,0.0 z "
-         id="path23253" />
+         transform="matrix(0.8,0,0,0.8,10,0)"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:1pt;stroke-opacity:1"
+         d="M 0,0 5,-5 -12.5,0 5,5 0,0 Z"
+         id="path23253"
+         inkscape:connector-curvature="0" />
     </marker>
     <marker
        inkscape:isstock="true"
        style="overflow:visible"
        id="marker23127"
-       refX="0.0"
-       refY="0.0"
+       refX="0"
+       refY="0"
        orient="auto"
        inkscape:stockid="Arrow1Lstart">
       <path
-         transform="scale(0.8) translate(12.5,0)"
-         style="fill-rule:evenodd;stroke:#008000;stroke-width:1pt;stroke-opacity:1;fill:#000000;fill-opacity:1"
-         d="M 0.0,0.0 L 5.0,-5.0 L -12.5,0.0 L 5.0,5.0 L 0.0,0.0 z "
-         id="path23129" />
+         transform="matrix(0.8,0,0,0.8,10,0)"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:1pt;stroke-opacity:1"
+         d="M 0,0 5,-5 -12.5,0 5,5 0,0 Z"
+         id="path23129"
+         inkscape:connector-curvature="0" />
     </marker>
     <marker
        inkscape:stockid="Arrow1Lstart"
        orient="auto"
-       refY="0.0"
-       refX="0.0"
+       refY="0"
+       refX="0"
        id="Arrow1Lstart"
        style="overflow:visible"
        inkscape:isstock="true"
        inkscape:collect="always">
       <path
          id="path4394"
-         d="M 0.0,0.0 L 5.0,-5.0 L -12.5,0.0 L 5.0,5.0 L 0.0,0.0 z "
-         style="fill-rule:evenodd;stroke:#008000;stroke-width:1pt;stroke-opacity:1;fill:#000000;fill-opacity:1"
-         transform="scale(0.8) translate(12.5,0)" />
+         d="M 0,0 5,-5 -12.5,0 5,5 0,0 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:1pt;stroke-opacity:1"
+         transform="matrix(0.8,0,0,0.8,10,0)"
+         inkscape:connector-curvature="0" />
     </marker>
     <marker
        inkscape:stockid="Arrow1Lend"
        orient="auto"
-       refY="0.0"
-       refX="0.0"
+       refY="0"
+       refX="0"
        id="marker22805"
-       style="overflow:visible;"
+       style="overflow:visible"
        inkscape:isstock="true">
       <path
          id="path22807"
-         d="M 0.0,0.0 L 5.0,-5.0 L -12.5,0.0 L 5.0,5.0 L 0.0,0.0 z "
-         style="fill-rule:evenodd;stroke:#008000;stroke-width:1pt;stroke-opacity:1;fill:#000000;fill-opacity:1"
-         transform="scale(0.8) rotate(180) translate(12.5,0)" />
+         d="M 0,0 5,-5 -12.5,0 5,5 0,0 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:1pt;stroke-opacity:1"
+         transform="matrix(-0.8,0,0,-0.8,-10,0)"
+         inkscape:connector-curvature="0" />
     </marker>
     <marker
        inkscape:isstock="true"
-       style="overflow:visible;"
+       style="overflow:visible"
        id="marker6763"
-       refX="0.0"
-       refY="0.0"
+       refX="0"
+       refY="0"
        orient="auto"
        inkscape:stockid="Arrow1Lend"
        inkscape:collect="always">
       <path
-         transform="scale(0.8) rotate(180) translate(12.5,0)"
-         style="fill-rule:evenodd;stroke:#008000;stroke-width:1pt;stroke-opacity:1;fill:#008000;fill-opacity:1"
-         d="M 0.0,0.0 L 5.0,-5.0 L -12.5,0.0 L 5.0,5.0 L 0.0,0.0 z "
-         id="path6765" />
+         transform="matrix(-0.8,0,0,-0.8,-10,0)"
+         style="fill:#008000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:1pt;stroke-opacity:1"
+         d="M 0,0 5,-5 -12.5,0 5,5 0,0 Z"
+         id="path6765"
+         inkscape:connector-curvature="0" />
     </marker>
     <marker
        inkscape:stockid="Arrow1Lend"
        orient="auto"
-       refY="0.0"
-       refX="0.0"
+       refY="0"
+       refX="0"
        id="marker6717"
-       style="overflow:visible;"
+       style="overflow:visible"
        inkscape:isstock="true">
       <path
          id="path6719"
-         d="M 0.0,0.0 L 5.0,-5.0 L -12.5,0.0 L 5.0,5.0 L 0.0,0.0 z "
-         style="fill-rule:evenodd;stroke:#000000;stroke-width:1pt;stroke-opacity:1;fill:#000000;fill-opacity:1"
-         transform="scale(0.8) rotate(180) translate(12.5,0)" />
+         d="M 0,0 5,-5 -12.5,0 5,5 0,0 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1pt;stroke-opacity:1"
+         transform="matrix(-0.8,0,0,-0.8,-10,0)"
+         inkscape:connector-curvature="0" />
     </marker>
     <marker
        inkscape:stockid="Arrow1Lend"
@@ -346,6 +353,36 @@
          d="M 0,0 5,-5 -12.5,0 5,5 0,0 Z"
          id="path6765-2-8" />
     </marker>
+    <marker
+       inkscape:stockid="Arrow1Lstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="marker23417-9"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path23419-5"
+         d="M 0,0 5,-5 -12.5,0 5,5 0,0 Z"
+         style="fill:#c87137;fill-opacity:1;fill-rule:evenodd;stroke:#c87137;stroke-width:1pt;stroke-opacity:1"
+         transform="matrix(0.8,0,0,0.8,10,0)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Lend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Lend-9-5-3-1-7-2-5-2-2-5"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4397-0-5-4-1-4-4-3-0-1-8"
+         d="M 0,0 5,-5 -12.5,0 5,5 0,0 Z"
+         style="fill:#c87137;fill-opacity:1;fill-rule:evenodd;stroke:#c87137;stroke-width:1pt;stroke-opacity:1"
+         transform="matrix(-0.8,0,0,-0.8,-10,0)" />
+    </marker>
   </defs>
   <sodipodi:namedview
      id="base"
@@ -354,17 +391,21 @@
      borderopacity="1.0"
      inkscape:pageopacity="0.0"
      inkscape:pageshadow="2"
-     inkscape:zoom="2"
-     inkscape:cx="253.89137"
-     inkscape:cy="923.6285"
+     inkscape:zoom="1"
+     inkscape:cx="251.08071"
+     inkscape:cy="521.23066"
      inkscape:document-units="px"
      inkscape:current-layer="layer1"
      showgrid="false"
      inkscape:window-width="1920"
-     inkscape:window-height="1031"
+     inkscape:window-height="1026"
      inkscape:window-x="0"
      inkscape:window-y="25"
-     inkscape:window-maximized="1" />
+     inkscape:window-maximized="1"
+     fit-margin-top="2"
+     fit-margin-left="2"
+     fit-margin-right="2"
+     fit-margin-bottom="2" />
   <metadata
      id="metadata7">
     <rdf:RDF>
@@ -373,14 +414,15 @@
         <dc:format>image/svg+xml</dc:format>
         <dc:type
            rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
-        <dc:title />
+        <dc:title></dc:title>
       </cc:Work>
     </rdf:RDF>
   </metadata>
   <g
      inkscape:label="Layer 1"
      inkscape:groupmode="layer"
-     id="layer1">
+     id="layer1"
+     transform="translate(-4.0098404,-1.9898711)">
     <ellipse
        style="fill:#6441a5;fill-opacity:0.34901961;fill-rule:evenodd;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
        id="path3336"
@@ -598,7 +640,7 @@
          id="tspan4289">Socket Server</tspan></text>
     <text
        xml:space="preserve"
-       style="font-style:normal;font-weight:normal;font-size:12.5px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;"
+       style="font-style:normal;font-weight:normal;font-size:12.5px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
        x="125.13116"
        y="496.83524"
        id="text4293"
@@ -669,7 +711,7 @@
          y="488.27744">tuturu.frankerfacez.com</tspan></text>
     <text
        xml:space="preserve"
-       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;text-anchor:middle;text-align:center;"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
        x="-591.36749"
        y="210.3251"
        id="text4682"
@@ -677,9 +719,9 @@
        transform="matrix(0.22471085,-0.97442549,0.97442549,0.22471085,0,0)"><tspan
          sodipodi:role="line"
          id="tspan4684"
-         x="-601.21124"
+         x="-591.36749"
          y="210.3251"
-         style="font-size:11.25px;fill:#008000;text-anchor:middle;text-align:center;"
+         style="font-size:11.25px;text-align:center;text-anchor:middle;fill:#008000"
          rotate="0 0 0 0"
          dy="0">TLS</tspan></text>
     <path
@@ -690,7 +732,7 @@
        sodipodi:nodetypes="cc" />
     <text
        xml:space="preserve"
-       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;text-anchor:middle;text-align:center;"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
        x="627.77789"
        y="-75.716248"
        id="text4682-9"
@@ -698,9 +740,9 @@
        transform="matrix(0.08046399,0.99675752,-0.99675752,0.08046399,0,0)"><tspan
          sodipodi:role="line"
          id="tspan4684-8"
-         x="617.93414"
+         x="627.77789"
          y="-75.716248"
-         style="font-size:11.25px;fill:#008000;text-anchor:middle;text-align:center;"
+         style="font-size:11.25px;text-align:center;text-anchor:middle;fill:#008000"
          rotate="0 0 0 0"
          dy="0">TLS</tspan></text>
     <text
@@ -733,7 +775,7 @@
        sodipodi:nodetypes="cc" />
     <text
        xml:space="preserve"
-       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;text-anchor:middle;text-align:center;"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
        x="681.88031"
        y="-36.647579"
        id="text4682-9-9"
@@ -743,9 +785,9 @@
        inkscape:transform-center-y="-21.804086"><tspan
          sodipodi:role="line"
          id="tspan4684-8-9"
-         x="672.03656"
+         x="681.88031"
          y="-36.647579"
-         style="font-size:11.25px;fill:#008000;text-anchor:middle;text-align:center;"
+         style="font-size:11.25px;text-align:center;text-anchor:middle;fill:#008000"
          rotate="0 0 0 0"
          dy="0">TLS</tspan></text>
     <path
@@ -802,7 +844,7 @@
        sodipodi:nodetypes="cc" />
     <text
        xml:space="preserve"
-       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;text-anchor:middle;text-align:center;"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
        x="694.75018"
        y="-213.68245"
        id="text4682-9-9-6"
@@ -812,14 +854,14 @@
        inkscape:transform-center-y="-21.804086"><tspan
          sodipodi:role="line"
          id="tspan4684-8-9-3"
-         x="684.90643"
+         x="694.75018"
          y="-213.68245"
-         style="font-size:11.25px;fill:#008000;text-anchor:middle;text-align:center;"
+         style="font-size:11.25px;text-align:center;text-anchor:middle;fill:#008000"
          rotate="0 0 0 0"
          dy="0">TLS</tspan></text>
     <text
        xml:space="preserve"
-       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;text-anchor:middle;text-align:center;"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
        x="697.33954"
        y="-256.61011"
        id="text4682-9-9-4"
@@ -829,14 +871,14 @@
        inkscape:transform-center-y="-21.804086"><tspan
          sodipodi:role="line"
          id="tspan4684-8-9-5"
-         x="687.49579"
+         x="697.33954"
          y="-256.61011"
-         style="font-size:11.25px;fill:#008000;text-anchor:middle;text-align:center;"
+         style="font-size:11.25px;text-align:center;text-anchor:middle;fill:#008000"
          rotate="0 0 0 0"
          dy="0">TLS</tspan></text>
     <text
        xml:space="preserve"
-       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;text-anchor:middle;text-align:center;"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
        x="-615.4494"
        y="560.20447"
        id="text4682-9-9-3"
@@ -846,14 +888,14 @@
        inkscape:transform-center-y="23.125079"><tspan
          sodipodi:role="line"
          id="tspan4684-8-9-37"
-         x="-625.29315"
+         x="-615.4494"
          y="560.20447"
-         style="font-size:11.25px;fill:#008000;text-anchor:middle;text-align:center;"
+         style="font-size:11.25px;text-align:center;text-anchor:middle;fill:#008000"
          rotate="0 0 0 0"
          dy="0">TLS</tspan></text>
     <text
        xml:space="preserve"
-       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;text-anchor:middle;text-align:center;"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
        x="-552.68439"
        y="663.12689"
        id="text4682-9-9-3-3"
@@ -863,14 +905,14 @@
        inkscape:transform-center-y="23.040781"><tspan
          sodipodi:role="line"
          id="tspan4684-8-9-37-1"
-         x="-562.52814"
+         x="-552.68439"
          y="663.12689"
-         style="font-size:11.25px;fill:#008000;text-anchor:middle;text-align:center;"
+         style="font-size:11.25px;text-align:center;text-anchor:middle;fill:#008000"
          rotate="0 0 0 0"
          dy="0">TLS</tspan></text>
     <text
        xml:space="preserve"
-       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;text-anchor:middle;text-align:center;"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#008000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
        x="-547.755"
        y="679.40869"
        id="text4682-9-9-3-5"
@@ -880,9 +922,9 @@
        inkscape:transform-center-y="23.107802"><tspan
          sodipodi:role="line"
          id="tspan4684-8-9-37-8"
-         x="-557.59875"
+         x="-547.755"
          y="679.40869"
-         style="font-size:11.25px;fill:#008000;text-anchor:middle;text-align:center;"
+         style="font-size:11.25px;text-align:center;text-anchor:middle;fill:#008000"
          rotate="0 0 0 0"
          dy="0">TLS</tspan></text>
     <text
@@ -910,7 +952,7 @@
    style="text-align:center;text-anchor:middle"
    id="tspan4338-2">catbag.franke</tspan>rfacez.com</tspan></text>
     <path
-       style="fill:#000000;fill-rule:evenodd;stroke:#008000;stroke-width:1.01131487px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend-9-5-3-1-7-2-5);marker-start:url(#marker23127)"
+       style="fill:#000000;fill-rule:evenodd;stroke:#008000;stroke-width:1.01131487px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-start:url(#marker23127);marker-end:url(#Arrow1Lend-9-5-3-1-7-2-5)"
        d="M 125.99509,423.67358 252.01374,279.85602"
        id="path4388-1-8-5-9-7-7-9"
        inkscape:connector-curvature="0"
@@ -928,13 +970,13 @@
          x="-142.66475"
          y="372.49307">NaCl / HTTP</tspan></text>
     <path
-       style="fill:#000000;fill-rule:evenodd;stroke:#008000;stroke-width:1.01131487px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend-9-5-3-1-7-2-5-2);marker-start:url(#Arrow1Lstart)"
+       style="fill:#000000;fill-rule:evenodd;stroke:#008000;stroke-width:1.01131487px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-start:url(#Arrow1Lstart);marker-end:url(#Arrow1Lend-9-5-3-1-7-2-5-2)"
        d="M 358.43958,398.54766 357.9365,315.74331"
        id="path4388-1-8-5-9-7-7-9-3"
        inkscape:connector-curvature="0"
        sodipodi:nodetypes="cc" />
     <path
-       style="fill:#000000;fill-rule:evenodd;stroke:#008000;stroke-width:1.01131487px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend-9-5-3-1-7-2-5-6);marker-start:url(#marker23251)"
+       style="fill:#000000;fill-rule:evenodd;stroke:#008000;stroke-width:1.01131487px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-start:url(#marker23251);marker-end:url(#Arrow1Lend-9-5-3-1-7-2-5-6)"
        d="M 598.85589,415.21516 472.53843,284.42857"
        id="path4388-1-8-5-9-7-7-9-7"
        inkscape:connector-curvature="0"
@@ -964,53 +1006,53 @@
          x="410.92709"
          y="366.58148">NaCl / HTTP</tspan></text>
     <ellipse
-       style="fill:#c7804a;fill-opacity:0.34901962;stroke:#000000;stroke-width:0.89510232;stroke-linecap:butt;stroke-opacity:1"
+       style="fill:#c7804a;fill-opacity:0.34901961;stroke:#000000;stroke-width:0.89510232;stroke-linecap:butt;stroke-opacity:1"
        id="path7045-6"
-       cx="366"
-       cy="71.362198"
+       cx="212"
+       cy="68.362198"
        rx="127.1953"
        ry="58.838161" />
     <path
        style="fill:#c87137;fill-rule:evenodd;stroke:#c87137;stroke-width:1.01131487px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-start:url(#marker23417);marker-end:url(#Arrow1Lend-9-5-3-1-7-2-5-2-2)"
-       d="m 361.73807,180.57084 -0.50308,-49.7471"
+       d="m 318.73807,186.07084 -85.00308,-59.2471"
        id="path4388-1-8-5-9-7-7-9-3-1"
        inkscape:connector-curvature="0"
        sodipodi:nodetypes="cc" />
     <text
        xml:space="preserve"
        style="font-style:normal;font-weight:normal;font-size:17.5px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       x="397.44247"
-       y="159.07924"
+       x="300.44247"
+       y="144.07924"
        id="text22558"
        sodipodi:linespacing="125%"><tspan
          sodipodi:role="line"
          id="tspan22560"
-         x="397.44247"
-         y="159.07924"
+         x="300.44247"
+         y="144.07924"
          style="font-size:10px;text-align:center;text-anchor:middle">(out of scope)</tspan></text>
     <text
        xml:space="preserve"
        style="font-style:normal;font-weight:normal;font-size:12.5px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       x="366.53732"
-       y="94.034554"
+       x="212.53732"
+       y="91.034554"
        id="text4293-0-2-6"
        sodipodi:linespacing="125%"><tspan
          sodipodi:role="line"
          id="tspan4295-6-2-6"
-         x="366.53732"
-         y="94.034554"><tspan
+         x="212.53732"
+         y="91.034554"><tspan
    style="text-align:center;text-anchor:middle"
    id="tspan4338-2-6">www.franke</tspan>rfacez.com</tspan></text>
     <text
        xml:space="preserve"
        style="font-style:normal;font-weight:normal;font-size:22.5px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       x="362.43375"
-       y="67.632027"
+       x="208.43375"
+       y="64.632027"
        id="text15347-1"
        sodipodi:linespacing="125%"><tspan
          sodipodi:role="line"
-         x="362.43375"
-         y="67.632027"
+         x="208.43375"
+         y="64.632027"
          id="tspan15351-4">Web Server</tspan></text>
     <path
        style="fill:none;fill-rule:evenodd;stroke:#008000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#marker6763-4)"
@@ -1024,5 +1066,62 @@
        id="path6849-9-8"
        inkscape:connector-curvature="0"
        sodipodi:nodetypes="cc" />
+    <ellipse
+       style="fill:#c7804a;fill-opacity:0.34901961;stroke:#000000;stroke-width:0.89499998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:5.36999989, 5.36999989;stroke-dashoffset:0;stroke-opacity:1"
+       id="path7045-6-2"
+       cx="564.5"
+       cy="69.362206"
+       rx="127.1953"
+       ry="58.838161" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:22.5px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="565.63745"
+       y="48.250267"
+       id="text15347-1-8"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         x="565.63745"
+         y="48.250267"
+         id="tspan15351-4-1">Various other</tspan><tspan
+         sodipodi:role="line"
+         x="565.63745"
+         y="76.375267"
+         id="tspan4507">services</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:12.5px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="564.77161"
+       y="92.02224"
+       id="text4293-0-2-6-8"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         x="564.77161"
+         y="92.02224"
+         id="tspan4530"><tspan
+   style="text-align:center;text-anchor:middle"
+   id="tspan4338-2-6-5">www.twitter</tspan>.com, bit.ly,</tspan><tspan
+         sodipodi:role="line"
+         x="564.77161"
+         y="107.64724"
+         id="tspan4536">www.speedrunslive.com</tspan></text>
+    <path
+       style="fill:#c87137;fill-rule:evenodd;stroke:#c87137;stroke-width:1.01131487px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-start:url(#marker23417-9);marker-end:url(#Arrow1Lend-9-5-3-1-7-2-5-2-2-5)"
+       d="m 449.50154,195.23575 70.99692,-70.2471"
+       id="path4388-1-8-5-9-7-7-9-3-1-6"
+       inkscape:connector-curvature="0"
+       sodipodi:nodetypes="cc" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:17.5px;line-height:125%;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="455.00244"
+       y="147.62099"
+       id="text22558-3"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan22560-4"
+         x="455.00244"
+         y="147.62099"
+         style="font-size:10px;text-align:center;text-anchor:middle">(out of scope)</tspan></text>
   </g>
 </svg>

From 189965c96703ce178868fc27dec551451d93a429 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 30 Mar 2016 17:33:45 -0700
Subject: [PATCH 144/176] Allow 'upgrade' anywhere in connection header

---
 socketserver/server/handlecore.go | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index ed0f41f4..0335e1f8 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/gorilla/websocket"
 	"io"
 	"io/ioutil"
 	"log"
@@ -12,6 +11,7 @@ import (
 	"net/url"
 	"os"
 	"os/signal"
+	"regexp"
 	"runtime"
 	"strconv"
 	"strings"
@@ -20,7 +20,8 @@ import (
 	"syscall"
 	"time"
 	"unicode/utf8"
-	"regexp"
+
+	"github.com/gorilla/websocket"
 )
 
 // SuccessCommand is a Reply Command to indicate success in reply to a C2S Command.
@@ -52,6 +53,7 @@ const defaultMinMemoryKB = 1024 * 24
 
 // DotTwitchDotTv is the .twitch.tv suffix.
 const DotTwitchDotTv = ".twitch.tv"
+
 var OriginRegexp = regexp.MustCompile(DotTwitchDotTv + "$")
 
 // ResponseSuccess is a Reply ClientMessage with the MessageID not yet filled out.
@@ -205,7 +207,7 @@ func HTTPHandleRootURL(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if r.Header.Get("Connection") == "Upgrade" {
+	if strings.Contains(strings.ToLower(r.Header.Get("Connection")), "upgrade") {
 		updateSysMem()
 
 		if Statistics.SysMemFreeKB > 0 && Statistics.SysMemFreeKB < Configuration.MinMemoryKBytes {

From 8b8bfe8ff2a19893022bb3d8ab2e24a5677121bd Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Fri, 1 Apr 2016 19:15:34 -0700
Subject: [PATCH 145/176] Update to the new twitch chat server addres

---
 socketserver/server/irc.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/socketserver/server/irc.go b/socketserver/server/irc.go
index 98234829..fe5bf9ff 100644
--- a/socketserver/server/irc.go
+++ b/socketserver/server/irc.go
@@ -113,7 +113,7 @@ func ircConnection() {
 		submitAuth(submittedUser, submittedChallenge)
 	})
 
-	err := c.ConnectTo("irc.twitch.tv")
+	err := c.ConnectTo("irc.chat.twitch.tv")
 	if err != nil {
 		log.Fatalln("Cannot connect to IRC:", err)
 	}

From e1e26f52888fa179a3bbb104894c28d4bfe4a62f Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Fri, 1 Apr 2016 19:22:55 -0700
Subject: [PATCH 146/176] add cbenni.com to origin regexp whitelist

---
 socketserver/server/handlecore.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 0335e1f8..f2f395b0 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -54,7 +54,9 @@ const defaultMinMemoryKB = 1024 * 24
 // DotTwitchDotTv is the .twitch.tv suffix.
 const DotTwitchDotTv = ".twitch.tv"
 
-var OriginRegexp = regexp.MustCompile(DotTwitchDotTv + "$")
+const dotCbenniDotCom = ".cbenni.com"
+
+var OriginRegexp = regexp.MustCompile("(" + DotTwitchDotTv + "|" + dotCbenniDotCom + ")" + "$")
 
 // ResponseSuccess is a Reply ClientMessage with the MessageID not yet filled out.
 var ResponseSuccess = ClientMessage{Command: SuccessCommand}

From 74270245ab79163da8d70aaa9ba5af79ac36e94b Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sun, 10 Apr 2016 17:19:30 -0700
Subject: [PATCH 147/176] lol changing a server into a script

---
 socketserver/cmd/mergecounts/mergecounts.go |  2 +-
 socketserver/cmd/statsweb/servers.go        |  2 +-
 socketserver/cmd/statsweb/statsweb.go       | 63 ++++++++++++++++-----
 3 files changed, 50 insertions(+), 17 deletions(-)

diff --git a/socketserver/cmd/mergecounts/mergecounts.go b/socketserver/cmd/mergecounts/mergecounts.go
index af3ec83b..4eff882f 100644
--- a/socketserver/cmd/mergecounts/mergecounts.go
+++ b/socketserver/cmd/mergecounts/mergecounts.go
@@ -11,7 +11,7 @@ import (
 )
 
 var SERVERS = []string{
-//	"https://catbag.frankerfacez.com",
+	"https://catbag.frankerfacez.com",
 	"https://andknuckles.frankerfacez.com",
 	"https://tuturu.frankerfacez.com",
 }
diff --git a/socketserver/cmd/statsweb/servers.go b/socketserver/cmd/statsweb/servers.go
index 659b430b..eef0e47b 100644
--- a/socketserver/cmd/statsweb/servers.go
+++ b/socketserver/cmd/statsweb/servers.go
@@ -171,7 +171,7 @@ func (si *serverInfo) GetHLL(at time.Time) (*hyperloglog.HyperLogLogPlus, error)
 	if err != nil {
 		// continue to download
 	} else {
-		fmt.Printf("opened hll %s:%s\n", si.subdomain, getHLLCacheKey(at))
+		//fmt.Printf("opened hll %s:%s\n", si.subdomain, getHLLCacheKey(at))
 		return loadHLLFromStream(reader)
 	}
 
diff --git a/socketserver/cmd/statsweb/statsweb.go b/socketserver/cmd/statsweb/statsweb.go
index 9fce5f6e..980f1cd8 100644
--- a/socketserver/cmd/statsweb/statsweb.go
+++ b/socketserver/cmd/statsweb/statsweb.go
@@ -1,17 +1,19 @@
 package main
 
 import (
-	"net/http"
-	"flag"
-	"github.com/clarkduvall/hyperloglog"
-	"time"
-	"bitbucket.org/stendec/frankerfacez/socketserver/server"
-	"net/url"
-	"fmt"
-	"strings"
-	"errors"
 	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
 	"sync"
+	"time"
+
+	"bitbucket.org/stendec/frankerfacez/socketserver/server"
+	"github.com/clarkduvall/hyperloglog"
 )
 
 var configLocation = flag.String("config", "./config.json", "Location of the configuration file. Defaults to ./config.json")
@@ -39,10 +41,41 @@ func main() {
 		allServers[i].Setup(v)
 	}
 
+	printEveryDay()
+	os.Exit(0)
 	http.HandleFunc("/api/get", ServeAPIGet)
 	http.ListenAndServe(config.ListenAddr, http.DefaultServeMux)
 }
 
+func printEveryDay() {
+	year := 2015
+	month := 12
+	day := 23
+	filter := serverFilterAll
+	var filter1, filter2, filter3 serverFilter
+	filter1.Mode = serverFilterModeWhitelist
+	filter2.Mode = serverFilterModeWhitelist
+	filter3.Mode = serverFilterModeWhitelist
+	filter1.Add(allServers[0].subdomain)
+	filter2.Add(allServers[1].subdomain)
+	filter3.Add(allServers[2].subdomain)
+	stopTime := time.Now()
+	var at time.Time
+	const timeFmt = "2006-01-02"
+	for ; stopTime.After(at); day++ {
+		at = time.Date(year, time.Month(month), day, 0, 0, 0, 0, server.CounterLocation)
+		hll, _ := hyperloglog.NewPlus(server.CounterPrecision)
+		hll1, _ := hyperloglog.NewPlus(server.CounterPrecision)
+		hll2, _ := hyperloglog.NewPlus(server.CounterPrecision)
+		hll3, _ := hyperloglog.NewPlus(server.CounterPrecision)
+		addSingleDate(at, filter, hll)
+		addSingleDate(at, filter1, hll1)
+		addSingleDate(at, filter2, hll2)
+		addSingleDate(at, filter3, hll3)
+		fmt.Printf("%s\t%d\t%d\t%d\t%d\n", at.Format(timeFmt), hll.Count(), hll1.Count(), hll2.Count(), hll3.Count())
+	}
+}
+
 const RequestURIName = "q"
 const separatorRange = "~"
 const separatorAdd = " "
@@ -54,15 +87,15 @@ const statusPartial = "partial"
 const statusOk = "ok"
 
 type apiResponse struct {
-	Status string `json:"status"`
+	Status    string            `json:"status"`
 	Responses []requestResponse `json:"resp"`
 }
 
 type requestResponse struct {
-	Status string `json:"status"`
+	Status  string `json:"status"`
 	Request string `json:"req"`
-	Error string `json:"error,omitempty"`
-	Count uint64 `json:"count,omitempty"`
+	Error   string `json:"error,omitempty"`
+	Count   uint64 `json:"count,omitempty"`
 }
 
 func ServeAPIGet(w http.ResponseWriter, r *http.Request) {
@@ -163,7 +196,7 @@ func ProcessSingleGetRequest(req string) (result requestResponse) {
 
 	addSplit := strings.Split(serverSplit[0], separatorAdd)
 
-	outerLoop:
+outerLoop:
 	for _, split1 := range addSplit {
 		if len(split1) == 0 {
 			continue
@@ -284,4 +317,4 @@ func addRange(start time.Time, end time.Time, filter serverFilter, dest *hyperlo
 func getHLL(ch chan hllAndError, si *serverInfo, at time.Time) {
 	hll, err := si.GetHLL(at)
 	ch <- hllAndError{hll: hll, err: err}
-}
\ No newline at end of file
+}

From d7ea9d610bfdec6ec037f4f9036ff7d7a51cb6a6 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 20 Apr 2016 13:45:59 -0700
Subject: [PATCH 148/176] Remove feature switch for not giving server time in
 reply

---
 socketserver/server/commands.go | 22 +++++++---------------
 1 file changed, 7 insertions(+), 15 deletions(-)

diff --git a/socketserver/server/commands.go b/socketserver/server/commands.go
index 6d868512..e65b3e0c 100644
--- a/socketserver/server/commands.go
+++ b/socketserver/server/commands.go
@@ -107,8 +107,6 @@ func callHandler(handler CommandHandler, conn *websocket.Conn, client *ClientInf
 	return handler(conn, client, cmsg)
 }
 
-var lastVersionWithoutReplyWithServerTime = VersionFromString("ffz_3.5.78")
-
 // C2SHello implements the `hello` C2S Command.
 // It calls SubscribeGlobal() and SubscribeDefaults() with the client, and fills out ClientInfo.Version and ClientInfo.ClientID.
 func C2SHello(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
@@ -130,19 +128,13 @@ func C2SHello(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg
 	SubscribeGlobal(client)
 	SubscribeDefaults(client)
 
-	if client.Version.After(&lastVersionWithoutReplyWithServerTime) {
-		jsTime := float64(time.Now().UnixNano()/1000) / 1000
-		return ClientMessage{
-			Arguments: []interface{}{
-				client.ClientID.String(),
-				jsTime,
-			},
-		}, nil
-	} else {
-		return ClientMessage{
-			Arguments: client.ClientID.String(),
-		}, nil
-	}
+	jsTime := float64(time.Now().UnixNano()/1000) / 1000
+	return ClientMessage{
+		Arguments: []interface{}{
+			client.ClientID.String(),
+			jsTime,
+		},
+	}, nil
 }
 
 func C2SPing(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {

From 841b6a018dcec7909a0d2859e464cf83d5a9867d Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 20 Apr 2016 14:11:12 -0700
Subject: [PATCH 149/176] Expose stats on frequency of client version numbers

---
 socketserver/server/stats.go | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/socketserver/server/stats.go b/socketserver/server/stats.go
index 8b4d8d9c..7a32ecab 100644
--- a/socketserver/server/stats.go
+++ b/socketserver/server/stats.go
@@ -39,6 +39,8 @@ type StatsData struct {
 	ClientConnectsTotal    uint64
 	ClientDisconnectsTotal uint64
 
+	ClientVersions map[string]uint64
+
 	DisconnectCodes map[string]uint64
 
 	CommandsIssuedTotal uint64
@@ -88,6 +90,7 @@ func newStatsData() *StatsData {
 		CommandsIssuedMap: make(map[Command]uint64),
 		DisconnectCodes:   make(map[string]uint64),
 		DisconnectReasons: make(map[string]uint64),
+		ClientVersions:    make(map[string]uint64),
 		StatsDataVersion:  StatsDataVersion,
 	}
 }
@@ -137,7 +140,14 @@ func updatePeriodicStats() {
 		ChatSubscriptionLock.RUnlock()
 
 		GlobalSubscriptionLock.RLock()
+
 		Statistics.CurrentClientCount = uint64(len(GlobalSubscriptionInfo))
+		versions := make(map[string]uint64)
+		for _, v := range GlobalSubscriptionInfo {
+			versions[v.VersionString]++
+		}
+		Statistics.ClientVersions = versions
+
 		GlobalSubscriptionLock.RUnlock()
 	}
 

From 69e5434cc2dfe1bd07dd86eecfccf1e60cfb5760 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Wed, 20 Apr 2016 14:28:35 -0700
Subject: [PATCH 150/176] Allow blank Origin header

---
 socketserver/server/handlecore.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index f2f395b0..0bd23a82 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -181,7 +181,7 @@ var SocketUpgrader = websocket.Upgrader{
 	ReadBufferSize:  1024,
 	WriteBufferSize: 1024,
 	CheckOrigin: func(r *http.Request) bool {
-		return OriginRegexp.MatchString(r.Header.Get("Origin"))
+		return r.Header.Get("Origin") == "" || OriginRegexp.MatchString(r.Header.Get("Origin"))
 	},
 }
 

From 922bda2218865c32cbff6b89a88453ed6a1fa43f Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Thu, 28 Apr 2016 14:36:59 -0700
Subject: [PATCH 151/176] run goimports

---
 socketserver/cmd/ffzsocketserver/console.go   |  7 ++++---
 .../cmd/ffzsocketserver/socketserver.go       |  3 ++-
 socketserver/cmd/mergecounts/mergecounts.go   |  5 +++--
 socketserver/cmd/statsweb/html.go             |  3 ++-
 socketserver/cmd/statsweb/servers.go          | 21 ++++++++++---------
 socketserver/server/backend.go                |  5 +++--
 socketserver/server/backend_test.go           |  3 ++-
 socketserver/server/commands.go               |  5 +++--
 socketserver/server/handlecore_test.go        |  3 ++-
 socketserver/server/irc.go                    |  3 ++-
 socketserver/server/subscriptions_test.go     |  5 +++--
 socketserver/server/testinfra_test.go         |  5 +++--
 socketserver/server/types.go                  | 14 ++++++-------
 socketserver/server/usercount.go              |  3 ++-
 socketserver/server/usercount_test.go         |  3 ++-
 socketserver/server/utils.go                  |  3 ++-
 16 files changed, 53 insertions(+), 38 deletions(-)

diff --git a/socketserver/cmd/ffzsocketserver/console.go b/socketserver/cmd/ffzsocketserver/console.go
index b3a77b0c..5d3063d9 100644
--- a/socketserver/cmd/ffzsocketserver/console.go
+++ b/socketserver/cmd/ffzsocketserver/console.go
@@ -1,13 +1,14 @@
 package main
 
 import (
-	"../../server"
 	"fmt"
-	"github.com/abiosoft/ishell"
-	"github.com/gorilla/websocket"
 	"runtime"
 	"strconv"
 	"strings"
+
+	"../../server"
+	"github.com/abiosoft/ishell"
+	"github.com/gorilla/websocket"
 )
 
 func commandLineConsole() {
diff --git a/socketserver/cmd/ffzsocketserver/socketserver.go b/socketserver/cmd/ffzsocketserver/socketserver.go
index 8b56d707..1d67bddc 100644
--- a/socketserver/cmd/ffzsocketserver/socketserver.go
+++ b/socketserver/cmd/ffzsocketserver/socketserver.go
@@ -1,7 +1,6 @@
 package main // import "bitbucket.org/stendec/frankerfacez/socketserver/cmd/ffzsocketserver"
 
 import (
-	"../../server"
 	"encoding/json"
 	"flag"
 	"fmt"
@@ -9,6 +8,8 @@ import (
 	"log"
 	"net/http"
 	"os"
+
+	"../../server"
 )
 
 import _ "net/http/pprof"
diff --git a/socketserver/cmd/mergecounts/mergecounts.go b/socketserver/cmd/mergecounts/mergecounts.go
index 4eff882f..8ce8b307 100644
--- a/socketserver/cmd/mergecounts/mergecounts.go
+++ b/socketserver/cmd/mergecounts/mergecounts.go
@@ -1,13 +1,14 @@
 package main
 
 import (
-	"../../server"
 	"encoding/gob"
 	"flag"
 	"fmt"
-	"github.com/clarkduvall/hyperloglog"
 	"net/http"
 	"os"
+
+	"../../server"
+	"github.com/clarkduvall/hyperloglog"
 )
 
 var SERVERS = []string{
diff --git a/socketserver/cmd/statsweb/html.go b/socketserver/cmd/statsweb/html.go
index a2bca5d4..4cbdf89c 100644
--- a/socketserver/cmd/statsweb/html.go
+++ b/socketserver/cmd/statsweb/html.go
@@ -1,10 +1,11 @@
 package main
 
 import (
-	"bitbucket.org/stendec/frankerfacez/socketserver/server"
 	"html/template"
 	"net/http"
 	"time"
+
+	"bitbucket.org/stendec/frankerfacez/socketserver/server"
 )
 
 type CalendarData struct {
diff --git a/socketserver/cmd/statsweb/servers.go b/socketserver/cmd/statsweb/servers.go
index eef0e47b..da3f3dfe 100644
--- a/socketserver/cmd/statsweb/servers.go
+++ b/socketserver/cmd/statsweb/servers.go
@@ -1,17 +1,18 @@
 package main
 
 import (
-	"time"
-	"io"
-	"fmt"
-	"os"
-	"net/http"
-	"errors"
-	"sync"
-	"github.com/hashicorp/golang-lru"
-	"github.com/clarkduvall/hyperloglog"
-	"bitbucket.org/stendec/frankerfacez/socketserver/server"
 	"encoding/gob"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"sync"
+	"time"
+
+	"bitbucket.org/stendec/frankerfacez/socketserver/server"
+	"github.com/clarkduvall/hyperloglog"
+	"github.com/hashicorp/golang-lru"
 )
 
 type serverFilter struct {
diff --git a/socketserver/server/backend.go b/socketserver/server/backend.go
index 55ccf9f1..5bb4d960 100644
--- a/socketserver/server/backend.go
+++ b/socketserver/server/backend.go
@@ -6,8 +6,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/pmylund/go-cache"
-	"golang.org/x/crypto/nacl/box"
 	"io/ioutil"
 	"log"
 	"net/http"
@@ -16,6 +14,9 @@ import (
 	"strings"
 	"sync"
 	"time"
+
+	"github.com/pmylund/go-cache"
+	"golang.org/x/crypto/nacl/box"
 )
 
 const bPathAnnounceStartup = "/startup"
diff --git a/socketserver/server/backend_test.go b/socketserver/server/backend_test.go
index 540571f6..b34edef1 100644
--- a/socketserver/server/backend_test.go
+++ b/socketserver/server/backend_test.go
@@ -1,10 +1,11 @@
 package server
 
 import (
-	. "gopkg.in/check.v1"
 	"net/http"
 	"net/url"
 	"testing"
+
+	. "gopkg.in/check.v1"
 )
 
 func Test(t *testing.T) { TestingT(t) }
diff --git a/socketserver/server/commands.go b/socketserver/server/commands.go
index e65b3e0c..ffe41293 100644
--- a/socketserver/server/commands.go
+++ b/socketserver/server/commands.go
@@ -4,13 +4,14 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/gorilla/websocket"
-	"github.com/satori/go.uuid"
 	"log"
 	"net/url"
 	"strconv"
 	"sync"
 	"time"
+
+	"github.com/gorilla/websocket"
+	"github.com/satori/go.uuid"
 )
 
 // Command is a string indicating which RPC is requested.
diff --git a/socketserver/server/handlecore_test.go b/socketserver/server/handlecore_test.go
index c6444c08..8ecf848b 100644
--- a/socketserver/server/handlecore_test.go
+++ b/socketserver/server/handlecore_test.go
@@ -2,8 +2,9 @@ package server
 
 import (
 	"fmt"
-	"github.com/gorilla/websocket"
 	"testing"
+
+	"github.com/gorilla/websocket"
 )
 
 func ExampleUnmarshalClientMessage() {
diff --git a/socketserver/server/irc.go b/socketserver/server/irc.go
index fe5bf9ff..6ec1f7f9 100644
--- a/socketserver/server/irc.go
+++ b/socketserver/server/irc.go
@@ -4,11 +4,12 @@ import (
 	"bytes"
 	"crypto/rand"
 	"encoding/base64"
-	irc "github.com/fluffle/goirc/client"
 	"log"
 	"strings"
 	"sync"
 	"time"
+
+	irc "github.com/fluffle/goirc/client"
 )
 
 type AuthCallback func(client *ClientInfo, successful bool)
diff --git a/socketserver/server/subscriptions_test.go b/socketserver/server/subscriptions_test.go
index 132dda74..9450d609 100644
--- a/socketserver/server/subscriptions_test.go
+++ b/socketserver/server/subscriptions_test.go
@@ -2,8 +2,6 @@ package server
 
 import (
 	"fmt"
-	"github.com/gorilla/websocket"
-	"github.com/satori/go.uuid"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
@@ -12,6 +10,9 @@ import (
 	"syscall"
 	"testing"
 	"time"
+
+	"github.com/gorilla/websocket"
+	"github.com/satori/go.uuid"
 )
 
 const TestOrigin = "http://www.twitch.tv"
diff --git a/socketserver/server/testinfra_test.go b/socketserver/server/testinfra_test.go
index bc7c5e0b..998ccffd 100644
--- a/socketserver/server/testinfra_test.go
+++ b/socketserver/server/testinfra_test.go
@@ -3,16 +3,17 @@ package server
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/gorilla/websocket"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
 	"os"
+	"strconv"
 	"sync"
 	"testing"
 	"time"
-	"strconv"
+
+	"github.com/gorilla/websocket"
 )
 
 const (
diff --git a/socketserver/server/types.go b/socketserver/server/types.go
index 3bbca724..44afc158 100644
--- a/socketserver/server/types.go
+++ b/socketserver/server/types.go
@@ -3,19 +3,19 @@ package server
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/satori/go.uuid"
 	"net"
 	"sync"
-	"time"
+
+	"github.com/satori/go.uuid"
 )
 
 const NegativeOne = ^uint64(0)
 
 type ConfigFile struct {
 	// Numeric server id known to the backend
-	ServerID      int
+	ServerID int
 	// Address to bind the HTTP server to on startup.
-	ListenAddr    string
+	ListenAddr string
 	// Address to bind the TLS server to on startup.
 	SSLListenAddr string
 	// URL to the backend server
@@ -24,15 +24,15 @@ type ConfigFile struct {
 	// Minimum memory to accept a new connection
 	MinMemoryKBytes uint64
 	// Maximum # of clients that can be connected. 0 to disable.
-	MaxClientCount  uint64
+	MaxClientCount uint64
 
 	// SSL/TLS
 	// Enable the use of SSL.
-	UseSSL             bool
+	UseSSL bool
 	// Path to certificate file.
 	SSLCertificateFile string
 	// Path to key file.
-	SSLKeyFile         string
+	SSLKeyFile string
 
 	UseESLogStashing bool
 	ESServer         string
diff --git a/socketserver/server/usercount.go b/socketserver/server/usercount.go
index 5390a3e8..0bbb7940 100644
--- a/socketserver/server/usercount.go
+++ b/socketserver/server/usercount.go
@@ -12,9 +12,10 @@ import (
 	"os"
 	"time"
 
+	"io"
+
 	"github.com/clarkduvall/hyperloglog"
 	"github.com/satori/go.uuid"
-	"io"
 )
 
 // UuidHash implements a hash for uuid.UUID by XORing the random bits.
diff --git a/socketserver/server/usercount_test.go b/socketserver/server/usercount_test.go
index 6c5a8d2f..2cf608a2 100644
--- a/socketserver/server/usercount_test.go
+++ b/socketserver/server/usercount_test.go
@@ -1,12 +1,13 @@
 package server
 
 import (
-	"github.com/satori/go.uuid"
 	"net/http/httptest"
 	"net/url"
 	"os"
 	"testing"
 	"time"
+
+	"github.com/satori/go.uuid"
 )
 
 func TestUniqueConnections(t *testing.T) {
diff --git a/socketserver/server/utils.go b/socketserver/server/utils.go
index d30ec2a0..1c895e6b 100644
--- a/socketserver/server/utils.go
+++ b/socketserver/server/utils.go
@@ -5,10 +5,11 @@ import (
 	"crypto/rand"
 	"encoding/base64"
 	"errors"
-	"golang.org/x/crypto/nacl/box"
 	"net/url"
 	"strconv"
 	"strings"
+
+	"golang.org/x/crypto/nacl/box"
 )
 
 func FillCryptoRandom(buf []byte) error {

From 801a57a1fab82b42c62735752e2487bc3b6c74d3 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Thu, 28 Apr 2016 14:37:35 -0700
Subject: [PATCH 152/176] usage stats changes

---
 socketserver/cmd/statsweb/servers.go | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/socketserver/cmd/statsweb/servers.go b/socketserver/cmd/statsweb/servers.go
index da3f3dfe..d4818c61 100644
--- a/socketserver/cmd/statsweb/servers.go
+++ b/socketserver/cmd/statsweb/servers.go
@@ -87,7 +87,7 @@ var serverFilterNone serverFilter = serverFilter{Mode: serverFilterModeWhitelist
 
 func cannotCacheHLL(at time.Time) bool {
 	now := time.Now()
-	now.Add(-25 * time.Hour)
+	now.Add(-72 * time.Hour)
 	return now.Before(at)
 }
 
@@ -127,15 +127,15 @@ func getHLLCacheKey(at time.Time) string {
 }
 
 type serverInfo struct {
-	subdomain    string
+	subdomain string
 
-	memcache     *lru.TwoQueueCache
+	memcache *lru.TwoQueueCache
 
 	FailedState  bool
 	FailureErr   error
 	failureCount int
 
-	lock         sync.Mutex
+	lock sync.Mutex
 }
 
 func (si *serverInfo) Setup(subdomain string) {
@@ -150,6 +150,7 @@ func (si *serverInfo) Setup(subdomain string) {
 // GetHLL gets the HLL from
 func (si *serverInfo) GetHLL(at time.Time) (*hyperloglog.HyperLogLogPlus, error) {
 	if cannotCacheHLL(at) {
+		fmt.Println(at)
 		err := si.ForceWrite()
 		if err != nil {
 			return nil, err
@@ -158,7 +159,8 @@ func (si *serverInfo) GetHLL(at time.Time) (*hyperloglog.HyperLogLogPlus, error)
 		if err != nil {
 			return nil, err
 		}
-		fmt.Printf("downloaded hll %s:%s\n", si.subdomain, getHLLCacheKey(at))
+		fmt.Printf("downloaded uncached hll %s:%s\n", si.subdomain, getHLLCacheKey(at))
+		defer si.DeleteHLL(at)
 		return loadHLLFromStream(reader)
 	}
 
@@ -213,6 +215,15 @@ func (si *serverInfo) PeekHLL(at time.Time) (*hyperloglog.HyperLogLogPlus, bool)
 	return nil, false
 }
 
+func (si *serverInfo) DeleteHLL(at time.Time) {
+	year, month, day := at.Date()
+	filename := fmt.Sprintf("%s/%s/%d-%d-%d.gob", config.GobFilesLocation, si.subdomain, year, month, day)
+	err := os.Remove(filename)
+	if err != nil {
+		fmt.Println(err)
+	}
+}
+
 func (si *serverInfo) OpenHLL(at time.Time) (io.ReadCloser, error) {
 	year, month, day := at.Date()
 	filename := fmt.Sprintf("%s/%s/%d-%d-%d.gob", config.GobFilesLocation, si.subdomain, year, month, day)
@@ -253,10 +264,10 @@ func (si *serverInfo) DownloadHLL(at time.Time) (io.ReadCloser, error) {
 	}
 
 	filename := fmt.Sprintf("%s/%s/%d-%d-%d.gob", config.GobFilesLocation, si.subdomain, year, month, day)
-	file, err := os.OpenFile(filename, os.O_CREATE | os.O_EXCL | os.O_RDWR, 0644)
+	file, err := os.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644)
 	if os.IsNotExist(err) {
 		os.MkdirAll(fmt.Sprintf("%s/%s", config.GobFilesLocation, si.subdomain), 0755)
-		file, err = os.OpenFile(filename, os.O_CREATE | os.O_EXCL | os.O_RDWR, 0644)
+		file, err = os.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644)
 	}
 	if err != nil {
 		resp.Body.Close()

From 822e298d5d46662836854e00d86b680abf009915 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Thu, 28 Apr 2016 14:39:20 -0700
Subject: [PATCH 153/176] Fix 'sub' commands after 'ready'

---
 socketserver/server/commands.go      | 21 +++++-----
 socketserver/server/irc.go           | 19 +++++-----
 socketserver/server/publisher.go     | 57 ++++++++++++++++++++++------
 socketserver/server/subscriptions.go |  5 ---
 socketserver/server/types.go         | 12 +-----
 5 files changed, 65 insertions(+), 49 deletions(-)

diff --git a/socketserver/server/commands.go b/socketserver/server/commands.go
index ffe41293..a112e6dc 100644
--- a/socketserver/server/commands.go
+++ b/socketserver/server/commands.go
@@ -174,15 +174,7 @@ func C2SReady(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg
 	//	}
 
 	client.Mutex.Lock()
-	if client.MakePendingRequests != nil {
-		if !client.MakePendingRequests.Stop() {
-			// Timer already fired, GetSubscriptionBacklog() has started
-			rmsg.Command = SuccessCommand
-			return
-		}
-	}
-	client.PendingSubscriptionsBacklog = nil
-	client.MakePendingRequests = nil
+	client.ReadyComplete = true
 	client.Mutex.Unlock()
 
 	client.MsgChannelKeepalive.Add(1)
@@ -204,13 +196,18 @@ func C2SSubscribe(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (
 
 	client.Mutex.Lock()
 	AddToSliceS(&client.CurrentChannels, channel)
-	if usePendingSubscrptionsBacklog {
-		client.PendingSubscriptionsBacklog = append(client.PendingSubscriptionsBacklog, channel)
-	}
 	client.Mutex.Unlock()
 
 	SubscribeChannel(client, channel)
 
+	if client.ReadyComplete {
+		client.MsgChannelKeepalive.Add(1)
+		go func() {
+			SendBacklogForChannel(client, channel)
+			client.MsgChannelKeepalive.Done()
+		}()
+	}
+
 	return ResponseSuccess, nil
 }
 
diff --git a/socketserver/server/irc.go b/socketserver/server/irc.go
index 6ec1f7f9..c0af288c 100644
--- a/socketserver/server/irc.go
+++ b/socketserver/server/irc.go
@@ -56,7 +56,7 @@ func authorizationJanitor_do() {
 		if !cullTime.After(v.EnteredAt) {
 			newPendingAuths = append(newPendingAuths, v)
 		} else {
-			v.Callback(v.Client, false)
+			go v.Callback(v.Client, false)
 		}
 	}
 
@@ -64,12 +64,13 @@ func authorizationJanitor_do() {
 }
 
 func (client *ClientInfo) StartAuthorization(callback AuthCallback) {
+	if callback == nil {
+		return // callback must not be nil
+	}
 	var nonce [32]byte
 	_, err := rand.Read(nonce[:])
 	if err != nil {
-		go func(client *ClientInfo, callback AuthCallback) {
-			callback(client, false)
-		}(client, callback)
+		go callback(client, false)
 		return
 	}
 	buf := bytes.NewBuffer(nil)
@@ -153,11 +154,9 @@ func submitAuth(user, challenge string) {
 	}
 	auth.Client.Mutex.Unlock()
 
-	if auth.Callback != nil {
-		if !usernameChanged {
-			auth.Callback(auth.Client, true)
-		} else {
-			auth.Callback(auth.Client, false)
-		}
+	if !usernameChanged {
+		auth.Callback(auth.Client, true)
+	} else {
+		auth.Callback(auth.Client, false)
 	}
 }
diff --git a/socketserver/server/publisher.go b/socketserver/server/publisher.go
index 45bfd9f3..f0531329 100644
--- a/socketserver/server/publisher.go
+++ b/socketserver/server/publisher.go
@@ -29,6 +29,9 @@ var S2CCommandsCacheInfo = map[Command]PushCommandCacheInfo{
 	"viewers":  {CacheTypeLastOnly, MsgTargetTypeChat},
 }
 
+var PersistentCachingCommands = []Command{"follow_sets", "follow_buttons"}
+var HourlyCachingCommands = []Command{"srl_race", "chatters", "viewers"}
+
 type BacklogCacheType int
 
 const (
@@ -101,8 +104,8 @@ func SendBacklogForNewClient(client *ClientInfo) {
 	client.Mutex.Unlock()
 
 	PersistentLSMLock.RLock()
-	for _, cmd := range GetCommandsOfType(PushCommandCacheInfo{CacheTypePersistent, MsgTargetTypeChat}) {
-		chanMap := CachedLastMessages[cmd]
+	for _, cmd := range GetCommandsOfType(CacheTypePersistent) {
+		chanMap := PersistentLastMessages[cmd]
 		if chanMap == nil {
 			continue
 		}
@@ -118,7 +121,7 @@ func SendBacklogForNewClient(client *ClientInfo) {
 	PersistentLSMLock.RUnlock()
 
 	CachedLSMLock.RLock()
-	for _, cmd := range GetCommandsOfType(PushCommandCacheInfo{CacheTypeLastOnly, MsgTargetTypeChat}) {
+	for _, cmd := range GetCommandsOfType(CacheTypeLastOnly) {
 		chanMap := CachedLastMessages[cmd]
 		if chanMap == nil {
 			continue
@@ -135,6 +138,36 @@ func SendBacklogForNewClient(client *ClientInfo) {
 	CachedLSMLock.RUnlock()
 }
 
+func SendBacklogForChannel(client *ClientInfo, channel string) {
+	PersistentLSMLock.RLock()
+	for _, cmd := range GetCommandsOfType(CacheTypePersistent) {
+		chanMap := PersistentLastMessages[cmd]
+		if chanMap == nil {
+			continue
+		}
+		if msg, ok := chanMap[channel]; ok {
+			msg := ClientMessage{MessageID: -1, Command: cmd, origArguments: msg.Data}
+			msg.parseOrigArguments()
+			client.MessageChannel <- msg
+		}
+	}
+	PersistentLSMLock.RUnlock()
+
+	CachedLSMLock.RLock()
+	for _, cmd := range GetCommandsOfType(CacheTypeLastOnly) {
+		chanMap := CachedLastMessages[cmd]
+		if chanMap == nil {
+			continue
+		}
+		if msg, ok := chanMap[channel]; ok {
+			msg := ClientMessage{MessageID: -1, Command: cmd, origArguments: msg.Data}
+			msg.parseOrigArguments()
+			client.MessageChannel <- msg
+		}
+	}
+	CachedLSMLock.RUnlock()
+}
+
 type timestampArray interface {
 	Len() int
 	GetTime(int) time.Time
@@ -144,13 +177,13 @@ func SaveLastMessage(which map[Command]map[string]LastSavedMessage, locker sync.
 	locker.Lock()
 	defer locker.Unlock()
 
-	chanMap, ok := CachedLastMessages[cmd]
+	chanMap, ok := which[cmd]
 	if !ok {
 		if deleting {
 			return
 		}
 		chanMap = make(map[string]LastSavedMessage)
-		CachedLastMessages[cmd] = chanMap
+		which[cmd] = chanMap
 	}
 
 	if deleting {
@@ -160,14 +193,14 @@ func SaveLastMessage(which map[Command]map[string]LastSavedMessage, locker sync.
 	}
 }
 
-func GetCommandsOfType(match PushCommandCacheInfo) []Command {
-	var ret []Command
-	for cmd, info := range S2CCommandsCacheInfo {
-		if info == match {
-			ret = append(ret, cmd)
-		}
+func GetCommandsOfType(match BacklogCacheType) []Command {
+	if match == CacheTypePersistent {
+		return PersistentCachingCommands
+	} else if match == CacheTypeLastOnly {
+		return HourlyCachingCommands
+	} else {
+		panic("unknown caching type")
 	}
-	return ret
 }
 
 func HTTPBackendDropBacklog(w http.ResponseWriter, r *http.Request) {
diff --git a/socketserver/server/subscriptions.go b/socketserver/server/subscriptions.go
index 890e2e19..0ea46b50 100644
--- a/socketserver/server/subscriptions.go
+++ b/socketserver/server/subscriptions.go
@@ -111,11 +111,6 @@ func UnsubscribeAll(client *ClientInfo) {
 		return // no need to remove from a high-contention list when the server is closing
 	}
 
-	client.Mutex.Lock()
-	client.PendingSubscriptionsBacklog = nil
-	client.PendingSubscriptionsBacklog = nil
-	client.Mutex.Unlock()
-
 	GlobalSubscriptionLock.Lock()
 	RemoveFromSliceCl(&GlobalSubscriptionInfo, client)
 	GlobalSubscriptionLock.Unlock()
diff --git a/socketserver/server/types.go b/socketserver/server/types.go
index 44afc158..3d360e26 100644
--- a/socketserver/server/types.go
+++ b/socketserver/server/types.go
@@ -104,14 +104,8 @@ type ClientInfo struct {
 	// Protected by Mutex.
 	CurrentChannels []string
 
-	// List of channels that we have not yet checked current chat-related channel info for.
-	// This lets us batch the backlog requests.
-	// Protected by Mutex.
-	PendingSubscriptionsBacklog []string
-
-	// A timer that, when fired, will make the pending backlog requests.
-	// Usually nil. Protected by Mutex.
-	MakePendingRequests *time.Timer
+	// True if the client has already sent the 'ready' command
+	ReadyComplete bool
 
 	// Server-initiated messages should be sent here
 	// This field will be nil before it is closed.
@@ -157,8 +151,6 @@ func (cv *ClientVersion) Equal(cv2 *ClientVersion) bool {
 	return cv.Major == cv2.Major && cv.Minor == cv2.Minor && cv.Revision == cv2.Revision
 }
 
-const usePendingSubscrptionsBacklog = false
-
 func (bct BacklogCacheType) Name() string {
 	switch bct {
 	case CacheTypeInvalid:

From ddedb404e1386705ae044bb7111d64fca520df40 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Thu, 28 Apr 2016 15:33:49 -0700
Subject: [PATCH 154/176] live debugging

---
 socketserver/server/publisher.go | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/socketserver/server/publisher.go b/socketserver/server/publisher.go
index f0531329..7f667f01 100644
--- a/socketserver/server/publisher.go
+++ b/socketserver/server/publisher.go
@@ -30,7 +30,7 @@ var S2CCommandsCacheInfo = map[Command]PushCommandCacheInfo{
 }
 
 var PersistentCachingCommands = []Command{"follow_sets", "follow_buttons"}
-var HourlyCachingCommands = []Command{"srl_race", "chatters", "viewers"}
+var HourlyCachingCommands = []Command{"chatters", "viewers"} /* srl_race */
 
 type BacklogCacheType int
 
@@ -75,13 +75,14 @@ type LastSavedMessage struct {
 
 // map is command -> channel -> data
 
-// CachedLastMessages is of CacheTypeLastOnly. Cleaned up by reaper goroutine every ~hour.
+// CachedLastMessages is of CacheTypeLastOnly.
+// Not actually cleaned up by reaper goroutine every ~hour.
 var CachedLastMessages = make(map[Command]map[string]LastSavedMessage)
 var CachedLSMLock sync.RWMutex
 
 // PersistentLastMessages is of CacheTypePersistent. Never cleaned.
-var PersistentLastMessages = make(map[Command]map[string]LastSavedMessage)
-var PersistentLSMLock sync.RWMutex
+var PersistentLastMessages = CachedLastMessages
+var PersistentLSMLock = CachedLSMLock
 
 // DumpBacklogData drops all /cached_pub data.
 func DumpBacklogData() {
@@ -89,9 +90,9 @@ func DumpBacklogData() {
 	CachedLastMessages = make(map[Command]map[string]LastSavedMessage)
 	CachedLSMLock.Unlock()
 
-	PersistentLSMLock.Lock()
-	PersistentLastMessages = make(map[Command]map[string]LastSavedMessage)
-	PersistentLSMLock.Unlock()
+	//PersistentLSMLock.Lock()
+	//PersistentLastMessages = make(map[Command]map[string]LastSavedMessage)
+	//PersistentLSMLock.Unlock()
 }
 
 // SendBacklogForNewClient sends any backlog data relevant to a new client.
@@ -177,13 +178,13 @@ func SaveLastMessage(which map[Command]map[string]LastSavedMessage, locker sync.
 	locker.Lock()
 	defer locker.Unlock()
 
-	chanMap, ok := which[cmd]
+	chanMap, ok := CachedLastMessages[cmd]
 	if !ok {
 		if deleting {
 			return
 		}
 		chanMap = make(map[string]LastSavedMessage)
-		which[cmd] = chanMap
+		CachedLastMessages[cmd] = chanMap
 	}
 
 	if deleting {

From 814f25ad53fc166dab0b797df17579b0c205df84 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sat, 21 May 2016 11:29:12 -0700
Subject: [PATCH 155/176] tests: Don't hardcode GetCommandsOfType

---
 socketserver/server/publisher.go | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/socketserver/server/publisher.go b/socketserver/server/publisher.go
index 7f667f01..88463c22 100644
--- a/socketserver/server/publisher.go
+++ b/socketserver/server/publisher.go
@@ -195,13 +195,13 @@ func SaveLastMessage(which map[Command]map[string]LastSavedMessage, locker sync.
 }
 
 func GetCommandsOfType(match BacklogCacheType) []Command {
-	if match == CacheTypePersistent {
-		return PersistentCachingCommands
-	} else if match == CacheTypeLastOnly {
-		return HourlyCachingCommands
-	} else {
-		panic("unknown caching type")
+	var ret []Command
+	for cmd, info := range S2CCommandsCacheInfo {
+		if info.Caching == match {
+			ret = append(ret, cmd)
+		}
 	}
+	return ret
 }
 
 func HTTPBackendDropBacklog(w http.ResponseWriter, r *http.Request) {

From 613d13652f143333ef7a09eec95fa0537e06ec3c Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sat, 21 May 2016 11:35:32 -0700
Subject: [PATCH 156/176] Allow opt-out of user tracking

---
 socketserver/server/commands.go | 25 ++++++++++++++++++++-----
 socketserver/server/types.go    |  2 ++
 2 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/socketserver/server/commands.go b/socketserver/server/commands.go
index a112e6dc..2c71fd0c 100644
--- a/socketserver/server/commands.go
+++ b/socketserver/server/commands.go
@@ -111,17 +111,32 @@ func callHandler(handler CommandHandler, conn *websocket.Conn, client *ClientInf
 // C2SHello implements the `hello` C2S Command.
 // It calls SubscribeGlobal() and SubscribeDefaults() with the client, and fills out ClientInfo.Version and ClientInfo.ClientID.
 func C2SHello(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg ClientMessage, err error) {
-	version, clientID, err := msg.ArgumentsAsTwoStrings()
-	if err != nil {
+	ary, ok := msg.Arguments.([]interface{})
+	if !ok {
+		err = ErrExpectedTwoStrings
+		return
+	}
+	if len(ary) != 2 {
+		err = ErrExpectedTwoStrings
+		return
+	}
+	version, ok := ary[0].(string)
+	if !ok {
+		err = ErrExpectedTwoStrings
 		return
 	}
 
 	client.VersionString = copyString(version)
 	client.Version = VersionFromString(version)
 
-	client.ClientID = uuid.FromStringOrNil(clientID)
-	if client.ClientID == uuid.Nil {
-		client.ClientID = uuid.NewV4()
+	if clientIDStr, ok := ary[1].(string); ok {
+		client.ClientID = uuid.FromStringOrNil(clientIDStr)
+		if client.ClientID == uuid.Nil {
+			client.ClientID = uuid.NewV4()
+		}
+	} else if _, ok := ary[1].(bool); ok {
+		// opt out
+		client.ClientID = AnonymousClientID
 	}
 
 	uniqueUserChannel <- client.ClientID
diff --git a/socketserver/server/types.go b/socketserver/server/types.go
index 3d360e26..e775acb5 100644
--- a/socketserver/server/types.go
+++ b/socketserver/server/types.go
@@ -11,6 +11,8 @@ import (
 
 const NegativeOne = ^uint64(0)
 
+var AnonymousClientID = uuid.FromStringOrNil("683b45e4-f853-4c45-bf96-7d799cc93e34")
+
 type ConfigFile struct {
 	// Numeric server id known to the backend
 	ServerID int

From 88b356177ae48afc16ba6a8c3dbe3598e72f58c3 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sat, 21 May 2016 11:38:48 -0700
Subject: [PATCH 157/176] missed error case

---
 socketserver/cmd/statsweb/statsweb.go | 6 ++++--
 socketserver/server/commands.go       | 3 +++
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/socketserver/cmd/statsweb/statsweb.go b/socketserver/cmd/statsweb/statsweb.go
index 980f1cd8..b36cc768 100644
--- a/socketserver/cmd/statsweb/statsweb.go
+++ b/socketserver/cmd/statsweb/statsweb.go
@@ -16,6 +16,8 @@ import (
 	"github.com/clarkduvall/hyperloglog"
 )
 
+var _ = os.Exit
+
 var configLocation = flag.String("config", "./config.json", "Location of the configuration file. Defaults to ./config.json")
 var genConfig = flag.Bool("genconf", false, "Generate a new configuration file.")
 
@@ -41,8 +43,8 @@ func main() {
 		allServers[i].Setup(v)
 	}
 
-	printEveryDay()
-	os.Exit(0)
+	//printEveryDay()
+	//os.Exit(0)
 	http.HandleFunc("/api/get", ServeAPIGet)
 	http.ListenAndServe(config.ListenAddr, http.DefaultServeMux)
 }
diff --git a/socketserver/server/commands.go b/socketserver/server/commands.go
index 2c71fd0c..8ad1d830 100644
--- a/socketserver/server/commands.go
+++ b/socketserver/server/commands.go
@@ -137,6 +137,9 @@ func C2SHello(conn *websocket.Conn, client *ClientInfo, msg ClientMessage) (rmsg
 	} else if _, ok := ary[1].(bool); ok {
 		// opt out
 		client.ClientID = AnonymousClientID
+	} else {
+		err = ErrExpectedTwoStrings
+		return
 	}
 
 	uniqueUserChannel <- client.ClientID

From d49c88a2227d9ddfe8d674c29462fdb35d8699ea Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Thu, 2 Jun 2016 08:16:16 -0700
Subject: [PATCH 158/176] Add Health to /stats, add irc reconnect

---
 socketserver/server/backend.go    | 19 ++++++++++++++++++-
 socketserver/server/handlecore.go | 11 ++++++++++-
 socketserver/server/irc.go        | 31 +++++++++++++++++++++++++------
 socketserver/server/stats.go      | 11 ++++++++++-
 4 files changed, 63 insertions(+), 9 deletions(-)

diff --git a/socketserver/server/backend.go b/socketserver/server/backend.go
index 5bb4d960..0294325c 100644
--- a/socketserver/server/backend.go
+++ b/socketserver/server/backend.go
@@ -37,7 +37,11 @@ var serverID int
 
 var messageBufferPool sync.Pool
 
+var lastBackendSuccess map[string]time.Time
+
 func setupBackend(config *ConfigFile) {
+	serverID = config.ServerID
+
 	backendHTTPClient.Timeout = 60 * time.Second
 	backendURL = config.BackendURL
 	if responseCache != nil {
@@ -48,13 +52,20 @@ func setupBackend(config *ConfigFile) {
 	announceStartupURL = fmt.Sprintf("%s%s", backendURL, bPathAnnounceStartup)
 	addTopicURL = fmt.Sprintf("%s%s", backendURL, bPathAddTopic)
 	postStatisticsURL = fmt.Sprintf("%s%s", backendURL, bPathAggStats)
+	epochTime := time.Unix(0, 0)
+	lastBackendSuccess = map[string]time.Time{
+		bPathAnnounceStartup: epochTime,
+		bPathAddTopic:        epochTime,
+		bPathAggStats:        epochTime,
+		bPathOtherCommand:    epochTime,
+	}
+	Statistics.Health.Backend = lastBackendSuccess
 
 	messageBufferPool.New = New4KByteBuffer
 
 	var theirPublic, ourPrivate [32]byte
 	copy(theirPublic[:], config.BackendPublicKey)
 	copy(ourPrivate[:], config.OurPrivateKey)
-	serverID = config.ServerID
 
 	box.Precompute(&backendSharedKey, &theirPublic, &ourPrivate)
 }
@@ -197,6 +208,8 @@ func SendRemoteCommand(remoteCommand, data string, auth AuthInfo) (responseStr s
 		responseCache.Set(getCacheKey(remoteCommand, data), responseStr, duration)
 	}
 
+	lastBackendSuccess[bPathOtherCommand] = time.Now()
+
 	return
 }
 
@@ -211,6 +224,8 @@ func SendAggregatedData(sealedForm url.Values) error {
 		return httpError(resp.StatusCode)
 	}
 
+	lastBackendSuccess[bPathAggStats] = time.Now()
+
 	return resp.Body.Close()
 }
 
@@ -271,6 +286,8 @@ func sendTopicNotice(topic string, added bool) error {
 		return ErrBackendNotOK{Code: resp.StatusCode, Response: respStr}
 	}
 
+	lastBackendSuccess[bPathAddTopic] = time.Now()
+
 	return nil
 }
 
diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 0bd23a82..aefd9e9e 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -94,6 +94,7 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 
 	serveMux.HandleFunc("/", HTTPHandleRootURL)
 	serveMux.Handle("/.well-known/", http.FileServer(http.Dir("/tmp/letsencrypt/")))
+	serveMux.HandleFunc("/healthcheck", HTTPSayOK)
 	serveMux.HandleFunc("/stats", HTTPShowStatistics)
 	serveMux.HandleFunc("/hll/", HTTPShowHLL)
 	serveMux.HandleFunc("/hll_force_write", HTTPWriteHLL)
@@ -113,6 +114,7 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 		log.Println("could not announce startup to backend:", err)
 	} else {
 		resp.Body.Close()
+		lastBackendSuccess[bPathAnnounceStartup] = time.Now()
 	}
 
 	if Configuration.UseESLogStashing {
@@ -167,7 +169,7 @@ func shutdownHandler() {
 func dumpStackOnCtrlZ() {
 	ch := make(chan os.Signal)
 	signal.Notify(ch, syscall.SIGTSTP)
-	for _ = range ch {
+	for range ch {
 		fmt.Println("Got ^Z")
 
 		buf := make([]byte, 10000)
@@ -176,6 +178,13 @@ func dumpStackOnCtrlZ() {
 	}
 }
 
+// HTTPSayOK replies with 200 and a body of "ok\n".
+func HTTPSayOK(w http.ResponseWriter, _ *http.Request) {
+	w.(interface {
+		WriteString(string) error
+	}).WriteString("ok\n")
+}
+
 // SocketUpgrader is the websocket.Upgrader currently in use.
 var SocketUpgrader = websocket.Upgrader{
 	ReadBufferSize:  1024,
diff --git a/socketserver/server/irc.go b/socketserver/server/irc.go
index c0af288c..a5a837dc 100644
--- a/socketserver/server/irc.go
+++ b/socketserver/server/irc.go
@@ -88,15 +88,38 @@ const AuthChannelName = "frankerfacezauthorizer"
 const AuthChannel = "#" + AuthChannelName
 const AuthCommand = "AUTH"
 
+var authIrcConnection *irc.Conn
+
 // is_init_func
 func ircConnection() {
-
 	c := irc.SimpleClient("justinfan123")
+	c.Config().Server = "irc.chat.twitch.tv"
+	authIrcConnection = c
+
+	var reconnect func(conn *irc.Conn)
+	connect := func(conn *irc.Conn) {
+		err := c.Connect()
+		if err != nil {
+			log.Println("irc: failed to connect to IRC:", err)
+			go reconnect(conn)
+		}
+	}
+
+	reconnect = func(conn *irc.Conn) {
+		time.Sleep(5 * time.Second)
+		log.Println("irc: Reconnecting…")
+		connect(conn)
+	}
 
 	c.HandleFunc(irc.CONNECTED, func(conn *irc.Conn, line *irc.Line) {
 		conn.Join(AuthChannel)
 	})
 
+	c.HandleFunc(irc.DISCONNECTED, func(conn *irc.Conn, line *irc.Line) {
+		log.Println("irc: Disconnected. Reconnecting in 5 seconds.")
+		go reconnect(conn)
+	})
+
 	c.HandleFunc(irc.PRIVMSG, func(conn *irc.Conn, line *irc.Line) {
 		channel := line.Args[0]
 		msg := line.Args[1]
@@ -115,11 +138,7 @@ func ircConnection() {
 		submitAuth(submittedUser, submittedChallenge)
 	})
 
-	err := c.ConnectTo("irc.chat.twitch.tv")
-	if err != nil {
-		log.Fatalln("Cannot connect to IRC:", err)
-	}
-
+	connect(c)
 }
 
 func submitAuth(user, challenge string) {
diff --git a/socketserver/server/stats.go b/socketserver/server/stats.go
index 7a32ecab..a330297d 100644
--- a/socketserver/server/stats.go
+++ b/socketserver/server/stats.go
@@ -21,6 +21,11 @@ type StatsData struct {
 
 	CachedStatsLastUpdate time.Time
 
+	Health struct {
+		IRC     bool
+		Backend map[string]time.Time
+	}
+
 	CurrentClientCount uint64
 
 	PubSubChannelCount int
@@ -76,7 +81,7 @@ func commandCounter() {
 }
 
 // StatsDataVersion is the version of the StatsData struct.
-const StatsDataVersion = 5
+const StatsDataVersion = 6
 const pageSize = 4096
 
 var cpuUsage struct {
@@ -154,6 +159,10 @@ func updatePeriodicStats() {
 	{
 		Statistics.Uptime = nowUpdate.Sub(Statistics.StartTime).String()
 	}
+
+	{
+		Statistics.Health.IRC = authIrcConnection.Connected()
+	}
 }
 
 var sysMemLastUpdate time.Time

From 80179abc36597eb828c7bc3272e28f5cbec03158 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Thu, 2 Jun 2016 08:28:26 -0700
Subject: [PATCH 159/176] start converting to backend as a struct

---
 socketserver/server/backend.go | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/socketserver/server/backend.go b/socketserver/server/backend.go
index 0294325c..ae085093 100644
--- a/socketserver/server/backend.go
+++ b/socketserver/server/backend.go
@@ -12,7 +12,6 @@ import (
 	"net/url"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/pmylund/go-cache"
@@ -24,6 +23,20 @@ const bPathAddTopic = "/topics"
 const bPathAggStats = "/stats"
 const bPathOtherCommand = "/cmd/"
 
+type backend struct {
+	HTTPClient http.Client
+	baseURL string
+	responseCache *cache.Cache
+
+	postStatsURL string
+	addTopicURL string
+	announceStartupURL string
+
+	sharedKey [32]byte
+	serverID int
+
+	lastSuccess map[string]time.Time
+}
 var backendHTTPClient http.Client
 var backendURL string
 var responseCache *cache.Cache
@@ -35,8 +48,6 @@ var announceStartupURL string
 var backendSharedKey [32]byte
 var serverID int
 
-var messageBufferPool sync.Pool
-
 var lastBackendSuccess map[string]time.Time
 
 func setupBackend(config *ConfigFile) {
@@ -61,8 +72,6 @@ func setupBackend(config *ConfigFile) {
 	}
 	Statistics.Health.Backend = lastBackendSuccess
 
-	messageBufferPool.New = New4KByteBuffer
-
 	var theirPublic, ourPrivate [32]byte
 	copy(theirPublic[:], config.BackendPublicKey)
 	copy(ourPrivate[:], config.OurPrivateKey)

From cd7faaba3814a87d40246332b0f065d23d685d33 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Thu, 2 Jun 2016 08:30:49 -0700
Subject: [PATCH 160/176] convert serverID

---
 socketserver/server/backend.go | 14 ++++++++++----
 socketserver/server/utils.go   |  2 +-
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/socketserver/server/backend.go b/socketserver/server/backend.go
index ae085093..deac6327 100644
--- a/socketserver/server/backend.go
+++ b/socketserver/server/backend.go
@@ -23,7 +23,7 @@ const bPathAddTopic = "/topics"
 const bPathAggStats = "/stats"
 const bPathOtherCommand = "/cmd/"
 
-type backend struct {
+type backendInfo struct {
 	HTTPClient http.Client
 	baseURL string
 	responseCache *cache.Cache
@@ -37,6 +37,9 @@ type backend struct {
 
 	lastSuccess map[string]time.Time
 }
+
+var Backend *backendInfo
+
 var backendHTTPClient http.Client
 var backendURL string
 var responseCache *cache.Cache
@@ -46,12 +49,13 @@ var addTopicURL string
 var announceStartupURL string
 
 var backendSharedKey [32]byte
-var serverID int
 
 var lastBackendSuccess map[string]time.Time
 
-func setupBackend(config *ConfigFile) {
-	serverID = config.ServerID
+func setupBackend(config *ConfigFile) *backendInfo {
+	b := new(backendInfo)
+	Backend = b
+	b.serverID = config.ServerID
 
 	backendHTTPClient.Timeout = 60 * time.Second
 	backendURL = config.BackendURL
@@ -77,6 +81,8 @@ func setupBackend(config *ConfigFile) {
 	copy(ourPrivate[:], config.OurPrivateKey)
 
 	box.Precompute(&backendSharedKey, &theirPublic, &ourPrivate)
+
+	return b
 }
 
 func getCacheKey(remoteCommand, data string) string {
diff --git a/socketserver/server/utils.go b/socketserver/server/utils.go
index 1c895e6b..b2618cd2 100644
--- a/socketserver/server/utils.go
+++ b/socketserver/server/utils.go
@@ -58,7 +58,7 @@ func SealRequest(form url.Values) (url.Values, error) {
 	retval := url.Values{
 		"nonce": []string{nonceString},
 		"msg":   []string{cipherString},
-		"id":    []string{strconv.Itoa(serverID)},
+		"id":    []string{strconv.Itoa(Backend.serverID)},
 	}
 
 	return retval, nil

From dd9e66c80d17d892b2ce5427f18ee60d2f62cd78 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Thu, 2 Jun 2016 08:36:02 -0700
Subject: [PATCH 161/176] convert httpClient

---
 socketserver/server/backend.go       | 27 +++++++++++++--------------
 socketserver/server/commands.go      |  6 +++---
 socketserver/server/handlecore.go    |  4 ++--
 socketserver/server/subscriptions.go |  4 ++--
 4 files changed, 20 insertions(+), 21 deletions(-)

diff --git a/socketserver/server/backend.go b/socketserver/server/backend.go
index deac6327..4b2ec2cb 100644
--- a/socketserver/server/backend.go
+++ b/socketserver/server/backend.go
@@ -40,7 +40,6 @@ type backendInfo struct {
 
 var Backend *backendInfo
 
-var backendHTTPClient http.Client
 var backendURL string
 var responseCache *cache.Cache
 
@@ -57,7 +56,7 @@ func setupBackend(config *ConfigFile) *backendInfo {
 	Backend = b
 	b.serverID = config.ServerID
 
-	backendHTTPClient.Timeout = 60 * time.Second
+	b.HTTPClient.Timeout = 60 * time.Second
 	backendURL = config.BackendURL
 	if responseCache != nil {
 		responseCache.Flush()
@@ -156,19 +155,19 @@ func (bfe ErrForwardedFromBackend) Error() string {
 var ErrAuthorizationNeeded = errors.New("Must authenticate Twitch username to use this command")
 
 // SendRemoteCommandCached performs a RPC call on the backend, but caches responses.
-func SendRemoteCommandCached(remoteCommand, data string, auth AuthInfo) (string, error) {
+func (backend *backendInfo) SendRemoteCommandCached(remoteCommand, data string, auth AuthInfo) (string, error) {
 	cached, ok := responseCache.Get(getCacheKey(remoteCommand, data))
 	if ok {
 		return cached.(string), nil
 	}
-	return SendRemoteCommand(remoteCommand, data, auth)
+	return backend.SendRemoteCommand(remoteCommand, data, auth)
 }
 
 // SendRemoteCommand performs a RPC call on the backend by POSTing to `/cmd/$remoteCommand`.
 // The form data is as follows: `clientData` is the JSON in the `data` parameter
 // (should be retrieved from ClientMessage.Arguments), and either `username` or
 // `usernameClaimed` depending on whether AuthInfo.UsernameValidates is true is AuthInfo.TwitchUsername.
-func SendRemoteCommand(remoteCommand, data string, auth AuthInfo) (responseStr string, err error) {
+func (backend *backendInfo) SendRemoteCommand(remoteCommand, data string, auth AuthInfo) (responseStr string, err error) {
 	destURL := fmt.Sprintf("%s/cmd/%s", backendURL, remoteCommand)
 
 	formData := url.Values{
@@ -187,7 +186,7 @@ func SendRemoteCommand(remoteCommand, data string, auth AuthInfo) (responseStr s
 		return "", err
 	}
 
-	resp, err := backendHTTPClient.PostForm(destURL, sealedForm)
+	resp, err := backend.HTTPClient.PostForm(destURL, sealedForm)
 	if err != nil {
 		return "", err
 	}
@@ -229,8 +228,8 @@ func SendRemoteCommand(remoteCommand, data string, auth AuthInfo) (responseStr s
 }
 
 // SendAggregatedData sends aggregated emote usage and following data to the backend server.
-func SendAggregatedData(sealedForm url.Values) error {
-	resp, err := backendHTTPClient.PostForm(postStatisticsURL, sealedForm)
+func (backend *backendInfo) SendAggregatedData(sealedForm url.Values) error {
+	resp, err := backend.HTTPClient.PostForm(postStatisticsURL, sealedForm)
 	if err != nil {
 		return err
 	}
@@ -259,19 +258,19 @@ func (noe ErrBackendNotOK) Error() string {
 // POST data:
 // channels=room.trihex
 // added=t
-func SendNewTopicNotice(topic string) error {
-	return sendTopicNotice(topic, true)
+func (backend *backendInfo) SendNewTopicNotice(topic string) error {
+	return backend.sendTopicNotice(topic, true)
 }
 
 // SendCleanupTopicsNotice notifies the backend that pub/sub topics have no subscribers anymore.
 // POST data:
 // channels=room.sirstendec,room.bobross,feature.foo
 // added=f
-func SendCleanupTopicsNotice(topics []string) error {
-	return sendTopicNotice(strings.Join(topics, ","), false)
+func (backend *backendInfo) SendCleanupTopicsNotice(topics []string) error {
+	return backend.sendTopicNotice(strings.Join(topics, ","), false)
 }
 
-func sendTopicNotice(topic string, added bool) error {
+func (backend *backendInfo) sendTopicNotice(topic string, added bool) error {
 	formData := url.Values{}
 	formData.Set("channels", topic)
 	if added {
@@ -285,7 +284,7 @@ func sendTopicNotice(topic string, added bool) error {
 		return err
 	}
 
-	resp, err := backendHTTPClient.PostForm(addTopicURL, sealedForm)
+	resp, err := backend.HTTPClient.PostForm(addTopicURL, sealedForm)
 	if err != nil {
 		return err
 	}
diff --git a/socketserver/server/commands.go b/socketserver/server/commands.go
index 8ad1d830..b780b579 100644
--- a/socketserver/server/commands.go
+++ b/socketserver/server/commands.go
@@ -396,7 +396,7 @@ func aggregateDataSender_do() {
 		return
 	}
 
-	err = SendAggregatedData(form)
+	err = Backend.SendAggregatedData(form)
 	if err != nil {
 		log.Println("error reporting aggregate data:", err)
 		return
@@ -533,7 +533,7 @@ func C2SHandleBunchedCommand(conn *websocket.Conn, client *ClientInfo, msg Clien
 	pendingBunchedRequests[br] = &bunchSubscriberList{Members: []bunchSubscriber{{Client: client, MessageID: msg.MessageID}}}
 
 	go func(request bunchedRequest) {
-		respStr, err := SendRemoteCommandCached(string(request.Command), request.Param, AuthInfo{})
+		respStr, err := Backend.SendRemoteCommandCached(string(request.Command), request.Param, AuthInfo{})
 
 		var msg ClientMessage
 		if err == nil {
@@ -581,7 +581,7 @@ const AuthorizationFailedErrorString = "Failed to verify your Twitch username."
 const AuthorizationNeededError = "You must be signed in to use that command."
 
 func doRemoteCommand(conn *websocket.Conn, msg ClientMessage, client *ClientInfo) {
-	resp, err := SendRemoteCommandCached(string(msg.Command), copyString(msg.origArguments), client.AuthInfo)
+	resp, err := Backend.SendRemoteCommandCached(string(msg.Command), copyString(msg.origArguments), client.AuthInfo)
 
 	if err == ErrAuthorizationNeeded {
 		if client.TwitchUsername == "" {
diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index aefd9e9e..fff48b74 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -80,7 +80,7 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 		config.MinMemoryKBytes = defaultMinMemoryKB
 	}
 
-	setupBackend(config)
+	Backend = setupBackend(config)
 
 	if serveMux == nil {
 		serveMux = http.DefaultServeMux
@@ -109,7 +109,7 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 	if err != nil {
 		log.Fatalln("Unable to seal requests:", err)
 	}
-	resp, err := backendHTTPClient.PostForm(announceStartupURL, announceForm)
+	resp, err := Backend.HTTPClient.PostForm(announceStartupURL, announceForm)
 	if err != nil {
 		log.Println("could not announce startup to backend:", err)
 	} else {
diff --git a/socketserver/server/subscriptions.go b/socketserver/server/subscriptions.go
index 0ea46b50..e55db08d 100644
--- a/socketserver/server/subscriptions.go
+++ b/socketserver/server/subscriptions.go
@@ -162,7 +162,7 @@ func pubsubJanitor_do() {
 	ChatSubscriptionLock.Unlock()
 
 	if len(cleanedUp) != 0 {
-		err := SendCleanupTopicsNotice(cleanedUp)
+		err := Backend.SendCleanupTopicsNotice(cleanedUp)
 		if err != nil {
 			log.Println("error reporting cleaned subs:", err)
 		}
@@ -186,7 +186,7 @@ func _subscribeWhileRlocked(channelName string, value chan<- ClientMessage) {
 		ChatSubscriptionLock.Unlock()
 
 		go func(topic string) {
-			err := SendNewTopicNotice(topic)
+			err := Backend.SendNewTopicNotice(topic)
 			if err != nil {
 				log.Println("error reporting new sub:", err)
 			}

From 24936a65432776c6aaba1b963b909d44d2f7780a Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Thu, 2 Jun 2016 08:39:01 -0700
Subject: [PATCH 162/176] convert baseURL

---
 socketserver/server/backend.go | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/socketserver/server/backend.go b/socketserver/server/backend.go
index 4b2ec2cb..82248b3c 100644
--- a/socketserver/server/backend.go
+++ b/socketserver/server/backend.go
@@ -40,7 +40,6 @@ type backendInfo struct {
 
 var Backend *backendInfo
 
-var backendURL string
 var responseCache *cache.Cache
 
 var postStatisticsURL string
@@ -57,15 +56,15 @@ func setupBackend(config *ConfigFile) *backendInfo {
 	b.serverID = config.ServerID
 
 	b.HTTPClient.Timeout = 60 * time.Second
-	backendURL = config.BackendURL
+	b.baseURL = config.BackendURL
 	if responseCache != nil {
 		responseCache.Flush()
 	}
 	responseCache = cache.New(60*time.Second, 120*time.Second)
 
-	announceStartupURL = fmt.Sprintf("%s%s", backendURL, bPathAnnounceStartup)
-	addTopicURL = fmt.Sprintf("%s%s", backendURL, bPathAddTopic)
-	postStatisticsURL = fmt.Sprintf("%s%s", backendURL, bPathAggStats)
+	announceStartupURL = fmt.Sprintf("%s%s", b.baseURL, bPathAnnounceStartup)
+	addTopicURL = fmt.Sprintf("%s%s", b.baseURL, bPathAddTopic)
+	postStatisticsURL = fmt.Sprintf("%s%s", b.baseURL, bPathAggStats)
 	epochTime := time.Unix(0, 0)
 	lastBackendSuccess = map[string]time.Time{
 		bPathAnnounceStartup: epochTime,
@@ -168,7 +167,7 @@ func (backend *backendInfo) SendRemoteCommandCached(remoteCommand, data string,
 // (should be retrieved from ClientMessage.Arguments), and either `username` or
 // `usernameClaimed` depending on whether AuthInfo.UsernameValidates is true is AuthInfo.TwitchUsername.
 func (backend *backendInfo) SendRemoteCommand(remoteCommand, data string, auth AuthInfo) (responseStr string, err error) {
-	destURL := fmt.Sprintf("%s/cmd/%s", backendURL, remoteCommand)
+	destURL := fmt.Sprintf("%s/cmd/%s", backend.baseURL, remoteCommand)
 
 	formData := url.Values{
 		"clientData": []string{data},

From 842dc6499151a33ddbfe8c2c354569c257e45c20 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Thu, 2 Jun 2016 08:39:11 -0700
Subject: [PATCH 163/176] all health timestamps in UTC

---
 socketserver/server/backend.go    | 8 ++++----
 socketserver/server/handlecore.go | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/socketserver/server/backend.go b/socketserver/server/backend.go
index 82248b3c..c0f9342b 100644
--- a/socketserver/server/backend.go
+++ b/socketserver/server/backend.go
@@ -65,7 +65,7 @@ func setupBackend(config *ConfigFile) *backendInfo {
 	announceStartupURL = fmt.Sprintf("%s%s", b.baseURL, bPathAnnounceStartup)
 	addTopicURL = fmt.Sprintf("%s%s", b.baseURL, bPathAddTopic)
 	postStatisticsURL = fmt.Sprintf("%s%s", b.baseURL, bPathAggStats)
-	epochTime := time.Unix(0, 0)
+	epochTime := time.Unix(0, 0).UTC()
 	lastBackendSuccess = map[string]time.Time{
 		bPathAnnounceStartup: epochTime,
 		bPathAddTopic:        epochTime,
@@ -221,7 +221,7 @@ func (backend *backendInfo) SendRemoteCommand(remoteCommand, data string, auth A
 		responseCache.Set(getCacheKey(remoteCommand, data), responseStr, duration)
 	}
 
-	lastBackendSuccess[bPathOtherCommand] = time.Now()
+	lastBackendSuccess[bPathOtherCommand] = time.Now().UTC()
 
 	return
 }
@@ -237,7 +237,7 @@ func (backend *backendInfo) SendAggregatedData(sealedForm url.Values) error {
 		return httpError(resp.StatusCode)
 	}
 
-	lastBackendSuccess[bPathAggStats] = time.Now()
+	lastBackendSuccess[bPathAggStats] = time.Now().UTC()
 
 	return resp.Body.Close()
 }
@@ -299,7 +299,7 @@ func (backend *backendInfo) sendTopicNotice(topic string, added bool) error {
 		return ErrBackendNotOK{Code: resp.StatusCode, Response: respStr}
 	}
 
-	lastBackendSuccess[bPathAddTopic] = time.Now()
+	lastBackendSuccess[bPathAddTopic] = time.Now().UTC()
 
 	return nil
 }
diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index fff48b74..dd840348 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -114,7 +114,7 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 		log.Println("could not announce startup to backend:", err)
 	} else {
 		resp.Body.Close()
-		lastBackendSuccess[bPathAnnounceStartup] = time.Now()
+		lastBackendSuccess[bPathAnnounceStartup] = time.Now().UTC()
 	}
 
 	if Configuration.UseESLogStashing {

From 10feaee47005608d41aa2235950d59ee6cfd742f Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Thu, 2 Jun 2016 08:41:58 -0700
Subject: [PATCH 164/176] fix test compile

---
 socketserver/server/backend_test.go | 31 +++++++++++++++--------------
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/socketserver/server/backend_test.go b/socketserver/server/backend_test.go
index b34edef1..754c44f8 100644
--- a/socketserver/server/backend_test.go
+++ b/socketserver/server/backend_test.go
@@ -59,7 +59,7 @@ func (s *BackendSuite) TestSendRemoteCommand(c *C) {
 	headersCacheInvalid := http.Header{"FFZ-Cache": []string{"NotANumber"}}
 	headersApplicationJson := http.Header{"Content-Type": []string{"application/json"}}
 
-	backend := NewTBackendRequestChecker(c,
+	mockBackend := NewTBackendRequestChecker(c,
 		TExpectedBackendRequest{200, PathTestCommand1, &url.Values{"clientData": []string{TestData1}, "authenticated": []string{"0"}, "username": []string{""}}, TestResponse1, nil},
 		TExpectedBackendRequest{200, PathTestCommand1, &url.Values{"clientData": []string{TestData1}, "authenticated": []string{"0"}, "username": []string{""}}, TestResponse2, nil},
 		TExpectedBackendRequest{200, PathTestCommand1, &url.Values{"clientData": []string{TestData1}, "authenticated": []string{"0"}, "username": []string{TestUsername}}, TestResponse1, nil},
@@ -73,57 +73,58 @@ func (s *BackendSuite) TestSendRemoteCommand(c *C) {
 		TExpectedBackendRequest{418, PathTestCommand1, &url.Values{"clientData": []string{TestData1}, "authenticated": []string{"0"}, "username": []string{TestUsername}}, TestErrorText, headersApplicationJson},
 		TExpectedBackendRequest{200, PathTestCommand2, &url.Values{"clientData": []string{TestData3}, "authenticated": []string{"0"}, "username": []string{TestUsername}}, TestResponse1, headersCacheInvalid},
 	)
-	_, _, _ = TSetup(SetupWantBackendServer, backend)
-	defer backend.Close()
+	_, _, _ = TSetup(SetupWantBackendServer, mockBackend)
+	defer mockBackend.Close()
 
 	var resp string
 	var err error
+	b := Backend
 
-	resp, err = SendRemoteCommand(TestCommand1, TestData1, AnonAuthInfo)
+	resp, err = b.SendRemoteCommand(TestCommand1, TestData1, AnonAuthInfo)
 	c.Check(resp, Equals, TestResponse1)
 	c.Check(err, IsNil)
 
-	resp, err = SendRemoteCommand(TestCommand1, TestData1, AnonAuthInfo)
+	resp, err = b.SendRemoteCommand(TestCommand1, TestData1, AnonAuthInfo)
 	c.Check(resp, Equals, TestResponse2)
 	c.Check(err, IsNil)
 
-	resp, err = SendRemoteCommand(TestCommand1, TestData1, NonValidatedAuthInfo)
+	resp, err = b.SendRemoteCommand(TestCommand1, TestData1, NonValidatedAuthInfo)
 	c.Check(resp, Equals, TestResponse1)
 	c.Check(err, IsNil)
 
-	resp, err = SendRemoteCommand(TestCommand1, TestData1, ValidatedAuthInfo)
+	resp, err = b.SendRemoteCommand(TestCommand1, TestData1, ValidatedAuthInfo)
 	c.Check(resp, Equals, TestResponse1)
 	c.Check(err, IsNil)
 	// cache save
-	resp, err = SendRemoteCommandCached(TestCommand2, TestData2, NonValidatedAuthInfo)
+	resp, err = b.SendRemoteCommandCached(TestCommand2, TestData2, NonValidatedAuthInfo)
 	c.Check(resp, Equals, TestResponse1)
 	c.Check(err, IsNil)
 
-	resp, err = SendRemoteCommandCached(TestCommand2, TestData2, NonValidatedAuthInfo) // cache hit
+	resp, err = b.SendRemoteCommandCached(TestCommand2, TestData2, NonValidatedAuthInfo) // cache hit
 	c.Check(resp, Equals, TestResponse1)
 	c.Check(err, IsNil)
 
-	resp, err = SendRemoteCommandCached(TestCommand2, TestData2, AnonAuthInfo) // cache hit
+	resp, err = b.SendRemoteCommandCached(TestCommand2, TestData2, AnonAuthInfo) // cache hit
 	c.Check(resp, Equals, TestResponse1)
 	c.Check(err, IsNil)
 	// cache miss - data is different
-	resp, err = SendRemoteCommandCached(TestCommand2, TestData1, NonValidatedAuthInfo)
+	resp, err = b.SendRemoteCommandCached(TestCommand2, TestData1, NonValidatedAuthInfo)
 	c.Check(resp, Equals, TestResponse2)
 	c.Check(err, IsNil)
 
-	resp, err = SendRemoteCommand(TestCommand1, TestData1, NonValidatedAuthInfo)
+	resp, err = b.SendRemoteCommand(TestCommand1, TestData1, NonValidatedAuthInfo)
 	c.Check(resp, Equals, "")
 	c.Check(err, Equals, ErrAuthorizationNeeded)
 
-	resp, err = SendRemoteCommand(TestCommand1, TestData1, NonValidatedAuthInfo)
+	resp, err = b.SendRemoteCommand(TestCommand1, TestData1, NonValidatedAuthInfo)
 	c.Check(resp, Equals, "")
 	c.Check(err, ErrorMatches, "backend http error: 503")
 
-	resp, err = SendRemoteCommand(TestCommand1, TestData1, NonValidatedAuthInfo)
+	resp, err = b.SendRemoteCommand(TestCommand1, TestData1, NonValidatedAuthInfo)
 	c.Check(resp, Equals, "")
 	c.Check(err, ErrorMatches, TestErrorText)
 
-	resp, err = SendRemoteCommand(TestCommand2, TestData3, NonValidatedAuthInfo)
+	resp, err = b.SendRemoteCommand(TestCommand2, TestData3, NonValidatedAuthInfo)
 	c.Check(resp, Equals, "")
 	c.Check(err, ErrorMatches, "The RPC server returned a non-integer cache duration: .*")
 }

From b94bcf743e6b3f2be353ad269ee2ab36f4db8c50 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Thu, 2 Jun 2016 08:47:07 -0700
Subject: [PATCH 165/176] convert (Un)SealRequest

---
 socketserver/server/backend.go    | 14 ++++++++++----
 socketserver/server/commands.go   |  8 +-------
 socketserver/server/handlecore.go |  2 +-
 socketserver/server/publisher.go  |  4 ++--
 socketserver/server/utils.go      | 12 ++++--------
 5 files changed, 18 insertions(+), 22 deletions(-)

diff --git a/socketserver/server/backend.go b/socketserver/server/backend.go
index c0f9342b..9b8c5995 100644
--- a/socketserver/server/backend.go
+++ b/socketserver/server/backend.go
@@ -92,8 +92,9 @@ func getCacheKey(remoteCommand, data string) string {
 // The POST arguments are `cmd`, `args`, `channel`, and `scope`.
 // The `scope` argument is required because no attempt is made to infer the scope from the command, unlike /cached_pub.
 func HTTPBackendUncachedPublish(w http.ResponseWriter, r *http.Request) {
+	b := Backend
 	r.ParseForm()
-	formData, err := UnsealRequest(r.Form)
+	formData, err := b.UnsealRequest(r.Form)
 	if err != nil {
 		w.WriteHeader(403)
 		fmt.Fprintf(w, "Error: %v", err)
@@ -180,7 +181,7 @@ func (backend *backendInfo) SendRemoteCommand(remoteCommand, data string, auth A
 		formData.Set("authenticated", "0")
 	}
 
-	sealedForm, err := SealRequest(formData)
+	sealedForm, err := backend.SealRequest(formData)
 	if err != nil {
 		return "", err
 	}
@@ -227,7 +228,12 @@ func (backend *backendInfo) SendRemoteCommand(remoteCommand, data string, auth A
 }
 
 // SendAggregatedData sends aggregated emote usage and following data to the backend server.
-func (backend *backendInfo) SendAggregatedData(sealedForm url.Values) error {
+func (backend *backendInfo) SendAggregatedData(form url.Values) error {
+	sealedForm, err := backend.SealRequest(form)
+	if err != nil {
+		return err
+	}
+
 	resp, err := backend.HTTPClient.PostForm(postStatisticsURL, sealedForm)
 	if err != nil {
 		return err
@@ -278,7 +284,7 @@ func (backend *backendInfo) sendTopicNotice(topic string, added bool) error {
 		formData.Set("added", "f")
 	}
 
-	sealedForm, err := SealRequest(formData)
+	sealedForm, err := backend.SealRequest(formData)
 	if err != nil {
 		return err
 	}
diff --git a/socketserver/server/commands.go b/socketserver/server/commands.go
index b780b579..3d0156a1 100644
--- a/socketserver/server/commands.go
+++ b/socketserver/server/commands.go
@@ -390,13 +390,7 @@ func aggregateDataSender_do() {
 		reportForm.Set("emotes", string(emoteJSON))
 	}
 
-	form, err := SealRequest(reportForm)
-	if err != nil {
-		log.Println("error reporting aggregate data:", err)
-		return
-	}
-
-	err = Backend.SendAggregatedData(form)
+	err = Backend.SendAggregatedData(reportForm)
 	if err != nil {
 		log.Println("error reporting aggregate data:", err)
 		return
diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index dd840348..da3c7a3d 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -103,7 +103,7 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 	serveMux.HandleFunc("/uncached_pub", HTTPBackendUncachedPublish)
 	serveMux.HandleFunc("/cached_pub", HTTPBackendCachedPublish)
 
-	announceForm, err := SealRequest(url.Values{
+	announceForm, err := Backend.SealRequest(url.Values{
 		"startup": []string{"1"},
 	})
 	if err != nil {
diff --git a/socketserver/server/publisher.go b/socketserver/server/publisher.go
index 88463c22..7a1726f9 100644
--- a/socketserver/server/publisher.go
+++ b/socketserver/server/publisher.go
@@ -206,7 +206,7 @@ func GetCommandsOfType(match BacklogCacheType) []Command {
 
 func HTTPBackendDropBacklog(w http.ResponseWriter, r *http.Request) {
 	r.ParseForm()
-	formData, err := UnsealRequest(r.Form)
+	formData, err := Backend.UnsealRequest(r.Form)
 	if err != nil {
 		w.WriteHeader(403)
 		fmt.Fprintf(w, "Error: %v", err)
@@ -225,7 +225,7 @@ func HTTPBackendDropBacklog(w http.ResponseWriter, r *http.Request) {
 // `scope` is implicit in the command
 func HTTPBackendCachedPublish(w http.ResponseWriter, r *http.Request) {
 	r.ParseForm()
-	formData, err := UnsealRequest(r.Form)
+	formData, err := Backend.UnsealRequest(r.Form)
 	if err != nil {
 		w.WriteHeader(403)
 		fmt.Fprintf(w, "Error: %v", err)
diff --git a/socketserver/server/utils.go b/socketserver/server/utils.go
index b2618cd2..9552e7bb 100644
--- a/socketserver/server/utils.go
+++ b/socketserver/server/utils.go
@@ -24,15 +24,11 @@ func FillCryptoRandom(buf []byte) error {
 	return nil
 }
 
-func New4KByteBuffer() interface{} {
-	return make([]byte, 0, 4096)
-}
-
 func copyString(s string) string {
 	return string([]byte(s))
 }
 
-func SealRequest(form url.Values) (url.Values, error) {
+func (backend *backendInfo) SealRequest(form url.Values) (url.Values, error) {
 	var nonce [24]byte
 	var err error
 
@@ -41,7 +37,7 @@ func SealRequest(form url.Values) (url.Values, error) {
 		return nil, err
 	}
 
-	cipherMsg := box.SealAfterPrecomputation(nil, []byte(form.Encode()), &nonce, &backendSharedKey)
+	cipherMsg := box.SealAfterPrecomputation(nil, []byte(form.Encode()), &nonce, &backend.sharedKey)
 
 	bufMessage := new(bytes.Buffer)
 	enc := base64.NewEncoder(base64.URLEncoding, bufMessage)
@@ -67,7 +63,7 @@ func SealRequest(form url.Values) (url.Values, error) {
 var ErrorShortNonce = errors.New("Nonce too short.")
 var ErrorInvalidSignature = errors.New("Invalid signature or contents")
 
-func UnsealRequest(form url.Values) (url.Values, error) {
+func (backend *backendInfo) UnsealRequest(form url.Values) (url.Values, error) {
 	var nonce [24]byte
 
 	nonceString := form.Get("nonce")
@@ -87,7 +83,7 @@ func UnsealRequest(form url.Values) (url.Values, error) {
 	cipherBuffer := new(bytes.Buffer)
 	cipherBuffer.ReadFrom(dec)
 
-	message, ok := box.OpenAfterPrecomputation(nil, cipherBuffer.Bytes(), &nonce, &backendSharedKey)
+	message, ok := box.OpenAfterPrecomputation(nil, cipherBuffer.Bytes(), &nonce, &backend.sharedKey)
 	if !ok {
 		Statistics.BackendVerifyFails++
 		return nil, ErrorInvalidSignature

From 6abd30d71c371220497088b0b0b8b39f1d5760be Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Thu, 2 Jun 2016 08:59:40 -0700
Subject: [PATCH 166/176] fix race condition with lastSuccess

convert responseCache
---
 socketserver/server/backend.go    | 44 ++++++++++++++++++-------------
 socketserver/server/handlecore.go |  8 +++---
 socketserver/server/stats.go      | 11 ++++++++
 3 files changed, 40 insertions(+), 23 deletions(-)

diff --git a/socketserver/server/backend.go b/socketserver/server/backend.go
index 9b8c5995..409f1f06 100644
--- a/socketserver/server/backend.go
+++ b/socketserver/server/backend.go
@@ -16,6 +16,7 @@ import (
 
 	"github.com/pmylund/go-cache"
 	"golang.org/x/crypto/nacl/box"
+	"sync"
 )
 
 const bPathAnnounceStartup = "/startup"
@@ -24,24 +25,23 @@ const bPathAggStats = "/stats"
 const bPathOtherCommand = "/cmd/"
 
 type backendInfo struct {
-	HTTPClient http.Client
-	baseURL string
+	HTTPClient    http.Client
+	baseURL       string
 	responseCache *cache.Cache
 
-	postStatsURL string
-	addTopicURL string
+	postStatsURL       string
+	addTopicURL        string
 	announceStartupURL string
 
 	sharedKey [32]byte
-	serverID int
+	serverID  int
 
-	lastSuccess map[string]time.Time
+	lastSuccess     map[string]time.Time
+	lastSuccessLock sync.Mutex
 }
 
 var Backend *backendInfo
 
-var responseCache *cache.Cache
-
 var postStatisticsURL string
 var addTopicURL string
 var announceStartupURL string
@@ -57,14 +57,13 @@ func setupBackend(config *ConfigFile) *backendInfo {
 
 	b.HTTPClient.Timeout = 60 * time.Second
 	b.baseURL = config.BackendURL
-	if responseCache != nil {
-		responseCache.Flush()
-	}
-	responseCache = cache.New(60*time.Second, 120*time.Second)
+	b.responseCache = cache.New(60*time.Second, 120*time.Second)
 
 	announceStartupURL = fmt.Sprintf("%s%s", b.baseURL, bPathAnnounceStartup)
 	addTopicURL = fmt.Sprintf("%s%s", b.baseURL, bPathAddTopic)
 	postStatisticsURL = fmt.Sprintf("%s%s", b.baseURL, bPathAggStats)
+
+
 	epochTime := time.Unix(0, 0).UTC()
 	lastBackendSuccess = map[string]time.Time{
 		bPathAnnounceStartup: epochTime,
@@ -72,7 +71,7 @@ func setupBackend(config *ConfigFile) *backendInfo {
 		bPathAggStats:        epochTime,
 		bPathOtherCommand:    epochTime,
 	}
-	Statistics.Health.Backend = lastBackendSuccess
+	b.lastSuccess = lastBackendSuccess
 
 	var theirPublic, ourPrivate [32]byte
 	copy(theirPublic[:], config.BackendPublicKey)
@@ -156,7 +155,7 @@ var ErrAuthorizationNeeded = errors.New("Must authenticate Twitch username to us
 
 // SendRemoteCommandCached performs a RPC call on the backend, but caches responses.
 func (backend *backendInfo) SendRemoteCommandCached(remoteCommand, data string, auth AuthInfo) (string, error) {
-	cached, ok := responseCache.Get(getCacheKey(remoteCommand, data))
+	cached, ok := backend.responseCache.Get(getCacheKey(remoteCommand, data))
 	if ok {
 		return cached.(string), nil
 	}
@@ -169,6 +168,7 @@ func (backend *backendInfo) SendRemoteCommandCached(remoteCommand, data string,
 // `usernameClaimed` depending on whether AuthInfo.UsernameValidates is true is AuthInfo.TwitchUsername.
 func (backend *backendInfo) SendRemoteCommand(remoteCommand, data string, auth AuthInfo) (responseStr string, err error) {
 	destURL := fmt.Sprintf("%s/cmd/%s", backend.baseURL, remoteCommand)
+	healthBucket := fmt.Sprintf("/cmd/%s", remoteCommand)
 
 	formData := url.Values{
 		"clientData": []string{data},
@@ -219,10 +219,14 @@ func (backend *backendInfo) SendRemoteCommand(remoteCommand, data string, auth A
 			return "", fmt.Errorf("The RPC server returned a non-integer cache duration: %v", err)
 		}
 		duration := time.Duration(durSecs) * time.Second
-		responseCache.Set(getCacheKey(remoteCommand, data), responseStr, duration)
+		backend.responseCache.Set(getCacheKey(remoteCommand, data), responseStr, duration)
 	}
 
-	lastBackendSuccess[bPathOtherCommand] = time.Now().UTC()
+	now := time.Now().UTC()
+	backend.lastSuccessLock.Lock()
+	defer backend.lastSuccessLock.Unlock()
+	backend.lastSuccess[bPathOtherCommand] = now
+	backend.lastSuccess[healthBucket] = now
 
 	return
 }
@@ -243,7 +247,9 @@ func (backend *backendInfo) SendAggregatedData(form url.Values) error {
 		return httpError(resp.StatusCode)
 	}
 
-	lastBackendSuccess[bPathAggStats] = time.Now().UTC()
+	backend.lastSuccessLock.Lock()
+	defer backend.lastSuccessLock.Unlock()
+	backend.lastSuccess[bPathAggStats] = time.Now().UTC()
 
 	return resp.Body.Close()
 }
@@ -305,7 +311,9 @@ func (backend *backendInfo) sendTopicNotice(topic string, added bool) error {
 		return ErrBackendNotOK{Code: resp.StatusCode, Response: respStr}
 	}
 
-	lastBackendSuccess[bPathAddTopic] = time.Now().UTC()
+	backend.lastSuccessLock.Lock()
+	defer backend.lastSuccessLock.Unlock()
+	backend.lastSuccess[bPathAddTopic] = time.Now().UTC()
 
 	return nil
 }
diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index da3c7a3d..66547586 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -114,11 +114,9 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 		log.Println("could not announce startup to backend:", err)
 	} else {
 		resp.Body.Close()
-		lastBackendSuccess[bPathAnnounceStartup] = time.Now().UTC()
-	}
-
-	if Configuration.UseESLogStashing {
-		// logstasher.Setup(Configuration.ESServer, Configuration.ESIndexPrefix, Configuration.ESHostName)
+		Backend.lastSuccessLock.Lock()
+		Backend.lastSuccess[bPathAnnounceStartup] = time.Now().UTC()
+		Backend.lastSuccessLock.Unlock()
 	}
 
 	janitorsOnce.Do(startJanitors)
diff --git a/socketserver/server/stats.go b/socketserver/server/stats.go
index a330297d..a9401248 100644
--- a/socketserver/server/stats.go
+++ b/socketserver/server/stats.go
@@ -97,6 +97,12 @@ func newStatsData() *StatsData {
 		DisconnectReasons: make(map[string]uint64),
 		ClientVersions:    make(map[string]uint64),
 		StatsDataVersion:  StatsDataVersion,
+		Health: struct {
+			IRC     bool
+			Backend map[string]time.Time
+		}{
+			Backend: make(map[string]time.Time),
+		},
 	}
 }
 
@@ -162,6 +168,11 @@ func updatePeriodicStats() {
 
 	{
 		Statistics.Health.IRC = authIrcConnection.Connected()
+		Backend.lastSuccessLock.Lock()
+		for k, v := range Backend.lastSuccess {
+			Statistics.Health.Backend[k] = v
+		}
+		Backend.lastSuccessLock.Unlock()
 	}
 }
 

From cdcb7fb07934abc55d5e5cf2b9ab65e1683fb97e Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Thu, 2 Jun 2016 09:03:43 -0700
Subject: [PATCH 167/176] convert backend urls

---
 socketserver/server/backend.go    | 17 ++++++-----------
 socketserver/server/handlecore.go |  2 +-
 2 files changed, 7 insertions(+), 12 deletions(-)

diff --git a/socketserver/server/backend.go b/socketserver/server/backend.go
index 409f1f06..f92e4690 100644
--- a/socketserver/server/backend.go
+++ b/socketserver/server/backend.go
@@ -29,7 +29,7 @@ type backendInfo struct {
 	baseURL       string
 	responseCache *cache.Cache
 
-	postStatsURL       string
+	postStatisticsURL  string
 	addTopicURL        string
 	announceStartupURL string
 
@@ -42,10 +42,6 @@ type backendInfo struct {
 
 var Backend *backendInfo
 
-var postStatisticsURL string
-var addTopicURL string
-var announceStartupURL string
-
 var backendSharedKey [32]byte
 
 var lastBackendSuccess map[string]time.Time
@@ -59,10 +55,9 @@ func setupBackend(config *ConfigFile) *backendInfo {
 	b.baseURL = config.BackendURL
 	b.responseCache = cache.New(60*time.Second, 120*time.Second)
 
-	announceStartupURL = fmt.Sprintf("%s%s", b.baseURL, bPathAnnounceStartup)
-	addTopicURL = fmt.Sprintf("%s%s", b.baseURL, bPathAddTopic)
-	postStatisticsURL = fmt.Sprintf("%s%s", b.baseURL, bPathAggStats)
-
+	b.announceStartupURL = fmt.Sprintf("%s%s", b.baseURL, bPathAnnounceStartup)
+	b.addTopicURL = fmt.Sprintf("%s%s", b.baseURL, bPathAddTopic)
+	b.postStatisticsURL = fmt.Sprintf("%s%s", b.baseURL, bPathAggStats)
 
 	epochTime := time.Unix(0, 0).UTC()
 	lastBackendSuccess = map[string]time.Time{
@@ -238,7 +233,7 @@ func (backend *backendInfo) SendAggregatedData(form url.Values) error {
 		return err
 	}
 
-	resp, err := backend.HTTPClient.PostForm(postStatisticsURL, sealedForm)
+	resp, err := backend.HTTPClient.PostForm(backend.postStatisticsURL, sealedForm)
 	if err != nil {
 		return err
 	}
@@ -295,7 +290,7 @@ func (backend *backendInfo) sendTopicNotice(topic string, added bool) error {
 		return err
 	}
 
-	resp, err := backend.HTTPClient.PostForm(addTopicURL, sealedForm)
+	resp, err := backend.HTTPClient.PostForm(backend.addTopicURL, sealedForm)
 	if err != nil {
 		return err
 	}
diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 66547586..1ed4fde4 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -109,7 +109,7 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 	if err != nil {
 		log.Fatalln("Unable to seal requests:", err)
 	}
-	resp, err := Backend.HTTPClient.PostForm(announceStartupURL, announceForm)
+	resp, err := Backend.HTTPClient.PostForm(Backend.announceStartupURL, announceForm)
 	if err != nil {
 		log.Println("could not announce startup to backend:", err)
 	} else {

From 89e62985832e2b449d6fc8124ff38c6689031a6f Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Thu, 2 Jun 2016 09:04:46 -0700
Subject: [PATCH 168/176] convert final touches

---
 socketserver/server/backend.go | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/socketserver/server/backend.go b/socketserver/server/backend.go
index f92e4690..98d281a8 100644
--- a/socketserver/server/backend.go
+++ b/socketserver/server/backend.go
@@ -42,10 +42,6 @@ type backendInfo struct {
 
 var Backend *backendInfo
 
-var backendSharedKey [32]byte
-
-var lastBackendSuccess map[string]time.Time
-
 func setupBackend(config *ConfigFile) *backendInfo {
 	b := new(backendInfo)
 	Backend = b
@@ -60,7 +56,7 @@ func setupBackend(config *ConfigFile) *backendInfo {
 	b.postStatisticsURL = fmt.Sprintf("%s%s", b.baseURL, bPathAggStats)
 
 	epochTime := time.Unix(0, 0).UTC()
-	lastBackendSuccess = map[string]time.Time{
+	lastBackendSuccess := map[string]time.Time{
 		bPathAnnounceStartup: epochTime,
 		bPathAddTopic:        epochTime,
 		bPathAggStats:        epochTime,
@@ -72,7 +68,7 @@ func setupBackend(config *ConfigFile) *backendInfo {
 	copy(theirPublic[:], config.BackendPublicKey)
 	copy(ourPrivate[:], config.OurPrivateKey)
 
-	box.Precompute(&backendSharedKey, &theirPublic, &ourPrivate)
+	box.Precompute(&b.sharedKey, &theirPublic, &ourPrivate)
 
 	return b
 }

From 4ae3cca6ac63d9e095695a8d6fbedfda11f8f555 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Thu, 2 Jun 2016 09:07:17 -0700
Subject: [PATCH 169/176] fix test compile

---
 socketserver/server/backend.go        | 3 +--
 socketserver/server/backend_test.go   | 5 +++--
 socketserver/server/testinfra_test.go | 6 +++---
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/socketserver/server/backend.go b/socketserver/server/backend.go
index 98d281a8..e8d827fa 100644
--- a/socketserver/server/backend.go
+++ b/socketserver/server/backend.go
@@ -82,9 +82,8 @@ func getCacheKey(remoteCommand, data string) string {
 // The POST arguments are `cmd`, `args`, `channel`, and `scope`.
 // The `scope` argument is required because no attempt is made to infer the scope from the command, unlike /cached_pub.
 func HTTPBackendUncachedPublish(w http.ResponseWriter, r *http.Request) {
-	b := Backend
 	r.ParseForm()
-	formData, err := b.UnsealRequest(r.Form)
+	formData, err := Backend.UnsealRequest(r.Form)
 	if err != nil {
 		w.WriteHeader(403)
 		fmt.Fprintf(w, "Error: %v", err)
diff --git a/socketserver/server/backend_test.go b/socketserver/server/backend_test.go
index 754c44f8..d1c85adb 100644
--- a/socketserver/server/backend_test.go
+++ b/socketserver/server/backend_test.go
@@ -12,19 +12,20 @@ func Test(t *testing.T) { TestingT(t) }
 
 func TestSealRequest(t *testing.T) {
 	TSetup(SetupNoServers, nil)
+	b := Backend
 
 	values := url.Values{
 		"QuickBrownFox": []string{"LazyDog"},
 	}
 
-	sealedValues, err := SealRequest(values)
+	sealedValues, err := b.SealRequest(values)
 	if err != nil {
 		t.Fatal(err)
 	}
 	// sealedValues.Encode()
 	// id=0&msg=KKtbng49dOLLyjeuX5AnXiEe6P0uZwgeP_7mMB5vhP-wMAAPZw%3D%3D&nonce=-wRbUnifscisWUvhm3gBEXHN5QzrfzgV
 
-	unsealedValues, err := UnsealRequest(sealedValues)
+	unsealedValues, err := b.UnsealRequest(sealedValues)
 	if err != nil {
 		t.Fatal(err)
 	}
diff --git a/socketserver/server/testinfra_test.go b/socketserver/server/testinfra_test.go
index 998ccffd..e52b7a22 100644
--- a/socketserver/server/testinfra_test.go
+++ b/socketserver/server/testinfra_test.go
@@ -123,7 +123,7 @@ func (backend *TBackendRequestChecker) ServeHTTP(w http.ResponseWriter, r *http.
 
 	r.ParseForm()
 
-	unsealedForm, err := UnsealRequest(r.PostForm)
+	unsealedForm, err := Backend.UnsealRequest(r.PostForm)
 	if err != nil {
 		backend.tb.Errorf("Failed to unseal backend request: %v", err)
 	}
@@ -276,7 +276,7 @@ func TSealForSavePubMsg(tb testing.TB, cmd Command, channel string, arguments in
 	}
 	form.Set("time", strconv.FormatInt(time.Now().Unix(), 10))
 
-	sealed, err := SealRequest(form)
+	sealed, err := Backend.SealRequest(form)
 	if err != nil {
 		tb.Error(err)
 		return nil, err
@@ -300,7 +300,7 @@ func TSealForUncachedPubMsg(tb testing.TB, cmd Command, channel string, argument
 	form.Set("time", time.Now().Format(time.UnixDate))
 	form.Set("scope", scope.String())
 
-	sealed, err := SealRequest(form)
+	sealed, err := Backend.SealRequest(form)
 	if err != nil {
 		tb.Error(err)
 		return nil, err

From 3b3457af1420944e9e2ca9670282a94202777f72 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sat, 4 Jun 2016 11:43:37 -0700
Subject: [PATCH 170/176] allow 2xx from backend

---
 socketserver/server/backend.go | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/socketserver/server/backend.go b/socketserver/server/backend.go
index e8d827fa..b7577f6f 100644
--- a/socketserver/server/backend.go
+++ b/socketserver/server/backend.go
@@ -232,7 +232,7 @@ func (backend *backendInfo) SendAggregatedData(form url.Values) error {
 	if err != nil {
 		return err
 	}
-	if resp.StatusCode != 200 {
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
 		resp.Body.Close()
 		return httpError(resp.StatusCode)
 	}
@@ -291,14 +291,12 @@ func (backend *backendInfo) sendTopicNotice(topic string, added bool) error {
 	}
 	defer resp.Body.Close()
 
-	respBytes, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-
-	respStr := string(respBytes)
-	if respStr != "ok" {
-		return ErrBackendNotOK{Code: resp.StatusCode, Response: respStr}
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		respBytes, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return ErrBackendNotOK{Code: resp.StatusCode, Response: fmt.Sprintf("(error reading non-2xx response): %s", err.Error()}
+		}
+		return ErrBackendNotOK{Code: resp.StatusCode, Response: string(respBytes)}
 	}
 
 	backend.lastSuccessLock.Lock()

From ec932b1c362fa5c4d4518a06fb7940118d8c9157 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Fri, 8 Jul 2016 12:46:16 -0700
Subject: [PATCH 171/176] remove MsgTargetType and CacheType

---
 socketserver/server/backend.go            |  27 ++--
 socketserver/server/publisher.go          | 143 ++--------------------
 socketserver/server/subscriptions_test.go |   8 +-
 socketserver/server/testinfra_test.go     |   4 +-
 socketserver/server/types.go              | 107 ----------------
 5 files changed, 24 insertions(+), 265 deletions(-)

diff --git a/socketserver/server/backend.go b/socketserver/server/backend.go
index b7577f6f..eb3dbe55 100644
--- a/socketserver/server/backend.go
+++ b/socketserver/server/backend.go
@@ -14,9 +14,10 @@ import (
 	"strings"
 	"time"
 
+	"sync"
+
 	"github.com/pmylund/go-cache"
 	"golang.org/x/crypto/nacl/box"
-	"sync"
 )
 
 const bPathAnnounceStartup = "/startup"
@@ -80,7 +81,7 @@ func getCacheKey(remoteCommand, data string) string {
 // HTTPBackendUncachedPublish handles the /uncached_pub route.
 // The backend can POST here to publish a message to clients with no caching.
 // The POST arguments are `cmd`, `args`, `channel`, and `scope`.
-// The `scope` argument is required because no attempt is made to infer the scope from the command, unlike /cached_pub.
+// If "scope" is "global", then "channel" is not used.
 func HTTPBackendUncachedPublish(w http.ResponseWriter, r *http.Request) {
 	r.ParseForm()
 	formData, err := Backend.UnsealRequest(r.Form)
@@ -95,14 +96,12 @@ func HTTPBackendUncachedPublish(w http.ResponseWriter, r *http.Request) {
 	channel := formData.Get("channel")
 	scope := formData.Get("scope")
 
-	target := MessageTargetTypeByName(scope)
-
 	if cmd == "" {
 		w.WriteHeader(422)
 		fmt.Fprintf(w, "Error: cmd cannot be blank")
 		return
 	}
-	if channel == "" && (target == MsgTargetTypeChat || target == MsgTargetTypeMultichat) {
+	if channel == "" && scope != "global" {
 		w.WriteHeader(422)
 		fmt.Fprintf(w, "Error: channel must be specified")
 		return
@@ -112,19 +111,11 @@ func HTTPBackendUncachedPublish(w http.ResponseWriter, r *http.Request) {
 	cm.parseOrigArguments()
 	var count int
 
-	switch target {
-	case MsgTargetTypeChat:
-		count = PublishToChannel(channel, cm)
-	case MsgTargetTypeMultichat:
-		count = PublishToMultiple(strings.Split(channel, ","), cm)
-	case MsgTargetTypeGlobal:
-		count = PublishToAll(cm)
-	case MsgTargetTypeInvalid:
-		fallthrough
+	switch scope {
 	default:
-		w.WriteHeader(422)
-		fmt.Fprint(w, "Invalid 'scope'. must be chat, multichat, channel, or global")
-		return
+		count = PublishToMultiple(strings.Split(channel, ","), cm)
+	case "global":
+		count = PublishToAll(cm)
 	}
 	fmt.Fprint(w, count)
 }
@@ -294,7 +285,7 @@ func (backend *backendInfo) sendTopicNotice(topic string, added bool) error {
 	if resp.StatusCode < 200 || resp.StatusCode > 299 {
 		respBytes, err := ioutil.ReadAll(resp.Body)
 		if err != nil {
-			return ErrBackendNotOK{Code: resp.StatusCode, Response: fmt.Sprintf("(error reading non-2xx response): %s", err.Error()}
+			return ErrBackendNotOK{Code: resp.StatusCode, Response: fmt.Sprintf("(error reading non-2xx response): %s", err.Error())}
 		}
 		return ErrBackendNotOK{Code: resp.StatusCode, Response: string(respBytes)}
 	}
diff --git a/socketserver/server/publisher.go b/socketserver/server/publisher.go
index 7a1726f9..305140f9 100644
--- a/socketserver/server/publisher.go
+++ b/socketserver/server/publisher.go
@@ -1,7 +1,6 @@
 package server
 
 import (
-	"errors"
 	"fmt"
 	"net/http"
 	"strconv"
@@ -10,64 +9,6 @@ import (
 	"time"
 )
 
-type PushCommandCacheInfo struct {
-	Caching BacklogCacheType
-	Target  MessageTargetType
-}
-
-// S2CCommandsCacheInfo details what the behavior is of each command that can be sent to /cached_pub.
-var S2CCommandsCacheInfo = map[Command]PushCommandCacheInfo{
-	/// Channel data
-	// follow_sets: extra emote sets included in the chat
-	// follow_buttons: extra follow buttons below the stream
-	"follow_sets":    {CacheTypePersistent, MsgTargetTypeChat},
-	"follow_buttons": {CacheTypePersistent, MsgTargetTypeChat},
-	"srl_race":       {CacheTypeLastOnly, MsgTargetTypeMultichat},
-
-	/// Chatter/viewer counts
-	"chatters": {CacheTypeLastOnly, MsgTargetTypeChat},
-	"viewers":  {CacheTypeLastOnly, MsgTargetTypeChat},
-}
-
-var PersistentCachingCommands = []Command{"follow_sets", "follow_buttons"}
-var HourlyCachingCommands = []Command{"chatters", "viewers"} /* srl_race */
-
-type BacklogCacheType int
-
-const (
-	// CacheTypeInvalid is the sentinel value.
-	CacheTypeInvalid BacklogCacheType = iota
-	// CacheTypeNever is a message that cannot be cached.
-	CacheTypeNever
-	// CacheTypeLastOnly means to save only the last copy of this message,
-	// and always send it when the backlog is requested.
-	CacheTypeLastOnly
-	// CacheTypePersistent means to save the last copy of this message,
-	// and always send it when the backlog is requested, but do not clean it periodically.
-	CacheTypePersistent
-)
-
-type MessageTargetType int
-
-const (
-	// MsgTargetTypeInvalid is the sentinel value.
-	MsgTargetTypeInvalid MessageTargetType = iota
-	// MsgTargetTypeChat is a message is targeted to all users in a particular chat.
-	MsgTargetTypeChat
-	// MsgTargetTypeMultichat is a message is targeted to all users in multiple chats.
-	MsgTargetTypeMultichat
-	// MsgTargetTypeGlobal is a message sent to all FFZ users.
-	MsgTargetTypeGlobal
-)
-
-// note: see types.go for methods on these
-
-// ErrorUnrecognizedCacheType is returned by BacklogCacheType.UnmarshalJSON()
-var ErrorUnrecognizedCacheType = errors.New("Invalid value for cachetype")
-
-// ErrorUnrecognizedTargetType is returned by MessageTargetType.UnmarshalJSON()
-var ErrorUnrecognizedTargetType = errors.New("Invalid value for message target")
-
 type LastSavedMessage struct {
 	Timestamp time.Time
 	Data      string
@@ -80,19 +21,11 @@ type LastSavedMessage struct {
 var CachedLastMessages = make(map[Command]map[string]LastSavedMessage)
 var CachedLSMLock sync.RWMutex
 
-// PersistentLastMessages is of CacheTypePersistent. Never cleaned.
-var PersistentLastMessages = CachedLastMessages
-var PersistentLSMLock = CachedLSMLock
-
 // DumpBacklogData drops all /cached_pub data.
 func DumpBacklogData() {
 	CachedLSMLock.Lock()
 	CachedLastMessages = make(map[Command]map[string]LastSavedMessage)
 	CachedLSMLock.Unlock()
-
-	//PersistentLSMLock.Lock()
-	//PersistentLastMessages = make(map[Command]map[string]LastSavedMessage)
-	//PersistentLSMLock.Unlock()
 }
 
 // SendBacklogForNewClient sends any backlog data relevant to a new client.
@@ -104,26 +37,8 @@ func SendBacklogForNewClient(client *ClientInfo) {
 	copy(curChannels, client.CurrentChannels)
 	client.Mutex.Unlock()
 
-	PersistentLSMLock.RLock()
-	for _, cmd := range GetCommandsOfType(CacheTypePersistent) {
-		chanMap := PersistentLastMessages[cmd]
-		if chanMap == nil {
-			continue
-		}
-		for _, channel := range curChannels {
-			msg, ok := chanMap[channel]
-			if ok {
-				msg := ClientMessage{MessageID: -1, Command: cmd, origArguments: msg.Data}
-				msg.parseOrigArguments()
-				client.MessageChannel <- msg
-			}
-		}
-	}
-	PersistentLSMLock.RUnlock()
-
 	CachedLSMLock.RLock()
-	for _, cmd := range GetCommandsOfType(CacheTypeLastOnly) {
-		chanMap := CachedLastMessages[cmd]
+	for cmd, chanMap := range CachedLastMessages {
 		if chanMap == nil {
 			continue
 		}
@@ -140,23 +55,8 @@ func SendBacklogForNewClient(client *ClientInfo) {
 }
 
 func SendBacklogForChannel(client *ClientInfo, channel string) {
-	PersistentLSMLock.RLock()
-	for _, cmd := range GetCommandsOfType(CacheTypePersistent) {
-		chanMap := PersistentLastMessages[cmd]
-		if chanMap == nil {
-			continue
-		}
-		if msg, ok := chanMap[channel]; ok {
-			msg := ClientMessage{MessageID: -1, Command: cmd, origArguments: msg.Data}
-			msg.parseOrigArguments()
-			client.MessageChannel <- msg
-		}
-	}
-	PersistentLSMLock.RUnlock()
-
 	CachedLSMLock.RLock()
-	for _, cmd := range GetCommandsOfType(CacheTypeLastOnly) {
-		chanMap := CachedLastMessages[cmd]
+	for cmd, chanMap := range CachedLastMessages {
 		if chanMap == nil {
 			continue
 		}
@@ -194,16 +94,6 @@ func SaveLastMessage(which map[Command]map[string]LastSavedMessage, locker sync.
 	}
 }
 
-func GetCommandsOfType(match BacklogCacheType) []Command {
-	var ret []Command
-	for cmd, info := range S2CCommandsCacheInfo {
-		if info.Caching == match {
-			ret = append(ret, cmd)
-		}
-	}
-	return ret
-}
-
 func HTTPBackendDropBacklog(w http.ResponseWriter, r *http.Request) {
 	r.ParseForm()
 	formData, err := Backend.UnsealRequest(r.Form)
@@ -245,33 +135,18 @@ func HTTPBackendCachedPublish(w http.ResponseWriter, r *http.Request) {
 	}
 	timestamp := time.Unix(timeNum, 0)
 
-	cacheinfo, ok := S2CCommandsCacheInfo[cmd]
-	if !ok {
-		w.WriteHeader(422)
-		fmt.Fprintf(w, "Caching semantics unknown for command '%s'. Post to /addcachedcommand first.", cmd)
-		return
-	}
-
 	var count int
 	msg := ClientMessage{MessageID: -1, Command: cmd, origArguments: json}
 	msg.parseOrigArguments()
 
-	if cacheinfo.Caching == CacheTypeLastOnly && cacheinfo.Target == MsgTargetTypeChat {
-		SaveLastMessage(CachedLastMessages, &CachedLSMLock, cmd, channel, timestamp, json, deleteMode)
-		count = PublishToChannel(channel, msg)
-	} else if cacheinfo.Caching == CacheTypePersistent && cacheinfo.Target == MsgTargetTypeChat {
-		SaveLastMessage(PersistentLastMessages, &PersistentLSMLock, cmd, channel, timestamp, json, deleteMode)
-		count = PublishToChannel(channel, msg)
-	} else if cacheinfo.Caching == CacheTypeLastOnly && cacheinfo.Target == MsgTargetTypeMultichat {
-		channels := strings.Split(channel, ",")
-		var dummyLock sync.Mutex
-		CachedLSMLock.Lock()
-		for _, channel := range channels {
-			SaveLastMessage(CachedLastMessages, &dummyLock, cmd, channel, timestamp, json, deleteMode)
-		}
-		CachedLSMLock.Unlock()
-		count = PublishToMultiple(channels, msg)
+	channels := strings.Split(channel, ",")
+	var dummyLock sync.Mutex
+	CachedLSMLock.Lock()
+	for _, channel := range channels {
+		SaveLastMessage(CachedLastMessages, &dummyLock, cmd, channel, timestamp, json, deleteMode)
 	}
+	CachedLSMLock.Unlock()
+	count = PublishToMultiple(channels, msg)
 
 	w.Write([]byte(strconv.Itoa(count)))
 }
diff --git a/socketserver/server/subscriptions_test.go b/socketserver/server/subscriptions_test.go
index 9450d609..fea4f533 100644
--- a/socketserver/server/subscriptions_test.go
+++ b/socketserver/server/subscriptions_test.go
@@ -33,9 +33,7 @@ func TestSubscriptionAndPublish(t *testing.T) {
 	const TestData3 = false
 	var TestData4 = []interface{}{"str1", "str2", "str3"}
 
-	S2CCommandsCacheInfo[TestCommandChan] = PushCommandCacheInfo{Caching: CacheTypeLastOnly, Target: MsgTargetTypeChat}
-	S2CCommandsCacheInfo[TestCommandMulti] = PushCommandCacheInfo{Caching: CacheTypeLastOnly, Target: MsgTargetTypeMultichat}
-	S2CCommandsCacheInfo[TestCommandGlobal] = PushCommandCacheInfo{Caching: CacheTypeLastOnly, Target: MsgTargetTypeGlobal}
+	t.Log("TestSubscriptionAndPublish")
 
 	var server *httptest.Server
 	var urls TURLs
@@ -195,7 +193,7 @@ func TestSubscriptionAndPublish(t *testing.T) {
 
 	// Publish message 4 - should go to clients 1, 2, 3
 
-	form, err = TSealForUncachedPubMsg(t, TestCommandGlobal, "", TestData4, MsgTargetTypeGlobal, false)
+	form, err = TSealForUncachedPubMsg(t, TestCommandGlobal, "", TestData4, "global", false)
 	if err != nil {
 		t.FailNow()
 	}
@@ -254,6 +252,8 @@ func TestRestrictedCommands(t *testing.T) {
 	var server *httptest.Server
 	var urls TURLs
 
+	t.Log("TestRestrictedCommands")
+
 	var backendExpected = NewTBackendRequestChecker(t,
 		TExpectedBackendRequest{200, bPathAnnounceStartup, &url.Values{"startup": []string{"1"}}, "", nil},
 		TExpectedBackendRequest{401, fmt.Sprintf("%s%s", bPathOtherCommand, TestCommandNeedsAuth), &url.Values{"authenticated": []string{"0"}, "username": []string{""}, "clientData": []string{TestRequestDataJSON}}, "", nil},
diff --git a/socketserver/server/testinfra_test.go b/socketserver/server/testinfra_test.go
index e52b7a22..5d8a4570 100644
--- a/socketserver/server/testinfra_test.go
+++ b/socketserver/server/testinfra_test.go
@@ -284,7 +284,7 @@ func TSealForSavePubMsg(tb testing.TB, cmd Command, channel string, arguments in
 	return sealed, nil
 }
 
-func TSealForUncachedPubMsg(tb testing.TB, cmd Command, channel string, arguments interface{}, scope MessageTargetType, deleteMode bool) (url.Values, error) {
+func TSealForUncachedPubMsg(tb testing.TB, cmd Command, channel string, arguments interface{}, scope string, deleteMode bool) (url.Values, error) {
 	form := url.Values{}
 	form.Set("cmd", string(cmd))
 	argsBytes, err := json.Marshal(arguments)
@@ -298,7 +298,7 @@ func TSealForUncachedPubMsg(tb testing.TB, cmd Command, channel string, argument
 		form.Set("delete", "1")
 	}
 	form.Set("time", time.Now().Format(time.UnixDate))
-	form.Set("scope", scope.String())
+	form.Set("scope", scope)
 
 	sealed, err := Backend.SealRequest(form)
 	if err != nil {
diff --git a/socketserver/server/types.go b/socketserver/server/types.go
index e775acb5..41c0d010 100644
--- a/socketserver/server/types.go
+++ b/socketserver/server/types.go
@@ -1,7 +1,6 @@
 package server
 
 import (
-	"encoding/json"
 	"fmt"
 	"net"
 	"sync"
@@ -152,109 +151,3 @@ func (cv *ClientVersion) After(cv2 *ClientVersion) bool {
 func (cv *ClientVersion) Equal(cv2 *ClientVersion) bool {
 	return cv.Major == cv2.Major && cv.Minor == cv2.Minor && cv.Revision == cv2.Revision
 }
-
-func (bct BacklogCacheType) Name() string {
-	switch bct {
-	case CacheTypeInvalid:
-		return ""
-	case CacheTypeNever:
-		return "never"
-	case CacheTypeLastOnly:
-		return "last"
-	case CacheTypePersistent:
-		return "persist"
-	}
-	panic("Invalid BacklogCacheType value")
-}
-
-var CacheTypesByName = map[string]BacklogCacheType{
-	"never":   CacheTypeNever,
-	"last":    CacheTypeLastOnly,
-	"persist": CacheTypePersistent,
-}
-
-func BacklogCacheTypeByName(name string) (bct BacklogCacheType) {
-	// CacheTypeInvalid is the zero value so it doesn't matter
-	bct, _ = CacheTypesByName[name]
-	return
-}
-
-// String implements Stringer
-func (bct BacklogCacheType) String() string { return bct.Name() }
-
-// MarshalJSON implements json.Marshaler
-func (bct BacklogCacheType) MarshalJSON() ([]byte, error) {
-	return json.Marshal(bct.Name())
-}
-
-// UnmarshalJSON implements json.Unmarshaler
-func (bct *BacklogCacheType) UnmarshalJSON(data []byte) error {
-	var str string
-	err := json.Unmarshal(data, &str)
-	if err != nil {
-		return err
-	}
-	if str == "" {
-		*bct = CacheTypeInvalid
-		return nil
-	}
-	newBct := BacklogCacheTypeByName(str)
-	if newBct != CacheTypeInvalid {
-		*bct = newBct
-		return nil
-	}
-	return ErrorUnrecognizedCacheType
-}
-
-func (mtt MessageTargetType) Name() string {
-	switch mtt {
-	case MsgTargetTypeInvalid:
-		return ""
-	case MsgTargetTypeChat:
-		return "chat"
-	case MsgTargetTypeMultichat:
-		return "multichat"
-	case MsgTargetTypeGlobal:
-		return "global"
-	}
-	panic("Invalid MessageTargetType value")
-}
-
-var TargetTypesByName = map[string]MessageTargetType{
-	"chat":      MsgTargetTypeChat,
-	"multichat": MsgTargetTypeMultichat,
-	"global":    MsgTargetTypeGlobal,
-}
-
-func MessageTargetTypeByName(name string) (mtt MessageTargetType) {
-	// MsgTargetTypeInvalid is the zero value so it doesn't matter
-	mtt, _ = TargetTypesByName[name]
-	return
-}
-
-// String implements Stringer
-func (mtt MessageTargetType) String() string { return mtt.Name() }
-
-// MarshalJSON implements json.Marshaler
-func (mtt MessageTargetType) MarshalJSON() ([]byte, error) {
-	return json.Marshal(mtt.Name())
-}
-
-// UnmarshalJSON implements json.Unmarshaler
-func (mtt *MessageTargetType) UnmarshalJSON(data []byte) error {
-	var str string
-	err := json.Unmarshal(data, &str)
-	if err != nil {
-		return err
-	}
-	if str == "" {
-		*mtt = MsgTargetTypeInvalid
-		return nil
-	}
-	newMtt := MessageTargetTypeByName(str)
-	if newMtt != MsgTargetTypeInvalid {
-		*mtt = newMtt
-		return nil
-	}
-	return ErrorUnrecognizedTargetType
-}

From f61fea6414e5369fd0d6ed7722aa9ffe46acf2de Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Fri, 8 Jul 2016 12:47:13 -0700
Subject: [PATCH 172/176] move /uncached_pub to publisher.go

---
 socketserver/server/backend.go   | 42 --------------------------------
 socketserver/server/publisher.go | 42 ++++++++++++++++++++++++++++++++
 2 files changed, 42 insertions(+), 42 deletions(-)

diff --git a/socketserver/server/backend.go b/socketserver/server/backend.go
index eb3dbe55..f41defde 100644
--- a/socketserver/server/backend.go
+++ b/socketserver/server/backend.go
@@ -78,48 +78,6 @@ func getCacheKey(remoteCommand, data string) string {
 	return fmt.Sprintf("%s/%s", remoteCommand, data)
 }
 
-// HTTPBackendUncachedPublish handles the /uncached_pub route.
-// The backend can POST here to publish a message to clients with no caching.
-// The POST arguments are `cmd`, `args`, `channel`, and `scope`.
-// If "scope" is "global", then "channel" is not used.
-func HTTPBackendUncachedPublish(w http.ResponseWriter, r *http.Request) {
-	r.ParseForm()
-	formData, err := Backend.UnsealRequest(r.Form)
-	if err != nil {
-		w.WriteHeader(403)
-		fmt.Fprintf(w, "Error: %v", err)
-		return
-	}
-
-	cmd := formData.Get("cmd")
-	json := formData.Get("args")
-	channel := formData.Get("channel")
-	scope := formData.Get("scope")
-
-	if cmd == "" {
-		w.WriteHeader(422)
-		fmt.Fprintf(w, "Error: cmd cannot be blank")
-		return
-	}
-	if channel == "" && scope != "global" {
-		w.WriteHeader(422)
-		fmt.Fprintf(w, "Error: channel must be specified")
-		return
-	}
-
-	cm := ClientMessage{MessageID: -1, Command: CommandPool.InternCommand(cmd), origArguments: json}
-	cm.parseOrigArguments()
-	var count int
-
-	switch scope {
-	default:
-		count = PublishToMultiple(strings.Split(channel, ","), cm)
-	case "global":
-		count = PublishToAll(cm)
-	}
-	fmt.Fprint(w, count)
-}
-
 // ErrForwardedFromBackend is an error returned by the backend server.
 type ErrForwardedFromBackend struct {
 	JSONError interface{}
diff --git a/socketserver/server/publisher.go b/socketserver/server/publisher.go
index 305140f9..0aa8e5f2 100644
--- a/socketserver/server/publisher.go
+++ b/socketserver/server/publisher.go
@@ -150,3 +150,45 @@ func HTTPBackendCachedPublish(w http.ResponseWriter, r *http.Request) {
 
 	w.Write([]byte(strconv.Itoa(count)))
 }
+
+// HTTPBackendUncachedPublish handles the /uncached_pub route.
+// The backend can POST here to publish a message to clients with no caching.
+// The POST arguments are `cmd`, `args`, `channel`, and `scope`.
+// If "scope" is "global", then "channel" is not used.
+func HTTPBackendUncachedPublish(w http.ResponseWriter, r *http.Request) {
+	r.ParseForm()
+	formData, err := Backend.UnsealRequest(r.Form)
+	if err != nil {
+		w.WriteHeader(403)
+		fmt.Fprintf(w, "Error: %v", err)
+		return
+	}
+
+	cmd := formData.Get("cmd")
+	json := formData.Get("args")
+	channel := formData.Get("channel")
+	scope := formData.Get("scope")
+
+	if cmd == "" {
+		w.WriteHeader(422)
+		fmt.Fprintf(w, "Error: cmd cannot be blank")
+		return
+	}
+	if channel == "" && scope != "global" {
+		w.WriteHeader(422)
+		fmt.Fprintf(w, "Error: channel must be specified")
+		return
+	}
+
+	cm := ClientMessage{MessageID: -1, Command: CommandPool.InternCommand(cmd), origArguments: json}
+	cm.parseOrigArguments()
+	var count int
+
+	switch scope {
+	default:
+		count = PublishToMultiple(strings.Split(channel, ","), cm)
+	case "global":
+		count = PublishToAll(cm)
+	}
+	fmt.Fprint(w, count)
+}

From 1dc92ed407b8229ae35c260da6ff91d926a4c440 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Fri, 8 Jul 2016 13:08:36 -0700
Subject: [PATCH 173/176] Add tests

---
 socketserver/server/handlecore.go     |  5 +-
 socketserver/server/publisher.go      | 55 ++++++++++++++++------
 socketserver/server/publisher_test.go | 66 +++++++++++++++++++++++++++
 3 files changed, 109 insertions(+), 17 deletions(-)
 create mode 100644 socketserver/server/publisher_test.go

diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 1ed4fde4..5b426c63 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -131,10 +131,11 @@ func startJanitors() {
 	loadUniqueUsers()
 
 	go authorizationJanitor()
-	go bunchCacheJanitor()
-	go pubsubJanitor()
 	go aggregateDataSender()
+	go bunchCacheJanitor()
+	go cachedMessageJanitor()
 	go commandCounter()
+	go pubsubJanitor()
 
 	go ircConnection()
 	go shutdownHandler()
diff --git a/socketserver/server/publisher.go b/socketserver/server/publisher.go
index 0aa8e5f2..d8aff0a3 100644
--- a/socketserver/server/publisher.go
+++ b/socketserver/server/publisher.go
@@ -10,7 +10,7 @@ import (
 )
 
 type LastSavedMessage struct {
-	Timestamp time.Time
+	Expires time.Time
 	Data      string
 }
 
@@ -21,6 +21,31 @@ type LastSavedMessage struct {
 var CachedLastMessages = make(map[Command]map[string]LastSavedMessage)
 var CachedLSMLock sync.RWMutex
 
+func cachedMessageJanitor() {
+	for {
+		time.Sleep(1*time.Hour)
+		cachedMessageJanitor_do()
+	}
+}
+
+func cachedMessageJanitor_do() {
+	CachedLSMLock.Lock()
+	defer CachedLSMLock.Unlock()
+
+	now := time.Now()
+
+	for cmd, chanMap := range CachedLastMessages {
+		for channel, msg := range chanMap {
+			if !msg.Expires.IsZero() && msg.Expires.Before(now) {
+				delete(chanMap, channel)
+			}
+		}
+		if len(chanMap) == 0 {
+			delete(CachedLastMessages, cmd)
+		}
+	}
+}
+
 // DumpBacklogData drops all /cached_pub data.
 func DumpBacklogData() {
 	CachedLSMLock.Lock()
@@ -74,10 +99,8 @@ type timestampArray interface {
 	GetTime(int) time.Time
 }
 
-func SaveLastMessage(which map[Command]map[string]LastSavedMessage, locker sync.Locker, cmd Command, channel string, timestamp time.Time, data string, deleting bool) {
-	locker.Lock()
-	defer locker.Unlock()
-
+// the CachedLSMLock must be held when calling this
+func saveLastMessage(cmd Command, channel string, expires time.Time, data string, deleting bool) {
 	chanMap, ok := CachedLastMessages[cmd]
 	if !ok {
 		if deleting {
@@ -90,7 +113,7 @@ func SaveLastMessage(which map[Command]map[string]LastSavedMessage, locker sync.
 	if deleting {
 		delete(chanMap, channel)
 	} else {
-		chanMap[channel] = LastSavedMessage{Timestamp: timestamp, Data: data}
+		chanMap[channel] = LastSavedMessage{Expires: expires, Data: data}
 	}
 }
 
@@ -126,24 +149,26 @@ func HTTPBackendCachedPublish(w http.ResponseWriter, r *http.Request) {
 	json := formData.Get("args")
 	channel := formData.Get("channel")
 	deleteMode := formData.Get("delete") != ""
-	timeStr := formData.Get("time")
-	timeNum, err := strconv.ParseInt(timeStr, 10, 64)
-	if err != nil {
-		w.WriteHeader(422)
-		fmt.Fprintf(w, "error parsing time: %v", err)
-		return
+	timeStr := formData.Get("expires")
+	var expires time.Time
+	if timeStr != "" {
+		timeNum, err := strconv.ParseInt(timeStr, 10, 64)
+		if err != nil {
+			w.WriteHeader(422)
+			fmt.Fprintf(w, "error parsing time: %v", err)
+			return
+		}
+		expires = time.Unix(timeNum, 0)
 	}
-	timestamp := time.Unix(timeNum, 0)
 
 	var count int
 	msg := ClientMessage{MessageID: -1, Command: cmd, origArguments: json}
 	msg.parseOrigArguments()
 
 	channels := strings.Split(channel, ",")
-	var dummyLock sync.Mutex
 	CachedLSMLock.Lock()
 	for _, channel := range channels {
-		SaveLastMessage(CachedLastMessages, &dummyLock, cmd, channel, timestamp, json, deleteMode)
+		saveLastMessage(cmd, channel, expires, json, deleteMode)
 	}
 	CachedLSMLock.Unlock()
 	count = PublishToMultiple(channels, msg)
diff --git a/socketserver/server/publisher_test.go b/socketserver/server/publisher_test.go
new file mode 100644
index 00000000..a8f62b37
--- /dev/null
+++ b/socketserver/server/publisher_test.go
@@ -0,0 +1,66 @@
+package server
+
+import (
+	"testing"
+	"time"
+)
+
+func TestExpiredCleanup(t *testing.T) {
+	const cmd = "test_command"
+	const channel = "trihex"
+	const channel2 = "twitch"
+	const channel3 = "360chrism"
+	const channel4 = "qa_partner"
+
+	DumpBacklogData()
+	defer DumpBacklogData()
+
+	var zeroTime time.Time
+	hourAgo := time.Now().Add(-1*time.Hour)
+	now := time.Now()
+	hourFromNow := time.Now().Add(1*time.Hour)
+
+	saveLastMessage(cmd, channel, hourAgo, "1", false)
+	saveLastMessage(cmd, channel2, now, "2", false)
+
+	if len(CachedLastMessages) != 1 {
+		t.Error("messages not saved")
+	}
+	if len(CachedLastMessages[cmd]) != 2{
+		t.Error("messages not saved")
+	}
+
+	time.Sleep(2*time.Millisecond)
+
+	cachedMessageJanitor_do()
+
+	if len(CachedLastMessages) != 0 {
+		t.Error("messages still present")
+	}
+
+	saveLastMessage(cmd, channel, hourAgo, "1", false)
+	saveLastMessage(cmd, channel2, now, "2", false)
+	saveLastMessage(cmd, channel3, hourFromNow, "3", false)
+	saveLastMessage(cmd, channel4, zeroTime, "4", false)
+
+	if len(CachedLastMessages[cmd]) != 4 {
+		t.Error("messages not saved")
+	}
+
+	time.Sleep(2*time.Millisecond)
+
+	cachedMessageJanitor_do()
+
+	if len(CachedLastMessages) != 1 {
+		t.Error("messages not saved")
+	}
+	if len(CachedLastMessages[cmd]) != 2 {
+		t.Error("messages not saved")
+	}
+	if CachedLastMessages[cmd][channel3].Data != "3" {
+		t.Error("saved wrong message")
+	}
+	if CachedLastMessages[cmd][channel4].Data != "4" {
+		t.Error("saved wrong message")
+	}
+}
\ No newline at end of file

From 328363ab6b78baa29e46c3f0967bb1d2cc23f88f Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Fri, 8 Jul 2016 14:20:35 -0700
Subject: [PATCH 174/176] update comments

---
 socketserver/server/publisher.go      | 7 +++++--
 socketserver/server/publisher_test.go | 2 +-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/socketserver/server/publisher.go b/socketserver/server/publisher.go
index d8aff0a3..c5120156 100644
--- a/socketserver/server/publisher.go
+++ b/socketserver/server/publisher.go
@@ -134,8 +134,11 @@ func HTTPBackendDropBacklog(w http.ResponseWriter, r *http.Request) {
 
 // HTTPBackendCachedPublish handles the /cached_pub route.
 // It publishes a message to clients, and then updates the in-server cache for the message.
-// notes:
-// `scope` is implicit in the command
+//
+// The 'channel' parameter is a comma-separated list of topics to publish the message to.
+// The 'args' parameter is the JSON-encoded command data.
+// If the 'delete' parameter is present, an entry is removed from the cache instead of publishing a message.
+// If the 'expires' parameter is not specified, the message will not expire (though it is only kept in-memory).
 func HTTPBackendCachedPublish(w http.ResponseWriter, r *http.Request) {
 	r.ParseForm()
 	formData, err := Backend.UnsealRequest(r.Form)
diff --git a/socketserver/server/publisher_test.go b/socketserver/server/publisher_test.go
index a8f62b37..ad667d1b 100644
--- a/socketserver/server/publisher_test.go
+++ b/socketserver/server/publisher_test.go
@@ -63,4 +63,4 @@ func TestExpiredCleanup(t *testing.T) {
 	if CachedLastMessages[cmd][channel4].Data != "4" {
 		t.Error("saved wrong message")
 	}
-}
\ No newline at end of file
+}

From 972437cd4a4eca9dcbc02269b93cf98eee504870 Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Fri, 8 Jul 2016 19:16:10 -0700
Subject: [PATCH 175/176] Add /get_sub_count

---
 socketserver/server/handlecore.go    |  1 +
 socketserver/server/publisher.go     | 17 +++++++++++++++++
 socketserver/server/subscriptions.go | 17 +++++++++++++++++
 3 files changed, 35 insertions(+)

diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go
index 5b426c63..723f4fb2 100644
--- a/socketserver/server/handlecore.go
+++ b/socketserver/server/handlecore.go
@@ -102,6 +102,7 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 	serveMux.HandleFunc("/drop_backlog", HTTPBackendDropBacklog)
 	serveMux.HandleFunc("/uncached_pub", HTTPBackendUncachedPublish)
 	serveMux.HandleFunc("/cached_pub", HTTPBackendCachedPublish)
+	serveMux.HandleFunc("/get_sub_count", HTTPGetSubscriberCount)
 
 	announceForm, err := Backend.SealRequest(url.Values{
 		"startup": []string{"1"},
diff --git a/socketserver/server/publisher.go b/socketserver/server/publisher.go
index c5120156..85912dd6 100644
--- a/socketserver/server/publisher.go
+++ b/socketserver/server/publisher.go
@@ -220,3 +220,20 @@ func HTTPBackendUncachedPublish(w http.ResponseWriter, r *http.Request) {
 	}
 	fmt.Fprint(w, count)
 }
+
+// HTTPGetSubscriberCount handles the /get_sub_count route.
+// It replies with the number of clients subscribed to a pub/sub topic.
+// A "global" option is not available, use fetch(/stats).CurrentClientCount instead.
+func HTTPGetSubscriberCount(w http.ResponseWriter, r *http.Request) {
+	r.ParseForm()
+	formData, err := Backend.UnsealRequest(r.Form)
+	if err != nil {
+		w.WriteHeader(403)
+		fmt.Fprintf(w, "Error: %v", err)
+		return
+	}
+
+	channel := formData.Get("channel")
+
+	fmt.Fprint(w, CountSubscriptions(strings.Split(channel, ",")))
+}
\ No newline at end of file
diff --git a/socketserver/server/subscriptions.go b/socketserver/server/subscriptions.go
index e55db08d..30bc4112 100644
--- a/socketserver/server/subscriptions.go
+++ b/socketserver/server/subscriptions.go
@@ -19,6 +19,23 @@ var ChatSubscriptionLock sync.RWMutex
 var GlobalSubscriptionInfo []*ClientInfo
 var GlobalSubscriptionLock sync.RWMutex
 
+func CountSubscriptions(channels []string) int {
+	ChatSubscriptionLock.RLock()
+	defer ChatSubscriptionLock.RUnlock()
+
+	count := 0
+	for _, channelName := range channels {
+		list := ChatSubscriptionInfo[channelName]
+		if list != nil {
+			list.RLock()
+			count += len(list.Members)
+			list.RUnlock()
+		}
+	}
+
+	return count
+}
+
 func SubscribeChannel(client *ClientInfo, channelName string) {
 	ChatSubscriptionLock.RLock()
 	_subscribeWhileRlocked(channelName, client.MessageChannel)

From 9d1b3e926d0bb529cf425bbe17dd3796e0ee717b Mon Sep 17 00:00:00 2001
From: Kane York <rikingcoding@gmail.com>
Date: Sat, 16 Jul 2016 15:29:12 -0700
Subject: [PATCH 176/176] stop panicing earlier than needed

---
 socketserver/server/usercount.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/socketserver/server/usercount.go b/socketserver/server/usercount.go
index 0bbb7940..1b9cbcbb 100644
--- a/socketserver/server/usercount.go
+++ b/socketserver/server/usercount.go
@@ -83,8 +83,6 @@ func loadHLL(at time.Time, dest *PeriodUniqueUsers) error {
 	dec := gob.NewDecoder(bytes.NewReader(fileBytes))
 	err = dec.Decode(dest.Counter)
 	if err != nil {
-		log.Panicln(err)
-
 		return err
 	}
 	return nil
@@ -160,6 +158,7 @@ func loadUniqueUsers() {
 		// file didn't finish writing
 		// errors in NewPlus are bad precisions
 		uniqueCounter.Counter, _ = hyperloglog.NewPlus(CounterPrecision)
+		log.Println("failed to load unique users data:", err)
 	} else if err != nil {
 		log.Panicln("failed to load unique users data:", err)
 	}