Add Health to /stats, add irc reconnect

socketserver/server/backend.go

@@ -37,7 +37,11 @@ var serverID int
 
 var messageBufferPool sync.Pool
 
+var lastBackendSuccess map[string]time.Time
+
 func setupBackend(config *ConfigFile) {
+	serverID = config.ServerID
+
 	backendHTTPClient.Timeout = 60 * time.Second
 	backendURL = config.BackendURL
 	if responseCache != nil {
@@ -48,13 +52,20 @@ func setupBackend(config *ConfigFile) {
 	announceStartupURL = fmt.Sprintf("%s%s", backendURL, bPathAnnounceStartup)
 	addTopicURL = fmt.Sprintf("%s%s", backendURL, bPathAddTopic)
 	postStatisticsURL = fmt.Sprintf("%s%s", backendURL, bPathAggStats)
+	epochTime := time.Unix(0, 0)
+	lastBackendSuccess = map[string]time.Time{
+		bPathAnnounceStartup: epochTime,
+		bPathAddTopic:        epochTime,
+		bPathAggStats:        epochTime,
+		bPathOtherCommand:    epochTime,
+	}
+	Statistics.Health.Backend = lastBackendSuccess
 
 	messageBufferPool.New = New4KByteBuffer
 
 	var theirPublic, ourPrivate [32]byte
 	copy(theirPublic[:], config.BackendPublicKey)
 	copy(ourPrivate[:], config.OurPrivateKey)
-	serverID = config.ServerID
 
 	box.Precompute(&backendSharedKey, &theirPublic, &ourPrivate)
 }
@@ -197,6 +208,8 @@ func SendRemoteCommand(remoteCommand, data string, auth AuthInfo) (responseStr s
 		responseCache.Set(getCacheKey(remoteCommand, data), responseStr, duration)
 	}
 
+	lastBackendSuccess[bPathOtherCommand] = time.Now()
+
 	return
 }
 
@@ -211,6 +224,8 @@ func SendAggregatedData(sealedForm url.Values) error {
 		return httpError(resp.StatusCode)
 	}
 
+	lastBackendSuccess[bPathAggStats] = time.Now()
+
 	return resp.Body.Close()
 }
 
@@ -271,6 +286,8 @@ func sendTopicNotice(topic string, added bool) error {
 		return ErrBackendNotOK{Code: resp.StatusCode, Response: respStr}
 	}
 
+	lastBackendSuccess[bPathAddTopic] = time.Now()
+
 	return nil
 }
 

socketserver/server/handlecore.go

@@ -94,6 +94,7 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 
 	serveMux.HandleFunc("/", HTTPHandleRootURL)
 	serveMux.Handle("/.well-known/", http.FileServer(http.Dir("/tmp/letsencrypt/")))
+	serveMux.HandleFunc("/healthcheck", HTTPSayOK)
 	serveMux.HandleFunc("/stats", HTTPShowStatistics)
 	serveMux.HandleFunc("/hll/", HTTPShowHLL)
 	serveMux.HandleFunc("/hll_force_write", HTTPWriteHLL)
@@ -113,6 +114,7 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) {
 		log.Println("could not announce startup to backend:", err)
 	} else {
 		resp.Body.Close()
+		lastBackendSuccess[bPathAnnounceStartup] = time.Now()
 	}
 
 	if Configuration.UseESLogStashing {
@@ -167,7 +169,7 @@ func shutdownHandler() {
 func dumpStackOnCtrlZ() {
 	ch := make(chan os.Signal)
 	signal.Notify(ch, syscall.SIGTSTP)
-	for _ = range ch {
+	for range ch {
 		fmt.Println("Got ^Z")
 
 		buf := make([]byte, 10000)
@@ -176,6 +178,13 @@ func dumpStackOnCtrlZ() {
 	}
 }
 
+// HTTPSayOK replies with 200 and a body of "ok\n".
+func HTTPSayOK(w http.ResponseWriter, _ *http.Request) {
+	w.(interface {
+		WriteString(string) error
+	}).WriteString("ok\n")
+}
+
 // SocketUpgrader is the websocket.Upgrader currently in use.
 var SocketUpgrader = websocket.Upgrader{
 	ReadBufferSize:  1024,

socketserver/server/irc.go

@@ -88,15 +88,38 @@ const AuthChannelName = "frankerfacezauthorizer"
 const AuthChannel = "#" + AuthChannelName
 const AuthCommand = "AUTH"
 
+var authIrcConnection *irc.Conn
+
 // is_init_func
 func ircConnection() {
-
 	c := irc.SimpleClient("justinfan123")
+	c.Config().Server = "irc.chat.twitch.tv"
+	authIrcConnection = c
+
+	var reconnect func(conn *irc.Conn)
+	connect := func(conn *irc.Conn) {
+		err := c.Connect()
+		if err != nil {
+			log.Println("irc: failed to connect to IRC:", err)
+			go reconnect(conn)
+		}
+	}
+
+	reconnect = func(conn *irc.Conn) {
+		time.Sleep(5 * time.Second)
+		log.Println("irc: Reconnecting…")
+		connect(conn)
+	}
 
 	c.HandleFunc(irc.CONNECTED, func(conn *irc.Conn, line *irc.Line) {
 		conn.Join(AuthChannel)
 	})
 
+	c.HandleFunc(irc.DISCONNECTED, func(conn *irc.Conn, line *irc.Line) {
+		log.Println("irc: Disconnected. Reconnecting in 5 seconds.")
+		go reconnect(conn)
+	})
+
 	c.HandleFunc(irc.PRIVMSG, func(conn *irc.Conn, line *irc.Line) {
 		channel := line.Args[0]
 		msg := line.Args[1]
@@ -115,11 +138,7 @@ func ircConnection() {
 		submitAuth(submittedUser, submittedChallenge)
 	})
 
-	err := c.ConnectTo("irc.chat.twitch.tv")
+	connect(c)
-	if err != nil {
-		log.Fatalln("Cannot connect to IRC:", err)
-	}
-
 }
 
 func submitAuth(user, challenge string) {

socketserver/server/stats.go

@@ -21,6 +21,11 @@ type StatsData struct {
 
 	CachedStatsLastUpdate time.Time
 
+	Health struct {
+		IRC     bool
+		Backend map[string]time.Time
+	}
+
 	CurrentClientCount uint64
 
 	PubSubChannelCount int
@@ -76,7 +81,7 @@ func commandCounter() {
 }
 
 // StatsDataVersion is the version of the StatsData struct.
-const StatsDataVersion = 5
+const StatsDataVersion = 6
 const pageSize = 4096
 
 var cpuUsage struct {
@@ -154,6 +159,10 @@ func updatePeriodicStats() {
 	{
 		Statistics.Uptime = nowUpdate.Sub(Statistics.StartTime).String()
 	}
+
+	{
+		Statistics.Health.IRC = authIrcConnection.Connected()
+	}
 }
 
 var sysMemLastUpdate time.Time