diff --git a/socketserver/server/backend.go b/socketserver/server/backend.go index 5bb4d960..0294325c 100644 --- a/socketserver/server/backend.go +++ b/socketserver/server/backend.go @@ -37,7 +37,11 @@ var serverID int var messageBufferPool sync.Pool +var lastBackendSuccess map[string]time.Time + func setupBackend(config *ConfigFile) { + serverID = config.ServerID + backendHTTPClient.Timeout = 60 * time.Second backendURL = config.BackendURL if responseCache != nil { @@ -48,13 +52,20 @@ func setupBackend(config *ConfigFile) { announceStartupURL = fmt.Sprintf("%s%s", backendURL, bPathAnnounceStartup) addTopicURL = fmt.Sprintf("%s%s", backendURL, bPathAddTopic) postStatisticsURL = fmt.Sprintf("%s%s", backendURL, bPathAggStats) + epochTime := time.Unix(0, 0) + lastBackendSuccess = map[string]time.Time{ + bPathAnnounceStartup: epochTime, + bPathAddTopic: epochTime, + bPathAggStats: epochTime, + bPathOtherCommand: epochTime, + } + Statistics.Health.Backend = lastBackendSuccess messageBufferPool.New = New4KByteBuffer var theirPublic, ourPrivate [32]byte copy(theirPublic[:], config.BackendPublicKey) copy(ourPrivate[:], config.OurPrivateKey) - serverID = config.ServerID box.Precompute(&backendSharedKey, &theirPublic, &ourPrivate) } @@ -197,6 +208,8 @@ func SendRemoteCommand(remoteCommand, data string, auth AuthInfo) (responseStr s responseCache.Set(getCacheKey(remoteCommand, data), responseStr, duration) } + lastBackendSuccess[bPathOtherCommand] = time.Now() + return } @@ -211,6 +224,8 @@ func SendAggregatedData(sealedForm url.Values) error { return httpError(resp.StatusCode) } + lastBackendSuccess[bPathAggStats] = time.Now() + return resp.Body.Close() } @@ -271,6 +286,8 @@ func sendTopicNotice(topic string, added bool) error { return ErrBackendNotOK{Code: resp.StatusCode, Response: respStr} } + lastBackendSuccess[bPathAddTopic] = time.Now() + return nil } diff --git a/socketserver/server/handlecore.go b/socketserver/server/handlecore.go index 0bd23a82..aefd9e9e 100644 --- a/socketserver/server/handlecore.go +++ b/socketserver/server/handlecore.go @@ -94,6 +94,7 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) { serveMux.HandleFunc("/", HTTPHandleRootURL) serveMux.Handle("/.well-known/", http.FileServer(http.Dir("/tmp/letsencrypt/"))) + serveMux.HandleFunc("/healthcheck", HTTPSayOK) serveMux.HandleFunc("/stats", HTTPShowStatistics) serveMux.HandleFunc("/hll/", HTTPShowHLL) serveMux.HandleFunc("/hll_force_write", HTTPWriteHLL) @@ -113,6 +114,7 @@ func SetupServerAndHandle(config *ConfigFile, serveMux *http.ServeMux) { log.Println("could not announce startup to backend:", err) } else { resp.Body.Close() + lastBackendSuccess[bPathAnnounceStartup] = time.Now() } if Configuration.UseESLogStashing { @@ -167,7 +169,7 @@ func shutdownHandler() { func dumpStackOnCtrlZ() { ch := make(chan os.Signal) signal.Notify(ch, syscall.SIGTSTP) - for _ = range ch { + for range ch { fmt.Println("Got ^Z") buf := make([]byte, 10000) @@ -176,6 +178,13 @@ func dumpStackOnCtrlZ() { } } +// HTTPSayOK replies with 200 and a body of "ok\n". +func HTTPSayOK(w http.ResponseWriter, _ *http.Request) { + w.(interface { + WriteString(string) error + }).WriteString("ok\n") +} + // SocketUpgrader is the websocket.Upgrader currently in use. var SocketUpgrader = websocket.Upgrader{ ReadBufferSize: 1024, diff --git a/socketserver/server/irc.go b/socketserver/server/irc.go index c0af288c..a5a837dc 100644 --- a/socketserver/server/irc.go +++ b/socketserver/server/irc.go @@ -88,15 +88,38 @@ const AuthChannelName = "frankerfacezauthorizer" const AuthChannel = "#" + AuthChannelName const AuthCommand = "AUTH" +var authIrcConnection *irc.Conn + // is_init_func func ircConnection() { - c := irc.SimpleClient("justinfan123") + c.Config().Server = "irc.chat.twitch.tv" + authIrcConnection = c + + var reconnect func(conn *irc.Conn) + connect := func(conn *irc.Conn) { + err := c.Connect() + if err != nil { + log.Println("irc: failed to connect to IRC:", err) + go reconnect(conn) + } + } + + reconnect = func(conn *irc.Conn) { + time.Sleep(5 * time.Second) + log.Println("irc: Reconnecting…") + connect(conn) + } c.HandleFunc(irc.CONNECTED, func(conn *irc.Conn, line *irc.Line) { conn.Join(AuthChannel) }) + c.HandleFunc(irc.DISCONNECTED, func(conn *irc.Conn, line *irc.Line) { + log.Println("irc: Disconnected. Reconnecting in 5 seconds.") + go reconnect(conn) + }) + c.HandleFunc(irc.PRIVMSG, func(conn *irc.Conn, line *irc.Line) { channel := line.Args[0] msg := line.Args[1] @@ -115,11 +138,7 @@ func ircConnection() { submitAuth(submittedUser, submittedChallenge) }) - err := c.ConnectTo("irc.chat.twitch.tv") - if err != nil { - log.Fatalln("Cannot connect to IRC:", err) - } - + connect(c) } func submitAuth(user, challenge string) { diff --git a/socketserver/server/stats.go b/socketserver/server/stats.go index 7a32ecab..a330297d 100644 --- a/socketserver/server/stats.go +++ b/socketserver/server/stats.go @@ -21,6 +21,11 @@ type StatsData struct { CachedStatsLastUpdate time.Time + Health struct { + IRC bool + Backend map[string]time.Time + } + CurrentClientCount uint64 PubSubChannelCount int @@ -76,7 +81,7 @@ func commandCounter() { } // StatsDataVersion is the version of the StatsData struct. -const StatsDataVersion = 5 +const StatsDataVersion = 6 const pageSize = 4096 var cpuUsage struct { @@ -154,6 +159,10 @@ func updatePeriodicStats() { { Statistics.Uptime = nowUpdate.Sub(Statistics.StartTime).String() } + + { + Statistics.Health.IRC = authIrcConnection.Connected() + } } var sysMemLastUpdate time.Time