Commit script to merge HLL counts

socketserver/cmd/mergecounts/mergecounts.go

@@ -0,0 +1,78 @@
+package main
+
+import (
+	"github.com/clarkduvall/hyperloglog"
+	"flag"
+	"fmt"
+	"../../server"
+	"net/http"
+	"encoding/gob"
+	"os"
+)
+
+var SERVERS = []string{
+	"https://catbag.frankerfacez.com",
+	"https://andknuckles.frankerfacez.com",
+	"https://tuturu.frankerfacez.com",
+}
+
+const folderPrefix = "/hll/"
+
+const HELP = `
+Usage: mergecounts [filename]
+
+Downloads the file /hll/filename from the 3 FFZ socket servers, merges the contents, and prints the total cardinality.
+
+Filename should be in one of the following formats:
+
+  daily-25-12-2015.gob
+  weekly-51-2015.gob
+  monthly-12-2015.gob
+`
+
+func main() {
+	flag.Parse()
+	if flag.NArg() < 1 {
+		fmt.Print(HELP)
+		os.Exit(2)
+		return
+	}
+
+	filename := flag.Arg(1)
+	hll, err := DownloadAll(filename)
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+		return
+	}
+
+	fmt.Println(hll.Count())
+}
+
+func DownloadAll(filename string) (*hyperloglog.HyperLogLogPlus, error) {
+	result, _ := hyperloglog.NewPlus(server.CounterPrecision)
+
+	for _, server := range SERVERS {
+		singleHLL, err := DownloadHLL(fmt.Sprintf("%s%s%s", server, folderPrefix, filename))
+		if err != nil {
+			return nil, err
+		}
+		result.Merge(singleHLL)
+	}
+
+	return result, nil
+}
+
+func DownloadHLL(url string) (*hyperloglog.HyperLogLogPlus, error) {
+	result, _ := hyperloglog.NewPlus(server.CounterPrecision)
+
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	dec := gob.NewDecoder(resp.Body)
+	dec.Decode(result)
+	resp.Body.Close()
+
+	return result, nil
+}

socketserver/server/usercount.go

@@ -46,7 +46,7 @@ const uniqCountDir = "./uniques"
 const usersDailyFmt = "daily-%d-%d-%d.gob"  // d-m-y
 const usersWeeklyFmt = "weekly-%d-%d.gob"   // w-y
 const usersMonthlyFmt = "monthly-%d-%d.gob" // m-y
-const counterPrecision uint8 = 12
+const CounterPrecision uint8 = 12
 
 var uniqueCounters [3]PeriodUniqueUsers
 var uniqueUserChannel chan uuid.UUID
@@ -100,7 +100,7 @@ func loadHLL(which int, at time.Time, dest *PeriodUniqueUsers) error {
 	}
 
 	if dest.Counter == nil {
-		dest.Counter, _ = hyperloglog.NewPlus(counterPrecision)
+		dest.Counter, _ = hyperloglog.NewPlus(CounterPrecision)
 	}
 
 	dec := gob.NewDecoder(bytes.NewReader(fileBytes))
@@ -184,7 +184,7 @@ func loadUniqueUsers() {
 		err := loadHLL(period, now, &uniqueCounters[period])
 		if err != nil && os.IsNotExist(err) {
 			// errors are bad precisions
-			uniqueCounters[period].Counter, _ = hyperloglog.NewPlus(counterPrecision)
+			uniqueCounters[period].Counter, _ = hyperloglog.NewPlus(CounterPrecision)
 		} else if err != nil && !os.IsNotExist(err) {
 			log.Panicln("failed to load unique users data:", err)
 		}
@@ -269,7 +269,7 @@ func rolloverCounters_do() {
 
 			uniqueCounters[period].Start, uniqueCounters[period].End = getCounterPeriod(period, now)
 			// errors are bad precisions, so we can ignore
-			uniqueCounters[period].Counter, _ = hyperloglog.NewPlus(counterPrecision)
+			uniqueCounters[period].Counter, _ = hyperloglog.NewPlus(CounterPrecision)
 		}
 	}