diff --git a/socketserver/cmd/mergecounts/mergecounts.go b/socketserver/cmd/mergecounts/mergecounts.go new file mode 100644 index 00000000..cb503c08 --- /dev/null +++ b/socketserver/cmd/mergecounts/mergecounts.go @@ -0,0 +1,78 @@ +package main + +import ( + "github.com/clarkduvall/hyperloglog" + "flag" + "fmt" + "../../server" + "net/http" + "encoding/gob" + "os" +) + +var SERVERS = []string{ + "https://catbag.frankerfacez.com", + "https://andknuckles.frankerfacez.com", + "https://tuturu.frankerfacez.com", +} + +const folderPrefix = "/hll/" + +const HELP = ` +Usage: mergecounts [filename] + +Downloads the file /hll/filename from the 3 FFZ socket servers, merges the contents, and prints the total cardinality. + +Filename should be in one of the following formats: + + daily-25-12-2015.gob + weekly-51-2015.gob + monthly-12-2015.gob +` + +func main() { + flag.Parse() + if flag.NArg() < 1 { + fmt.Print(HELP) + os.Exit(2) + return + } + + filename := flag.Arg(1) + hll, err := DownloadAll(filename) + if err != nil { + fmt.Println(err) + os.Exit(1) + return + } + + fmt.Println(hll.Count()) +} + +func DownloadAll(filename string) (*hyperloglog.HyperLogLogPlus, error) { + result, _ := hyperloglog.NewPlus(server.CounterPrecision) + + for _, server := range SERVERS { + singleHLL, err := DownloadHLL(fmt.Sprintf("%s%s%s", server, folderPrefix, filename)) + if err != nil { + return nil, err + } + result.Merge(singleHLL) + } + + return result, nil +} + +func DownloadHLL(url string) (*hyperloglog.HyperLogLogPlus, error) { + result, _ := hyperloglog.NewPlus(server.CounterPrecision) + + resp, err := http.Get(url) + if err != nil { + return nil, err + } + dec := gob.NewDecoder(resp.Body) + dec.Decode(result) + resp.Body.Close() + + return result, nil +} diff --git a/socketserver/server/usercount.go b/socketserver/server/usercount.go index 44ad68e3..ef9522c9 100644 --- a/socketserver/server/usercount.go +++ b/socketserver/server/usercount.go @@ -46,7 +46,7 @@ const uniqCountDir = "./uniques" const usersDailyFmt = "daily-%d-%d-%d.gob" // d-m-y const usersWeeklyFmt = "weekly-%d-%d.gob" // w-y const usersMonthlyFmt = "monthly-%d-%d.gob" // m-y -const counterPrecision uint8 = 12 +const CounterPrecision uint8 = 12 var uniqueCounters [3]PeriodUniqueUsers var uniqueUserChannel chan uuid.UUID @@ -100,7 +100,7 @@ func loadHLL(which int, at time.Time, dest *PeriodUniqueUsers) error { } if dest.Counter == nil { - dest.Counter, _ = hyperloglog.NewPlus(counterPrecision) + dest.Counter, _ = hyperloglog.NewPlus(CounterPrecision) } dec := gob.NewDecoder(bytes.NewReader(fileBytes)) @@ -184,7 +184,7 @@ func loadUniqueUsers() { err := loadHLL(period, now, &uniqueCounters[period]) if err != nil && os.IsNotExist(err) { // errors are bad precisions - uniqueCounters[period].Counter, _ = hyperloglog.NewPlus(counterPrecision) + uniqueCounters[period].Counter, _ = hyperloglog.NewPlus(CounterPrecision) } else if err != nil && !os.IsNotExist(err) { log.Panicln("failed to load unique users data:", err) } @@ -269,7 +269,7 @@ func rolloverCounters_do() { uniqueCounters[period].Start, uniqueCounters[period].End = getCounterPeriod(period, now) // errors are bad precisions, so we can ignore - uniqueCounters[period].Counter, _ = hyperloglog.NewPlus(counterPrecision) + uniqueCounters[period].Counter, _ = hyperloglog.NewPlus(CounterPrecision) } }