package util import ( "backend/Type" "backend/common" "encoding/json" "fmt" "log" "sort" "strconv" "strings" "time" ) func ScheduleDailyTask() { // Statistics(common.US_APP_ID) // 每天凌晨1点执行统计任务 go task(common.US_APP_ID) } // 1759047187 func task(AppId int) { AppConfig, _ := GetAppConfig(AppId) for { loc, err := time.LoadLocation(AppConfig.Tz) if err != nil { log.Printf("failed to load location: %v", err) return } now := time.Now().In(loc) next := now.AddDate(0, 0, 1) next = time.Date(next.Year(), next.Month(), next.Day(), 0, 5, 0, 0, loc) duration := time.Duration(next.Sub(now).Seconds()) * time.Second // 每天凌晨0:05点执行 time.Sleep(duration) // 执行统计函数 Statistics(AppId) } } func Statistics(AppId int) { log.Println("start statistics") AppConfig, err := GetAppConfig(AppId) if err != nil { log.Printf("failed to get app config: %v", err) return } LogDb := MPool.GetTopicDB(AppConfig.Topic) defer LogDb.Close() for i := 0; i <= 30; i++ { remain(-i, LogDb, AppConfig) } log.Println("end statistics") //SendInfo(LogDb) } func remain(Day int, LogDb *Db, AppConfig *Type.App) { defer func() { if err := recover(); err != nil { log.Printf("remain panic: %v", err) } }() loc, _ := time.LoadLocation(AppConfig.Tz) now, _ := GetZeroTimestamp(AppConfig.Tz, Day) Date := time.Unix(now, 0).In(loc).Format("2006-01-02") SecondRemain, _ := DayRemain(now, 1, LogDb) // 计算次留 ThirdRemain, _ := DayRemain(now, 2, LogDb) // 计算三留 SeventhRemain, _ := DayRemain(now, 6, LogDb) // 计算七留 FourteenthRemain, _ := DayRemain(now, 13, LogDb) // 计算14留 ThirtiethRemain, _ := DayRemain(now, 29, LogDb) // 计算30留 Register, _ := DayRemain(now, 0, LogDb) // 计算注册 PerOnlineTime := GetPerOnlineTime(LogDb, now) PerOrderNum := GetPerOrderNum(LogDb, now) Login := Login(now, LogDb) // 计算登录 Recharge := Recharge(now, LogDb) // 计算充值 Db := MPool.GetTopicDB(AppConfig.Topic) defer Db.Close() if Db == nil { log.Printf("failed to get mysql database") return } ext := make(map[string]interface{}) ext["PerOnlineTime"] = fmt.Sprintf("%.2f", PerOnlineTime) ext["PerOrderNum"] = fmt.Sprintf("%.2f", float64(PerOrderNum)/float64(Login)) extStr, _ := json.Marshal(ext) _, err := Db.Exec("INSERT INTO `remain` (`Date`, `SecondRemain`, `ThirdRemain`, `SeventhRemain`, `FourteenthRemain`, `ThirtiethRemain`, `Register`, `Login`, `Recharge`, `Ext`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?,?) ON DUPLICATE KEY UPDATE SecondRemain = ?, ThirdRemain = ?, SeventhRemain = ?, FourteenthRemain = ?, ThirtiethRemain = ?, Register = ?, Login = ?, Recharge = ?, Ext = ?", Date, SecondRemain, ThirdRemain, SeventhRemain, FourteenthRemain, ThirtiethRemain, Register, Login, Recharge, string(extStr), SecondRemain, ThirdRemain, SeventhRemain, FourteenthRemain, ThirtiethRemain, Register, Login, Recharge, string(extStr)) if err != nil { log.Printf("failed to insert data: %v", err) } log.Printf("remain %s success", Date) } func DayRemain(now int64, Day int, LogDb *Db) (int, error) { if LogDb == nil { log.Println("failed to get mysql database") return 0, fmt.Errorf("failed to get mysql database") } Timestamp := now + int64(Day)*86400 // log.Print("secondRemain:", Timestamp) type data struct { Uid int `db:"Uid"` } dataList := []data{} err := LogDb.Select(&dataList, "SELECT Uid FROM log_login WHERE Event = 'register' and Timestamp >= ? and Timestamp <= ?", now, now+86399) if err != nil { log.Fatalf("failed to select data: %v", err) } if Day == 0 { return len(dataList), nil } if len(dataList) == 0 { return 0, nil } UidStr := "" for _, v := range dataList { UidStr += fmt.Sprintf("%d,", v.Uid) } if len(UidStr) > 0 { UidStr = UidStr[:len(UidStr)-1] } DataList2 := []data{} Sql := fmt.Sprintf("SELECT `Uid` FROM `log_login` WHERE `Event` = 'Login_log' and `Uid` in ( %s ) and Timestamp >= ? and Timestamp <= ? group by `Uid`", UidStr) err = LogDb.Select(&DataList2, Sql, Timestamp, Timestamp+86399) if err != nil { log.Fatalf("failed to select data: %v", err) } if len(UidStr) < len(DataList2) { log.Printf("UidStr:%d, DataList2:%d", len(UidStr), len(DataList2)) } return len(DataList2), nil } func Login(Now int64, LogDb *Db) int { if LogDb == nil { log.Println("failed to get mysql database") return 0 } var Login int err := LogDb.Get(&Login, "SELECT count(distinct Uid) as count FROM log_login WHERE Event = 'Login_log' and Timestamp >= ? and Timestamp <= ?", Now, Now+86399) if err != nil { log.Fatalf("failed to get login data: %v", err) } return Login } func Recharge(Now int64, LogDb *Db) float64 { if LogDb == nil { log.Println("failed to get mysql database") return 0 } var Recharge float64 err := LogDb.Get(&Recharge, "SELECT IFNULL(Sum(Price),0.0) as sum FROM log_order WHERE `OrderId`!='gm' and Timestamp >= ? and Timestamp <= ?", Now, Now+86399) if err != nil { log.Fatalf("failed to get recharge data: %v", err) } return Recharge } func GetPerOrderNum(db *Db, Zero int64) int { var num int err := db.Get(&num, "SELECT count(`Event`) FROM log_event WHERE `Event` = 'order_finish' and Timestamp >= ? and Timestamp <= ?", Zero, Zero+86399) if err != nil { log.Printf("failed to GetPerOrderNum data: %v", err) return 0 } return num } func GetPerOnlineTime(db *Db, Zero int64) float64 { type data struct { Uid int `db:"Uid"` Timestamp int64 `db:"Timestamp"` Event string `db:"Event"` } dataList := []data{} err := db.Select(&dataList, "SELECT `Uid`, `Timestamp`, `Event` FROM log_login WHERE (`Event` = 'Login_log' or `Event` = 'Login_Out') and Timestamp >= ? and Timestamp <= ? ORDER BY Timestamp ASC", Zero, Zero+86399) if err != nil { log.Printf("failed to GetPerOnlineTime data: %v", err) return 0 } d1 := make(map[int][]*data) for _, v := range dataList { if _, ok := d1[v.Uid]; !ok { d1[v.Uid] = make([]*data, 0) } d1[v.Uid] = append(d1[v.Uid], &v) } d2 := make(map[int]int64) d3 := make(map[int][]int64) for _, v := range d1 { s := -1 LoginTime := Zero for i := 0; i < len(v); i++ { if i == 0 && v[i].Event == "Login_Out" { d2[v[i].Uid] += v[i].Timestamp - Zero - 300 d3[v[i].Uid] = append(d3[v[i].Uid], v[i].Timestamp-Zero-300) continue } if i == len(v)-1 && v[i].Event == "Login_log" { d2[v[i].Uid] += Zero + 86399 - v[i].Timestamp - 300 continue } if v[i].Event == "Login_log" && s == -1 { s = i LoginTime = v[i].Timestamp continue } if v[i].Event == "Login_Out" { d2[v[i].Uid] += v[i].Timestamp - LoginTime - 300 d3[v[i].Uid] = append(d3[v[i].Uid], v[i].Timestamp-LoginTime-300) s = -1 } } } //log.Println(d2) var sum int64 for _, v := range d2 { sum += v } if len(d2) == 0 { return 0 } return float64(sum/int64(len(d2))) / 60 } func GetOperation(App *Type.App) (*Type.Operation, error) { AppConfig, err := GetAppConfig(App.AppId) if err != nil { return nil, fmt.Errorf("failed to get app config: %v", err) } Db := MPool.GetTopicDB(AppConfig.Topic) defer Db.Close() Retain := []*Type.Retain{} StartDate, _ := GetDateStr(App.Tz, -14) EndDate, _ := GetDateStr(App.Tz, -1) err = Db.Select(&Retain, "SELECT `Date`, `Register`, `SecondRemain`, `ThirdRemain`, `SeventhRemain`, `ThirtiethRemain`, `Recharge`, `Login`, `Ext` FROM remain where `Date` >= ? and `Date` <= ? order by `Date` desc", StartDate, EndDate) if err != nil { return nil, fmt.Errorf("failed to select data: %v", err) } var Register int err = Db.Get(&Register, "SELECT count(`Uid`) as count FROM log_login WHERE Event = 'register' and Timestamp >= ?", App.OpenTime) if err != nil { return nil, fmt.Errorf("failed to select data: %v", err) } var Recharge float64 err = Db.Get(&Recharge, "SELECT IFNULL(SUM(Price), 0) as sum FROM log_order where `OrderId` != 'gm' and Timestamp >= ?", App.OpenTime) if err != nil { return nil, fmt.Errorf("failed to select data: %v", err) } InactiveTime, _ := GetZeroTimestamp(App.Tz, -8) var InactiveUsers int err = Db.Get(&InactiveUsers, "SELECT count(distinct Uid) as count from (SELECT Uid, MAX(Timestamp) as LastLogin FROM log_login WHERE Event = 'Login_log' GROUP BY Uid) as lt where lastlogin < ? and lastlogin > ?", InactiveTime, App.OpenTime) if err != nil { return nil, fmt.Errorf("failed to select data: %v", err) } var RechargeUserNum int err = Db.Get(&RechargeUserNum, "SELECT count(distinct Uid) as count FROM log_order where Timestamp >= ? ", App.OpenTime) if err != nil { return nil, fmt.Errorf("failed to select data: %v", err) } type ExtStruct struct { PerOnlineTime string `json:"PerOnlineTime"` PerOrderNum string `json:"PerOrderNum"` } for _, v := range Retain { if v.Ext == "" { continue } var d ExtStruct err := json.Unmarshal([]byte(v.Ext), &d) if err != nil { fmt.Printf("err :%s", err.Error()) continue } value, err := strconv.ParseFloat(d.PerOnlineTime, 64) if err == nil { v.PerOnlineTime = value } value, err = strconv.ParseFloat(d.PerOrderNum, 64) if err == nil { v.PerOrderNum = value } } ChurnRate := 100 * float64(InactiveUsers) / float64(Register) R := &Type.Operation{ Retain: Retain, Register: Register, Recharge: Decimal(Recharge, 2), ChurnRate: Decimal(ChurnRate, 2), RechargeUserNum: RechargeUserNum, Date: time.Now().In(time.UTC).Format("2006-01-02"), ARPU: FloatDiv(Recharge, Register, 3), } return R, nil } func GetDailyHeat(AppId int) (*Type.HeatData, error) { AppConfig, err := GetAppConfig(AppId) if err != nil { return nil, err } LogDb := MPool.GetTopicDB(AppConfig.Topic) defer LogDb.Close() if LogDb == nil { return nil, fmt.Errorf("failed to get mysql database") } now, _ := GetZeroTimestamp(AppConfig.Tz, 0) value := make([]int, 0, 24) value2 := make([]int, 0, 24) key := make([]string, 0, 24) for i := 0; i < 24; i++ { key = append(key, fmt.Sprintf("%02d:00", i)) } type r struct { Sum1 int `db:"sum1"` Sum2 int `db:"sum2"` } var result r for i := 0; i < 24; i++ { err := LogDb.Get(&result, "select count(*) as sum1, count(distinct Uid) as sum2 from log_event where Timestamp >= ? and Timestamp <= ?", now-int64(i)*3600-3600, now-int64(i)*3600) if err != nil { return nil, fmt.Errorf("failed to get heat count: %v", err) } value = append(value, result.Sum1) value2 = append(value2, result.Sum2) key[i] = fmt.Sprintf("%02d:00", GetHour(now-int64(i)*3600, AppConfig.Tz)) } if len(value) < 2 { return nil, nil } // 反转key和value for i, j := 0, len(value)-1; i < j; i, j = i+1, j-1 { key[i], key[j] = key[j], key[i] value2[i], value2[j] = value2[j], value2[i] value[i], value[j] = value[j], value[i] } return &Type.HeatData{ Key: key, Value: value, Value2: value2, }, nil } func GetLevelData(AppId int) (*Type.LevelData, error) { AppConfig, err := GetAppConfig(AppId) if err != nil { return nil, err } LogDb := MPool.GetTopicDB(AppConfig.Topic) defer LogDb.Close() if LogDb == nil { return nil, fmt.Errorf("failed to get mysql database") } type data struct { Param string `db:"param"` Uid int `db:"Uid"` Timestamp int64 `db:"Timestamp"` } dataList := []data{} err = LogDb.Select(&dataList, "SELECT `param`, `Uid`, `Timestamp` FROM log_event WHERE `Event` = 'level_up'") if err != nil { return nil, fmt.Errorf("failed to select data: %v", err) } type Users struct { Uid int `db:"Uid"` } InactiveUsers := []Users{} ZeroTimestamp, _ := GetZeroTimestamp(AppConfig.Tz, 0) err = LogDb.Select(&InactiveUsers, "SELECT Uid from (SELECT Uid, MAX(Timestamp) as LastLogin FROM log_login WHERE Event = 'Login_log' GROUP BY Uid) as lt where lastlogin < ?", ZeroTimestamp-7*86400) if err != nil { return nil, fmt.Errorf("failed to select data: %v", err) } d0 := make([]int, 0) for _, v := range InactiveUsers { d0 = append(d0, v.Uid) } d1 := make(map[int]data) for _, v := range dataList { if _, ok := d1[v.Uid]; !ok { d1[v.Uid] = v } if v.Timestamp > d1[v.Uid].Timestamp { d1[v.Uid] = v } } type data2 struct { Num int Churn int } d2 := make(map[int]*data2) for _, v := range d1 { var jd struct { NextLevel int `json:"after_level"` } err := json.Unmarshal([]byte(v.Param), &jd) if err != nil { continue } if _, ok := d2[jd.NextLevel]; !ok { d2[jd.NextLevel] = &data2{ Num: 0, Churn: 0, } } d2[jd.NextLevel].Num++ if InArray(v.Uid, d0) { d2[jd.NextLevel].Churn++ } } key := make([]int, 0, len(d2)) for k := range d2 { key = append(key, k) } sort.Slice(key, func(i, j int) bool { return key[i] < key[j] }) value := make([]int, 0, len(d2)) value2 := make([]int, 0, len(d2)) for _, v := range key { value = append(value, d2[v].Num) value2 = append(value2, d2[v].Churn) } key1 := make([]string, 0, len(d2)) for _, v := range key { key1 = append(key1, strconv.Itoa(v)) } return &Type.LevelData{ Key: key1, Value: value, Value2: value2, }, nil } func GetDecorateData(AppId int) (*Type.DecorateData, error) { AppConfig, err := GetAppConfig(AppId) if err != nil { return nil, err } LogDb := MPool.GetTopicDB(AppConfig.Topic) defer LogDb.Close() if LogDb == nil { return nil, fmt.Errorf("failed to get mysql database") } type data struct { Param string `db:"param"` Uid int `db:"Uid"` Timestamp int64 `db:"Timestamp"` } dataList := []data{} err = LogDb.Select(&dataList, "SELECT `param`, `Uid`, `Timestamp` FROM log_event WHERE `Event` = 'scene_reward'") if err != nil { return nil, fmt.Errorf("failed to select data: %v", err) } type Users struct { Uid int `db:"Uid"` } InactiveUsers := []Users{} ZeroTimestamp, _ := GetZeroTimestamp(AppConfig.Tz, 0) err = LogDb.Select(&InactiveUsers, "SELECT Uid from (SELECT Uid, MAX(Timestamp) as LastLogin FROM log_login WHERE Event = 'Login_log' GROUP BY Uid) as lt where lastlogin < ?", ZeroTimestamp-7*86400) if err != nil { return nil, fmt.Errorf("failed to select data: %v", err) } d0 := make([]int, 0) for _, v := range InactiveUsers { d0 = append(d0, v.Uid) } d1 := make(map[int]data) for _, v := range dataList { if _, ok := d1[v.Uid]; !ok { d1[v.Uid] = v } if v.Timestamp > d1[v.Uid].Timestamp { d1[v.Uid] = v } } type data2 struct { Num int Churn int } d2 := make(map[string]*data2) for _, v := range d1 { var jd struct { Step string `json:"deco_step_id"` } err := json.Unmarshal([]byte(v.Param), &jd) if err != nil { continue } if _, ok := d2[jd.Step]; !ok { d2[jd.Step] = &data2{ Num: 0, Churn: 0, } } d2[jd.Step].Num++ if InArray(v.Uid, d0) { d2[jd.Step].Churn++ } } key := make([]string, 0, len(d2)) for k := range d2 { key = append(key, k) } sort.Slice(key, func(i, j int) bool { a := strings.Split(key[i], "_") b := strings.Split(key[j], "_") aa1, _ := strconv.Atoi(a[0]) aa2, _ := strconv.Atoi(a[1]) bb1, _ := strconv.Atoi(b[0]) bb2, _ := strconv.Atoi(b[1]) if aa1 == bb1 { return aa2 < bb2 } return aa1 < bb1 }) value := make([]int, 0, len(d2)) value2 := make([]int, 0, len(d2)) for _, v := range key { value = append(value, d2[v].Num) value2 = append(value2, d2[v].Churn) } return &Type.DecorateData{ Key: key, Value: value, Value2: value2, }, nil } func GetOrderData(AppId int) (interface{}, error) { AppConfig, err := GetAppConfig(AppId) if err != nil { return nil, err } LogDb := MPool.GetTopicDB(AppConfig.Topic) defer LogDb.Close() if LogDb == nil { return nil, fmt.Errorf("failed to get mysql database") } type data struct { Param string `db:"param"` Uid int `db:"Uid"` Timestamp int64 `db:"Timestamp"` } dataList := []data{} err = LogDb.Select(&dataList, "SELECT `param`, `Uid`, `Timestamp` FROM log_event WHERE `Event` = 'order_finish'") if err != nil { return nil, fmt.Errorf("failed to select data: %v", err) } type Users struct { Uid int `db:"Uid"` } InactiveUsers := []Users{} ZeroTimestamp, _ := GetZeroTimestamp(AppConfig.Tz, 0) err = LogDb.Select(&InactiveUsers, "SELECT Uid from (SELECT Uid, MAX(Timestamp) as LastLogin FROM log_login WHERE Event = 'Login_log' GROUP BY Uid) as lt where lastlogin < ?", ZeroTimestamp-7*86400) if err != nil { return nil, fmt.Errorf("failed to select data: %v", err) } d0 := make([]int, 0) for _, v := range InactiveUsers { d0 = append(d0, v.Uid) } d1 := make(map[int]data) for _, v := range dataList { if _, ok := d1[v.Uid]; !ok { d1[v.Uid] = v } if v.Timestamp > d1[v.Uid].Timestamp { d1[v.Uid] = v } } type data2 struct { Num int Churn int } d2 := make(map[int]*data2) for _, v := range d1 { var jd struct { Step int `json:"order_id"` } err := json.Unmarshal([]byte(v.Param), &jd) if err != nil { continue } if _, ok := d2[jd.Step]; !ok { d2[jd.Step] = &data2{ Num: 0, Churn: 0, } } d2[jd.Step].Num++ if InArray(v.Uid, d0) { d2[jd.Step].Churn++ } } key := make([]int, 0, len(d2)) for k := range d2 { key = append(key, k) } sort.Slice(key, func(i, j int) bool { return i < j }) value := make([]int, 0, len(d2)) value2 := make([]int, 0, len(d2)) for _, v := range key { value = append(value, d2[v].Num) value2 = append(value2, d2[v].Churn) } for i := 1; i < 30; i++ { _, ok := d2[i] if !ok { continue } fmt.Printf("key:%d, value:%.2f%%, value2:%d\n", i+1, float64(d2[i].Num)/13, d2[i].Churn) } return map[string]interface{}{ "Key": key, "Value": value, "Value2": value2, }, nil }