admin_backend/util/statistics.go
2026-01-14 10:33:55 +08:00

627 lines
17 KiB
Go

package util
import (
"backend/Type"
"backend/common"
"encoding/json"
"fmt"
"log"
"sort"
"strconv"
"strings"
"time"
)
func ScheduleDailyTask() {
// Statistics(common.US_APP_ID)
// 每天凌晨1点执行统计任务
go task(common.US_APP_ID)
}
// 1759047187
func task(AppId int) {
AppConfig, _ := GetAppConfig(AppId)
for {
loc, err := time.LoadLocation(AppConfig.Tz)
if err != nil {
log.Printf("failed to load location: %v", err)
return
}
now := time.Now().In(loc)
next := now.AddDate(0, 0, 1)
next = time.Date(next.Year(), next.Month(), next.Day(), 0, 5, 0, 0, loc)
duration := time.Duration(next.Sub(now).Seconds()) * time.Second // 每天凌晨0:05点执行
time.Sleep(duration)
// 执行统计函数
Statistics(AppId)
}
}
func Statistics(AppId int) {
log.Println("start statistics")
AppConfig, err := GetAppConfig(AppId)
if err != nil {
log.Printf("failed to get app config: %v", err)
return
}
LogDb := MPool.GetTopicDB(AppConfig.Topic)
defer LogDb.Close()
for i := 0; i <= 30; i++ {
remain(-i, LogDb, AppConfig)
}
log.Println("end statistics")
//SendInfo(LogDb)
}
func remain(Day int, LogDb *Db, AppConfig *Type.App) {
defer func() {
if err := recover(); err != nil {
log.Printf("remain panic: %v", err)
}
}()
loc, _ := time.LoadLocation(AppConfig.Tz)
now, _ := GetZeroTimestamp(AppConfig.Tz, Day)
Date := time.Unix(now, 0).In(loc).Format("2006-01-02")
SecondRemain, _ := DayRemain(now, 1, LogDb) // 计算次留
ThirdRemain, _ := DayRemain(now, 2, LogDb) // 计算三留
SeventhRemain, _ := DayRemain(now, 6, LogDb) // 计算七留
FourteenthRemain, _ := DayRemain(now, 13, LogDb) // 计算14留
ThirtiethRemain, _ := DayRemain(now, 29, LogDb) // 计算30留
Register, _ := DayRemain(now, 0, LogDb) // 计算注册
PerOnlineTime := GetPerOnlineTime(LogDb, now)
PerOrderNum := GetPerOrderNum(LogDb, now)
Login := Login(now, LogDb) // 计算登录
Recharge := Recharge(now, LogDb) // 计算充值
Db := MPool.GetTopicDB(AppConfig.Topic)
defer Db.Close()
if Db == nil {
log.Printf("failed to get mysql database")
return
}
ext := make(map[string]interface{})
ext["PerOnlineTime"] = fmt.Sprintf("%.2f", PerOnlineTime)
ext["PerOrderNum"] = fmt.Sprintf("%.2f", float64(PerOrderNum)/float64(Login))
extStr, _ := json.Marshal(ext)
_, err := Db.Exec("INSERT INTO `remain` (`Date`, `SecondRemain`, `ThirdRemain`, `SeventhRemain`, `FourteenthRemain`, `ThirtiethRemain`, `Register`, `Login`, `Recharge`, `Ext`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?,?) ON DUPLICATE KEY UPDATE SecondRemain = ?, ThirdRemain = ?, SeventhRemain = ?, FourteenthRemain = ?, ThirtiethRemain = ?, Register = ?, Login = ?, Recharge = ?, Ext = ?", Date, SecondRemain, ThirdRemain, SeventhRemain, FourteenthRemain, ThirtiethRemain, Register, Login, Recharge, string(extStr), SecondRemain, ThirdRemain, SeventhRemain, FourteenthRemain, ThirtiethRemain, Register, Login, Recharge, string(extStr))
if err != nil {
log.Printf("failed to insert data: %v", err)
}
log.Printf("remain %s success", Date)
}
func DayRemain(now int64, Day int, LogDb *Db) (int, error) {
if LogDb == nil {
log.Println("failed to get mysql database")
return 0, fmt.Errorf("failed to get mysql database")
}
Timestamp := now + int64(Day)*86400
// log.Print("secondRemain:", Timestamp)
type data struct {
Uid int `db:"Uid"`
}
dataList := []data{}
err := LogDb.Select(&dataList, "SELECT Uid FROM log_login WHERE Event = 'register' and Timestamp >= ? and Timestamp <= ?", now, now+86399)
if err != nil {
log.Fatalf("failed to select data: %v", err)
}
if Day == 0 {
return len(dataList), nil
}
if len(dataList) == 0 {
return 0, nil
}
UidStr := ""
for _, v := range dataList {
UidStr += fmt.Sprintf("%d,", v.Uid)
}
if len(UidStr) > 0 {
UidStr = UidStr[:len(UidStr)-1]
}
DataList2 := []data{}
Sql := fmt.Sprintf("SELECT `Uid` FROM `log_login` WHERE `Event` = 'Login_log' and `Uid` in ( %s ) and Timestamp >= ? and Timestamp <= ? group by `Uid`", UidStr)
err = LogDb.Select(&DataList2, Sql, Timestamp, Timestamp+86399)
if err != nil {
log.Fatalf("failed to select data: %v", err)
}
if len(UidStr) < len(DataList2) {
log.Printf("UidStr:%d, DataList2:%d", len(UidStr), len(DataList2))
}
return len(DataList2), nil
}
func Login(Now int64, LogDb *Db) int {
if LogDb == nil {
log.Println("failed to get mysql database")
return 0
}
var Login int
err := LogDb.Get(&Login, "SELECT count(distinct Uid) as count FROM log_login WHERE Event = 'Login_log' and Timestamp >= ? and Timestamp <= ?", Now, Now+86399)
if err != nil {
log.Fatalf("failed to get login data: %v", err)
}
return Login
}
func Recharge(Now int64, LogDb *Db) float64 {
if LogDb == nil {
log.Println("failed to get mysql database")
return 0
}
var Recharge float64
err := LogDb.Get(&Recharge, "SELECT IFNULL(Sum(Price),0.0) as sum FROM log_order WHERE `OrderId`!='gm' and Timestamp >= ? and Timestamp <= ?", Now, Now+86399)
if err != nil {
log.Fatalf("failed to get recharge data: %v", err)
}
return Recharge
}
func GetPerOrderNum(db *Db, Zero int64) int {
var num int
err := db.Get(&num, "SELECT count(`Event`) FROM log_event WHERE `Event` = 'order_finish' and Timestamp >= ? and Timestamp <= ?", Zero, Zero+86399)
if err != nil {
log.Printf("failed to GetPerOrderNum data: %v", err)
return 0
}
return num
}
func GetPerOnlineTime(db *Db, Zero int64) float64 {
type data struct {
Uid int `db:"Uid"`
Timestamp int64 `db:"Timestamp"`
Event string `db:"Event"`
}
dataList := []data{}
err := db.Select(&dataList, "SELECT `Uid`, `Timestamp`, `Event` FROM log_login WHERE (`Event` = 'Login_log' or `Event` = 'Login_Out') and Timestamp >= ? and Timestamp <= ? ORDER BY Timestamp ASC", Zero, Zero+86399)
if err != nil {
log.Printf("failed to GetPerOnlineTime data: %v", err)
return 0
}
d1 := make(map[int][]*data)
for _, v := range dataList {
if _, ok := d1[v.Uid]; !ok {
d1[v.Uid] = make([]*data, 0)
}
d1[v.Uid] = append(d1[v.Uid], &v)
}
d2 := make(map[int]int64)
d3 := make(map[int][]int64)
for _, v := range d1 {
s := -1
LoginTime := Zero
for i := 0; i < len(v); i++ {
if i == 0 && v[i].Event == "Login_Out" {
d2[v[i].Uid] += v[i].Timestamp - Zero - 300
d3[v[i].Uid] = append(d3[v[i].Uid], v[i].Timestamp-Zero-300)
continue
}
if i == len(v)-1 && v[i].Event == "Login_log" {
d2[v[i].Uid] += Zero + 86399 - v[i].Timestamp - 300
continue
}
if v[i].Event == "Login_log" && s == -1 {
s = i
LoginTime = v[i].Timestamp
continue
}
if v[i].Event == "Login_Out" {
d2[v[i].Uid] += v[i].Timestamp - LoginTime - 300
d3[v[i].Uid] = append(d3[v[i].Uid], v[i].Timestamp-LoginTime-300)
s = -1
}
}
}
//log.Println(d2)
var sum int64
for _, v := range d2 {
sum += v
}
if len(d2) == 0 {
return 0
}
return float64(sum/int64(len(d2))) / 60
}
func GetOperation(App *Type.App) (*Type.Operation, error) {
AppConfig, err := GetAppConfig(App.AppId)
if err != nil {
return nil, fmt.Errorf("failed to get app config: %v", err)
}
Db := MPool.GetTopicDB(AppConfig.Topic)
defer Db.Close()
Retain := []*Type.Retain{}
StartDate, _ := GetDateStr(App.Tz, -14)
EndDate, _ := GetDateStr(App.Tz, -1)
err = Db.Select(&Retain, "SELECT `Date`, `Register`, `SecondRemain`, `ThirdRemain`, `SeventhRemain`, `ThirtiethRemain`, `Recharge`, `Login`, `Ext` FROM remain where `Date` >= ? and `Date` <= ? order by `Date` desc", StartDate, EndDate)
if err != nil {
return nil, fmt.Errorf("failed to select data: %v", err)
}
var Register int
err = Db.Get(&Register, "SELECT count(`Uid`) as count FROM log_login WHERE Event = 'register' and Timestamp >= ?", App.OpenTime)
if err != nil {
return nil, fmt.Errorf("failed to select data: %v", err)
}
var Recharge float64
err = Db.Get(&Recharge, "SELECT IFNULL(SUM(Price), 0) as sum FROM log_order where `OrderId` != 'gm' and Timestamp >= ?", App.OpenTime)
if err != nil {
return nil, fmt.Errorf("failed to select data: %v", err)
}
InactiveTime, _ := GetZeroTimestamp(App.Tz, -8)
var InactiveUsers int
err = Db.Get(&InactiveUsers, "SELECT count(distinct Uid) as count from (SELECT Uid, MAX(Timestamp) as LastLogin FROM log_login WHERE Event = 'Login_log' GROUP BY Uid) as lt where lastlogin < ? and lastlogin > ?", InactiveTime, App.OpenTime)
if err != nil {
return nil, fmt.Errorf("failed to select data: %v", err)
}
var RechargeUserNum int
err = Db.Get(&RechargeUserNum, "SELECT count(distinct Uid) as count FROM log_order where Timestamp >= ? ", App.OpenTime)
if err != nil {
return nil, fmt.Errorf("failed to select data: %v", err)
}
type ExtStruct struct {
PerOnlineTime string `json:"PerOnlineTime"`
PerOrderNum string `json:"PerOrderNum"`
}
for _, v := range Retain {
if v.Ext == "" {
continue
}
var d ExtStruct
err := json.Unmarshal([]byte(v.Ext), &d)
if err != nil {
fmt.Printf("err :%s", err.Error())
continue
}
value, err := strconv.ParseFloat(d.PerOnlineTime, 64)
if err == nil {
v.PerOnlineTime = value
}
value, err = strconv.ParseFloat(d.PerOrderNum, 64)
if err == nil {
v.PerOrderNum = value
}
}
ChurnRate := 100 * float64(InactiveUsers) / float64(Register)
R := &Type.Operation{
Retain: Retain,
Register: Register,
Recharge: Decimal(Recharge, 2),
ChurnRate: Decimal(ChurnRate, 2),
RechargeUserNum: RechargeUserNum,
Date: time.Now().In(time.UTC).Format("2006-01-02"),
ARPU: FloatDiv(Recharge, Register, 3),
}
return R, nil
}
func GetDailyHeat(AppId int) (*Type.HeatData, error) {
AppConfig, err := GetAppConfig(AppId)
if err != nil {
return nil, err
}
LogDb := MPool.GetTopicDB(AppConfig.Topic)
defer LogDb.Close()
if LogDb == nil {
return nil, fmt.Errorf("failed to get mysql database")
}
now, _ := GetZeroTimestamp(AppConfig.Tz, 0)
value := make([]int, 0, 24)
value2 := make([]int, 0, 24)
key := make([]string, 0, 24)
for i := 0; i < 24; i++ {
key = append(key, fmt.Sprintf("%02d:00", i))
}
type r struct {
Sum1 int `db:"sum1"`
Sum2 int `db:"sum2"`
}
var result r
for i := 0; i < 24; i++ {
err := LogDb.Get(&result, "select count(*) as sum1, count(distinct Uid) as sum2 from log_event where Timestamp >= ? and Timestamp <= ?", now-int64(i)*3600-3600, now-int64(i)*3600)
if err != nil {
return nil, fmt.Errorf("failed to get heat count: %v", err)
}
value = append(value, result.Sum1)
value2 = append(value2, result.Sum2)
key[i] = fmt.Sprintf("%02d:00", GetHour(now-int64(i)*3600, AppConfig.Tz))
}
if len(value) < 2 {
return nil, nil
}
// 反转key和value
for i, j := 0, len(value)-1; i < j; i, j = i+1, j-1 {
key[i], key[j] = key[j], key[i]
value2[i], value2[j] = value2[j], value2[i]
value[i], value[j] = value[j], value[i]
}
return &Type.HeatData{
Key: key,
Value: value,
Value2: value2,
}, nil
}
func GetLevelData(AppId int) (*Type.LevelData, error) {
AppConfig, err := GetAppConfig(AppId)
if err != nil {
return nil, err
}
LogDb := MPool.GetTopicDB(AppConfig.Topic)
defer LogDb.Close()
if LogDb == nil {
return nil, fmt.Errorf("failed to get mysql database")
}
type data struct {
Param string `db:"param"`
Uid int `db:"Uid"`
Timestamp int64 `db:"Timestamp"`
}
dataList := []data{}
err = LogDb.Select(&dataList, "SELECT `param`, `Uid`, `Timestamp` FROM log_event WHERE `Event` = 'level_up'")
if err != nil {
return nil, fmt.Errorf("failed to select data: %v", err)
}
type Users struct {
Uid int `db:"Uid"`
}
InactiveUsers := []Users{}
ZeroTimestamp, _ := GetZeroTimestamp(AppConfig.Tz, 0)
err = LogDb.Select(&InactiveUsers, "SELECT Uid from (SELECT Uid, MAX(Timestamp) as LastLogin FROM log_login WHERE Event = 'Login_log' GROUP BY Uid) as lt where lastlogin < ?", ZeroTimestamp-7*86400)
if err != nil {
return nil, fmt.Errorf("failed to select data: %v", err)
}
d0 := make([]int, 0)
for _, v := range InactiveUsers {
d0 = append(d0, v.Uid)
}
d1 := make(map[int]data)
for _, v := range dataList {
if _, ok := d1[v.Uid]; !ok {
d1[v.Uid] = v
}
if v.Timestamp > d1[v.Uid].Timestamp {
d1[v.Uid] = v
}
}
type data2 struct {
Num int
Churn int
}
d2 := make(map[int]*data2)
for _, v := range d1 {
var jd struct {
NextLevel int `json:"after_level"`
}
err := json.Unmarshal([]byte(v.Param), &jd)
if err != nil {
continue
}
if _, ok := d2[jd.NextLevel]; !ok {
d2[jd.NextLevel] = &data2{
Num: 0,
Churn: 0,
}
}
d2[jd.NextLevel].Num++
if InArray(v.Uid, d0) {
d2[jd.NextLevel].Churn++
}
}
key := make([]int, 0, len(d2))
for k := range d2 {
key = append(key, k)
}
sort.Slice(key, func(i, j int) bool {
return key[i] < key[j]
})
value := make([]int, 0, len(d2))
value2 := make([]int, 0, len(d2))
for _, v := range key {
value = append(value, d2[v].Num)
value2 = append(value2, d2[v].Churn)
}
key1 := make([]string, 0, len(d2))
for _, v := range key {
key1 = append(key1, strconv.Itoa(v))
}
return &Type.LevelData{
Key: key1,
Value: value,
Value2: value2,
}, nil
}
func GetDecorateData(AppId int) (*Type.DecorateData, error) {
AppConfig, err := GetAppConfig(AppId)
if err != nil {
return nil, err
}
LogDb := MPool.GetTopicDB(AppConfig.Topic)
defer LogDb.Close()
if LogDb == nil {
return nil, fmt.Errorf("failed to get mysql database")
}
type data struct {
Param string `db:"param"`
Uid int `db:"Uid"`
Timestamp int64 `db:"Timestamp"`
}
dataList := []data{}
err = LogDb.Select(&dataList, "SELECT `param`, `Uid`, `Timestamp` FROM log_event WHERE `Event` = 'scene_reward'")
if err != nil {
return nil, fmt.Errorf("failed to select data: %v", err)
}
type Users struct {
Uid int `db:"Uid"`
}
InactiveUsers := []Users{}
ZeroTimestamp, _ := GetZeroTimestamp(AppConfig.Tz, 0)
err = LogDb.Select(&InactiveUsers, "SELECT Uid from (SELECT Uid, MAX(Timestamp) as LastLogin FROM log_login WHERE Event = 'Login_log' GROUP BY Uid) as lt where lastlogin < ?", ZeroTimestamp-7*86400)
if err != nil {
return nil, fmt.Errorf("failed to select data: %v", err)
}
d0 := make([]int, 0)
for _, v := range InactiveUsers {
d0 = append(d0, v.Uid)
}
d1 := make(map[int]data)
for _, v := range dataList {
if _, ok := d1[v.Uid]; !ok {
d1[v.Uid] = v
}
if v.Timestamp > d1[v.Uid].Timestamp {
d1[v.Uid] = v
}
}
type data2 struct {
Num int
Churn int
}
d2 := make(map[string]*data2)
for _, v := range d1 {
var jd struct {
Step string `json:"deco_step_id"`
}
err := json.Unmarshal([]byte(v.Param), &jd)
if err != nil {
continue
}
if _, ok := d2[jd.Step]; !ok {
d2[jd.Step] = &data2{
Num: 0,
Churn: 0,
}
}
d2[jd.Step].Num++
if InArray(v.Uid, d0) {
d2[jd.Step].Churn++
}
}
key := make([]string, 0, len(d2))
for k := range d2 {
key = append(key, k)
}
sort.Slice(key, func(i, j int) bool {
a := strings.Split(key[i], "_")
b := strings.Split(key[j], "_")
aa1, _ := strconv.Atoi(a[0])
aa2, _ := strconv.Atoi(a[1])
bb1, _ := strconv.Atoi(b[0])
bb2, _ := strconv.Atoi(b[1])
if aa1 == bb1 {
return aa2 < bb2
}
return aa1 < bb1
})
value := make([]int, 0, len(d2))
value2 := make([]int, 0, len(d2))
for _, v := range key {
value = append(value, d2[v].Num)
value2 = append(value2, d2[v].Churn)
}
return &Type.DecorateData{
Key: key,
Value: value,
Value2: value2,
}, nil
}
func GetOrderData(AppId int) (interface{}, error) {
AppConfig, err := GetAppConfig(AppId)
if err != nil {
return nil, err
}
LogDb := MPool.GetTopicDB(AppConfig.Topic)
defer LogDb.Close()
if LogDb == nil {
return nil, fmt.Errorf("failed to get mysql database")
}
type data struct {
Param string `db:"param"`
Uid int `db:"Uid"`
Timestamp int64 `db:"Timestamp"`
}
dataList := []data{}
err = LogDb.Select(&dataList, "SELECT `param`, `Uid`, `Timestamp` FROM log_event WHERE `Event` = 'order_finish'")
if err != nil {
return nil, fmt.Errorf("failed to select data: %v", err)
}
type Users struct {
Uid int `db:"Uid"`
}
InactiveUsers := []Users{}
ZeroTimestamp, _ := GetZeroTimestamp(AppConfig.Tz, 0)
err = LogDb.Select(&InactiveUsers, "SELECT Uid from (SELECT Uid, MAX(Timestamp) as LastLogin FROM log_login WHERE Event = 'Login_log' GROUP BY Uid) as lt where lastlogin < ?", ZeroTimestamp-7*86400)
if err != nil {
return nil, fmt.Errorf("failed to select data: %v", err)
}
d0 := make([]int, 0)
for _, v := range InactiveUsers {
d0 = append(d0, v.Uid)
}
d1 := make(map[int]data)
for _, v := range dataList {
if _, ok := d1[v.Uid]; !ok {
d1[v.Uid] = v
}
if v.Timestamp > d1[v.Uid].Timestamp {
d1[v.Uid] = v
}
}
type data2 struct {
Num int
Churn int
}
d2 := make(map[int]*data2)
for _, v := range d1 {
var jd struct {
Step int `json:"order_id"`
}
err := json.Unmarshal([]byte(v.Param), &jd)
if err != nil {
continue
}
if _, ok := d2[jd.Step]; !ok {
d2[jd.Step] = &data2{
Num: 0,
Churn: 0,
}
}
d2[jd.Step].Num++
if InArray(v.Uid, d0) {
d2[jd.Step].Churn++
}
}
key := make([]int, 0, len(d2))
for k := range d2 {
key = append(key, k)
}
sort.Slice(key, func(i, j int) bool {
return i < j
})
value := make([]int, 0, len(d2))
value2 := make([]int, 0, len(d2))
for _, v := range key {
value = append(value, d2[v].Num)
value2 = append(value2, d2[v].Churn)
}
for i := 1; i < 30; i++ {
_, ok := d2[i]
if !ok {
continue
}
fmt.Printf("key:%d, value:%.2f%%, value2:%d\n", i+1, float64(d2[i].Num)/13, d2[i].Churn)
}
return map[string]interface{}{
"Key": key,
"Value": value,
"Value2": value2,
}, nil
}