initial work

Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>

app/cmd.go

@@ -0,0 +1,118 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"runtime"
+	"strings"
+	"time"
+)
+
+var memInfo runtime.MemStats
+
+type Info struct {
+	ch     chan struct{}
+	room   string
+	Server string `json:"server"`
+	Proxy  string `json:"proxy"`
+	Online string `json:"online"`
+	Rid    int64  `json:"rid"`
+	Start  int64  `json:"start"`
+	Last   int64  `json:"last"`
+	Income int64  `json:"income"`
+	Dons   int64  `json:"dons"`
+	Tips   int64  `json:"tips"`
+}
+
+func updateFileRooms() string {
+	for {
+		rooms.Json <- ""
+		s := <-rooms.Json
+		err := os.WriteFile(conf.Conn["start"], []byte(s), 0644)
+		if err != nil {
+			fmt.Println(err)
+		}
+		time.Sleep(10 * time.Second)
+	}
+}
+
+func listHandler(w http.ResponseWriter, _ *http.Request) {
+	dat, err := os.ReadFile(conf.Conn["start"])
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	fmt.Fprint(w, string(dat))
+}
+
+func debugHandler(w http.ResponseWriter, _ *http.Request) {
+	ws.Count <- 0
+	l := <-ws.Count
+	runtime.ReadMemStats(&memInfo)
+	j, err := json.Marshal(struct {
+		Goroutines int
+		WebSocket  int
+		Uptime     int64
+		Alloc      uint64
+		HeapSys    uint64
+	}{
+		Goroutines: runtime.NumGoroutine(),
+		Alloc:      memInfo.Alloc,
+		HeapSys:    memInfo.HeapSys,
+		Uptime:     uptime,
+		WebSocket:  l,
+	})
+	if err == nil {
+		fmt.Fprint(w, string(j))
+	}
+}
+
+func cmdHandler(w http.ResponseWriter, r *http.Request) {
+	if !conf.List[r.Header.Get("X-REAL-IP")] {
+		fmt.Fprint(w, "403")
+		return
+	}
+	params := r.URL.Query()
+	if len(params["room"]) > 0 && len(params["server"]) > 0 && len(params["proxy"]) > 0 {
+		now := time.Now().Unix()
+		workerData := Info{
+			room:   params["room"][0],
+			Server: params["server"][0],
+			Proxy:  params["proxy"][0],
+			Online: "0",
+			Start:  now,
+			Last:   now,
+			Rid:    0,
+			Income: 0,
+			Dons:   0,
+			Tips:   0,
+		}
+		startRoom(workerData)
+	}
+	if len(params["exit"]) > 0 {
+		rooms.Stop <- strings.Join(params["exit"], "")
+	}
+	fmt.Fprint(w, string("ok"))
+}
+
+func startRoom(workerData Info) {
+	rooms.Check <- workerData.room
+	testRoom := <-rooms.Check
+	if testRoom == workerData.room {
+		fmt.Println("Already track:", workerData.room)
+		return
+	}
+
+	rid, ok := getRoomInfo(workerData.room)
+	if !ok {
+		fmt.Println("No room in MySQL:", workerData.room)
+		return
+	}
+
+	workerData.Rid = rid
+	workerData.ch = make(chan struct{})
+
+	go xWorker(workerData, url.URL{Scheme: "wss", Host: workerData.Server + ".bcccdn.com", Path: "/websocket"})
+}

app/conf.go

@@ -0,0 +1,34 @@
+package main
+
+type Conf struct {
+	Conn  map[string]string
+	Proxy map[string]string
+	List  map[string]bool
+}
+
+var conf = &Conf{
+	Conn:  make(map[string]string),
+	Proxy: make(map[string]string),
+	List:  make(map[string]bool),
+}
+
+func startConfig() {
+	// DB
+	conf.Conn = map[string]string{
+		"mysql": "user:passwd@unix(/var/run/mysqld/mysqld.sock)/base?interpolateParams=true",
+		"click": "tcp://127.0.0.1:9000/base?compress=true&debug=false",
+		"start": "/tmp/bongaStart.txt",
+	}
+
+	// 3proxy
+	conf.Proxy = map[string]string{
+		"us": "ip:port",
+		"fi": "ip:port",
+	}
+
+	// allow ips
+	conf.List = map[string]bool{
+		"::1":       true,
+		"127.0.0.1": true,
+	}
+}

app/echo.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"net/http"
+
+	"github.com/gorilla/websocket"
+)
+
+var (
+	wsClients = make(map[*websocket.Conn]struct{})
+
+	ws = struct {
+		Count chan int
+		Send  chan []byte
+		Add   chan *websocket.Conn
+	}{
+		Count: make(chan int, 100),
+		Send:  make(chan []byte, 100),
+		Add:   make(chan *websocket.Conn, 100),
+	}
+)
+
+func broadcast() {
+	for {
+		select {
+		case conn := <-ws.Add:
+			wsClients[conn] = struct{}{}
+
+		case <-ws.Count:
+			ws.Count <- len(wsClients)
+
+		case message := <-ws.Send:
+			for conn := range wsClients {
+				if err := conn.WriteMessage(1, message); err != nil {
+					conn.Close()
+					delete(wsClients, conn)
+				}
+			}
+		}
+	}
+}
+
+func wsHandler(w http.ResponseWriter, r *http.Request) {
+	conn, err := websocket.Upgrade(w, r, w.Header(), 1024, 1024)
+	if err != nil {
+		return
+	}
+	ws.Add <- conn
+}

app/main.go

@@ -0,0 +1,132 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"math/rand"
+	"net"
+	"net/http"
+	"os"
+	"time"
+
+	_ "github.com/ClickHouse/clickhouse-go"
+	_ "github.com/go-sql-driver/mysql"
+	"github.com/jmoiron/sqlx"
+	jsoniter "github.com/json-iterator/go"
+)
+
+type Rooms struct {
+	Count chan int
+	Json  chan string
+	Check chan string
+	Stop  chan string
+	Del   chan string
+	Add   chan Info
+}
+
+var (
+	Mysql, Clickhouse *sqlx.DB
+
+	json = jsoniter.ConfigCompatibleWithStandardLibrary
+
+	save = make(chan saveData, 100)
+	slog = make(chan saveLog, 100)
+
+	rooms = &Rooms{
+		Count: make(chan int),
+		Json:  make(chan string),
+		Check: make(chan string),
+		Stop:  make(chan string),
+		Del:   make(chan string),
+		Add:   make(chan Info),
+	}
+)
+
+func main() {
+	rand.Seed(time.Now().UnixNano())
+
+	startConfig()
+
+	initMysql()
+	initClickhouse()
+
+	go mapRooms()
+	go announceCount()
+	go saveDB()
+	go saveLogs()
+	go broadcast()
+
+	http.HandleFunc("/bongacams/ws/", wsHandler)
+	http.HandleFunc("/bongacams/cmd/", cmdHandler)
+	http.HandleFunc("/bongacams/list/", listHandler)
+	http.HandleFunc("/bongacams/debug/", debugHandler)
+
+	go fastStart()
+
+	const SOCK = "/tmp/bongacams.sock"
+	os.Remove(SOCK)
+	unixListener, err := net.Listen("unix", SOCK)
+	if err != nil {
+		log.Fatal("Listen (UNIX socket): ", err)
+	}
+	defer unixListener.Close()
+	os.Chmod(SOCK, 0777)
+	log.Fatal(http.Serve(unixListener, nil))
+}
+
+func initMysql() {
+	db, err := sqlx.Connect("mysql", conf.Conn["mysql"])
+	if err != nil {
+		panic(err)
+	}
+	Mysql = db
+}
+
+func initClickhouse() {
+	db, err := sqlx.Connect("clickhouse", conf.Conn["click"])
+	if err != nil {
+		panic(err)
+	}
+	Clickhouse = db
+}
+
+func randInt(min int, max int) int {
+	return min + rand.Intn(max-min)
+}
+
+func fastStart() {
+	defer func() {
+		go updateFileRooms()
+	}()
+	dat, err := os.ReadFile(conf.Conn["start"])
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	list := make(map[string]Info)
+	if err := json.Unmarshal(dat, &list); err != nil {
+		fmt.Println(err.Error())
+		return
+	}
+	now := time.Now().Unix()
+	for k, v := range list {
+		if now > v.Last+60*20 {
+			continue
+		}
+		fmt.Println("fastStart:", k, v.Server, v.Proxy)
+		workerData := Info{
+			room:   k,
+			Server: v.Server,
+			Proxy:  v.Proxy,
+			Online: v.Online,
+			Start:  v.Start,
+			Last:   now,
+			Rid:    v.Rid,
+			Income: v.Income,
+			Dons:   v.Dons,
+			Tips:   v.Tips,
+		}
+		startRoom(workerData)
+		time.Sleep(2 * time.Second)
+	}
+}

app/save.go

@@ -0,0 +1,193 @@
+package main
+
+import (
+	"fmt"
+	"time"
+)
+
+type saveData struct {
+	Room   string
+	From   string
+	Rid    int64
+	Amount int64
+	Now    int64
+}
+
+type saveLog struct {
+	Rid int64
+	Now int64
+	Mes string
+}
+
+type DonatorCache struct {
+	Id   int64
+	Last int64
+}
+
+func getDonId(name string) int64 {
+	var id int64
+	err := Mysql.Get(&id, "SELECT id FROM donator WHERE name=?", name)
+	if err != nil {
+		res, _ := Mysql.Exec("INSERT INTO donator (`name`) VALUES (?)", name)
+		id, _ = res.LastInsertId()
+	}
+	return id
+}
+
+func getRoomInfo(name string) (int64, bool) {
+	var id int64
+	result := true
+	err := Mysql.Get(&id, "SELECT id FROM room WHERE name=?", name)
+	if err != nil {
+		result = false
+	}
+	return id, result
+}
+
+func getSumTokens() int64 {
+	r := struct {
+		Date string
+		Sum  int64
+	}{}
+	err := Clickhouse.Get(&r, "SELECT toStartOfHour(toDateTime(`unix`)) as date, SUM(`token`) as sum FROM `stat` WHERE time = today() GROUP BY date ORDER BY date DESC LIMIT 1")
+	if err == nil && r.Sum > 0 {
+		return r.Sum
+	}
+	return 0
+}
+
+func saveDB() {
+	hours, _, _ := time.Now().Clock()
+
+	bulk := make(map[int]saveData)
+	update := make(map[int64]int64)
+	data := make(map[string]*DonatorCache)
+	index := make(map[string]int64)
+
+	index = map[string]int64{"hours": int64(hours), "tokens": getSumTokens(), "last": time.Now().Unix()}
+
+	for {
+		select {
+		case m := <-save:
+			//fmt.Println("Save channel:", len(save), cap(save))
+
+			now := time.Now().Unix()
+
+			if _, ok := data[m.From]; ok {
+				data[m.From].Last = now
+			} else {
+				data[m.From] = &DonatorCache{Id: getDonId(m.From), Last: now}
+			}
+
+			num := len(bulk)
+
+			bulk[num] = m
+
+			update[m.Rid] = m.Now
+
+			if num > 512 {
+
+				tx, err := Mysql.Begin()
+				if err == nil {
+					st, _ := tx.Prepare("INSERT INTO `stat` (`did`, `rid`, `token`, `time`) VALUES (?, ?, ?, ?)")
+					for _, v := range bulk {
+						st.Exec(data[v.From].Id, v.Rid, v.Amount, v.Now)
+					}
+					tx.Commit()
+					st.Close()
+				}
+
+				tx, err = Mysql.Begin()
+				if err == nil {
+					st, _ := tx.Prepare("UPDATE `room` SET `last` = ? WHERE `id` = ?")
+					for k, v := range update {
+						st.Exec(v, k)
+					}
+					tx.Commit()
+					st.Close()
+				}
+
+				tx, err = Clickhouse.Begin()
+				if err == nil {
+					st, _ := tx.Prepare("INSERT INTO stat VALUES (?, ?, ?, ?, ?)")
+					for _, v := range bulk {
+						st.Exec(uint32(data[v.From].Id), uint32(v.Rid), uint32(v.Amount), time.Unix(v.Now, 0), uint32(v.Now))
+					}
+					tx.Commit()
+					st.Close()
+				}
+
+				bulk = make(map[int]saveData)
+				update = make(map[int64]int64)
+			}
+
+			if m.Amount > 99 {
+				msg, err := json.Marshal(struct {
+					Room    string `json:"room"`
+					Donator string `json:"donator"`
+					Amount  int64  `json:"amount"`
+				}{
+					Room:    m.Room,
+					Donator: m.From,
+					Amount:  m.Amount,
+				})
+				if err == nil {
+					ws.Send <- msg
+				}
+			}
+
+			hours, minutes, seconds := time.Now().Clock()
+			if int64(hours) == index["hours"] {
+				index["tokens"] += m.Amount
+			} else {
+				index = map[string]int64{"hours": int64(hours), "tokens": 0, "last": 0}
+			}
+
+			if minutes >= 5 && now > index["last"]+30 {
+				seconds += minutes * 60
+				msg, err := json.Marshal(struct {
+					Index int64 `json:"index"`
+				}{Index: index["tokens"] / int64(seconds) * 3600 / 1000 * 5 / 100})
+				if err == nil {
+					ws.Send <- msg
+				}
+				index["last"] = now
+			}
+
+			if randInt(0, 10000) == 777 { // 0.001%
+				l := len(data)
+				for k, v := range data {
+					if now > v.Last+60*60*48 {
+						delete(data, k)
+					}
+				}
+				fmt.Println("Clean map:", l, "=>", len(data))
+			}
+		}
+	}
+}
+
+func saveLogs() {
+	bulk := make(map[int]saveLog)
+	for {
+		select {
+		case m := <-slog:
+			if len(m.Mes) > 0 {
+				num := len(bulk)
+				bulk[num] = m
+				if num > 2048 {
+					tx, err := Mysql.Begin()
+					if err == nil {
+						st, _ := tx.Prepare("INSERT INTO `logs` (`rid`, `time`, `mes`) VALUES (?, ?, ?)")
+						for _, v := range bulk {
+							st.Exec(v.Rid, v.Now, v.Mes)
+						}
+						tx.Commit()
+						st.Close()
+					}
+					bulk = make(map[int]saveLog)
+				}
+			}
+		}
+	}
+}

app/worker.go

@@ -0,0 +1,276 @@
+package main
+
+import (
+	"fmt"
+	"github.com/gorilla/websocket"
+	jsoniter "github.com/json-iterator/go"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+)
+
+var uptime = time.Now().Unix()
+
+type AuthResponse struct {
+	Status    string `json:"status"`
+	LocalData struct {
+		DataKey string `json:"dataKey"`
+	} `json:"localData"`
+	UserData struct {
+		Username    string `json:"username"`
+		DisplayName string `json:"displayName"`
+		Location    string `json:"location"`
+		Chathost    string `json:"chathost"`
+		IsRu        bool   `json:"isRu"`
+	} `json:"userData"`
+}
+
+type ServerResponse struct {
+	TS   int64               `json:"ts"`
+	Type string              `json:"type"`
+	Body jsoniter.RawMessage `json:"body"`
+}
+
+type DonateResponse struct {
+	F struct {
+		Username string `json:"username"`
+	} `json:"f"`
+	A int64 `json:"a"`
+}
+
+func mapRooms() {
+
+	data := make(map[string]*Info)
+
+	for {
+		select {
+		case m := <-rooms.Add:
+			data[m.room] = &Info{Server: m.Server, Proxy: m.Proxy, Start: m.Start, Last: m.Last, Online: m.Online, Income: m.Income, Dons: m.Dons, Tips: m.Tips, ch: m.ch}
+
+		case s := <-rooms.Json:
+			j, err := json.Marshal(data)
+			if err == nil {
+				s = string(j)
+			}
+			rooms.Json <- s
+
+		case <-rooms.Count:
+			rooms.Count <- len(data)
+
+		case key := <-rooms.Del:
+			delete(data, key)
+
+		case room := <-rooms.Check:
+			if _, ok := data[room]; !ok {
+				room = ""
+			}
+			rooms.Check <- room
+
+		case room := <-rooms.Stop:
+			if _, ok := data[room]; ok {
+				close(data[room].ch)
+			}
+		}
+	}
+}
+
+func announceCount() {
+	for {
+		time.Sleep(30 * time.Second)
+		rooms.Count <- 0
+		l := <-rooms.Count
+		msg, err := json.Marshal(struct {
+			Count int `json:"count"`
+		}{Count: l})
+		if err == nil {
+			ws.Send <- msg
+		}
+	}
+}
+
+func getAMF(room string) (bool, *AuthResponse) {
+
+	v := &AuthResponse{}
+
+	req, err := http.NewRequest(http.MethodPost, "https://rt.bongocams.com/tools/amf.php?res=771840&t=1654437233142", strings.NewReader(`method=getRoomData&args[]=`+room))
+	if err != nil {
+		fmt.Println(err.Error())
+		return false, v
+	}
+	req.Header.Add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
+	req.Header.Add("X-Requested-With", "XMLHttpRequest")
+	req.Header.Add("Accept", "application/json")
+	req.Header.Add("Referrer", "https://bongacams.com")
+	req.Header.Add("User-agent", "curl/7.79.1")
+
+	rsp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		fmt.Println(err.Error())
+		return false, v
+	}
+	defer rsp.Body.Close()
+
+	if err = json.NewDecoder(rsp.Body).Decode(v); err != nil {
+		fmt.Println(err.Error())
+		return false, v
+	}
+
+	return true, v
+}
+
+func xWorker(workerData Info, u url.URL) {
+	fmt.Println("Start", workerData.room, "server", workerData.Server, "proxy", workerData.Proxy)
+
+	rooms.Add <- workerData
+
+	defer func() {
+		rooms.Del <- workerData.room
+	}()
+
+	ok, v := getAMF(workerData.room)
+	if !ok {
+		fmt.Println("exit: no amf parms")
+		return
+	}
+
+	Dialer := *websocket.DefaultDialer
+
+	if _, ok := conf.Proxy[workerData.Proxy]; ok {
+		Dialer = websocket.Dialer{
+			Proxy: http.ProxyURL(&url.URL{
+				Scheme: "http", // or "https" depending on your proxy
+				Host:   conf.Proxy[workerData.Proxy],
+				Path:   "/",
+			}),
+			HandshakeTimeout: 45 * time.Second, // https://pkg.go.dev/github.com/gorilla/websocket
+		}
+	}
+
+	c, _, err := Dialer.Dial(u.String(), nil)
+	if err != nil {
+		fmt.Println(err.Error(), workerData.room)
+		return
+	}
+
+	defer c.Close()
+
+	c.SetReadDeadline(time.Now().Add(60 * time.Second))
+
+	if err = c.WriteMessage(websocket.TextMessage, []byte(fmt.Sprintf(`{"id":%d,"name":"joinRoom","args":["%s",{"username":"%s","displayName":"%s","location":"%s","chathost":"%s","isRu":%t,"isPerformer":false,"hasStream":false,"isLogged":false,"isPayable":false,"showType":"public"},"%s"]}`, 1, v.UserData.Chathost, v.UserData.Username, v.UserData.DisplayName, v.UserData.Location, v.UserData.Chathost, v.UserData.IsRu, v.LocalData.DataKey))); err != nil {
+		fmt.Println(err.Error())
+		return
+	}
+
+	_, message, err := c.ReadMessage()
+	if err != nil {
+		fmt.Println(err.Error(), workerData.room)
+		return
+	}
+
+	slog <- saveLog{workerData.Rid, time.Now().Unix(), string(message)}
+
+	if string(message) == `{"id":1,"result":{"audioAvailable":false,"freeShow":false},"error":null}` {
+		fmt.Println("room offline, exit", workerData.room)
+		return
+	}
+
+	if err = c.WriteMessage(websocket.TextMessage, []byte(fmt.Sprintf(`{"id":%d,"name":"ChatModule.connect","args":["public-chat"]}`, 2))); err != nil {
+		fmt.Println(err.Error(), workerData.room)
+		return
+	}
+
+	_, message, err = c.ReadMessage()
+	if err != nil {
+		fmt.Println(err.Error(), workerData.room)
+		return
+	}
+
+	slog <- saveLog{workerData.Rid, time.Now().Unix(), string(message)}
+
+	quit := make(chan struct{})
+	pid := 3
+
+	defer func() {
+		close(quit)
+	}()
+
+	go func() {
+		ticker := time.NewTicker(30 * time.Second)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-quit:
+				return
+			case <-ticker.C:
+				if err = c.WriteMessage(websocket.TextMessage, []byte(fmt.Sprintf(`{"id":%d,"name":"ping"}`, pid))); err != nil {
+					fmt.Println(err.Error(), workerData.room)
+					rooms.Stop <- workerData.room
+					return
+				}
+				pid++
+				break
+			}
+		}
+	}()
+
+	dons := make(map[string]struct{})
+
+	for {
+		select {
+		case <-workerData.ch:
+			fmt.Println("Exit room:", workerData.room)
+			return
+		default:
+		}
+
+		c.SetReadDeadline(time.Now().Add(30 * time.Minute))
+		_, message, err := c.ReadMessage()
+		if err != nil {
+			fmt.Println(err.Error())
+			return
+		}
+
+		now := time.Now().Unix()
+
+		slog <- saveLog{workerData.Rid, now, string(message)}
+
+		m := &ServerResponse{}
+
+		if err = json.Unmarshal(message, m); err != nil {
+			fmt.Println(err.Error(), workerData.room)
+			continue
+		}
+
+		workerData.Last = now
+		rooms.Add <- workerData
+
+		if m.Type == "ServerMessageEvent:PERFORMER_STATUS_CHANGE" && string(m.Body) == `"offline"` {
+			fmt.Println(m.Type, workerData.room)
+			return
+		}
+
+		if m.Type == "ServerMessageEvent:ROOM_CLOSE" {
+			fmt.Println(m.Type, workerData.room)
+			return
+		}
+
+		if m.Type == "ServerMessageEvent:INCOMING_TIP" {
+			d := &DonateResponse{}
+			if err = json.Unmarshal(m.Body, d); err == nil {
+				//fmt.Println(d.F.Username, "send", d.A, "tokens")
+
+				workerData.Tips++
+				if _, ok := dons[d.F.Username]; !ok {
+					dons[d.F.Username] = struct{}{}
+					workerData.Dons++
+				}
+
+				save <- saveData{workerData.room, d.F.Username, workerData.Rid, d.A, now}
+
+				workerData.Income += d.A
+				rooms.Add <- workerData
+			}
+		}
+	}
+}