Lbenedar/Foundry-Scrapping-API
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add init world data

Browse Source
This commit is contained in:
lbenedar
2026-04-08 18:19:20 +03:00
parent e5deb5d76d
commit 468e027418
9 changed files with 464 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
internal/foundry/foundry.go
View File

@@ -21,12 +21,11 @@ type Foundry struct {
isAuth bool
logger *slog.Logger
config requests.Config
ws *webSocketUtil
}
func NewFoundry() *Foundry {
foundry := &Foundry{config: requests.Config{SessionID: ""}, isAuth: false, ws: NewWebSocketUtil()}
foundry := &Foundry{isAuth: false, ws: NewWebSocketUtil()}
// err := foundry.config.GetSessionId()
// if err != nil {
@@ -51,14 +50,14 @@ func (foundry *Foundry) checkAuthRespAnswer(r io.Reader, respLength int) error {
}
func (foundry *Foundry) Authenticate() error {
if foundry.config.SessionID == "" {
err := foundry.config.GetSessionId()
if foundry.ws.config.SessionID == "" {
err := foundry.ws.config.GetSessionId()
if err != nil {
return err
}
}
resp, err := foundry.config.PostAuthenticationData()
resp, err := foundry.ws.config.PostAuthenticationData()
if err != nil {
return err
}
@@ -73,9 +72,9 @@ func (foundry *Foundry) Authenticate() error {
}
func (foundry *Foundry) StartListen() error {
foundry.ws.channels = *types.InitWsChannels()
foundry.ws.readChan = *types.InitWsChannels()
wsChannels := &foundry.ws.channels
wsChannels := &foundry.ws.readChan
defer wsChannels.Close()
go foundry.ListenAndServeWS()
@@ -115,19 +114,19 @@ func (foundry *Foundry) ConnectToFoundry() error {
if foundry == nil {
return ErrorFoundryNotInit
}
ok, err := (*foundry).config.GetSessionToken()
ok, err := (*foundry).ws.config.GetSessionToken()
if err != nil {
return err
}
if (*foundry).config.Password != "" && !ok {
if (*foundry).ws.config.Password != "" && !ok {
err := (*foundry).Authenticate()
if err != nil {
return err
}
}
foundry.logger.Info("Successefully connected to Foundry", "host", foundry.config.Host)
foundry.logger.Info("Successefully connected to Foundry", "host", foundry.ws.config.Host)
return nil
}
@@ -137,14 +136,14 @@ func (foundry *Foundry) ConnectToWebSocket() error {
}
query := url.Values{}
query.Add("session", foundry.config.SessionID)
query.Add("session", foundry.ws.config.SessionID)
query.Add("EIO", "4")
query.Add("transport", "websocket")
u := url.URL{Scheme: "ws", Host: foundry.config.Host, Path: fmt.Sprintf("%s/", requests.SocketPath), RawQuery: query.Encode()}
u := url.URL{Scheme: "ws", Host: foundry.ws.config.Host, Path: fmt.Sprintf("%s/", requests.SocketPath), RawQuery: query.Encode()}
wsHeader := http.Header{}
wsHeader.Set("Cookie", fmt.Sprintf("session=%s", foundry.config.SessionID))
wsHeader.Set("Cookie", fmt.Sprintf("session=%s", foundry.ws.config.SessionID))
wsConn, _, err := websocket.DefaultDialer.Dial(u.String(), wsHeader)
if err != nil {
@@ -173,15 +172,15 @@ func (foundry *Foundry) HandleWSRequest(msgType string) ([]byte, error) {
}
func (foundry *Foundry) HasSessionId() bool {
return foundry.config.SessionID != ""
return foundry.ws.config.SessionID != ""
}
func (foundry *Foundry) SetUpSessionId() error {
return foundry.config.GetSessionId()
return foundry.ws.config.GetSessionId()
}
func (foundry *Foundry) SetConfig(config *requests.Config) {
foundry.config = *config
foundry.ws.config = *config
}
func (foundry *Foundry) SetLogger(slogger *slog.Logger) {

2
internal/foundry/models/db/world.go
View File

@@ -34,7 +34,7 @@ func (m FoundryStateModel) InsertWorld(world *World, stateId int64) error {
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
RETURNING id, created_at`
args := []any{stateId, world.TextId, world.Title, world.Description, world.System, world.System, world.CoreVersion, world.SystemVersion, world.PlayTime, world.NextSession}
args := []any{stateId, world.TextId, world.Title, world.Description, world.System, world.CoreVersion, world.SystemVersion, world.PlayTime, world.NextSession}
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()

14
internal/foundry/requests/config.go
View File

@@ -2,6 +2,7 @@ package requests
import (
"errors"
"fmt"
"net/http"
"strings"
@@ -71,3 +72,16 @@ func (conf *Config) SetSessionTokenFromHeader(resp http.Header) (bool, error) {
return false, nil
}
func (conf *Config) GetRequestHeader() *http.Header {
header := http.Header{}
header.Set("Cookie", fmt.Sprintf("session=%s", conf.SessionID))
header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
header.Set("Connection", "keep-alive")
header.Set("Content-Type", "application/x-www-form-urlencoded")
header.Set("Host", conf.Host)
header.Set("Origin", fmt.Sprintf("http://%s", conf.Host))
header.Set("Referer", fmt.Sprintf("http://%s%s", conf.Host, AuthPath))
return &header
}

56
internal/foundry/requests/post_requests.go
View File

@@ -17,17 +17,55 @@ func (conf *Config) PostAuthenticationData() (*http.Response, error) {
return nil, err
}
header := http.Header{}
header.Set("Cookie", fmt.Sprintf("session=%s", conf.SessionID))
header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
header.Set("Connection", "keep-alive")
header := conf.GetRequestHeader()
header.Set("Content-Length", fmt.Sprintf("%d", len(authData.Encode())))
header.Set("Content-Type", "application/x-www-form-urlencoded")
header.Set("Host", conf.Host)
header.Set("Origin", fmt.Sprintf("http://%s", conf.Host))
header.Set("Referer", fmt.Sprintf("http://%s%s", conf.Host, AuthPath))
req.Header = header
req.Header = header.Clone()
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
return resp, nil
}
func (conf *Config) PostLaunchWorld(world string) (*http.Response, error) {
launchData := url.Values{}
launchData.Add("world", world)
launchData.Add("action", "launchWorld")
req, err := http.NewRequest("POST", fmt.Sprintf("http://%s%s", conf.Host, SetupPath), bytes.NewBuffer([]byte(launchData.Encode())))
if err != nil {
return nil, err
}
header := conf.GetRequestHeader()
header.Set("Content-Length", fmt.Sprintf("%d", len(launchData.Encode())))
req.Header = header.Clone()
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
return resp, nil
}
func (conf *Config) PostReturnToSetup() (*http.Response, error) {
launchData := url.Values{}
launchData.Add("action", "shutdown")
req, err := http.NewRequest("POST", fmt.Sprintf("http://%s%s", conf.Host, JoinPath), bytes.NewBuffer([]byte(launchData.Encode())))
if err != nil {
return nil, err
}
header := conf.GetRequestHeader()
header.Set("Content-Length", fmt.Sprintf("%d", len(launchData.Encode())))
req.Header = header.Clone()
client := &http.Client{}
resp, err := client.Do(req)

9
internal/foundry/types/exchange_channels.go Normal file
View File

@@ -0,0 +1,9 @@
package types
import "sync"
type ExchangeChannels struct {
Msgs map[int](chan []byte)
ProgressMsg map[string](chan struct{})
ProgressMutex sync.Mutex
}

88
internal/foundry/types/utils.go Normal file
View File

@@ -0,0 +1,88 @@
package types
import (
"errors"
"fmt"
"reflect"
"strings"
)
func findByTag(obj any, tag string) (int, error) {
structType := reflect.TypeOf(obj).Elem()
for i := 0; i < structType.NumField(); i++ {
fieldTags := structType.Field(i).Tag.Get("json")
if tag == strings.Split(fieldTags, ",")[0] {
return i, nil
}
}
return -1, fmt.Errorf("No such field: %s in obj", tag)
}
func setField(obj any, name string, value any) error {
structValue := reflect.ValueOf(obj).Elem()
idx, err := findByTag(obj, name)
if err != nil {
return err
}
structFieldValue := structValue.Field(idx)
if !structFieldValue.IsValid() {
return fmt.Errorf("No such field: %s in obj", name)
}
if !structFieldValue.CanSet() {
return fmt.Errorf("Cannot set %s field value", name)
}
structFieldType := structFieldValue.Type()
val := reflect.ValueOf(value)
if structFieldType != val.Type() {
return errors.New("Provided value type didn't match obj field type")
}
if value == nil {
structFieldValue.Set(reflect.Zero(structFieldType))
} else {
structFieldValue.Set(val)
}
return nil
}
func isPointer(obj any) bool {
iv := reflect.ValueOf(obj)
if !iv.IsValid() {
return false
}
return iv.Kind() == reflect.Pointer
}
func FillStruct(m map[string]any, obj any) error {
if !isPointer(obj) {
return errors.New("Passed obj is not pointer")
}
for k, v := range m {
if v == nil {
continue
}
err := setField(obj, k, v)
if err != nil {
return err
}
}
return nil
}
func Dereference(input any) any {
v := reflect.ValueOf(input)
// Check if the value is a pointer
if v.Kind() == reflect.Pointer {
// v.Elem() returns the value the pointer points to
return v.Elem().Interface()
}
return input
}

72
internal/foundry/types/ws_action.go Normal file
View File

@@ -0,0 +1,72 @@
package types
import "fmt"
const (
WsSessionType = "session"
WsProgressType = "progress"
WsUserActivityType = "userActivity"
WsShutdownType = "shutdown"
)
var WsMsgActions = map[string]func() any{
"session": func() any { return &WsSessionMsg{} },
"progress": func() any { return &WsProgressMsg{} },
// "userActivity": WsUserActivityCursor{},
"shutdown": func() any { return &WsShutdownMsg{} },
}
func GetWsMsgAction(dataType string) any {
if factory, ok := WsMsgActions[dataType]; ok {
return factory()
}
return nil
}
type WsSessionMsg struct {
SessionId string `json:"sessionId"`
UserId *string `json:"userId,omitempty"`
}
func (msg WsSessionMsg) Action() error {
fmt.Println("WsSessionMsg")
return nil
}
type WsProgressMsg struct {
Id string `json:"id"`
Message string `json:"message"`
Pct float64 `json:"pct"`
HasChanged bool `json:"hasChanged,omitempty"`
Act string `json:"action"`
Step string `json:"step"`
}
func (msg WsProgressMsg) Action() error {
fmt.Println("WsProgressMsg")
return nil
}
type WsUserActivityMsg struct {
Cursor WsUserActivityCursor `json:"cursor"`
}
type WsUserActivityCursor struct {
X int `json:"x"`
Y int `json:"y"`
}
func (msg WsUserActivityMsg) Action() error {
fmt.Println("WsUserActivityMsg")
return nil
}
type WsShutdownMsg struct {
World string `json:"world"`
UserId *string `json:"userId,omitempty"`
}
func (msg WsShutdownMsg) Action() error {
fmt.Println("WsShutdownMsg")
return nil
}

14
internal/foundry/types/ws_channels.go
View File

@@ -1,31 +1,31 @@
package types
type Channels struct {
type ReadChannels struct {
done chan struct{}
msg chan *WsMessage
err chan error
}
func (channels Channels) Err() chan error {
func (channels ReadChannels) Err() chan error {
return channels.err
}
func (channels Channels) Msg() chan *WsMessage {
func (channels ReadChannels) Msg() chan *WsMessage {
return channels.msg
}
func (channels Channels) Done() chan struct{} {
func (channels ReadChannels) Done() chan struct{} {
return channels.done
}
func (channels *Channels) Close() {
func (channels *ReadChannels) Close() {
close(channels.done)
close(channels.err)
close(channels.msg)
}
func InitWsChannels() *Channels {
channels := Channels{
func InitWsChannels() *ReadChannels {
channels := ReadChannels{
done: make(chan struct{}),
err: make(chan error, 10),
msg: make(chan *WsMessage, 10),

236
internal/foundry/websocket.go
View File

@@ -3,6 +3,8 @@ package foundry
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"log/slog"
"strconv"
@@ -17,18 +19,28 @@ import (
"github.com/gorilla/websocket"
)
//TODO: refactor all code
type TransportCode int
type FoundryMode int
const (
WriterCode = TransportCode(0)
ReaderCode = TransportCode(1)
)
// const (
// SetupMode = FoundryMode(0)
// WorldMode = FoundryMode(1)
// )
// var
const (
RespSessionData = "0"
RespPingCode = "2"
RespSessionId = "40"
RespCreateSessionCode = "42"
RespServerChangeCode = "42"
RespDataCode = "43"
ReqPongCode = "3"
@@ -40,15 +52,15 @@ var RequestCodes = []string{
RespSessionData,
RespPingCode,
RespSessionId,
RespCreateSessionCode,
RespServerChangeCode,
RespDataCode,
}
var CodesRespToReq = map[string]string{
RespSessionData: ReqCreateSessionCode,
RespPingCode: ReqPongCode,
RespCreateSessionCode: ReqDataCode,
//ReqDataCode: RespCreateSessionCode,
RespServerChangeCode: ReqDataCode,
//ReqDataCode: RespServerChangeCode,
}
type webSocketUtil struct {
@@ -56,15 +68,23 @@ type webSocketUtil struct {
currWsId int
isReadReady bool
logger *slog.Logger
config requests.Config
msgMap map[int](chan []byte)
chanMutex sync.Mutex
channels types.Channels
readChan types.ReadChannels
exchangeChan types.ExchangeChannels
models db_model.Models
}
func NewWebSocketUtil() *webSocketUtil {
return &webSocketUtil{currWsId: 0, isReadReady: false, msgMap: make(map[int]chan []byte)}
return &webSocketUtil{
currWsId: 0,
isReadReady: false,
exchangeChan: types.ExchangeChannels{
Msgs: make(map[int]chan []byte),
ProgressMsg: map[string]chan struct{}{},
},
config: requests.Config{SessionID: ""}}
}
func parseCode(msg *string) string {
@@ -100,9 +120,6 @@ func parseWsRespMessage(msg string) (*types.WsMessage, error) {
data := &types.WsMessage{}
data.Code = parseCode(&msg)
if data.Code != RespDataCode {
return data, nil
}
i := len(data.Code)
if i == 0 {
@@ -111,8 +128,11 @@ func parseWsRespMessage(msg string) (*types.WsMessage, error) {
data.Id, i = parseId(&msg, i)
if i == 0 {
return nil, ErrorMsgNotHaveNumber
i = len(data.Code)
}
// if i == 0 {
// return nil, ErrorMsgNotHaveNumber
// }
data.MsgJson = msg[i:]
return data, nil
@@ -125,7 +145,7 @@ func (ws *webSocketUtil) ReceiveMessage(id int) ([]byte, error) {
for {
select {
case msg, ok := <-ws.msgMap[id]:
case msg, ok := <-ws.exchangeChan.Msgs[id]:
if !ok {
time.Sleep(5 * time.Microsecond)
continue
@@ -151,18 +171,18 @@ func (ws *webSocketUtil) closeMsgChannel(id int, timeout time.Duration) bool {
defer ws.chanMutex.Unlock()
ok := true
if _, ok = ws.msgMap[id]; !ok {
if _, ok = ws.exchangeChan.Msgs[id]; !ok {
return false
}
select {
case _, ok = <-ws.msgMap[id]:
case _, ok = <-ws.exchangeChan.Msgs[id]:
if ok {
close(ws.msgMap[id])
delete(ws.msgMap, id)
close(ws.exchangeChan.Msgs[id])
delete(ws.exchangeChan.Msgs, id)
}
default:
close(ws.msgMap[id])
delete(ws.msgMap, id)
close(ws.exchangeChan.Msgs[id])
delete(ws.exchangeChan.Msgs, id)
}
ws.logger.Debug("WS: Channel has been closed\n", "id", id, "timeout", timeout.String())
@@ -196,22 +216,22 @@ func (ws *webSocketUtil) ListenWebSocket() {
for {
_, message, err := ws.wsConn.ReadMessage()
if err != nil {
ws.channels.Err() <- &FoundryError{Type: ReaderCode, Err: err, IsFatal: true}
ws.readChan.Err() <- &FoundryError{Type: ReaderCode, Err: err, IsFatal: true}
return
}
data, err := parseWsRespMessage(string(message))
if err != nil {
ws.channels.Err() <- &FoundryError{Type: ReaderCode, Err: err}
ws.readChan.Err() <- &FoundryError{Type: ReaderCode, Err: err}
continue
}
ws.channels.Msg() <- data
ws.readChan.Msg() <- data
}
}
func (ws *webSocketUtil) ServeWebSocket() {
for {
select {
case message := <-ws.channels.Msg():
case message := <-ws.readChan.Msg():
ws.logger.Debug("WS: Data has been received\n", "msgCode", message.Code)
switch message.Code {
@@ -219,19 +239,25 @@ func (ws *webSocketUtil) ServeWebSocket() {
err := ws.sendOnlyCodeRequest(message.Code)
if err != nil {
ws.isReadReady = false
ws.channels.Err() <- &FoundryError{Type: WriterCode, Err: err}
ws.readChan.Err() <- &FoundryError{Type: WriterCode, Err: err}
continue
}
case RespCreateSessionCode:
case RespServerChangeCode:
ws.isReadReady = true
go ws.GetInitialData()
dataType, data, err := ws.SplitToTypeAndData([]byte(message.MsgJson))
if err != nil {
ws.readChan.Err() <- &FoundryError{Type: WriterCode, Err: err}
continue
}
// ws.logger.Debug("Parsed server change message", "dataType", dataType, "data", data)
ws.ProcessAction(dataType, data)
case RespDataCode:
ws.msgMap[message.Id] = make(chan []byte, 1)
ws.msgMap[message.Id] <- []byte(message.MsgJson)
ws.exchangeChan.Msgs[message.Id] = make(chan []byte, 1)
ws.exchangeChan.Msgs[message.Id] <- []byte(message.MsgJson)
go ws.closeMsgChannel(message.Id, 5*time.Second)
default:
}
case <-ws.channels.Done():
case <-ws.readChan.Done():
return
}
}
@@ -273,26 +299,170 @@ func (ws *webSocketUtil) InsertJsonDataToDB(statePath string) error {
}
func (ws *webSocketUtil) GetInitialData() {
err := ws.InitSetupData()
if err != nil {
ws.readChan.Err() <- &FoundryError{Type: WriterCode, Err: err}
return
}
idState, err := ws.models.FoundryState.GetIdByType(db_model.SetupState)
if err != nil {
ws.readChan.Err() <- &FoundryError{Type: WriterCode, Err: err}
return
}
ws.logger.Debug("Id state", "id", idState)
err = ws.InitWorldsData(idState)
if err != nil {
ws.readChan.Err() <- &FoundryError{Type: WriterCode, Err: err}
return
}
}
func (ws *webSocketUtil) InitSetupData() error {
ws.models.FoundryState.DeleteAll()
ws.models.FoundryState.DeleteAllSeq()
for k := range requests.PathToSetupState {
err := ws.InsertJsonDataToDB(k)
if err != nil {
ws.channels.Err() <- &FoundryError{Type: WriterCode, Err: err}
break
return err
}
}
// for k := range requests.PathToWorldState {
return nil
}
// }
func (ws *webSocketUtil) InitWorldsData(idState int64) error {
worlds, err := ws.models.FoundryState.GetWorlds(idState)
if err != nil {
return err
}
ws.logger.Debug("Worlds", "len", len(worlds))
for i := range worlds {
worldName := worlds[i].TextId
ws.InitWorldData(worldName)
_, err = ws.config.PostReturnToSetup()
if err != nil {
return err
}
}
return nil
}
func (ws *webSocketUtil) InitWorldData(worldName string) error {
ws.logger.Debug("World name", "name", worldName)
err := ws.LaunchWorld(worldName)
if err != nil {
return err
}
for k := range requests.PathToWorldState {
err := ws.InsertJsonDataToDB(k)
if err != nil {
return err
}
}
return nil
}
func (ws *webSocketUtil) LaunchWorld(worldName string) error {
resp, err := ws.config.PostLaunchWorld(worldName)
if err != nil {
return err
}
if resp.StatusCode != 200 {
return errors.New("World is not found")
}
ws.exchangeChan.ProgressMsg[worldName] = make(chan struct{})
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
select {
case <-ws.exchangeChan.ProgressMsg[worldName]:
ws.logger.Debug("World has been started(from GetInitialData)", "world", worldName)
case <-ctx.Done():
close(ws.exchangeChan.ProgressMsg[worldName])
err := ctx.Err()
if err != nil {
return err
}
return ErrorTimeout
}
return nil
}
func (ws *webSocketUtil) CreateNewDataModels(db *sql.DB) {
ws.models = db_model.NewModels(db)
}
func (ws *webSocketUtil) SplitToTypeAndData(dataJson []byte) (string, any, error) {
var rawItems []json.RawMessage
err := json.Unmarshal(dataJson, &rawItems)
if err != nil {
return "", nil, err
}
var dataType string
err = json.Unmarshal(rawItems[0], &dataType)
if err != nil {
return "", nil, err
}
var data map[string]any
err = json.Unmarshal(rawItems[1], &data)
if err != nil {
return "", nil, err
}
actionDataPtr := types.GetWsMsgAction(dataType)
if actionDataPtr == nil {
return dataType, nil, nil
}
err = types.FillStruct(data, actionDataPtr)
if err != nil {
return "", nil, err
}
return dataType, actionDataPtr, nil
}
func (ws *webSocketUtil) ProcessAction(dataType string, data any) {
switch tp := data.(type) {
case *types.WsSessionMsg:
ws.ProcessSessionAction(tp)
case *types.WsProgressMsg:
ws.ProcessProgressAction(tp)
case *types.WsShutdownMsg:
ws.ProcessShutdownAction(tp)
default:
ws.logger.Debug("default")
}
}
func (ws *webSocketUtil) ProcessSessionAction(data *types.WsSessionMsg) {
ws.logger.Debug("ProcessSessionAction", "data", data)
go ws.GetInitialData()
}
func (ws *webSocketUtil) ProcessProgressAction(data *types.WsProgressMsg) {
if !data.HasChanged {
ws.logger.Debug("World has been launched", "world", data.Id)
select {
case <-ws.exchangeChan.ProgressMsg[data.Id]:
ws.logger.Warn("Channel for the world is closed", "world", data.Id)
default:
close(ws.exchangeChan.ProgressMsg[data.Id])
}
}
}
func (ws *webSocketUtil) ProcessShutdownAction(data *types.WsShutdownMsg) {
ws.logger.Debug("ProcessShutdownAction", "data", data)
}
func (ws *webSocketUtil) CreateWSMessageByPage(page string) *types.WsMessage {
msgToSend := &types.WsMessage{Code: CodesRespToReq[RespCreateSessionCode], Id: ws.currWsId, MsgJson: fmt.Sprintf("[\"%s\"]", requests.WsTypeData[page])}
msgToSend := &types.WsMessage{Code: CodesRespToReq[RespServerChangeCode], Id: ws.currWsId, MsgJson: fmt.Sprintf("[\"%s\"]", requests.WsTypeData[page])}
ws.currWsId++
ws.logger.Debug("WS: Data to send\n", "msgToSend", msgToSend.ToString())