Lbenedar/Foundry-Scrapping-API
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

finish setup data insertion, add world insertion

Browse Source
This commit is contained in:
lbenedar
2026-04-22 18:18:43 +03:00
parent 21abe68858
commit f46ff8333e
66 changed files with 2257 additions and 676 deletions
Download Patch File Download Diff File Expand all files Collapse all files

150
internal/foundry/temp_models/db/utils.go
View File

@@ -4,14 +4,15 @@ import (
"context"
"errors"
"fmt"
"sync"
"time"
"github.com/jmoiron/sqlx"
"golang.org/x/sync/errgroup"
)
var (
ErrNoQuery = errors.New("Query has not been set")
ErrNoQuery = errors.New("Query has not been set")
ErrorRecordNotFound = errors.New("Record not found")
)
type AllowedIds interface {
@@ -25,75 +26,40 @@ type InsertId[T AllowedIds] struct {
query string
}
type SyncDB struct {
errChan chan error
wg sync.WaitGroup
}
func NewSyncDB() *SyncDB {
return &SyncDB{
wg: sync.WaitGroup{},
errChan: make(chan error),
}
}
func (s *SyncDB) Wait() error {
return WaitSync(&s.wg, s.errChan)
}
func WaitSync(wg *sync.WaitGroup, errChan chan error) error {
wgDone := make(chan struct{})
go func() {
wg.Wait()
close(wgDone)
}()
select {
case <-wgDone:
return nil
case err := <-errChan:
close(wgDone)
return err
}
}
type Insertable[T AllowedIds] interface {
Query(data *InsertId[T])
Insert(tx *sqlx.Tx, relId *InsertId[T]) error
InsertCtx(ctx context.Context, tx *sqlx.Tx, relId *InsertId[T]) error
}
func InsertWithCtx[T AllowedIds, I Insertable[T]](tx *sqlx.Tx, data I, relId *InsertId[T]) error {
func InsertWithCtx[T AllowedIds, I Insertable[T]](tx *sqlx.Tx, data I, relId InsertId[T]) error {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
data.Query(relId)
return data.InsertCtx(ctx, tx, relId)
data.Query(&relId)
return data.InsertCtx(ctx, tx, &relId)
}
func InsertWithCtxParallel[T AllowedIds, I Insertable[T]](syncDb *SyncDB, tx *sqlx.Tx, data I, relId *InsertId[T]) {
syncDb.wg.Go(func() {
func InsertWithCtxParallel[T AllowedIds, I Insertable[T]](g *errgroup.Group, ctx context.Context, tx *sqlx.Tx, data I, relId InsertId[T]) {
g.Go(func() error {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
data.Query(relId)
err := data.InsertCtx(ctx, tx, relId)
if err != nil {
syncDb.errChan <- err
}
data.Query(&relId)
err := data.InsertCtx(ctx, tx, &relId)
return err
})
}
func InsertSlice[T AllowedIds, I Insertable[T]](tx *sqlx.Tx, data []I, relId *InsertId[T]) error {
func InsertSlice[T AllowedIds, I Insertable[T]](tx *sqlx.Tx, data []I, relId InsertId[T]) error {
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
if len(data) > 0 {
data[0].Query(relId)
data[0].Query(&relId)
}
for i := range data {
err := data[i].InsertCtx(ctx, tx, relId)
err := data[i].InsertCtx(ctx, tx, &relId)
if err != nil {
return err
}
@@ -101,32 +67,25 @@ func InsertSlice[T AllowedIds, I Insertable[T]](tx *sqlx.Tx, data []I, relId *In
return nil
}
func InsertSliceParallel[T AllowedIds, I Insertable[T]](syncDb *SyncDB, tx *sqlx.Tx, data []I, relId *InsertId[T]) {
syncDb.wg.Go(func() {
wg := sync.WaitGroup{}
errChan := make(chan error)
defer close(errChan)
func InsertSliceParallel[T AllowedIds, I Insertable[T]](g *errgroup.Group, tx *sqlx.Tx, data []I, relId InsertId[T]) {
g.Go(func() error {
wg, _ := errgroup.WithContext(context.Background())
var err error
if len(data) > 0 {
data[0].Query(relId)
data[0].Query(&relId)
}
for i := range data {
wg.Go(func() {
wg.Go(func() error {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
err = data[i].InsertCtx(ctx, tx, relId)
if err != nil {
errChan <- err
}
err = data[i].InsertCtx(ctx, tx, &relId)
return err
})
}
err = WaitSync(&wg, errChan)
if err != nil {
syncDb.errChan <- err
}
return wg.Wait()
})
}
@@ -147,30 +106,69 @@ func InsertSimpleSlice[T AllowedIds, I any](tx *sqlx.Tx, data []I, relId *Insert
return nil
}
func InsertSimpleSliceParallel[T AllowedIds, I any](syncDb *SyncDB, tx *sqlx.Tx, data []I, relId *InsertId[T]) {
syncDb.wg.Go(func() {
wg := sync.WaitGroup{}
errChan := make(chan error)
defer close(errChan)
func InsertSimpleSliceParallel[T AllowedIds, I any](g *errgroup.Group, tx *sqlx.Tx, data []I, relId *InsertId[T]) {
g.Go(func() error {
wg, _ := errgroup.WithContext(context.Background())
query := fmt.Sprintf(`
INSERT INTO %s (%s, value)
VALUES ($1, $2)`, relId.tableName, relId.fieldName)
for i := range data {
wg.Go(func() {
wg.Go(func() error {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
_, err := tx.ExecContext(ctx, query, relId.id, data[i])
if err != nil {
errChan <- err
}
return err
})
}
err := WaitSync(&wg, errChan)
if err != nil {
syncDb.errChan <- err
}
return wg.Wait()
})
}
func DeleteAll(db *sqlx.DB) error {
query := `
DELETE FROM setup`
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
result, err := db.ExecContext(ctx, query)
if err != nil {
return err
}
rowsAffected, err := result.RowsAffected()
if err != nil {
return err
}
if rowsAffected == 0 {
return ErrorRecordNotFound
}
return nil
}
func DeleteAllSeq(db *sqlx.DB) error {
query := `
DELETE FROM sqlite_sequence`
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
result, err := db.ExecContext(ctx, query)
if err != nil {
return err
}
rowsAffected, err := result.RowsAffected()
if err != nil {
return err
}
if rowsAffected == 0 {
return ErrorRecordNotFound
}
return nil
}