Lbenedar/Foundry-Scrapping-API
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

finish setup insert method, refactor sql migration file

Browse Source
This commit is contained in:
lbenedar
2026-04-17 17:33:21 +03:00
parent 1b7c7e9ae3
commit 21abe68858
47 changed files with 2054 additions and 1056 deletions
Download Patch File Download Diff File Expand all files Collapse all files

118
internal/foundry/temp_models/db/utils.go
View File

@@ -2,6 +2,7 @@ package db
import (
"context"
"errors"
"fmt"
"sync"
"time"
@@ -9,6 +10,10 @@ import (
"github.com/jmoiron/sqlx"
)
var (
ErrNoQuery = errors.New("Query has not been set")
)
type AllowedIds interface {
~uint | ~string
}
@@ -17,14 +22,22 @@ type InsertId[T AllowedIds] struct {
id T
fieldName string
tableName string
query string
}
type SyncDbOperations struct {
type SyncDB struct {
errChan chan error
wg sync.WaitGroup
}
func (s *SyncDbOperations) Wait() error {
func NewSyncDB() *SyncDB {
return &SyncDB{
wg: sync.WaitGroup{},
errChan: make(chan error),
}
}
func (s *SyncDB) Wait() error {
return WaitSync(&s.wg, s.errChan)
}
@@ -46,6 +59,7 @@ func WaitSync(wg *sync.WaitGroup, errChan chan error) error {
}
type Insertable[T AllowedIds] interface {
Query(data *InsertId[T])
Insert(tx *sqlx.Tx, relId *InsertId[T]) error
InsertCtx(ctx context.Context, tx *sqlx.Tx, relId *InsertId[T]) error
}
@@ -54,13 +68,30 @@ func InsertWithCtx[T AllowedIds, I Insertable[T]](tx *sqlx.Tx, data I, relId *In
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
data.Query(relId)
return data.InsertCtx(ctx, tx, relId)
}
func InsertWithCtxParallel[T AllowedIds, I Insertable[T]](syncDb *SyncDB, tx *sqlx.Tx, data I, relId *InsertId[T]) {
syncDb.wg.Go(func() {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
data.Query(relId)
err := data.InsertCtx(ctx, tx, relId)
if err != nil {
syncDb.errChan <- err
}
})
}
func InsertSlice[T AllowedIds, I Insertable[T]](tx *sqlx.Tx, data []I, relId *InsertId[T]) error {
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
if len(data) > 0 {
data[0].Query(relId)
}
for i := range data {
err := data[i].InsertCtx(ctx, tx, relId)
if err != nil {
@@ -70,41 +101,42 @@ func InsertSlice[T AllowedIds, I Insertable[T]](tx *sqlx.Tx, data []I, relId *In
return nil
}
func InsertSliceParallel[T AllowedIds, I Insertable[T]](syncDb *SyncDbOperations, tx *sqlx.Tx, data []I, relId *InsertId[T]) {
syncDb.wg.Add(1)
func InsertSliceParallel[T AllowedIds, I Insertable[T]](syncDb *SyncDB, tx *sqlx.Tx, data []I, relId *InsertId[T]) {
syncDb.wg.Go(func() {
wg := sync.WaitGroup{}
errChan := make(chan error)
defer close(errChan)
wg := sync.WaitGroup{}
errChan := make(chan error)
defer close(errChan)
var err error
if len(data) > 0 {
data[0].Query(relId)
}
for i := range data {
wg.Go(func() {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
for i := range data {
go func() {
wg.Add(1)
err = data[i].InsertCtx(ctx, tx, relId)
if err != nil {
errChan <- err
}
})
}
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
errChan <- data[i].InsertCtx(ctx, tx, relId)
wg.Done()
}()
}
err := WaitSync(&wg, errChan)
if err != nil {
syncDb.errChan <- err
}
syncDb.wg.Done()
err = WaitSync(&wg, errChan)
if err != nil {
syncDb.errChan <- err
}
})
}
func InsertSimpleSliceToTable[T AllowedIds, I Insertable[T]](tx *sqlx.Tx, data []I, relId *InsertId[T]) error {
query := fmt.Sprintf(`
INSERT INTO %s (%s, value)
VALUES (:id, :value)`, relId.tableName, relId.fieldName)
func InsertSimpleSlice[T AllowedIds, I any](tx *sqlx.Tx, data []I, relId *InsertId[T]) error {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
query := fmt.Sprintf(`
INSERT INTO %s (%s, value)
VALUES ($1, $2)`, relId.tableName, relId.fieldName)
for i := range data {
_, err := tx.ExecContext(ctx, query, relId.id, data[i])
if err != nil {
@@ -114,3 +146,31 @@ func InsertSimpleSliceToTable[T AllowedIds, I Insertable[T]](tx *sqlx.Tx, data [
return nil
}
func InsertSimpleSliceParallel[T AllowedIds, I any](syncDb *SyncDB, tx *sqlx.Tx, data []I, relId *InsertId[T]) {
syncDb.wg.Go(func() {
wg := sync.WaitGroup{}
errChan := make(chan error)
defer close(errChan)
query := fmt.Sprintf(`
INSERT INTO %s (%s, value)
VALUES ($1, $2)`, relId.tableName, relId.fieldName)
for i := range data {
wg.Go(func() {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
_, err := tx.ExecContext(ctx, query, relId.id, data[i])
if err != nil {
errChan <- err
}
})
}
err := WaitSync(&wg, errChan)
if err != nil {
syncDb.errChan <- err
}
})
}