Revert "switch to batch inserts"

This reverts commit d6245debb9
This commit is contained in:
Jonas Leder 2025-05-01 15:13:44 +02:00
commit c375529258

64
main.go
View file

@ -13,8 +13,6 @@ import (
_ "github.com/nakagami/firebirdsql"
)
const batchSize = 100
func main() {
err := godotenv.Load()
if err != nil {
@ -128,6 +126,7 @@ func getSortedTablesByDependency(db *sql.DB) ([]string, error) {
}
if len(sorted) < len(tableSet) {
// Append remaining tables not in sorted (no FKs)
existing := map[string]bool{}
for _, t := range sorted {
existing[t] = true
@ -143,7 +142,6 @@ func getSortedTablesByDependency(db *sql.DB) ([]string, error) {
}
func migrateTable(src *sql.DB, dst *sql.DB, table string) error {
log.Printf("Selecting from source table: %s", table)
rows, err := src.Query(fmt.Sprintf("SELECT * FROM %s", table))
if err != nil {
return err
@ -155,20 +153,32 @@ func migrateTable(src *sql.DB, dst *sql.DB, table string) error {
return err
}
namedParams := make([]string, len(columns))
for i := range namedParams {
namedParams[i] = fmt.Sprintf("@p%d", i+1)
}
tx, err := dst.Begin()
if err != nil {
return err
}
defer tx.Rollback()
log.Printf("Deleting from destination table: %s", table)
_, err = tx.Exec(fmt.Sprintf("DELETE FROM [%s]", table))
if err != nil {
return fmt.Errorf("failed to clear table: %w", err)
}
colList := join(columns, ", ")
var batch [][]interface{}
insertSQL := fmt.Sprintf("INSERT INTO [%s] (%s) VALUES (%s)",
table,
join(columns, ", "),
strings.Join(namedParams, ", "))
stmt, err := tx.Prepare(insertSQL)
if err != nil {
return err
}
defer stmt.Close()
for rows.Next() {
values := make([]interface{}, len(columns))
@ -179,6 +189,7 @@ func migrateTable(src *sql.DB, dst *sql.DB, table string) error {
if err := rows.Scan(pointers...); err != nil {
return err
}
for i, v := range values {
if t, ok := v.(time.Time); ok {
if t.Before(time.Date(1753, 1, 1, 0, 0, 0, 0, time.UTC)) {
@ -186,49 +197,16 @@ func migrateTable(src *sql.DB, dst *sql.DB, table string) error {
}
}
}
batch = append(batch, values)
if len(batch) == batchSize {
log.Printf("Inserting batch into destination table: %s", table)
if err := insertBatch(tx, table, colList, batch); err != nil {
return err
}
batch = batch[:0]
}
}
if len(batch) > 0 {
log.Printf("Inserting final batch into destination table: %s", table)
if err := insertBatch(tx, table, colList, batch); err != nil {
return err
_, err = stmt.Exec(values...)
if err != nil {
return fmt.Errorf("insert failed: %w", err)
}
}
return tx.Commit()
}
func insertBatch(tx *sql.Tx, table, colList string, batch [][]interface{}) error {
if len(batch) == 0 {
return nil
}
rowPlaceholders := make([]string, len(batch))
args := []interface{}{}
argCounter := 1
for i, row := range batch {
placeholders := make([]string, len(row))
for j := range row {
placeholders[j] = fmt.Sprintf("@p%d", argCounter)
argCounter++
}
rowPlaceholders[i] = fmt.Sprintf("(%s)", strings.Join(placeholders, ", "))
args = append(args, row...)
}
query := fmt.Sprintf("INSERT INTO [%s] (%s) VALUES %s", table, colList, strings.Join(rowPlaceholders, ", "))
_, err := tx.Exec(query, args...)
return err
}
func join(items []string, sep string) string {
out := ""
for i, item := range items {