pref: 优化批量导入数据库

features/udm_user/repo/repo_udm_auth_user.go

@@ -1,7 +1,6 @@
 package repo
 
 import (
-	"fmt"
 	"strconv"
 	"strings"
 
@@ -182,66 +181,39 @@ func (r *RepoUdmAuthUser) ClearAndInsert(neID string, authArr []model.UdmAuthUse
 		}
 		num += results
 	}
-
-	// // 清空指定ne_id
-	// _, err := datasource.ExecDB("", "DELETE FROM u_auth_user WHERE ne_id = ?", []any{neID})
-	// if err != nil {
-	// 	log.Errorf("TRUNCATE err => %v", err)
-	// 	return num
-	// }
-
-	// for _, u := range authArr {
-	// 	u.NeID = neID
-	// 	results, err := datasource.DefaultDB().Table("u_auth_user").Insert(u)
-	// 	if err != nil {
-	// 		return num
-	// 	}
-	// 	num += results
-	// }
 	return num
 }
 
 // Insert 新增实体
-func (r *RepoUdmAuthUser) Insert(neID string, authUser model.UdmAuthUser) int64 {
+func (r *RepoUdmAuthUser) Insert(authUser model.UdmAuthUser) int64 {
-	authUser.NeID = neID
-	authUser.Status = "1"
 	results, err := datasource.DefaultDB().Table("u_auth_user").Insert(authUser)
 	if err != nil {
+		log.Errorf("Insert err => %v", err)
 		return results
 	}
 	return results
 }
 
 // Insert 批量添加
-func (r *RepoUdmAuthUser) Inserts(neID string, authUser model.UdmAuthUser, num string) int64 {
+func (r *RepoUdmAuthUser) Inserts(authUsers []model.UdmAuthUser) int64 {
-	var insertNum int64
+	var num int64
-
+	n := len(authUsers)
-	imsiVlen := len(authUser.Imsi)
+	batchSize := 5000
-	imsiV, err := strconv.Atoi(authUser.Imsi)
+	for i := 0; i < n; i += batchSize {
-	if err != nil {
+		end := i + batchSize
-		return 0
+		if end > n {
-	}
+			end = n
-
-	numV, err := strconv.Atoi(num)
-	if err != nil {
-		return 0
-	}
-
-	authUser.NeID = neID
-	authUser.Status = "1"
-	for i := 0; i < numV; i++ {
-		imsi := fmt.Sprint(imsiV + i)
-		if len(imsi) < imsiVlen {
-			imsi = fmt.Sprintf("%0*s", imsiVlen, imsi)
 		}
-		authUser.Imsi = imsi
+		batch := authUsers[i:end]
-		results, err := datasource.DefaultDB().Table("u_auth_user").Insert(authUser)
-		if err == nil {
-			insertNum += results
-		}
-	}
 
-	return insertNum
+		// 调用 InsertMulti 函数将批量数据插入数据库
+		results, err := datasource.DefaultDB().Table("u_auth_user").InsertMulti(batch)
+		if err != nil {
+			continue
+		}
+		num += results
+	}
+	return num
 }
 
 // Update 修改更新

features/udm_user/repo/repo_udm_sub_user.go

@@ -213,8 +213,7 @@ func (r *RepoUdmSubUser) ClearAndInsert(neID string, subArr []model.UdmSubUser)
 }
 
 // Insert 新增实体
-func (r *RepoUdmSubUser) Insert(neID string, subUser model.UdmSubUser) int64 {
+func (r *RepoUdmSubUser) Insert(subUser model.UdmSubUser) int64 {
-	subUser.NeID = neID
 	results, err := datasource.DefaultDB().Table("u_sub_user").Insert(subUser)
 	if err != nil {
 		return results
@@ -223,57 +222,25 @@ func (r *RepoUdmSubUser) Insert(neID string, subUser model.UdmSubUser) int64 {
 }
 
 // Insert 批量添加
-func (r *RepoUdmSubUser) Inserts(neID string, subUser model.UdmSubUser, num string) int64 {
+func (r *RepoUdmSubUser) Inserts(subUser []model.UdmSubUser) int64 {
-	var insertNum int64
+	var num int64
-
+	n := len(subUser)
-	imsiVlen := len(subUser.Imsi)
+	batchSize := 5000
-	imsiV, err := strconv.Atoi(subUser.Imsi)
+	for i := 0; i < n; i += batchSize {
-	if err != nil {
+		end := i + batchSize
-		return 0
+		if end > n {
-	}
+			end = n
-	msisdnVlen := len(subUser.Msisdn)
-	msisdnV, err := strconv.Atoi(subUser.Msisdn)
-	if err != nil {
-		return 0
-	}
-
-	numV, err := strconv.Atoi(num)
-	if err != nil {
-		return 0
-	}
-
-	subUser.NeID = neID
-	for i := 0; i < numV; i++ {
-		msisdn := fmt.Sprint(msisdnV + i)
-		if len(msisdn) < msisdnVlen {
-			msisdn = fmt.Sprintf("%0*s", msisdnVlen, msisdn)
 		}
-		subUser.Msisdn = msisdn
+		batch := subUser[i:end]
 
-		imsi := fmt.Sprint(imsiV + i)
+		// 调用 InsertMulti 函数将批量数据插入数据库
-		if len(imsi) < imsiVlen {
+		results, err := datasource.DefaultDB().Table("u_sub_user").InsertMulti(batch)
-			imsi = fmt.Sprintf("%0*s", imsiVlen, imsi)
+		if err != nil {
-		}
+			continue
-		subUser.Imsi = imsi
-
-		// IP会自动递增
-		if subUser.StaticIp != "" {
-			parts := strings.Split(subUser.StaticIp, ".")
-			lastPart := parts[3]
-			lastNum, _ := strconv.Atoi(lastPart)
-			lastNum += i
-			newLastPart := strconv.Itoa(lastNum)
-			parts[3] = newLastPart
-			newIP := strings.Join(parts, ".")
-			subUser.StaticIp = newIP
-		}
-		results, err := datasource.DefaultDB().Table("u_sub_user").Insert(subUser)
-		if err == nil {
-			insertNum += results
 		}
+		num += results
 	}
-
+	return num
-	return insertNum
 }
 
 // Insert4G 批量添加4G用户

features/udm_user/service/service_redis_data.go

@@ -60,7 +60,6 @@ func redisUdmAuthUserList() []model.UdmAuthUser {
 
 // redisUdmSubUserList UDM签约用户
 func redisUdmSubUserList() []model.UdmSubUser {
-	// phoneAndImsiArr := phoneImsiList()
 	user := []model.UdmSubUser{}
 	udmsdArr, err := redis.GetKeys("udmuser", "udm-sd:*")
 	if err != nil {

features/udm_user/service/service_udm_auth_user.go

@@ -1,6 +1,9 @@
 package service
 
 import (
+	"fmt"
+	"strconv"
+
 	"ems.agt/features/udm_user/model"
 	"ems.agt/features/udm_user/repo"
 )
@@ -40,17 +43,43 @@ func (r *ServiceUdmAuthUser) List(authUser model.UdmAuthUser) []model.UdmAuthUse
 // Insert UDM鉴权用户-新增单个
 // imsi长度15，ki长度32，opc长度0或者32
 func (r *ServiceUdmAuthUser) Insert(neID string, authUser model.UdmAuthUser) int64 {
-	return r.repoAuthUser.Insert(neID, authUser)
+	authUser.NeID = neID
+	authUser.Status = "1"
+	return r.repoAuthUser.Insert(authUser)
 }
 
 // Insert UDM鉴权用户-批量添加
 func (r *ServiceUdmAuthUser) Inserts(neID string, authUser model.UdmAuthUser, num string) int64 {
-	return r.repoAuthUser.Inserts(neID, authUser, num)
+	var arr []model.UdmAuthUser
+
+	imsiVlen := len(authUser.Imsi)
+	imsiV, err := strconv.Atoi(authUser.Imsi)
+	if err != nil {
+		return 0
+	}
+
+	numV, err := strconv.Atoi(num)
+	if err != nil {
+		return 0
+	}
+
+	authUser.NeID = neID
+	authUser.Status = "1"
+	for i := 0; i < numV; i++ {
+		imsi := fmt.Sprint(imsiV + i)
+		if len(imsi) < imsiVlen {
+			imsi = fmt.Sprintf("%0*s", imsiVlen, imsi)
+		}
+		authUser.Imsi = imsi
+		arr = append(arr, authUser)
+	}
+
+	return r.repoAuthUser.Inserts(arr)
 }
 
 // InsertCSV UDM鉴权用户-批量添加
 func (r *ServiceUdmAuthUser) InsertCSV(neID string, data []map[string]string) int64 {
-	var num int64
+	var arr []model.UdmAuthUser
 	for _, v := range data {
 		var authUser model.UdmAuthUser
 		authUser.NeID = neID
@@ -70,14 +99,14 @@ func (r *ServiceUdmAuthUser) InsertCSV(neID string, data []map[string]string) in
 		if s, ok := v["opc"]; ok {
 			authUser.Opc = s
 		}
-		num += r.repoAuthUser.Insert(neID, authUser)
+		arr = append(arr, authUser)
 	}
-	return num
+	return r.repoAuthUser.Inserts(arr)
 }
 
 // InsertTxt UDM鉴权用户-批量添加
 func (r *ServiceUdmAuthUser) InsertTxt(neID string, data [][]string) int64 {
-	var num int64
+	var arr []model.UdmAuthUser
 	for _, v := range data {
 		if len(v) < 4 {
 			continue
@@ -92,9 +121,10 @@ func (r *ServiceUdmAuthUser) InsertTxt(neID string, data [][]string) int64 {
 		if len(v) == 5 {
 			authUser.Opc = v[4]
 		}
-		num += r.repoAuthUser.Insert(neID, authUser)
+		arr = append(arr, authUser)
 	}
-	return num
+
+	return r.repoAuthUser.Inserts(arr)
 }
 
 // Insert UDM鉴权用户-修改更新

features/udm_user/service/service_udm_sub_user.go

@@ -1,6 +1,10 @@
 package service
 
 import (
+	"fmt"
+	"strconv"
+	"strings"
+
 	"ems.agt/features/udm_user/model"
 	"ems.agt/features/udm_user/repo"
 )
@@ -40,17 +44,64 @@ func (r *ServiceUdmSubUser) List(subUser model.UdmSubUser) []model.UdmSubUser {
 // Insert UDM签约用户-新增单个
 // imsi长度15，ki长度32，opc长度0或者32
 func (r *ServiceUdmSubUser) Insert(neID string, subUser model.UdmSubUser) int64 {
-	return r.repoSunUser.Insert(neID, subUser)
+	subUser.NeID = neID
+	return r.repoSunUser.Insert(subUser)
 }
 
 // Insert UDM签约用户-批量添加
 func (r *ServiceUdmSubUser) Inserts(neID string, subUser model.UdmSubUser, num string) int64 {
-	return r.repoSunUser.Inserts(neID, subUser, num)
+	var arr []model.UdmSubUser
+
+	imsiVlen := len(subUser.Imsi)
+	imsiV, err := strconv.Atoi(subUser.Imsi)
+	if err != nil {
+		return 0
+	}
+	msisdnVlen := len(subUser.Msisdn)
+	msisdnV, err := strconv.Atoi(subUser.Msisdn)
+	if err != nil {
+		return 0
+	}
+
+	numV, err := strconv.Atoi(num)
+	if err != nil {
+		return 0
+	}
+
+	subUser.NeID = neID
+	for i := 0; i < numV; i++ {
+		msisdn := fmt.Sprint(msisdnV + i)
+		if len(msisdn) < msisdnVlen {
+			msisdn = fmt.Sprintf("%0*s", msisdnVlen, msisdn)
+		}
+		subUser.Msisdn = msisdn
+
+		imsi := fmt.Sprint(imsiV + i)
+		if len(imsi) < imsiVlen {
+			imsi = fmt.Sprintf("%0*s", imsiVlen, imsi)
+		}
+		subUser.Imsi = imsi
+
+		// IP会自动递增
+		if subUser.StaticIp != "" {
+			parts := strings.Split(subUser.StaticIp, ".")
+			lastPart := parts[3]
+			lastNum, _ := strconv.Atoi(lastPart)
+			lastNum += i
+			newLastPart := strconv.Itoa(lastNum)
+			parts[3] = newLastPart
+			newIP := strings.Join(parts, ".")
+			subUser.StaticIp = newIP
+		}
+		arr = append(arr, subUser)
+	}
+
+	return r.repoSunUser.Inserts(arr)
 }
 
 // InsertCSV UDM签约用户-批量添加
 func (r *ServiceUdmSubUser) InsertCSV(neID string, data []map[string]string) int64 {
-	var num int64
+	var arr []model.UdmSubUser
 	for _, v := range data {
 		var subUser model.UdmSubUser
 		subUser.NeID = neID
@@ -87,14 +138,14 @@ func (r *ServiceUdmSubUser) InsertCSV(neID string, data []map[string]string) int
 		if s, ok := v["eps_dat"]; ok {
 			subUser.EpsDat = s
 		}
-		num += r.repoSunUser.Insert(neID, subUser)
+		arr = append(arr, subUser)
 	}
-	return num
+	return r.repoSunUser.Inserts(arr)
 }
 
 // InsertTxt UDM签约用户-批量添加
 func (r *ServiceUdmSubUser) InsertTxt(neID string, data [][]string) int64 {
-	var num int64
+	var arr []model.UdmSubUser
 	for _, v := range data {
 		if len(v) < 10 {
 			continue
@@ -114,9 +165,9 @@ func (r *ServiceUdmSubUser) InsertTxt(neID string, data [][]string) int64 {
 		if len(v) == 11 {
 			subUser.EpsDat = v[10]
 		}
-		num += r.repoSunUser.Insert(neID, subUser)
+		arr = append(arr, subUser)
 	}
-	return num
+	return r.repoSunUser.Inserts(arr)
 }
 
 // Insert UDM签约用户-批量添加4G用户