OMC/be.ems
1
0
Fork 0
Code Issues Releases 3 Activity

pref: 优化批量导入数据库

Browse Source
This commit is contained in:
TsMask
2023-09-23 14:35:12 +08:00
parent 56944ad13d
commit b35ac3e50b
5 changed files with 132 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
features/udm_user/repo/repo_udm_auth_user.go
View File

@@ -1,7 +1,6 @@
package repo
import (
"fmt"
"strconv"
"strings"
@@ -182,66 +181,39 @@ func (r *RepoUdmAuthUser) ClearAndInsert(neID string, authArr []model.UdmAuthUse
}
num += results
}
// // 清空指定ne_id
// _, err := datasource.ExecDB("", "DELETE FROM u_auth_user WHERE ne_id = ?", []any{neID})
// if err != nil {
// log.Errorf("TRUNCATE err => %v", err)
// return num
// }
// for _, u := range authArr {
// u.NeID = neID
// results, err := datasource.DefaultDB().Table("u_auth_user").Insert(u)
// if err != nil {
// return num
// }
// num += results
// }
return num
}
// Insert 新增实体
func (r *RepoUdmAuthUser) Insert(neID string, authUser model.UdmAuthUser) int64 {
authUser.NeID = neID
authUser.Status = "1"
func (r *RepoUdmAuthUser) Insert(authUser model.UdmAuthUser) int64 {
results, err := datasource.DefaultDB().Table("u_auth_user").Insert(authUser)
if err != nil {
log.Errorf("Insert err => %v", err)
return results
}
return results
}
// Insert 批量添加
func (r *RepoUdmAuthUser) Inserts(neID string, authUser model.UdmAuthUser, num string) int64 {
var insertNum int64
imsiVlen := len(authUser.Imsi)
imsiV, err := strconv.Atoi(authUser.Imsi)
if err != nil {
return 0
}
numV, err := strconv.Atoi(num)
if err != nil {
return 0
}
authUser.NeID = neID
authUser.Status = "1"
for i := 0; i < numV; i++ {
imsi := fmt.Sprint(imsiV + i)
if len(imsi) < imsiVlen {
imsi = fmt.Sprintf("%0*s", imsiVlen, imsi)
func (r *RepoUdmAuthUser) Inserts(authUsers []model.UdmAuthUser) int64 {
var num int64
n := len(authUsers)
batchSize := 5000
for i := 0; i < n; i += batchSize {
end := i + batchSize
if end > n {
end = n
}
authUser.Imsi = imsi
results, err := datasource.DefaultDB().Table("u_auth_user").Insert(authUser)
if err == nil {
insertNum += results
}
}
batch := authUsers[i:end]
return insertNum
// 调用 InsertMulti 函数将批量数据插入数据库
results, err := datasource.DefaultDB().Table("u_auth_user").InsertMulti(batch)
if err != nil {
continue
}
num += results
}
return num
}
// Update 修改更新

65
features/udm_user/repo/repo_udm_sub_user.go
View File

@@ -213,8 +213,7 @@ func (r *RepoUdmSubUser) ClearAndInsert(neID string, subArr []model.UdmSubUser)
}
// Insert 新增实体
func (r *RepoUdmSubUser) Insert(neID string, subUser model.UdmSubUser) int64 {
subUser.NeID = neID
func (r *RepoUdmSubUser) Insert(subUser model.UdmSubUser) int64 {
results, err := datasource.DefaultDB().Table("u_sub_user").Insert(subUser)
if err != nil {
return results
@@ -223,57 +222,25 @@ func (r *RepoUdmSubUser) Insert(neID string, subUser model.UdmSubUser) int64 {
}
// Insert 批量添加
func (r *RepoUdmSubUser) Inserts(neID string, subUser model.UdmSubUser, num string) int64 {
var insertNum int64
imsiVlen := len(subUser.Imsi)
imsiV, err := strconv.Atoi(subUser.Imsi)
if err != nil {
return 0
}
msisdnVlen := len(subUser.Msisdn)
msisdnV, err := strconv.Atoi(subUser.Msisdn)
if err != nil {
return 0
}
numV, err := strconv.Atoi(num)
if err != nil {
return 0
}
subUser.NeID = neID
for i := 0; i < numV; i++ {
msisdn := fmt.Sprint(msisdnV + i)
if len(msisdn) < msisdnVlen {
msisdn = fmt.Sprintf("%0*s", msisdnVlen, msisdn)
func (r *RepoUdmSubUser) Inserts(subUser []model.UdmSubUser) int64 {
var num int64
n := len(subUser)
batchSize := 5000
for i := 0; i < n; i += batchSize {
end := i + batchSize
if end > n {
end = n
}
subUser.Msisdn = msisdn
batch := subUser[i:end]
imsi := fmt.Sprint(imsiV + i)
if len(imsi) < imsiVlen {
imsi = fmt.Sprintf("%0*s", imsiVlen, imsi)
}
subUser.Imsi = imsi
// IP会自动递增
if subUser.StaticIp != "" {
parts := strings.Split(subUser.StaticIp, ".")
lastPart := parts[3]
lastNum, _ := strconv.Atoi(lastPart)
lastNum += i
newLastPart := strconv.Itoa(lastNum)
parts[3] = newLastPart
newIP := strings.Join(parts, ".")
subUser.StaticIp = newIP
}
results, err := datasource.DefaultDB().Table("u_sub_user").Insert(subUser)
if err == nil {
insertNum += results
// 调用 InsertMulti 函数将批量数据插入数据库
results, err := datasource.DefaultDB().Table("u_sub_user").InsertMulti(batch)
if err != nil {
continue
}
num += results
}
return insertNum
return num
}
// Insert4G 批量添加4G用户

1
features/udm_user/service/service_redis_data.go
View File

@@ -60,7 +60,6 @@ func redisUdmAuthUserList() []model.UdmAuthUser {
// redisUdmSubUserList UDM签约用户
func redisUdmSubUserList() []model.UdmSubUser {
// phoneAndImsiArr := phoneImsiList()
user := []model.UdmSubUser{}
udmsdArr, err := redis.GetKeys("udmuser", "udm-sd:*")
if err != nil {

46
features/udm_user/service/service_udm_auth_user.go
View File

@@ -1,6 +1,9 @@
package service
import (
"fmt"
"strconv"
"ems.agt/features/udm_user/model"
"ems.agt/features/udm_user/repo"
)
@@ -40,17 +43,43 @@ func (r *ServiceUdmAuthUser) List(authUser model.UdmAuthUser) []model.UdmAuthUse
// Insert UDM鉴权用户-新增单个
// imsi长度15，ki长度32，opc长度0或者32
func (r *ServiceUdmAuthUser) Insert(neID string, authUser model.UdmAuthUser) int64 {
return r.repoAuthUser.Insert(neID, authUser)
authUser.NeID = neID
authUser.Status = "1"
return r.repoAuthUser.Insert(authUser)
}
// Insert UDM鉴权用户-批量添加
func (r *ServiceUdmAuthUser) Inserts(neID string, authUser model.UdmAuthUser, num string) int64 {
return r.repoAuthUser.Inserts(neID, authUser, num)
var arr []model.UdmAuthUser
imsiVlen := len(authUser.Imsi)
imsiV, err := strconv.Atoi(authUser.Imsi)
if err != nil {
return 0
}
numV, err := strconv.Atoi(num)
if err != nil {
return 0
}
authUser.NeID = neID
authUser.Status = "1"
for i := 0; i < numV; i++ {
imsi := fmt.Sprint(imsiV + i)
if len(imsi) < imsiVlen {
imsi = fmt.Sprintf("%0*s", imsiVlen, imsi)
}
authUser.Imsi = imsi
arr = append(arr, authUser)
}
return r.repoAuthUser.Inserts(arr)
}
// InsertCSV UDM鉴权用户-批量添加
func (r *ServiceUdmAuthUser) InsertCSV(neID string, data []map[string]string) int64 {
var num int64
var arr []model.UdmAuthUser
for _, v := range data {
var authUser model.UdmAuthUser
authUser.NeID = neID
@@ -70,14 +99,14 @@ func (r *ServiceUdmAuthUser) InsertCSV(neID string, data []map[string]string) in
if s, ok := v["opc"]; ok {
authUser.Opc = s
}
num += r.repoAuthUser.Insert(neID, authUser)
arr = append(arr, authUser)
}
return num
return r.repoAuthUser.Inserts(arr)
}
// InsertTxt UDM鉴权用户-批量添加
func (r *ServiceUdmAuthUser) InsertTxt(neID string, data [][]string) int64 {
var num int64
var arr []model.UdmAuthUser
for _, v := range data {
if len(v) < 4 {
continue
@@ -92,9 +121,10 @@ func (r *ServiceUdmAuthUser) InsertTxt(neID string, data [][]string) int64 {
if len(v) == 5 {
authUser.Opc = v[4]
}
num += r.repoAuthUser.Insert(neID, authUser)
arr = append(arr, authUser)
}
return num
return r.repoAuthUser.Inserts(arr)
}
// Insert UDM鉴权用户-修改更新

67
features/udm_user/service/service_udm_sub_user.go
View File

@@ -1,6 +1,10 @@
package service
import (
"fmt"
"strconv"
"strings"
"ems.agt/features/udm_user/model"
"ems.agt/features/udm_user/repo"
)
@@ -40,17 +44,64 @@ func (r *ServiceUdmSubUser) List(subUser model.UdmSubUser) []model.UdmSubUser {
// Insert UDM签约用户-新增单个
// imsi长度15，ki长度32，opc长度0或者32
func (r *ServiceUdmSubUser) Insert(neID string, subUser model.UdmSubUser) int64 {
return r.repoSunUser.Insert(neID, subUser)
subUser.NeID = neID
return r.repoSunUser.Insert(subUser)
}
// Insert UDM签约用户-批量添加
func (r *ServiceUdmSubUser) Inserts(neID string, subUser model.UdmSubUser, num string) int64 {
return r.repoSunUser.Inserts(neID, subUser, num)
var arr []model.UdmSubUser
imsiVlen := len(subUser.Imsi)
imsiV, err := strconv.Atoi(subUser.Imsi)
if err != nil {
return 0
}
msisdnVlen := len(subUser.Msisdn)
msisdnV, err := strconv.Atoi(subUser.Msisdn)
if err != nil {
return 0
}
numV, err := strconv.Atoi(num)
if err != nil {
return 0
}
subUser.NeID = neID
for i := 0; i < numV; i++ {
msisdn := fmt.Sprint(msisdnV + i)
if len(msisdn) < msisdnVlen {
msisdn = fmt.Sprintf("%0*s", msisdnVlen, msisdn)
}
subUser.Msisdn = msisdn
imsi := fmt.Sprint(imsiV + i)
if len(imsi) < imsiVlen {
imsi = fmt.Sprintf("%0*s", imsiVlen, imsi)
}
subUser.Imsi = imsi
// IP会自动递增
if subUser.StaticIp != "" {
parts := strings.Split(subUser.StaticIp, ".")
lastPart := parts[3]
lastNum, _ := strconv.Atoi(lastPart)
lastNum += i
newLastPart := strconv.Itoa(lastNum)
parts[3] = newLastPart
newIP := strings.Join(parts, ".")
subUser.StaticIp = newIP
}
arr = append(arr, subUser)
}
return r.repoSunUser.Inserts(arr)
}
// InsertCSV UDM签约用户-批量添加
func (r *ServiceUdmSubUser) InsertCSV(neID string, data []map[string]string) int64 {
var num int64
var arr []model.UdmSubUser
for _, v := range data {
var subUser model.UdmSubUser
subUser.NeID = neID
@@ -87,14 +138,14 @@ func (r *ServiceUdmSubUser) InsertCSV(neID string, data []map[string]string) int
if s, ok := v["eps_dat"]; ok {
subUser.EpsDat = s
}
num += r.repoSunUser.Insert(neID, subUser)
arr = append(arr, subUser)
}
return num
return r.repoSunUser.Inserts(arr)
}
// InsertTxt UDM签约用户-批量添加
func (r *ServiceUdmSubUser) InsertTxt(neID string, data [][]string) int64 {
var num int64
var arr []model.UdmSubUser
for _, v := range data {
if len(v) < 10 {
continue
@@ -114,9 +165,9 @@ func (r *ServiceUdmSubUser) InsertTxt(neID string, data [][]string) int64 {
if len(v) == 11 {
subUser.EpsDat = v[10]
}
num += r.repoSunUser.Insert(neID, subUser)
arr = append(arr, subUser)
}
return num
return r.repoSunUser.Inserts(arr)
}
// Insert UDM签约用户-批量添加4G用户