optimize migration proces

config/general-setting.yml

@@ -182,7 +182,7 @@ redirectUrl:
 - http://localhost:4200/oauth2redirect
 - https://cmpd.telmessenger.com/oauth2redirect
 - https://app.insomnia.rest/oauth/redirect
-dataKey: RMyvjTZXDMBnCr3jymw9oHGUUzIJTHCULv9x%2B6xE4De56nu2OlXjkTWuk5%2B3QLfyE5PJQUBl9SIAuZOGmpIJBTHvNBFOUN%2BtwT0FzDUIKfD%2BdOcU4lJzNzo9mc2cMilw
+dataKey: aY6tKotSoUoMIi7JiTBZq8CkB0I1UEuOu79K3g%2FQejKhSsmYjMq0dfnpcMVIWAWdZ51b%2B3Lj8llKQkPMaX%2BvlR0SRfOKo5W87r2XIwSHJNxf1H5pKL1mL84ZXNBh9%2BYF
 
 #database: 
 #  type: sqlserver

src/main/kotlin/com/datacomsolusindo/migration/MigrationEntity.kt

@@ -33,6 +33,7 @@ import java.io.File
 import java.nio.charset.StandardCharsets
 import java.time.LocalDate
 import java.util.UUID
+import kotlin.collections.set
 import kotlin.time.measureTimedValue
 
 
@@ -203,12 +204,25 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
             }
         }
 
+        val value = process.value.map { m ->
+            val dt = m as MutableMap<String, Any?>
+            dt["password"]?.toString()?.let {
+                dt["password"] = if (it.isBlank()) ""
+                else cpDecrypt.decrypt(it)?.let { p ->
+                    tempPassword[p] ?: run {
+                        val pass = passwordEncoder.encode(p)
+                        tempPassword[p] = pass
+                        pass
+                    }
+                } ?: ""
+            }
+            dt
+        }
         logger.info(
             "finish prepare data [${process.value.size}] migration class ${clazz.simpleName} " +
                     "takes time ${process.duration.inWholeMilliseconds}ms"
         )
-
-        return process.value
+        return value
     }
 
     private fun readQueryDataToMap(file: File): List<Map<String, Any?>> {
@@ -352,7 +366,7 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
                 val data = if (clazz.simpleName == "PhoneUser") {
                     finalizer["pbx.list"]?.toString()?.let {
                         it.split(";").forEach { fi ->
-                            findId(Pbx::class.java, fi)?.let { id -> phoneUserPbxIds.add(id) }
+                            findUidByCode(Pbx::class.java, fi)?.let { id -> phoneUserPbxIds.add(id) }
                         }
                     }
                     finalizer["pbx_id"]?.toString()?.let {
@@ -467,10 +481,10 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
             if (t.contains("_")) {
                 val isParent = t.startsWith("parent")
                 val value = if (t == "pbx_id") {
-                    findId(Pbx::class.java, u ?: "PBX01")
+                    findUidByCode(Pbx::class.java, u ?: "PBX01")
                 } else u?.toString()?.let { code ->
                     val clazzEntity = clazzEntity(if (isParent) className else t.split("_")[0])
-                    clazzEntity?.let { findId(it, code) }
+                    clazzEntity?.let { findUidByCode(it, code) }
                 }
 
                 if (value != null) {
@@ -488,10 +502,16 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
             }
         }
 
-        mapFinalize["password"]?.toString()?.let {
-            mapFinalize["password"] = if (it.isBlank()) ""
-            else cpDecrypt.decrypt(it)?.let { p -> passwordEncoder.encode(p) } ?: ""
-        }
+//        mapFinalize["password"]?.toString()?.let {
+//            mapFinalize["password"] = if (it.isBlank()) ""
+//            else cpDecrypt.decrypt(it)?.let { p ->
+//                tempPassword[p] ?: run {
+//                    val pass = passwordEncoder.encode(p)
+//                    tempPassword[p] = pass
+//                    pass
+//                }
+//            } ?: ""
+//        }
 
         if (className == "transaction") {
             val to = mapFinalize["extTransferTo"]?.toString() ?: ""
@@ -512,7 +532,7 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
             val pinPassword = map["loginPin"]?.toString()?.toInt()?.let { i ->
                 if (i == 1) {
                     map["phoneUser_id"]?.toString()?.let { uid ->
-                        findPin(uid)?.let { phoneUserPbx ->
+                        findPinPhonePbx(uid)?.let { phoneUserPbx ->
                             mapFinalize["pinext_uid"] = phoneUserPbx.first
                             ToolAes.decrypt(phoneUserPbx.second)
                         }
@@ -521,11 +541,19 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
             }
             mapFinalize["password"] = when {
                 pinPassword != null -> {
-                    passwordEncoder.encode(pinPassword)
+                    tempPassword[pinPassword] ?: run {
+                        val pass = passwordEncoder.encode(pinPassword)
+                        tempPassword[pinPassword] = pass
+                        pass
+                    }
                 }
 
                 else -> if (mapFinalize["password"].toString().isEmpty()) {
-                    passwordEncoder.encode("12345")
+                    tempPassword["12345"] ?: run {
+                        val pass = passwordEncoder.encode("12345")
+                        tempPassword["12345"] = pass
+                        pass
+                    }
                 } else mapFinalize["password"]
             }
         }
@@ -564,61 +592,27 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
         ).firstOrNull()?.get("uid")
     }
 
-    private fun toSnakeCase(input: String): String {
-        return input
-            .replace(Regex("([a-z0-9])([A-Z])"), "$1_$2")
-            .lowercase()
-    }
-
-    val tempFindId: MutableMap<String, String?> = mutableMapOf()
-
-    private fun <T : BaseEntity> findId(clazz: Class<T>, value: Any): String? {
-        val id = tempDataUid["${clazz.simpleName}_${value}"]
-        return id
-//        return tempFindId["$clazz-$value"] ?: run {
-//            val id = try {
-//                val query = "SELECT uid FROM ${toSnakeCase(clazz.simpleName)} WHERE code = :code "
-//                val id = apiService.em.createNativeQuery(query, String::class.java)
-//                    .setParameter("code", value)
-//                    .singleResult as String
-//                id
-//            } catch (_: Exception) {
-//                null
-//            }
-//            tempFindId["${clazz.simpleName}-$value"] = id
-//            id
-//        }
+    private fun <T : BaseEntity> findUidByCode(clazz: Class<T>, value: Any): String? {
+        val tmpData = temporaryData[clazz.simpleName] ?: run {
+            val data = apiService.findListAll(clazz)
+                .associateBy { it["code"]?.toString() ?: it["id"]!!.toString() }
+            temporaryData[clazz.simpleName] = data
+            data
+        }
+        return tmpData[value.toString()]?.get("uid")?.toString() ?: tempDataParent["${clazz.simpleName}_$value"]
     }
 
-    val tempPhonePbxPin: MutableMap<String, Pair<String, String>?> = mutableMapOf()
-
-    private fun findPin(phoneUserCode: String): Pair<String, String>? {
-        return tempPhonePbxPin[phoneUserCode] ?: run {
-            val map = try {
-                val query = """
-            SELECT TOP 1 pin, pbx_uid 
-            FROM phoneuserpbx 
-            LEFT JOIN phone_user ph ON ph.uid = phoneuserpbx.phone_user_uid 
-            WHERE ph.code = :code
-        """.trimIndent()
-
-                val result = apiService.em
-                    .createNativeQuery(query)
-                    .setParameter("code", phoneUserCode)
-                    .singleResult as Array<*>
-
-                val pin = result[0]?.toString()
-                val pbxUid = result[1]?.toString()
-
-                if (pin != null && pbxUid != null) {
-                    Pair(pbxUid, pin)
-                } else null
-            } catch (_: Exception) {
-                null
-            }
-            tempPhonePbxPin[phoneUserCode] = map
-            map
+    private fun findPinPhonePbx(phoneUserCode: String): Pair<String, String>? {
+        val tmpData = temporaryData[PhoneUserPbx::class.java.simpleName] ?: run {
+            val data = apiService.findListAll(
+                PhoneUserPbx::class.java,
+                listOf("pin", "pbx.uid", "phoneUser.code")
+            ).associateBy { it["phoneUser.code"]!!.toString() }
+            temporaryData[PhoneUserPbx::class.java.simpleName] = data
+            data
         }
+
+        return tmpData[phoneUserCode]?.let { it["pbx.uid"].toString() to it["pin"].toString() }
     }
 
 }
@@ -664,9 +658,6 @@ class SecurityConfig {
 
 }
 
-val tempDataUid: MutableMap<String, String?> = mutableMapOf()
-val tempFieldColumn: MutableMap<String, String?> = mutableMapOf()
-
 @Service
 @Transactional
 class QueryNativeService(val apiService: ApiService) {
@@ -709,12 +700,16 @@ class QueryNativeService(val apiService: ApiService) {
         structure?.let { sqlNative.setParameter("structure", structure) }
         finalMap.forEach { (t, u) -> sqlNative.setParameter(t.replace("_id", "_uid"), u) }
         sqlNative.executeUpdate()
-        finalMap["code"]?.let {
-            tempDataUid["${clazz.simpleName}_$it"] = uid
-        }
-        cpId?.let {
-            tempDataUid["${clazz.simpleName}_$it"] = uid
+
+        if (clazz.simpleName.lowercase() == "organization" || clazz.simpleName.lowercase() == "costcenter") {
+            finalMap["code"]?.let {
+                tempDataParent["${clazz.simpleName}_$it"] = uid
+            }
+            cpId?.let {
+                tempDataParent["${clazz.simpleName}_$it"] = uid
+            }
         }
+
         return uid
     }
 

src/main/kotlin/com/datacomsolusindo/migration/MigrationService.kt

@@ -1,7 +1,6 @@
 package com.datacomsolusindo.migration
 
 import com.datacomsolusindo.cpx_shared_code.entity.BaseEntity
-import com.datacomsolusindo.cpx_shared_code.entity.PhoneUser
 import com.datacomsolusindo.cpx_shared_code.utility.SimpleLogger
 import org.springframework.beans.factory.config.YamlPropertiesFactoryBean
 import org.springframework.core.io.FileSystemResource
@@ -25,6 +24,9 @@ data class PreparedMigration(
     val groupFile: File?
 )
 
+val temporaryData: MutableMap<String, Map<String, Map<String, Any?>>> = mutableMapOf()
+val tempDataParent: MutableMap<String, String?> = mutableMapOf()
+val tempPassword: MutableMap<String, String?> = mutableMapOf()
 
 @Service
 class MigrationService(val migrationEntity: MigrationEntity) {
@@ -119,6 +121,7 @@ class MigrationService(val migrationEntity: MigrationEntity) {
             while (true) {
                 val data = queueInsertData.take()
                 try {
+                    logger.info("started process migration ${data.first.simpleName} data ${data.second.size}")
                     val process = measureTimedValue { migrationEntity.insertData(data.first, data.second) }
                     logger.info(
                         "finished process migration ${data.first.simpleName} " +