penyesuan migration auto update, perbaikan parent tidak valid

config/general-setting.yml

@@ -182,7 +182,7 @@ redirectUrl:
 - http://localhost:4200/oauth2redirect
 - https://cmpd.telmessenger.com/oauth2redirect
 - https://app.insomnia.rest/oauth/redirect
-dataKey: jRA4iyjy1ZCaIdlJBe6Cm%2BRPGb%2BwcO85%2FWR7Z9%2B8AFX7dJUSJjVnyr4JXSEyayg6i74%2F7IcBsn9iY%2BJ1ArmTtLSCD0plJfr%2B1S6%2ByBesS3dVTDSwlrEAuv2Pm%2BhYLmQ6f560DMvqCuahhKkkybkl5g%3D%3D
+dataKey: jRA4iyjy1ZCaIdlJBe6Cm%2BRPGb%2BwcO85%2FWR7Z9%2B8AFVJ6uwomb2YRYofgQlAjM3gQofyhQQc1IbUhh%2BvBsYk1AO%2BKXSrp4POmJMYfmO8hryzt2OTo%2BpH9Bf3bH049SQ1fK97a9hq42fBI7Pj%2FX01zg%3D%3D
 
 #database: 
 #  type: sqlserver

config/migration.yml

@@ -53,6 +53,7 @@ migration:
     table: trunk
     history: trunkhistory
     group: null
+    unique: code
     attribute:
       id: history.trunk_id
       code: trunk_code
@@ -79,6 +80,7 @@ migration:
     table: organization
     history: organizationhistory
     group: null
+    unique: code
     attribute:
       id: history.organization_id
       code: organization_code

src/main/kotlin/com/datacomsolusindo/migration/General.kt

@@ -0,0 +1,39 @@
+package com.datacomsolusindo.migration
+
+import com.datacomsolusindo.cpx_shared_code.entity.Account
+import com.datacomsolusindo.cpx_shared_code.entity.Area
+import com.datacomsolusindo.cpx_shared_code.entity.BaseEntity
+import com.datacomsolusindo.cpx_shared_code.entity.CdrModifier
+import com.datacomsolusindo.cpx_shared_code.entity.Corcos
+import com.datacomsolusindo.cpx_shared_code.entity.CostCenter
+import com.datacomsolusindo.cpx_shared_code.entity.Organization
+import com.datacomsolusindo.cpx_shared_code.entity.Pbx
+import com.datacomsolusindo.cpx_shared_code.entity.PhoneUser
+import com.datacomsolusindo.cpx_shared_code.entity.Provider
+import com.datacomsolusindo.cpx_shared_code.entity.Rate
+import com.datacomsolusindo.cpx_shared_code.entity.Transaction
+import com.datacomsolusindo.cpx_shared_code.entity.Trunk
+import com.datacomsolusindo.cpx_shared_code.entity.WebUser
+
+object General {
+
+    fun clazzEntity(migrationTarget: String): Class<out BaseEntity>? {
+        return when (migrationTarget) {
+            "organization" -> Organization::class.java
+            "costCenter" -> CostCenter::class.java
+            "pbx" -> Pbx::class.java
+            "trunk" -> Trunk::class.java
+            "callTransaction" -> Transaction::class.java
+            "account" -> Account::class.java
+            "area" -> Area::class.java
+            "cdrModifier" -> CdrModifier::class.java
+            "phoneUser" -> PhoneUser::class.java
+            "provider" -> Provider::class.java
+            "rate" -> Rate::class.java
+            "webUser" -> WebUser::class.java
+            "corcos" -> Corcos::class.java
+            else -> null
+        }
+    }
+
+}

src/main/kotlin/com/datacomsolusindo/migration/MigrationEntity.kt

@@ -90,41 +90,70 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
         groupData: MutableList<MutableMap<String, Any?>>? = null
     ) {
         val fieldMapping = fields.mapValues { it.value.substringAfterLast(".") }
+        val hasParent = fieldMapping.any { it.key == "parent_id" }
         val uniqueField = fieldMapping["code"] ?: "id"
         val uniqueFieldId = fieldMapping["id"]
 
         val historyIndex = buildHistoryIndex(historyData, uniqueField, uniqueFieldId)
+        val historyIndexById = buildHistoryIndex(historyData, uniqueField, uniqueFieldId, byId = true)
         val groupIndex = buildGroupIndex(groupData, uniqueFieldId ?: uniqueField)
 
         val fieldRoots = fields.filterValues { !it.contains(".") }
         val joinRoots = fields.filterValues { it.contains(".") }
 
-        val chunkData = rootData.chunked(1000)
-        logger.info("data migration class ${clazz.simpleName} chunk data ${chunkData.size}")
-        chunkData.forEachIndexed { _, data ->
-            val dataMap = data.map { row ->
+        if (hasParent) {
+            val finalizeData = rootData.map { row ->
                 buildRow(
                     row,
                     fieldRoots,
                     joinRoots,
                     historyIndex,
+                    historyIndexById,
                     groupIndex,
                     uniqueField,
                     uniqueFieldId
                 )
-            }.map { postProcessPassword(it) }
-            queueInsertData.put((clazz as Class<out BaseEntity>) to dataMap)
+            }
+
+            val chunkData = (if (finalizeData.any { it.any { a -> a.key == "structure" } }) {
+                finalizeData.sortedBy { f -> f["structure"].toString().length }
+            } else finalizeData).chunked(1000)
+
+            logger.info("data migration class ${clazz.simpleName} chunk data ${chunkData.size}")
+            chunkData.forEachIndexed { _, data ->
+                queueInsertData.put((clazz as Class<out BaseEntity>) to data)
+            }
+        } else {
+            val chunkData = rootData.chunked(1000)
+            logger.info("data migration class ${clazz.simpleName} chunk data ${chunkData.size}")
+            chunkData.forEachIndexed { _, data ->
+                val dataMap = data.map { row ->
+                    buildRow(
+                        row,
+                        fieldRoots,
+                        joinRoots,
+                        historyIndex,
+                        historyIndexById,
+                        groupIndex,
+                        uniqueField,
+                        uniqueFieldId
+                    )
+                }.map { postProcessPassword(it) }
+                queueInsertData.put((clazz as Class<out BaseEntity>) to dataMap)
+            }
         }
+
     }
 
     private fun buildHistoryIndex(
         historyData: List<Map<String, Any?>>?,
         uniqueField: String,
-        uniqueFieldId: String?
+        uniqueFieldId: String?,
+        byId: Boolean = false
     ): Map<Any?, Map<String, Any?>>? {
         if (historyData == null || uniqueFieldId == null) return null
         return historyData
-            .groupBy { it[uniqueField] }
+            .groupBy { if (byId) it[uniqueFieldId] else it[uniqueField] }
             .mapValues { (_, items) ->
                 items.maxByOrNull {
                     it[uniqueFieldId.removePrefix("history.")]
@@ -143,6 +172,7 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
         fieldRoots: Map<String, String>,
         joinRoots: Map<String, String>,
         historyIndex: Map<Any?, Map<String, Any?>>?,
+        historyIndexById: Map<Any?, Map<String, Any?>>?,
         groupIndex: Map<String?, Map<String, Any?>>?,
         uniqueField: String,
         uniqueFieldId: String?
@@ -152,13 +182,20 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
         fieldRoots.forEach { (target, source) -> data[target] = row[source] }
         // Join fields
         joinRoots.forEach { (target, sourceFull) ->
-
             val value = when {
                 sourceFull.startsWith("history.") -> {
                     val key = sourceFull.removePrefix("history.")
-                    historyIndex
+                    val valueHistories = historyIndex
                         ?.get(row[uniqueField])
                         ?.get(key.substringAfterLast("."))
+                    valueHistories?.let { valHis ->
+                        if (target.contains("_")) {
+                            val codeKey = uniqueField.removePrefix("history.")
+                            val joinCode = historyIndexById?.get(valHis)?.get(codeKey)
+                            joinCode
+                        } else valHis
+                    }
+
                 }
 
                 sourceFull.startsWith("group.") -> {
@@ -230,25 +267,70 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
         return map
     }
 
+//    private fun splitValues(input: String): List<String> {
+//        val result = mutableListOf<String>()
+//        var depth = 0
+//        var current = StringBuilder()
+//        for (c in input) {
+//            when (c) {
+//                '(' -> depth++
+//                ')' -> depth--
+//                ',' -> {
+//                    if (depth == 0) {
+//                        result.add(current.toString())
+//                        current = StringBuilder()
+//                        continue
+//                    }
+//                }
+//            }
+//            current.append(c)
+//        }
+//        result.add(current.toString())
+//        return result
+//    }
+
     private fun splitValues(input: String): List<String> {
         val result = mutableListOf<String>()
         var depth = 0
+        var inQuotes = false
         var current = StringBuilder()
-        for (c in input) {
+
+        var i = 0
+        while (i < input.length) {
+            val c = input[i]
             when (c) {
-                '(' -> depth++
-                ')' -> depth--
+                '\'' -> {
+                    // toggle flag ketika ada tanda kutip tunggal
+                    inQuotes = !inQuotes
+                    current.append(c)
+                }
+
+                '(' -> {
+                    if (!inQuotes) depth++
+                    current.append(c)
+                }
+
+                ')' -> {
+                    if (!inQuotes) depth--
+                    current.append(c)
+                }
+
                 ',' -> {
-                    if (depth == 0) {
-                        result.add(current.toString())
+                    // hanya split jika BUKAN sedang di dalam quotes dan depth = 0
+                    if (!inQuotes && depth == 0) {
+                        result.add(current.toString().trim())
                         current = StringBuilder()
-                        continue
+                    } else {
+                        current.append(c)
                     }
                 }
+
+                else -> current.append(c)
             }
-            current.append(c)
+            i++
         }
-        result.add(current.toString())
+
+        result.add(current.toString().trim())
         return result
     }
 
@@ -291,41 +373,21 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
         rootData: MutableList<MutableMap<String, Any?>>,
         historyData: MutableList<MutableMap<String, Any?>>?,
         groupData: MutableList<MutableMap<String, Any?>>?
-    ) = dataToMapWithDataSource(clazz, fields,  rootData, historyData, groupData)
-
-    fun clazzEntity(migrationTarget: String): Class<out BaseEntity>? {
-        return when (migrationTarget) {
-            "organization" -> Organization::class.java
-            "costCenter" -> CostCenter::class.java
-            "pbx" -> Pbx::class.java
-            "trunk" -> Trunk::class.java
-            "callTransaction" -> Transaction::class.java
-            "account" -> Account::class.java
-            "area" -> Area::class.java
-            "cdrModifier" -> CdrModifier::class.java
-            "phoneUser" -> PhoneUser::class.java
-            "provider" -> Provider::class.java
-            "rate" -> Rate::class.java
-            "webUser" -> WebUser::class.java
-            "corcos" -> Corcos::class.java
-            else -> null
-        }
-    }
+    ) = dataToMapWithDataSource(clazz, fields, rootData, historyData, groupData)
 
     fun <T : BaseEntity> insertData(clazz: Class<T>, dataMap: List<MutableMap<String, Any?>>): MutableList<Any> {
         failed.clear()
-        val sortingData = if (dataMap.any { it.any { a -> a.key == "structure" } }) {
-            dataMap.sortedBy { f -> f["structure"].toString().length }
-        } else dataMap
-        sortingData.forEach { map ->
+        dataMap.forEach { map ->
             try {
                 val finalizer = finalizeMap(clazz.simpleName.camelCase(), map)
                 val phoneUserPin = finalizer["pin"]
                 val phoneUserExtension = finalizer["extension"]
 
                 // budget
-                val budgetAnnual = (finalizer["budget.maxCost"]?.toString() ?: finalizer["budget__maxCost"]?.toString())?.toDoubleOrNull()
-                val warningAnnual = (finalizer["budget.warnCost"]?.toString() ?: finalizer["budget__warnCost"]?.toString())?.toDoubleOrNull()
+                val budgetAnnual = (finalizer["budget.maxCost"]?.toString()
+                    ?: finalizer["budget__maxCost"]?.toString())?.toDoubleOrNull()
+                val warningAnnual = (finalizer["budget.warnCost"]?.toString()
+                    ?: finalizer["budget__warnCost"]?.toString())?.toDoubleOrNull()
                 val budgetMaxCost = budgetAnnual?.let { max ->
                     List(12) { max }.joinToString(";")
                 }
@@ -337,7 +399,7 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
 
                 // phoneUserPbx
                 val phoneUserPbxIds: MutableList<Any> = mutableListOf()
-                val data = if (clazz.simpleName == "PhoneUser") {
+                val finalMapEntity = if (clazz.simpleName == "PhoneUser") {
                     (finalizer["pbx.list"]?.toString() ?: finalizer["pbx__list"]?.toString())?.let {
                         it.split(";").forEach { fi ->
                             findUidByCode(Pbx::class.java, fi)?.let { id -> phoneUserPbxIds.add(id) }
@@ -371,28 +433,28 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
                     if (phoneUserPbxIds.isEmpty()) {
                         finalMap["expiredDate"] = finalMap["expiredDate"] ?: LocalDate.now().atStartOfDay()
                     }
-                    queryNativeService.insertDataWithNativeQuery(clazz, finalMap)
-                } else {
-                    queryNativeService.insertDataWithNativeQuery(clazz, finalizer)
-                }
-
-                // create budget
-                if (BudgetUserType.entries.any { a -> a.name.snakeToCamel() == clazz.simpleName.camelCase() }) {
-                    queryNativeService.insertDataWithNativeQuery(
-                        Budget::class.java, mutableMapOf(
-                            "userType" to BudgetUserType.valueOf(
-                                clazz.simpleName.camelCase().camelToSnake().uppercase()
-                            ).ordinal,
-                            "userUid" to data,
-                            "type" to BudgetType.FLAT.ordinal,
-                            "annualCost" to (budgetAnnual?.let { it * 12 }?.toInt() ?: 0),
-                            "accumulate" to 0,
-                            "maxCost" to (budgetMaxCost ?: "0;0;0;0;0;0;0;0;0;0;0;0"),
-                            "warnCostPercentage" to (budgetWarnCost ?: "0;0;0;0;0;0;0;0;0;0;0;0"),
-                            "tempCost" to "0;0;0;0;0;0;0;0;0;0;0;0",
-                            "maxAutoCalculate" to "1;1;1;1;1;1;1;1;1;1;1;1"
+                    finalMap
+                } else finalizer
+
+                queryNativeService.insertDataWithNativeQuery(clazz, finalMapEntity, functionAfter = { uid ->
+                    // create budget
+                    if (BudgetUserType.entries.any { a -> a.name.snakeToCamel() == clazz.simpleName.camelCase() }) {
+                        queryNativeService.insertDataWithNativeQuery(
+                            Budget::class.java, mutableMapOf(
+                                "userType" to BudgetUserType.valueOf(
+                                    clazz.simpleName.camelCase().camelToSnake().uppercase()
+                                ).ordinal,
+                                "userUid" to uid,
+                                "type" to BudgetType.FLAT.ordinal,
+                                "annualCost" to (budgetAnnual?.let { it * 12 }?.toInt() ?: 0),
+                                "accumulate" to 0,
+                                "maxCost" to (budgetMaxCost ?: "0;0;0;0;0;0;0;0;0;0;0;0"),
+                                "warnCostPercentage" to (budgetWarnCost ?: "0;0;0;0;0;0;0;0;0;0;0;0"),
+                                "tempCost" to "0;0;0;0;0;0;0;0;0;0;0;0",
+                                "maxAutoCalculate" to "1;1;1;1;1;1;1;1;1;1;1;1"
+                            )
                         )
-                    )
+                    }
 
                     // create phoneUserPbx
                     if (phoneUserPbxIds.isNotEmpty()) {
@@ -402,12 +464,12 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
                                     "pin" to phoneUserPin,
                                     "extension" to phoneUserExtension,
                                     "pbx_id" to pbxId,
-                                    "phoneUser_id" to data,
+                                    "phoneUser_id" to uid,
                                 )
                             )
                         }
                     }
-                }
+                })
             } catch (e: Exception) {
                 failed.add(map)
                 logger.error("failed insert data migration", e)
@@ -460,7 +522,7 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
                 val value = if (t == "pbx_id") {
                     findUidByCode(Pbx::class.java, u ?: "PBX01")
                 } else u?.toString()?.let { code ->
-                    val clazzEntity = clazzEntity(if (isParent) className else t.split("_")[0])
+                    val clazzEntity = General.clazzEntity(if (isParent) className else t.split("_")[0])
                     clazzEntity?.let { findUidByCode(it, code) }
                 }
 
@@ -559,28 +621,34 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
     }
 
     private fun <T : BaseEntity> findUidByCode(clazz: Class<T>, value: Any): String? {
-        return tempDataParent["${clazz.simpleName}_$value"] ?: run {
-            val tmpData = temporaryData[clazz.simpleName] ?: run {
-                val data = apiService.findListAll(clazz)
-                    .associateBy { it["code"]?.toString() ?: it["id"]!!.toString() }
-                temporaryData[clazz.simpleName] = data
-                data
+        return temporaryDataByCode["${clazz.simpleName};$value"] ?: run {
+            val uid = temporaryDataEntity["${clazz.simpleName};$value"]?.get("uid") ?: run {
+                apiService.findListPage(
+                    clazz, filter = FilterData.filter("code", FilterData.FILTEROP.EQ, "$value")
+                ).firstOrNull()?.let { data ->
+                    temporaryDataEntity["${clazz.simpleName};$value"] = data
+                    temporaryDataByCode["${clazz.simpleName};$value"] = data["uid"].toString()
+                    data["uid"].toString()
+                }
             }
-            tmpData[value.toString()]?.get("uid")?.toString()
+            uid?.toString()
         }
     }
 
     private fun findPinPhonePbx(phoneUserCode: String): Pair<String, String>? {
-        val tmpData = temporaryData[PhoneUserPbx::class.java.simpleName] ?: run {
-            val data = apiService.findListAll(
+        val key = "${PhoneUserPbx::class.java.simpleName};$phoneUserCode"
+        val tmpData = temporaryDataEntity[key] ?: run {
+            apiService.findListAll(
                 PhoneUserPbx::class.java,
                 listOf("pin", "pbx.uid", "phoneUser.code")
-            ).associateBy { it["phoneUser.code"]!!.toString() }
-            temporaryData[PhoneUserPbx::class.java.simpleName] = data
-            data
+            ).forEach { dt ->
+                temporaryDataEntity["${PhoneUserPbx::class.java.simpleName};${dt["phoneUser.code"]!!.toString()}"] = dt
+            }
+            //.associateBy { it["phoneUser.code"]!!.toString() }
+            temporaryDataEntity[key]
         }
 
-        return tmpData[phoneUserCode]?.let { it["pbx.uid"].toString() to it["pin"].toString() }
+        return tmpData?.let { it["pbx.uid"].toString() to it["pin"].toString() }
     }
 
 }
@@ -628,11 +696,26 @@ class SecurityConfig {
 
 @Service
 @Transactional
-class QueryNativeService(val apiService: ApiService) {
+class QueryNativeService(
+    val apiService: ApiService,
+    val migrationSettingService: MigrationSettingService
+) {
+
+    fun <T> insertDataWithNativeQuery(
+        clazz: Class<T>,
+        mapData: MutableMap<String, Any?>,
+        functionAfter: ((uid: String) -> Unit)? = null
+    ): String? {
+
+        val fieldUnique = migrationSettingService.schema.firstOrNull {
+            it.table.equals(clazz.simpleName, ignoreCase = true)
+        }?.unique
+        val fieldKey = fieldUnique?.split(";")?.mapNotNull { m -> mapData[m]?.toString() }?.joinToString(";")
+
+        val uidFromDb = temporaryDataEntity["${clazz.simpleName};$fieldKey"]?.get("uid")?.toString()
+        val uid = uidFromDb ?: ULID.random()
+        val isUpdate = uidFromDb != null
 
-    fun <T> insertDataWithNativeQuery(clazz: Class<T>, mapData: MutableMap<String, Any?>): String? {
-        val uid = ULID.random()
-        val cpId = mapData["cpid"]
         val fields = mutableListOf("uid")
         val finalMap = mapData.filterNot { it.key == "cpid" }
         finalMap.keys.forEach {
@@ -641,7 +724,10 @@ class QueryNativeService(val apiService: ApiService) {
         }
         val structure = finalMap["parent_id"]?.toString()?.let {
             fields.add("structure")
-            "${EntityUtility(apiService, Organization::class.java).parentStructure(it)}|$uid"
+            val parentUid = EntityUtility(
+                apiService, General.clazzEntity(clazz.simpleName.camelCase())!!
+            ).parentStructure(it)
+            "$parentUid|$uid"
         }
 
         val tableName = when (clazz.simpleName.lowercase()) {
@@ -654,28 +740,38 @@ class QueryNativeService(val apiService: ApiService) {
             else -> clazz.simpleName.camelToSnake().lowercase()
         }
 
-        val tempField = fields.joinToString() { it.camelToSnake() }
-
-        val query = "INSERT INTO $tableName ($tempField) " +
-                "VALUES (${fields.joinToString() { ":$it" }})"
+        val query = fieldUnique?.let { fu ->
+            val uniqueOn = fu.split(";").joinToString(" AND ") { m -> "t.$m = s.$m" }
+            """
+    MERGE INTO $tableName AS t
+    USING (VALUES (${fields.joinToString() { ":$it" }})) 
+        AS s(${fields.joinToString() { it.camelToSnake() }})
+    ON $uniqueOn
+    WHEN MATCHED THEN
+        UPDATE SET 
+            ${fields.joinToString(",\n") { "t.${it.camelToSnake()} = s.${it.camelToSnake()}" }}
+    WHEN NOT MATCHED THEN
+        INSERT (${fields.joinToString { it.camelToSnake() }})
+        VALUES (${fields.joinToString { "s.${it.camelToSnake()}" }});
+""".trimIndent()
+        } ?: ("INSERT INTO $tableName (${fields.joinToString() { it.camelToSnake() }}) " +
+                "VALUES (${fields.joinToString() { ":$it" }})")
+
+//        val tempField = fields.joinToString() { it.camelToSnake() }
+//        val query = "INSERT INTO $tableName ($tempField) " +
+//                "VALUES (${fields.joinToString() { ":$it" }})"
 
         val sqlNative = apiService.em.createNativeQuery(query)
 
         if (clazz.simpleName.lowercase() != "transaction") {
             sqlNative.setParameter("uid", uid)
         }
-
         structure?.let { sqlNative.setParameter("structure", structure) }
         finalMap.forEach { (t, u) -> sqlNative.setParameter(t.replace("_id", "_uid"), u) }
         sqlNative.executeUpdate()
 
-        if (clazz.simpleName.lowercase() == "organization" || clazz.simpleName.lowercase() == "costcenter") {
-            finalMap["code"]?.let {
-                tempDataParent["${clazz.simpleName}_$it"] = uid
-            }
-            cpId?.let {
-                tempDataParent["${clazz.simpleName}_$it"] = uid
-            }
+        if (isUpdate) {
+            functionAfter?.invoke(uid)
         }
 
         return uid

src/main/kotlin/com/datacomsolusindo/migration/MigrationService.kt

@@ -7,7 +7,6 @@ import org.springframework.core.io.FileSystemResource
 import org.springframework.stereotype.Service
 import org.yaml.snakeyaml.DumperOptions
 import org.yaml.snakeyaml.Yaml
-import java.io.File
 import java.io.FileInputStream
 import java.io.FileWriter
 import java.nio.file.Files
@@ -17,19 +16,11 @@ import java.time.LocalDateTime
 import java.time.format.DateTimeFormatter
 import java.util.concurrent.Executors
 import java.util.concurrent.LinkedBlockingQueue
+import kotlin.time.measureTime
 import kotlin.time.measureTimedValue
 
-
-data class PreparedMigration(
-    val clazz: Class<out BaseEntity>,
-    val fields: Map<String, String>,
-    val rootFile: File,
-    val historyFile: File?,
-    val groupFile: File?
-)
-
-val temporaryData: MutableMap<String, Map<String, Map<String, Any?>>> = mutableMapOf()
-val tempDataParent: MutableMap<String, String?> = mutableMapOf()
+val temporaryDataEntity: MutableMap<String, Map<String, Any?>> = mutableMapOf()
+val temporaryDataByCode: MutableMap<String, String?> = mutableMapOf()
 val tempPassword: MutableMap<String, String?> = mutableMapOf()
 
 val executorPrepareData = Executors.newSingleThreadExecutor()
@@ -88,7 +79,7 @@ class MigrationService(
                 val rootTable = sourceDataMigrationService.getData(
                     it.table, it.attribute.filterNot { i ->
                         i.value.toString().startsWith("history.") || i.value.toString().startsWith("group.")
-                    }.map { m -> m.value.toString() }) ?: mutableListOf()
+                    }.map { m -> m.value.toString() })
                 val rootGroup = it.group?.let { group ->
                     sourceDataMigrationService.getData(
                         group, it.attribute.filter { i -> i.value.toString().startsWith("group.") }
@@ -99,7 +90,8 @@ class MigrationService(
                         history, it.attribute.filter { i -> i.value.toString().startsWith("history.") }
                             .map { m -> m.value.toString().removePrefix("history.") })
                 } ?: mutableListOf()
-                migrationEntity.clazzEntity(it.target)?.let { clazz ->
+                General.clazzEntity(it.target)?.let { clazz ->
+                    loadTemporaryDataEntity(clazz, it.unique)
                     migrationEntity.executeWithDataSource(
                         clazz, it.attribute as MutableMap<String, String>, rootTable, rootHistory, rootGroup
                     )
@@ -131,8 +123,8 @@ class MigrationService(
                 executorPrepareData.submit {
                     val filename = mi.fileName.toString()
                     val migrationTarget = filename.split("_")[2].replace(".txt", "")
-                    val fields = migrationSettingService.schema.firstOrNull { fi ->
-                        fi.target == migrationTarget}?.attribute?.let { it as MutableMap<String, String> } ?: mutableMapOf()
+                    val schema = migrationSettingService.schema.firstOrNull { fi -> fi.target == migrationTarget }
+                    val fields = schema?.attribute?.let { it as MutableMap<String, String> } ?: mutableMapOf()
                     //loadYamlAsMap(migrationTarget)
 
                     if (fields.isNotEmpty()) {
@@ -147,7 +139,8 @@ class MigrationService(
                                 f.fileName.toString().endsWith(toHistoryName(filename, "group"))
                             }?.let { addFile -> renameToTemp(addFile) }
                         } else null
-                        migrationEntity.clazzEntity(migrationTarget)?.let { clazz ->
+                        General.clazzEntity(migrationTarget)?.let { clazz ->
+                            loadTemporaryDataEntity(clazz, schema?.unique)
                             migrationEntity.execute(
                                 clazz,
                                 fields,
@@ -162,6 +155,25 @@ class MigrationService(
         }
     }
 
+    private fun loadTemporaryDataEntity(clazz: Class<out BaseEntity>, uniqueField: String? = null) {
+        val simple = clazz.simpleName
+        val process = measureTimedValue {
+            uniqueField?.let { field ->
+                val fields = field.split(";")
+                val data = migrationEntity.apiService.findListAll(clazz)
+                data.forEach {
+                    val fieldKey = fields.mapNotNull { m -> it[m]?.toString() }.joinToString(";")
+                    val key = "$simple;$fieldKey"
+                    temporaryDataEntity[key] = it
+                }
+                data
+            }
+        }
+        logger.info("load temporary data entity $simple " +
+                "from database ${process.value?.size ?: 0} " +
+                "takes time ${process.duration.inWholeMilliseconds}ms")
+    }
+
     fun startInsertWorker() {
         executorInsertData.submit {
             while (true) {

src/main/kotlin/com/datacomsolusindo/migration/MigrationSetting.kt

@@ -17,5 +17,6 @@ class MigrationSchema(
     val table: String,
     val history: String? = null,
     val group: String? = null,
+    val unique: String? = null,
     val attribute: MutableMap<String, Any>
 )