penyesuaian migration cp2

.gitignore

@@ -32,4 +32,6 @@ build/
 ### VS Code ###
 .vscode/
 
-log/
+log/
+migration/
+config/general-setting.yml

config/general-setting.yml

@@ -182,7 +182,7 @@ redirectUrl:
 - http://localhost:4200/oauth2redirect
 - https://cmpd.telmessenger.com/oauth2redirect
 - https://app.insomnia.rest/oauth/redirect
-dataKey: RMyvjTZXDMBnCr3jymw9oEib1gkpn7D%2FUH%2FdpJG9aJKNSkY9oU9nYuXGxkgsI6zBBLEAetoMH4gd7KdidD2Yp7Tieldm5LHjFFV2JTjcXibjwZdDx0GR8LhkB400WpwV
+dataKey: RMyvjTZXDMBnCr3jymw9oNuRqTbZK882k7b5dwgV633EPY0VorJuycbVMivlSmMyf42cVzUYXAaNmi9DXMHupycNfeTE1kOB7zcYMW4h9GCc2nUTOEDXcNgA%2FyteyLT4
 
 #database: 
 #  type: sqlserver

config/migration.yml

@@ -1,5 +1,6 @@
 migration:
   type: FILE
+  autoCreateCorcos: true
   sourceDatabase:
     type: sqlserver
     host: 192.168.100.25
@@ -119,10 +120,10 @@ migration:
       budget__warnCost: history.warn_cost
       pbx_id: history.pbx_code
       pbx__list: group.pbx_list
-#      corcosNormal_id: history.pbx_0_day
-#      corcosReducing_id: history.pbx_1_day
-#      corcosBlock_id: history.pbx_1_day
-#      pbx__default: "01"
+      corcosNormal_id: history.pbx_0_day
+      corcosReducing_id: history.pbx_1_day
+      corcosBlock_id: history.pbx_1_day
+      pbx__default: "01"
   - target: callTransaction
     table: calltransaction_20260127
     history: null
@@ -156,6 +157,7 @@ migration:
     table: usersetup
     history: null
     group: null
+    unique: username
     attribute:
       username: login_id
       password: login_password

src/main/kotlin/com/datacomsolusindo/migration/General.kt

@@ -39,6 +39,7 @@ object General {
     fun toTableName(migrationTarget: String): String {
         return when (migrationTarget) {
             "callTransaction" -> "calltransaction"
+            "corcosNormal", "corcosReducing", "corcosBlock" -> "corcos"
             else -> migrationTarget.camelToSnake()
         }
     }

src/main/kotlin/com/datacomsolusindo/migration/MigrationApplication.kt

@@ -51,13 +51,13 @@ fun main(args: Array<String>) {
 class AppEvent(
     val migrationService: MigrationService,
     val migrationFileService: MigrationFileService,
-    val workerMigrationService: WorkerMigrationService
+    val workerMigrationService: WorkerMigrationService,
+    val migrationSettingService: MigrationSettingService
 ) {
     private val logger = SimpleLogger.getLogger(this::class.java)
 
     @EventListener(ApplicationReadyEvent::class)
     fun doSomethingAfterStartup() {
-        println("runtime max ram: ${Runtime.getRuntime().maxMemory()}");
         logger.info("started service migration data with support mandiri menghidupi")
         logger.info("build information 20260520.1")
 //        val folder = Paths.get("migration")

src/main/kotlin/com/datacomsolusindo/migration/MigrationSetting.kt

@@ -10,6 +10,7 @@ class MigrationSettingService {
     var sourceDatabase: MutableMap<String, Any> = mutableMapOf()
     var target: Array<String>? = arrayOf()
     var schema: Array<MigrationSchema> = arrayOf()
+    var autoCreateCorcos: Boolean = false
 }
 
 class MigrationSchema(

src/main/kotlin/com/datacomsolusindo/migration/data/FinalizedDataService.kt

@@ -19,14 +19,16 @@ import kotlin.time.measureTimedValue
 
 val temporaryPrepareCallTransactionCallTo: MutableMap<String, Map<String, Any?>?> = mutableMapOf()
 val temporaryPrepareCallTransactionCallFrom: MutableMap<String, String?> = mutableMapOf()
+var temporaryHlrData: MutableMap<String, Map<String, Any?>?> = mutableMapOf()
 
 
 @Service
 class FinalizedDataService(
     private val cpDecrypt: CpDecrypt,
     private val passwordEncoder: PasswordEncoder,
-    private val apiService: ApiService,
-    private val insertDataService: InsertDataService
+    val apiService: ApiService,
+    private val insertDataService: InsertDataService,
+    private val migrationSettingService: MigrationSettingService
 ) {
 
     private val logger = SimpleLogger.getLogger(this::class.java)
@@ -318,22 +320,28 @@ class FinalizedDataService(
             if (temporaryDataByCode.any { a -> a.key == keyCode }) {
                 temporaryDataByCode[keyCode]
             } else {
-                try {
-                    val fd =
-                        if (table == "trunk") " AND (subscribed_no != 'new trunk' OR subscribed_no IS NULL) " else " "
+                 var dataUid = try {
+                    val fd = if (table == "trunk") " AND (subscribed_no != 'new trunk' OR subscribed_no IS NULL) " else " "
                     apiService.transaction { em ->
                         val result = em.createNativeQuery("SELECT $select FROM $tableName WHERE $field = :$field $fd")
                             .setParameter(field, value.toString())
                             .resultList
                             .first() as String?
-                        temporaryDataByCode[keyCode] = result
                         result
                     }
                 } catch (e: Exception) {
-                    temporaryDataByCode[keyCode] = null
                     logger.info("failed find uid $tableName $field $value")
                     null
                 }
+                if (tableName == "corcos" && dataUid == null && migrationSettingService.autoCreateCorcos) {
+                    dataUid = insertDataService.insertDataWithNativeQuery(tableName, "command", mutableMapOf(
+                        "command" to value,
+                        "name" to value,
+                        "description" to "auto create corcos"
+                    ))
+                }
+                temporaryDataByCode[keyCode] = dataUid
+                dataUid
             }
 //            temporaryDataByCode["$tableName;$select;$value"] ?: run {
 //                try {
@@ -457,8 +465,8 @@ class FinalizedDataService(
                         ?.mapNotNull { findUidByCode("pbx", it) } ?: listOf()
                     if (pbxId.isBlank() && pbxGroup.isEmpty()) {
                         finalizer["pbx__default"]?.toString()?.let { pbx ->
-                            if (pbx.isNotBlank()) {
-                                phoneUserPbxIds.add(pbx)
+                            findUidByCode("pbx", pbx)?.let { pbxDefault ->
+                                phoneUserPbxIds.add(pbxDefault)
                             }
                         }
                     } else {
@@ -545,7 +553,7 @@ class FinalizedDataService(
 
             1
         } catch (e: Exception) {
-            println(Util.mapper.writeValueAsString(map))
+//            println(Util.mapper.writeValueAsString(map))
             logger.error("failed insert data $table", e)
             0
         }
@@ -617,7 +625,7 @@ class FinalizedDataService(
         val rawData = buildRawData.joinToString(",")
 
         data["rawData"] = rawData
-//        getCallTo(buildRawData[12].toString())?.forEach { (t, u) -> data[t] = u }
+        getCallTo(buildRawData[12].toString())?.forEach { (t, u) -> data[t] = u }
 //
 //        data["area_uid"]?.toString()?.let { area ->
 //            val domainFrom = getCallFrom(area, buildRawData[10]?.toString(), buildRawData[1]?.toString())
@@ -639,31 +647,23 @@ class FinalizedDataService(
         return if (temporaryPrepareCallTransactionCallTo.any { a -> a.key == keyCode })
             temporaryPrepareCallTransactionCallTo[keyCode]
         else {
-            try {
-                apiService.transaction { em ->
-                    val hlr = em.createNativeQuery(
-                        "SELECT TOP 1 prefix, provider_uid, area_uid, phone_type, zone_uid, domain \n" +
-                                "FROM hlr\n" +
-//                            "WHERE '$number' LIKE prefix + '%'\n" +
-                                "WHERE '$number' LIKE prefix + '%'\n" +
-                                "ORDER BY LEN(prefix) DESC"
-                    ).singleResult as Array<Any?>
-                    val result = mapOf(
-                        "prefix" to hlr[0].toString(),
-                        "provider_to_uid" to hlr[1].toString(),
-                        "area_uid" to hlr[2].toString(),
-                        "phone_type" to hlr[3].toString(),
-                        "zone_uid" to hlr[4].toString(),
-                        "domain" to hlr[5].toString()
-                    )
-                    temporaryPrepareCallTransactionCallTo[keyCode] = result
-                    result
-                }
-            } catch (e: Exception) {
-                temporaryPrepareCallTransactionCallTo[keyCode] = null
-                logger.info("failed get call to attribute number $number")
-                null
+            val matchedPrefix = temporaryHlrData.entries
+                .filter { number.startsWith(it.key) }
+                .maxByOrNull { it.key.length }
+                ?.value
+
+            val result = matchedPrefix?.let {
+                mapOf(
+                    "prefix" to it["prefix"]?.toString(),
+                    "provider_to_uid" to it["provider.uid"]?.toString(),
+                    "area_uid" to it["area.uid"]?.toString(),
+                    "phone_type" to it["phoneType"]?.let { i -> (i as PhoneType).ordinal },
+                    "zone_uid" to it["zone.uid"]?.toString(),
+                    "domain" to it["domain"]?.toString()
+                )
             }
+            temporaryPrepareCallTransactionCallTo[keyCode] = result
+            result
         }
     }
 
@@ -719,6 +719,12 @@ class FinalizedDataService(
             val keyCode = "account;uid;${it["code"] ?: "#"}"
             temporaryDataByCode[keyCode] = it["uid"]?.toString()
         }
+
+        // hlr-data
+        temporaryHlrData = apiService.findListAll(
+            Hlr::class.java,
+            select = listOf("prefix", "provider.uid", "area.uid", "phoneType", "zone.uid", "domain")
+        ).distinctBy { it["prefix"] }.associateBy { it["prefix"].toString() }.toMutableMap()
     }
 
 
@@ -774,7 +780,7 @@ class InsertDataService(
         }
 
         val fieldKey = fieldUnique?.split(";")?.mapNotNull { m -> mapData[m]?.toString() }?.joinToString(";")
-        val uidFromDb = temporaryDataEntity["$table;$fieldKey"]?.get("uid")?.toString()
+        val uidFromDb = temporaryDataByCode["$table;uid;$fieldKey"]
         val uid = uidFromDb ?: ULID.random()
         val isUpdate = uidFromDb != null
 
@@ -782,13 +788,6 @@ class InsertDataService(
             finalMap.remove("uid")
         }
 
-//        val structure = finalMap["parent_id"]?.toString()?.let {
-//            fields.add("structure")
-//            val parentUid = EntityUtility(
-//                apiService, General.clazzEntity(className.camelCase())!!
-//            ).parentStructure(it)
-//            "$parentUid|$uid"
-//        }
         val structure = finalMap["parent_id"]?.let {
             findParentStructure(table, it)?.let { parentUid ->
                 fields.add("structure")
@@ -796,7 +795,11 @@ class InsertDataService(
             }
         }
 
-        val query = if (table == "calltransaction") "INSERT INTO $table (${fields.joinToString() { it.camelToSnake() }}) " +
+        if (isUpdate) {
+            fields.remove("uid")
+        }
+        val query =
+            if (table == "calltransaction") "INSERT INTO $table (${fields.joinToString() { it.camelToSnake() }}) " +
                     "VALUES (${fields.joinToString() { ":$it" }})"
             else {
                 fieldUnique?.let { fu ->
@@ -828,7 +831,7 @@ class InsertDataService(
         finalMap.forEach { (t, u) -> sqlNative.setParameter(t.replace("_id", "_uid"), u) }
         sqlNative.executeUpdate()
 
-//        functionAfter?.invoke(uid, isUpdate)
+        functionAfter?.invoke(uid, isUpdate)
 
         return uid
     }

src/main/kotlin/com/datacomsolusindo/migration/service/MigrationFileService.kt

@@ -1,13 +1,9 @@
 package com.datacomsolusindo.migration.service
 
 import com.datacomsolusindo.cpx_shared_code.utility.SimpleLogger
-import com.datacomsolusindo.cpx_shared_code.utility.Util
-import com.datacomsolusindo.migration.General
-import com.datacomsolusindo.migration.MigrationSettingService
-import com.datacomsolusindo.migration.camelToSnake
+import com.datacomsolusindo.migration.*
 import com.datacomsolusindo.migration.data.FinalizedDataService
 import com.datacomsolusindo.migration.model.MigrationFileData
-import com.datacomsolusindo.migration.queueInsertMigrationData
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.async
 import kotlinx.coroutines.coroutineScope
@@ -21,8 +17,11 @@ import java.nio.file.Paths
 import java.nio.file.StandardCopyOption
 import java.time.LocalDateTime
 import java.time.format.DateTimeFormatter
+import kotlin.system.exitProcess
 import kotlin.time.measureTimedValue
 
+val runningTableMigration : MutableList<String> = mutableListOf()
+
 @Service
 class MigrationFileService(
     private val migrationSettingService: MigrationSettingService,
@@ -60,7 +59,11 @@ class MigrationFileService(
 
     fun execute() {
         val migrationFile = prepareFile()
+        if (migrationFile.isEmpty()) {
+            exitProcess(1)
+        }
         runBlocking {
+            runningTableMigration.addAll(migrationFile.map { it.table })
             migrationFile.map {
                 val pathMigrationFile = renameToTemp(it.migrationFile)
                 val pathHistoryFile = it.historyFile?.let { i -> renameToTemp(i) }
@@ -77,6 +80,9 @@ class MigrationFileService(
                 it.groupData = groupData
                 it
             }.forEach { fi ->
+                if (fi.table != "calltransaction") {
+                    fi.unique?.let { unique -> loadTemporaryDataEntity(fi.table, unique) }
+                }
                 val finalizedData = finalizedDataService.processData(fi)
                 queueInsertMigrationData.add(fi to finalizedData)
             }
@@ -165,7 +171,8 @@ class MigrationFileService(
         )
 
         return if (file.name.contains("trunk_history")) {
-            val validTrunk = result.value.filter { f -> f.get("trunk_number")?.toString()?.let { t -> t != "new trunk" } ?: true }
+            val validTrunk =
+                result.value.filter { f -> f.get("trunk_number")?.toString()?.let { t -> t != "new trunk" } ?: true }
             validTrunk
         } else result.value
 
@@ -252,4 +259,27 @@ class MigrationFileService(
             ?: value
     }
 
+    private fun loadTemporaryDataEntity(table: String, uniqueField: String) {
+        val process = measureTimedValue {
+            finalizedDataService.apiService.transaction { em ->
+                val data = em.createNativeQuery(
+                    "SELECT uid,${uniqueField.split(";").joinToString(",")} \n" +
+                            "FROM $table "
+                ).resultList as List<Any>
+                data.forEach {
+                    it as Array<Any?>
+                    val fieldKey = List(uniqueField.split(";").size) { index -> it[index + 1] }.joinToString(";")
+                    val key = "$table;uid;$fieldKey"
+                    temporaryDataByCode[key] = it[0]?.toString()
+                }
+                data
+            }
+        }
+        logger.info(
+            "load temporary data entity $table " +
+                    "from database ${process.value.size} " +
+                    "takes time ${process.duration.inWholeMilliseconds}ms"
+        )
+    }
+
 }

src/main/kotlin/com/datacomsolusindo/migration/service/WorkerMigrationService.kt

@@ -10,6 +10,7 @@ import com.datacomsolusindo.migration.queueInsertData
 import com.datacomsolusindo.migration.queueInsertMigrationData
 import org.springframework.stereotype.Service
 import java.util.concurrent.Executors
+import kotlin.system.exitProcess
 import kotlin.time.measureTimedValue
 
 
@@ -57,7 +58,7 @@ class WorkerMigrationService(
                     finalizedDataService.prepareDataCallTransaction()
                 }
                 val data = migrationData.second
-            //.take(100)
+//                    .take(1)
                 try {
                     val chunkData = data.chunked(500)
                     logger.info("migration table ${attribute.table} with total data ${data.size} and chunk data ${chunkData.size}")
@@ -85,7 +86,11 @@ class WorkerMigrationService(
                                     "takes time ${process.duration.inWholeMilliseconds}ms"
                         )
                     }
-                    println("MIGRATION DATA END---")
+                    //println("MIGRATION DATA END---")
+                    runningTableMigration.remove(attribute.table)
+                    if (runningTableMigration.isEmpty()) {
+                        exitProcess(1)
+                    }
                 } catch (ex: Exception) {
                     logger.error("Insert failed", ex)
                 }