update service queue prepare data split, default trunk pbx

config/general-setting.yml

@@ -182,13 +182,4 @@ redirectUrl:
 - http://localhost:4200/oauth2redirect
 - https://cmpd.telmessenger.com/oauth2redirect
 - https://app.insomnia.rest/oauth/redirect
-dataKey: RMyvjTZXDMBnCr3jymw9oNuRqTbZK882k7b5dwgV633EPY0VorJuycbVMivlSmMyf42cVzUYXAaNmi9DXMHupycNfeTE1kOB7zcYMW4h9GCc2nUTOEDXcNgA%2FyteyLT4
-
-#database: 
-#  type: sqlserver
-#  host: 127.0.0.1
-#  port: 1433
-#  name: dbname
-#  username: sa
-#  password: password
-#  properties: encrypt=true;trustServerCertificate=true;
+dataKey: aY6tKotSoUoMIi7JiTBZq0KhtC4bslM0qvnU1EZgmMJ9rsAaBP8fCV3cugOaHB%2BO0rdqCT8BLvRpW3zZRMyaZN%2BOgLPchzBSS4WXWMJ2e%2BWfVKtx1U0WBWevGkoDU3IK

config/migration.yml

@@ -65,6 +65,7 @@ migration:
       appliedDate: history.app_datetime
       expiredDate: history.exp_datetime
       subscribedNo: history.trunk_number
+      pbx__default: "PBX01"
   - target: costCenter
     table: costcenter
     history: costcenterhistory

src/main/kotlin/com/datacomsolusindo/migration/MigrationApplication.kt

@@ -64,6 +64,7 @@ class AppEvent(
 //        migrationService.startInsertWorker()
 //        migrationService.scanAndQueue(folder)
 
+        migrationFileService.queueProcessingPrepareDataMigration()
         workerMigrationService.queueProcessingInsertDataMigration()
         migrationFileService.execute()
     }

src/main/kotlin/com/datacomsolusindo/migration/MigrationService.kt

@@ -30,7 +30,6 @@ val tempPassword: MutableMap<String, String?> = mutableMapOf()
 //val executorPrepareData = Executors.newSingleThreadExecutor()
 //val executorInsertData = Executors.newSingleThreadExecutor()
 val queueInsertData = LinkedBlockingQueue<Pair<Class<out BaseEntity>, List<MutableMap<String, Any?>>>>(20)
-val queueInsertMigrationData = LinkedBlockingQueue<Pair<MigrationFileData, List<MutableMap<String, Any?>>>>(20)
 
 @Service
 class MigrationService(

src/main/kotlin/com/datacomsolusindo/migration/data/FinalizedDataService.kt

@@ -459,6 +459,16 @@ class FinalizedDataService(
                     callMap
                 }
 
+                "trunk" -> {
+                    if (finalizer["pbx_id"] == null) {
+                        finalizer["pbx_id"] = finalizer["pbx__default"]?.toString()?.let { pbx ->
+                            findUidByCode("pbx", pbx)
+                        }
+                    }
+                    finalizer.remove("pbx__default")
+                    finalizer
+                }
+
                 "phone_user" -> {
                     val pbxId = finalizer["pbx.id"]?.toString() ?: finalizer["pbx.list"]?.toString() ?: ""
                     val pbxGroup = finalizer["pbx__list"]?.toString()?.split(";")

src/main/kotlin/com/datacomsolusindo/migration/service/MigrationFileService.kt

@@ -17,6 +17,7 @@ import java.nio.file.Paths
 import java.nio.file.StandardCopyOption
 import java.time.LocalDateTime
 import java.time.format.DateTimeFormatter
+import kotlin.io.path.name
 import kotlin.system.exitProcess
 import kotlin.time.measureTimedValue
 
@@ -63,31 +64,59 @@ class MigrationFileService(
             exitProcess(1)
         }
         runBlocking {
-            runningTableMigration.addAll(migrationFile.map { it.table })
-            migrationFile.map {
-                val pathMigrationFile = renameToTemp(it.migrationFile)
-                val pathHistoryFile = it.historyFile?.let { i -> renameToTemp(i) }
-                val pathGroupFile = it.groupFile?.let { i -> renameToTemp(i) }
-                val (migrationData, historyData, groupData) =
-                    prepareMigrationData(
-                        pathMigrationFile!!.toFile(),
-                        pathHistoryFile?.toFile(),
-                        pathGroupFile?.toFile()
-                    )
+            runningTableMigration.addAll(migrationFile.map { it.migrationFile.name })
+            migrationFile.forEach { mi -> queuePrepareMigrationData.add(mi) }
+//            migrationFile.map {
+//                val pathMigrationFile = renameToTemp(it.migrationFile)
+//                val pathHistoryFile = it.historyFile?.let { i -> renameToTemp(i) }
+//                val pathGroupFile = it.groupFile?.let { i -> renameToTemp(i) }
+//                val (migrationData, historyData, groupData) =
+//                    prepareMigrationData(
+//                        pathMigrationFile!!.toFile(),
+//                        pathHistoryFile?.toFile(),
+//                        pathGroupFile?.toFile()
+//                    )
+//
+//                it.rootData = migrationData
+//                it.historyData = historyData
+//                it.groupData = groupData
+//                it
+//            }.forEach { fi ->
+//                if (fi.table != "calltransaction") {
+//                    fi.unique?.let { unique -> loadTemporaryDataEntity(fi.table, unique) }
+//                }
+//                val finalizedData = finalizedDataService.processData(fi)
+//                queueInsertMigrationData.add(fi to finalizedData)
+//            }
+        }
+    }
 
-                it.rootData = migrationData
-                it.historyData = historyData
-                it.groupData = groupData
-                it
-            }.forEach { fi ->
-                if (fi.table != "calltransaction") {
-                    fi.unique?.let { unique -> loadTemporaryDataEntity(fi.table, unique) }
+    fun queueProcessingPrepareDataMigration() {
+        executorPrepareData.submit {
+            while (true) {
+                val migrationData = queuePrepareMigrationData.take()
+                migrationData.let {
+                    val pathMigrationFile = renameToTemp(it.migrationFile)
+                    val pathHistoryFile = it.historyFile?.let { i -> renameToTemp(i) }
+                    val pathGroupFile = it.groupFile?.let { i -> renameToTemp(i) }
+                    val (migrationData, historyData, groupData) =
+                        prepareMigrationData(
+                            pathMigrationFile!!.toFile(),
+                            pathHistoryFile?.toFile(),
+                            pathGroupFile?.toFile()
+                        )
+
+                    it.rootData = migrationData
+                    it.historyData = historyData
+                    it.groupData = groupData
+                    if (it.table != "calltransaction") {
+                        it.unique?.let { unique -> loadTemporaryDataEntity(it.table, unique) }
+                    }
+                    val finalizedData = finalizedDataService.processData(it)
+                    queueInsertMigrationData.add(it to finalizedData)
                 }
-                val finalizedData = finalizedDataService.processData(fi)
-                queueInsertMigrationData.add(fi to finalizedData)
             }
         }
-
     }
 
     private fun prepareFile(): MutableList<MigrationFileData> {
@@ -133,20 +162,20 @@ class MigrationFileService(
         return migrationFileData
     }
 
-    suspend fun prepareMigrationData(
+    fun prepareMigrationData(
         migrationFile: File,
         historyFile: File?,
         groupFile: File?
-    ) = coroutineScope {
-        val migrationDeferred = async(Dispatchers.IO) { readQueryDataToMap(migrationFile) }
-        val historyDeferred = async(Dispatchers.IO) { historyFile?.let { readQueryDataToMap(it) } }
-        val groupDeferred = async(Dispatchers.IO) { groupFile?.let { readQueryDataToMap(it) } }
+    ): Triple<List<Map<String, Any?>>, List<Map<String, Any?>>?, List<Map<String, Any?>>?> {
+        val migrationDeferred = readQueryDataToMap(migrationFile) //async(Dispatchers.IO) { readQueryDataToMap(migrationFile) }
+        val historyDeferred = historyFile?.let { readQueryDataToMap(it) } //async(Dispatchers.IO) { historyFile?.let { readQueryDataToMap(it) } }
+        val groupDeferred = groupFile?.let { readQueryDataToMap(it) } //async(Dispatchers.IO) { groupFile?.let { readQueryDataToMap(it) } }
 
-        val migrationData = migrationDeferred.await()
-        val historyData = historyDeferred.await()
-        val groupData = groupDeferred.await()
+        val migrationData = migrationDeferred//.await()
+        val historyData = historyDeferred//.await()
+        val groupData = groupDeferred//.await()
 
-        Triple(migrationData, historyData, groupData)
+        return Triple(migrationData, historyData, groupData)
     }
 
     private fun readQueryDataToMap(file: File): List<Map<String, Any?>> {

src/main/kotlin/com/datacomsolusindo/migration/service/WorkerMigrationService.kt

@@ -2,20 +2,23 @@ package com.datacomsolusindo.migration.service
 
 import com.datacomsolusindo.cpx_shared_code.utility.CacheData
 import com.datacomsolusindo.cpx_shared_code.utility.SimpleLogger
-import com.datacomsolusindo.cpx_shared_code.utility.Util
 import com.datacomsolusindo.migration.MigrationEntity
 import com.datacomsolusindo.migration.data.FinalizedDataService
 import com.datacomsolusindo.migration.data.InsertDataService
+import com.datacomsolusindo.migration.model.MigrationFileData
 import com.datacomsolusindo.migration.queueInsertData
-import com.datacomsolusindo.migration.queueInsertMigrationData
 import org.springframework.stereotype.Service
 import java.util.concurrent.Executors
+import java.util.concurrent.LinkedBlockingQueue
+import kotlin.io.path.name
 import kotlin.system.exitProcess
 import kotlin.time.measureTimedValue
 
 
 val executorPrepareData = Executors.newSingleThreadExecutor()
 val executorInsertData = Executors.newSingleThreadExecutor()
+val queuePrepareMigrationData = LinkedBlockingQueue<MigrationFileData>(100)
+val queueInsertMigrationData = LinkedBlockingQueue<Pair<MigrationFileData, List<MutableMap<String, Any?>>>>(100)
 
 @Service
 class WorkerMigrationService(
@@ -87,7 +90,7 @@ class WorkerMigrationService(
                         )
                     }
                     //println("MIGRATION DATA END---")
-                    runningTableMigration.remove(attribute.table)
+                    runningTableMigration.remove(attribute.migrationFile.name)
                     if (runningTableMigration.isEmpty()) {
                         exitProcess(1)
                     }