update migration datasource

config/general-setting.yml

@@ -182,7 +182,7 @@ redirectUrl:
 - http://localhost:4200/oauth2redirect
 - https://cmpd.telmessenger.com/oauth2redirect
 - https://app.insomnia.rest/oauth/redirect
-dataKey: RMyvjTZXDMBnCr3jymw9oEKiJUoCWfNcVgeIqzSvkOSyrrTcWRDz8u2NZSZUGeZ7a%2F8b89eIG3GcoHX0NtFZGeaqFUNFasg1zxDUSJKy7HpgxW7ca69yD0ZUlMQvlOrI
+dataKey: aY6tKotSoUoMIi7JiTBZq%2Bt0qdYZz7AmZrTkYTSE5zO%2BuTlZFkXv0r4J7d1UwmZRRPglM6vrDhjKye0quA3seEFIxy1gdafb2t300b8GfzhcRMB5EFzVg%2FZDy5QL8OnE
 
 #database: 
 #  type: sqlserver

config/migration.yml

@@ -1,101 +1,26 @@
-organization:
-  id: history.organization_id
-  code: organization_code
-  name: organization_name
-  description: description
-  email: organization_email
-  emailOnOverBudget: sendemail_overbudget
-  parent_id: history.parent_organization_id
-  structure: history.organization_id_structure
-  appliedDate: history.app_datetime
-  expiredDate: history.exp_datetime
-  cpid: history.organization_id
-costCenter:
-  id: history.costcenter_id
-  code: costcenter_code
-  name: costcenter_name
-  description: description
-  email: costcenter_email
-  parent_id: history.parent_costcenter_id
-  structure: history.costcenter_id_structure
-  appliedDate: history.app_datetime
-  expiredDate: history.exp_datetime
-pbx:
-  code: pbx_code
-  name: description
-  ipPbx: ip_pbx
-  pbxSupporter: pbx_supporter
-  sharePath: share_path
-#  appliedDate: ext_start
-#  expiredDate: ext_end
-trunk:
-  id: history.trunk_id
-  code: trunk_code
-  name: description
-  pbx_id: location_code
-  direction: cap_direction
-  appliedDate: history.app_datetime
-  expiredDate: history.exp_datetime
-callTransaction:
-  direction: direction
-  startOfCall: start_of_call
-  duration: duration
-  extension: ext_used
-  pin: pin
-  phoneUser_id: phoneuser_code
-  extTransferFrom: ext_transfer_from
-  extTransferTo: ext_transfer_to
-  number: dialed_number
-  callerNumber: caller_number
-  accessNumber: access_number
-  organization_id: organization_code
-  costCenter_id: costcenter_code
-  cost: cost_1
-  service: cost_service
-  tax: cost_tax1
-  discount: cost_adjustment
-  currency: currency
-  pbx_id: pbx_code
-  terminationCode: call_termination
-  redirectReason: redirect_reason
-  account_id: account_code
-  trunk_id: trunk_code
-webUser:
-  username: login_id
-  password: login_password
-  name: description
-  phoneUser_id: phoneuser_code
-  loginPin: used_pin
-phoneUser:
-  id: history.phoneuser_id
-  code: phoneuser_code
-  name: phoneuser_name
-  email: phoneuser_email
-  whatsapp: phoneuser_mobile
-  position: position
-  emailOnOverBudget: sendemail_overbudget
-  pin: history.phoneuser_pin
-  extension: history.default_extension
-  organization_id: history.organization_code
-  costCenter_id: history.costcenter_code
-  pbx_id: history.pbx_code
-  pbx.list: group.pbx_list
-  appliedDate: history.app_datetime
-  expiredDate: history.exp_datetime
-  budget.maxCost: history.max_cost
-  budget.warnCost: history.warn_cost
-corcos:
-  command: command
-  name: short_desc
-  description: description
-provider:
-  code: provider_prefix
-  name: description
-  description: description
-account:
-  id: history.account_id
-  code: account_code
-  name: history.account_number
-  description: description
-  appliedDate: history.app_datetime
-  expiredDate: history.exp_datetime
+migration:
+  type: DATABASE
+  sourceDatabase:
+    type: sqlserver
+    host: 192.168.100.25
+    port: 1433
+    name: callpartner
+    username: sa
+    password: D4tacom!
+    properties: encrypt=true;trustServerCertificate=true;
+  schema:
+    - table: organization
+      history: organizationhistory
+      group:
+      attribute:
+        id: history.organization_id
+        code: organization_code
+        name: organization_name
+        description: description
+        email: organization_email
+        emailOnOverBudget: sendemail_overbudget
+        parent_id: history.parent_organization_id
+        structure: history.organization_id_structure
+        appliedDate: history.app_datetime
+        expiredDate: history.exp_datetime
+        cpid: history.organization_id

config/migration__.yml

@@ -0,0 +1,101 @@
+organization:
+  id: history.organization_id
+  code: organization_code
+  name: organization_name
+  description: description
+  email: organization_email
+  emailOnOverBudget: sendemail_overbudget
+  parent_id: history.parent_organization_id
+  structure: history.organization_id_structure
+  appliedDate: history.app_datetime
+  expiredDate: history.exp_datetime
+  cpid: history.organization_id
+costCenter:
+  id: history.costcenter_id
+  code: costcenter_code
+  name: costcenter_name
+  description: description
+  email: costcenter_email
+  parent_id: history.parent_costcenter_id
+  structure: history.costcenter_id_structure
+  appliedDate: history.app_datetime
+  expiredDate: history.exp_datetime
+pbx:
+  code: pbx_code
+  name: description
+  ipPbx: ip_pbx
+  pbxSupporter: pbx_supporter
+  sharePath: share_path
+#  appliedDate: ext_start
+#  expiredDate: ext_end
+trunk:
+  id: history.trunk_id
+  code: trunk_code
+  name: description
+  pbx_id: location_code
+  direction: cap_direction
+  appliedDate: history.app_datetime
+  expiredDate: history.exp_datetime
+callTransaction:
+  direction: direction
+  startOfCall: start_of_call
+  duration: duration
+  extension: ext_used
+  pin: pin
+  phoneUser_id: phoneuser_code
+  extTransferFrom: ext_transfer_from
+  extTransferTo: ext_transfer_to
+  number: dialed_number
+  callerNumber: caller_number
+  accessNumber: access_number
+  organization_id: organization_code
+  costCenter_id: costcenter_code
+  cost: cost_1
+  service: cost_service
+  tax: cost_tax1
+  discount: cost_adjustment
+  currency: currency
+  pbx_id: pbx_code
+  terminationCode: call_termination
+  redirectReason: redirect_reason
+  account_id: account_code
+  trunk_id: trunk_code
+webUser:
+  username: login_id
+  password: login_password
+  name: description
+  phoneUser_id: phoneuser_code
+  loginPin: used_pin
+phoneUser:
+  id: history.phoneuser_id
+  code: phoneuser_code
+  name: phoneuser_name
+  email: phoneuser_email
+  whatsapp: phoneuser_mobile
+  position: position
+  emailOnOverBudget: sendemail_overbudget
+  pin: history.phoneuser_pin
+  extension: history.default_extension
+  organization_id: history.organization_code
+  costCenter_id: history.costcenter_code
+  pbx_id: history.pbx_code
+  pbx.list: group.pbx_list
+  appliedDate: history.app_datetime
+  expiredDate: history.exp_datetime
+  budget.maxCost: history.max_cost
+  budget.warnCost: history.warn_cost
+corcos:
+  command: command
+  name: short_desc
+  description: description
+provider:
+  code: provider_prefix
+  name: description
+  description: description
+account:
+  id: history.account_id
+  code: account_code
+  name: history.account_number
+  description: description
+  appliedDate: history.app_datetime
+  expiredDate: history.exp_datetime

src/main/kotlin/com/datacomsolusindo/migration/MigrationApplication.kt

@@ -2,6 +2,7 @@ package com.datacomsolusindo.migration
 
 import com.datacomsolusindo.cpx_shared_code.utility.AppListener
 import com.datacomsolusindo.cpx_shared_code.utility.SimpleLogger
+import com.datacomsolusindo.cpx_shared_code.utility.Util
 import kotlinx.coroutines.runBlocking
 import org.springframework.boot.autoconfigure.SpringBootApplication
 import org.springframework.boot.autoconfigure.domain.EntityScan
@@ -37,22 +38,25 @@ fun main(args: Array<String>) {
         .listeners(AppListener("migration"))
         .run(
             *args,
-            "--spring.config.location=classpath:/application.yml, ./config/general-setting.yml"
+            "--spring.config.location=classpath:/application.yml, " +
+                    "./config/general-setting.yml, " +
+                    "./config/migration.yml"
         )
 }
 
 @Suppress("UNCHECKED_CAST")
 @Component
-class AppEvent(val migrationService: MigrationService) {
+class AppEvent(
+    val migrationService: MigrationService
+) {
     private val logger = SimpleLogger.getLogger(this::class.java)
 
     @EventListener(ApplicationReadyEvent::class)
     fun doSomethingAfterStartup() {
         logger.info("started service migration data with support mandiri menghidupi")
-        val folder = Paths.get("migration")
-        migrationService.startInsertWorker()
-        migrationService.scanAndQueue(folder)
-
+//        val folder = Paths.get("migration")
+//        migrationService.startInsertWorker()
+//        migrationService.scanAndQueue(folder)
 //        exitProcess(1)
     }
 

src/main/kotlin/com/datacomsolusindo/migration/MigrationData.kt

@@ -0,0 +1,91 @@
+package com.datacomsolusindo.migration
+
+import com.datacomsolusindo.cpx_shared_code.utility.SimpleLogger
+import com.datacomsolusindo.cpx_shared_code.utility.Util
+import org.springframework.beans.factory.annotation.Qualifier
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
+import org.springframework.boot.context.properties.ConfigurationProperties
+import org.springframework.boot.jdbc.DataSourceBuilder
+import org.springframework.context.annotation.Bean
+import org.springframework.context.annotation.Configuration
+import org.springframework.context.annotation.Primary
+import org.springframework.jdbc.core.JdbcTemplate
+import org.springframework.stereotype.Service
+import javax.sql.DataSource
+import kotlin.time.measureTimedValue
+
+@Configuration
+class PrimaryDbConfig {
+
+    @Bean
+    @Primary
+    @ConfigurationProperties("spring.datasource")
+    fun dataSource(): DataSource {
+        val db = Util.generalConfig!!["database"] as Map<String, Any>
+        return DataSourceBuilder.create()
+            .driverClassName("com.microsoft.sqlserver.jdbc.SQLServerDriver")
+            .url(
+                "jdbc:sqlserver://${db["host"]}:${db["port"]};" +
+                        "databaseName=${db["name"]};${db["properties"]}"
+            )
+            .username(db["username"].toString())
+            .password(db["password"].toString())
+            .build()
+    }
+
+}
+
+@Configuration
+class MigrationSqlServerConfig(
+    private val migrationSettingService: MigrationSettingService
+) {
+
+    @Bean
+    @ConditionalOnProperty(name = ["migration.type"], havingValue = "DATABASE")
+    fun migrationDataSource(): DataSource {
+
+        val db = migrationSettingService.sourceDatabase
+
+        return DataSourceBuilder.create()
+            .driverClassName("com.microsoft.sqlserver.jdbc.SQLServerDriver")
+            .url(
+                "jdbc:sqlserver://${db["host"]}:${db["port"]};" +
+                        "databaseName=${db["name"]};${db["properties"]}"
+            )
+            .username(db["username"].toString())
+            .password(db["password"].toString())
+            .build()
+    }
+
+    @Bean
+    @ConditionalOnProperty(name = ["migration.type"], havingValue = "DATABASE")
+    fun migrationJdbcTemplate(
+        @Qualifier("migrationDataSource") ds: DataSource
+    ): JdbcTemplate {
+        return JdbcTemplate(ds)
+    }
+}
+
+@Service
+class SourceDataMigrationService(@param:Qualifier("migrationJdbcTemplate") private val migrationJdbcTemplate: JdbcTemplate?) {
+    private val logger = SimpleLogger.getLogger(this::class.java)
+
+    fun getData(table: String, column: List<String>): MutableList<MutableMap<String, Any?>> {
+        val process = measureTimedValue {
+            if (table.isBlank()) mutableListOf() else {
+                try {
+                    migrationJdbcTemplate?.queryForList(
+                        "SELECT * FROM $table"
+                    ) ?: mutableListOf()
+                } catch (e: Exception) {
+                    logger.error("failed get data $table", e)
+                    mutableListOf()
+                }
+            }
+        }
+        logger.info("get source data from table $table ${process.value.size} " +
+                "takes time ${process.duration.inWholeMilliseconds}ms")
+        return process.value
+    }
+
+}

src/main/kotlin/com/datacomsolusindo/migration/MigrationEntity.kt

@@ -130,7 +130,102 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
 //        return map.value
 //    }
 
-    fun <T> dataToMap(
+//    fun <T> dataToMap(
+//        clazz: Class<T>,
+//        fields: Map<String, String>,
+//        unique: String,
+//        rootFile: File,
+//        historyFile: File? = null,
+//        groupFile: File? = null
+//    ): List<MutableMap<String, Any?>> {
+//
+//        logger.info("prepare data migration class ${clazz.simpleName}")
+//
+//        val process = measureTimedValue {
+//
+//            // --- Read Files ---
+//            val historyData = historyFile?.let { readQueryDataToMap(it) }
+//            val groupData = groupFile?.let { readQueryDataToMap(it) }
+//            val rootData = readQueryDataToMap(rootFile)
+//
+//            // --- Pre-calc field names (avoid split(".") repeatedly) ---
+//            val fieldMapping = fields.mapValues { it.value.substringAfterLast(".") }
+//            val uniqueField = fieldMapping[unique] ?: "id"
+//            val uniqueFieldId = fieldMapping["id"]
+//
+//            // --- Create Indexes (O(n)) ---
+//            val historyIndex = historyData?.groupBy { it[uniqueField] }?.mapValues { (_, items) ->
+//                items.maxByOrNull {
+//                    it[uniqueFieldId!!.removePrefix("history.")].toString().toInt()
+//                }
+//            }
+//            //?.associateBy { it[uniqueField]?.toString() }
+//            val groupIndex = groupData?.associateBy { it[uniqueFieldId ?: uniqueField]?.toString() }
+//
+//            val fieldRoots = fields.filter { !it.value.contains(".") }
+//            val joinRoots = fields.filter { it.value.contains(".") }
+//                .toList().sortedByDescending { it.second }.toMap()
+//
+//            rootData.mapIndexed { index, row ->
+//                val data = mutableMapOf<String, Any?>()
+//
+//                // --- Direct Fields ---
+//                fieldRoots.forEach { (targetKey, sourceKey) ->
+//                    data[targetKey] = row[sourceKey]
+//                }
+//
+//                // --- Join Fields ---
+//                joinRoots.forEach { (targetKey, sourceKeyFull) ->
+//
+//                    val value = when {
+//                        sourceKeyFull.startsWith("history.") -> {
+//                            val sourceKey = sourceKeyFull.removePrefix("history.")
+//                            historyIndex
+//                                ?.get(row[uniqueField]?.toString())
+//                                ?.get(sourceKey.substringAfterLast("."))
+//                        }
+//
+//                        sourceKeyFull.startsWith("group.") -> {
+//                            val sourceKey = sourceKeyFull.removePrefix("group.")
+//                            val idKey = historyIndex
+//                                ?.get(row[uniqueField]?.toString())
+//                                ?.get(uniqueFieldId) ?: row[uniqueFieldId]
+//                            groupIndex
+//                                ?.get(idKey.toString())
+//                                ?.get(sourceKey.substringAfterLast("."))
+//                        }
+//
+//                        else -> row[sourceKeyFull]
+//                    }
+//
+//                    data[targetKey] = value
+//                }
+//                data
+//            }
+//        }
+//
+//        val value = process.value.map { m ->
+//            val dt = m as MutableMap<String, Any?>
+//            dt["password"]?.toString()?.let {
+//                dt["password"] = if (it.isBlank()) ""
+//                else cpDecrypt.decrypt(it)?.let { p ->
+//                    tempPassword[p] ?: run {
+//                        val pass = passwordEncoder.encode(p)
+//                        tempPassword[p] = pass
+//                        pass
+//                    }
+//                } ?: ""
+//            }
+//            dt
+//        }
+//        logger.info(
+//            "finish prepare data [${process.value.size}] migration class ${clazz.simpleName} " +
+//                    "takes time ${process.duration.inWholeMilliseconds}ms"
+//        )
+//        return value
+//    }
+
+    private fun <T> dataToMap(
         clazz: Class<T>,
         fields: Map<String, String>,
         unique: String,
@@ -138,91 +233,149 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
         historyFile: File? = null,
         groupFile: File? = null
     ): List<MutableMap<String, Any?>> {
+        return processData(
+            clazz,
+            fields,
+            unique,
+            rootData = readQueryDataToMap(rootFile) as MutableList<MutableMap<String, Any?>>,
+            historyData = historyFile?.let { readQueryDataToMap(it) as MutableList<MutableMap<String, Any?>> },
+            groupData = groupFile?.let { readQueryDataToMap(it) as MutableList<MutableMap<String, Any?>> }
+        )
+    }
 
-        logger.info("prepare data migration class ${clazz.simpleName}")
+    private fun <T> dataToMapWithDataSource(
+        clazz: Class<T>,
+        fields: Map<String, String>,
+        unique: String,
+        rootData: MutableList<MutableMap<String, Any?>>,
+        historyData: MutableList<MutableMap<String, Any?>>? = null,
+        groupData: MutableList<MutableMap<String, Any?>>? = null
+    ): List<MutableMap<String, Any?>> {
+        return processData(
+            clazz,
+            fields,
+            unique,
+            rootData,
+            historyData,
+            groupData
+        )
+    }
 
-        val process = measureTimedValue {
+    private fun <T> processData(
+        clazz: Class<T>,
+        fields: Map<String, String>,
+        unique: String,
+        rootData: MutableList<MutableMap<String, Any?>>,
+        historyData: MutableList<MutableMap<String, Any?>>? = null,
+        groupData: MutableList<MutableMap<String, Any?>>? = null
+    ): List<MutableMap<String, Any?>> {
 
-            // --- Read Files ---
-            val historyData = historyFile?.let { readQueryDataToMap(it) }
-            val groupData = groupFile?.let { readQueryDataToMap(it) }
-            val rootData = readQueryDataToMap(rootFile)
+        logger.info("prepare data migration class ${clazz.simpleName}")
 
-            // --- Pre-calc field names (avoid split(".") repeatedly) ---
+        val process = measureTimedValue {
             val fieldMapping = fields.mapValues { it.value.substringAfterLast(".") }
             val uniqueField = fieldMapping[unique] ?: "id"
             val uniqueFieldId = fieldMapping["id"]
 
-            // --- Create Indexes (O(n)) ---
-            val historyIndex = historyData?.groupBy { it[uniqueField] }?.mapValues { (_, items) ->
-                items.maxByOrNull {
-                    it[uniqueFieldId!!.removePrefix("history.")].toString().toInt()
-                }
+            val historyIndex = buildHistoryIndex(historyData, uniqueField, uniqueFieldId)
+            val groupIndex = buildGroupIndex(groupData, uniqueFieldId ?: uniqueField)
+
+            val fieldRoots = fields.filterValues { !it.contains(".") }
+            val joinRoots = fields.filterValues { it.contains(".") }
+
+            rootData.map { row ->
+                buildRow(
+                    row,
+                    fieldRoots,
+                    joinRoots,
+                    historyIndex,
+                    groupIndex,
+                    uniqueField,
+                    uniqueFieldId
+                )
             }
-            //?.associateBy { it[uniqueField]?.toString() }
-            val groupIndex = groupData?.associateBy { it[uniqueFieldId ?: uniqueField]?.toString() }
-
-            val fieldRoots = fields.filter { !it.value.contains(".") }
-            val joinRoots = fields.filter { it.value.contains(".") }
-                .toList().sortedByDescending { it.second }.toMap()
-
-            rootData.mapIndexed { index, row ->
-                val data = mutableMapOf<String, Any?>()
+        }
 
-                // --- Direct Fields ---
-                fieldRoots.forEach { (targetKey, sourceKey) ->
-                    data[targetKey] = row[sourceKey]
-                }
+        logger.info(
+            "finish prepare data [${process.value.size}] migration class ${clazz.simpleName} " +
+                    "takes time ${process.duration.inWholeMilliseconds}ms"
+        )
 
-                // --- Join Fields ---
-                joinRoots.forEach { (targetKey, sourceKeyFull) ->
+        return process.value.map { postProcessPassword(it) }
+    }
 
-                    val value = when {
-                        sourceKeyFull.startsWith("history.") -> {
-                            val sourceKey = sourceKeyFull.removePrefix("history.")
-                            historyIndex
-                                ?.get(row[uniqueField]?.toString())
-                                ?.get(sourceKey.substringAfterLast("."))
-                        }
+    private fun buildHistoryIndex(
+        historyData: List<Map<String, Any?>>?,
+        uniqueField: String,
+        uniqueFieldId: String?
+    ): Map<Any?, Map<String, Any?>>? {
+        if (historyData == null || uniqueFieldId == null) return null
+        return historyData
+            .groupBy { it[uniqueField] }
+            .mapValues { (_, items) ->
+                items.maxByOrNull {
+                    it[uniqueFieldId.removePrefix("history.")]
+                        ?.toString()
+                        ?.toIntOrNull() ?: 0
+                } as Map<String, Any?>
+            }
+    }
 
-                        sourceKeyFull.startsWith("group.") -> {
-                            val sourceKey = sourceKeyFull.removePrefix("group.")
-                            val idKey = historyIndex
-                                ?.get(row[uniqueField]?.toString())
-                                ?.get(uniqueFieldId) ?: row[uniqueFieldId]
-                            groupIndex
-                                ?.get(idKey.toString())
-                                ?.get(sourceKey.substringAfterLast("."))
-                        }
+    private fun buildGroupIndex(groupData: List<Map<String, Any?>>?, key: String): Map<String?, Map<String, Any?>>? {
+        return groupData?.associateBy { it[key]?.toString() }
+    }
 
-                        else -> row[sourceKeyFull]
-                    }
+    private fun buildRow(
+        row: Map<String, Any?>,
+        fieldRoots: Map<String, String>,
+        joinRoots: Map<String, String>,
+        historyIndex: Map<Any?, Map<String, Any?>>?,
+        groupIndex: Map<String?, Map<String, Any?>>?,
+        uniqueField: String,
+        uniqueFieldId: String?
+    ): MutableMap<String, Any?> {
+        val data = mutableMapOf<String, Any?>()
+        // Direct fields
+        fieldRoots.forEach { (target, source) -> data[target] = row[source] }
+        // Join fields
+        joinRoots.forEach { (target, sourceFull) ->
+
+            val value = when {
+                sourceFull.startsWith("history.") -> {
+                    val key = sourceFull.removePrefix("history.")
+                    historyIndex
+                        ?.get(row[uniqueField])
+                        ?.get(key.substringAfterLast("."))
+                }
 
-                    data[targetKey] = value
+                sourceFull.startsWith("group.") -> {
+                    val key = sourceFull.removePrefix("group.")
+                    val idKey = historyIndex
+                        ?.get(row[uniqueField])
+                        ?.get(uniqueFieldId)
+                        ?: row[uniqueFieldId]
+                    groupIndex
+                        ?.get(idKey?.toString())
+                        ?.get(key.substringAfterLast("."))
                 }
-                data
+                else -> row[sourceFull]
             }
+            data[target] = value
         }
+        return data
+    }
 
-        val value = process.value.map { m ->
-            val dt = m as MutableMap<String, Any?>
-            dt["password"]?.toString()?.let {
-                dt["password"] = if (it.isBlank()) ""
-                else cpDecrypt.decrypt(it)?.let { p ->
-                    tempPassword[p] ?: run {
-                        val pass = passwordEncoder.encode(p)
-                        tempPassword[p] = pass
-                        pass
-                    }
-                } ?: ""
-            }
-            dt
+    private fun postProcessPassword(data: MutableMap<String, Any?>): MutableMap<String, Any?> {
+        val raw = data["password"]?.toString() ?: return data
+        data["password"] = when {
+            raw.isBlank() -> ""
+            else -> cpDecrypt.decrypt(raw)?.let { plain ->
+                tempPassword[plain] ?: passwordEncoder.encode(plain).also {
+                    tempPassword[plain] = it
+                }
+            } ?: ""
         }
-        logger.info(
-            "finish prepare data [${process.value.size}] migration class ${clazz.simpleName} " +
-                    "takes time ${process.duration.inWholeMilliseconds}ms"
-        )
-        return value
+        return data
     }
 
     private fun readQueryDataToMap(file: File): List<Map<String, Any?>> {
@@ -309,6 +462,7 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
     }
 
     private val failed: MutableList<Any> = mutableListOf()
+
     fun <T : BaseEntity> execute(
         clazz: Class<T>,
         fields: Map<String, String>,
@@ -320,6 +474,17 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
         return data
     }
 
+    fun <T : BaseEntity> executeWithDataSource(
+        clazz: Class<T>,
+        fields: Map<String, String>,
+        rootData: MutableList<MutableMap<String, Any?>>,
+        historyData: MutableList<MutableMap<String, Any?>>?,
+        groupData: MutableList<MutableMap<String, Any?>>?
+    ): List<MutableMap<String, Any?>> {
+        val data = dataToMapWithDataSource(clazz, fields, "code", rootData, historyData, groupData)
+        return data
+    }
+
     fun clazzEntity(migrationTarget: String): Class<out BaseEntity>? {
         return when (migrationTarget) {
             "organization" -> Organization::class.java
@@ -503,17 +668,6 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
             }
         }
 
-//        mapFinalize["password"]?.toString()?.let {
-//            mapFinalize["password"] = if (it.isBlank()) ""
-//            else cpDecrypt.decrypt(it)?.let { p ->
-//                tempPassword[p] ?: run {
-//                    val pass = passwordEncoder.encode(p)
-//                    tempPassword[p] = pass
-//                    pass
-//                }
-//            } ?: ""
-//        }
-
         if (className == "transaction") {
             val to = mapFinalize["extTransferTo"]?.toString() ?: ""
             val from = mapFinalize["extTransferFrom"]?.toString() ?: ""
@@ -594,13 +748,15 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
     }
 
     private fun <T : BaseEntity> findUidByCode(clazz: Class<T>, value: Any): String? {
-        val tmpData = temporaryData[clazz.simpleName] ?: run {
-            val data = apiService.findListAll(clazz)
-                .associateBy { it["code"]?.toString() ?: it["id"]!!.toString() }
-            temporaryData[clazz.simpleName] = data
-            data
+        return tempDataParent["${clazz.simpleName}_$value"] ?: run {
+            val tmpData = temporaryData[clazz.simpleName] ?: run {
+                val data = apiService.findListAll(clazz)
+                    .associateBy { it["code"]?.toString() ?: it["id"]!!.toString() }
+                temporaryData[clazz.simpleName] = data
+                data
+            }
+            tmpData[value.toString()]?.get("uid")?.toString()
         }
-        return tmpData[value.toString()]?.get("uid")?.toString() ?: tempDataParent["${clazz.simpleName}_$value"]
     }
 
     private fun findPinPhonePbx(phoneUserCode: String): Pair<String, String>? {

src/main/kotlin/com/datacomsolusindo/migration/MigrationService.kt

@@ -2,6 +2,7 @@ package com.datacomsolusindo.migration
 
 import com.datacomsolusindo.cpx_shared_code.entity.BaseEntity
 import com.datacomsolusindo.cpx_shared_code.utility.SimpleLogger
+import com.datacomsolusindo.cpx_shared_code.utility.Util
 import org.springframework.beans.factory.config.YamlPropertiesFactoryBean
 import org.springframework.core.io.FileSystemResource
 import org.springframework.stereotype.Service
@@ -29,7 +30,11 @@ val tempDataParent: MutableMap<String, String?> = mutableMapOf()
 val tempPassword: MutableMap<String, String?> = mutableMapOf()
 
 @Service
-class MigrationService(val migrationEntity: MigrationEntity) {
+class MigrationService(
+    val migrationEntity: MigrationEntity,
+    val sourceDataMigrationService: SourceDataMigrationService,
+    val migrationSettingService: MigrationSettingService
+) {
 
     private val logger = SimpleLogger.getLogger(this::class.java)
     private val TEMP = "temp_"
@@ -64,6 +69,44 @@ class MigrationService(val migrationEntity: MigrationEntity) {
     }
 
     fun scanAndQueue(folder: Path) {
+        if (migrationSettingService.type.uppercase() == "FILE")
+            scanAndQueueWithFile(folder)
+        else
+            scanAndQueueWithDataSource()
+    }
+
+    private fun scanAndQueueWithDataSource() {
+        logger.info("service migration prepare data migration with data source")
+        migrationSettingService.schema.forEach {
+            val rootTable = sourceDataMigrationService.getData(
+                it.table, it.attribute.filterNot { i ->
+                    i.value.toString().startsWith("history.") || i.value.toString().startsWith("group.")
+                }.map { m -> m.value.toString() }) ?: mutableListOf()
+            val rootGroup = it.group?.let { group ->
+                 sourceDataMigrationService.getData(
+                    group, it.attribute.filter { i -> i.value.toString().startsWith("group.") }
+                        .map { m -> m.value.toString().removePrefix("group.") })
+            } ?: mutableListOf()
+            val rootHistory = it.history?.let { history ->
+                sourceDataMigrationService.getData(
+                    history, it.attribute.filter { i -> i.value.toString().startsWith("history.") }
+                        .map { m -> m.value.toString().removePrefix("history.") })
+            } ?: mutableListOf()
+            migrationEntity.clazzEntity(it.table)?.let { clazz ->
+                queueInsertData.put(
+                    clazz to migrationEntity.executeWithDataSource(
+                        clazz,
+                        it.attribute as MutableMap<String, String>,
+                        rootTable,
+                        rootHistory,
+                        rootGroup
+                    )
+                )
+            }
+        }
+    }
+
+    private fun scanAndQueueWithFile(folder: Path) {
         logger.info("service migration read files")
         Files.list(folder).use { stream ->
             val tempFiles = stream

src/main/kotlin/com/datacomsolusindo/migration/MigrationSetting.kt

@@ -0,0 +1,19 @@
+package com.datacomsolusindo.migration
+
+import org.springframework.boot.context.properties.ConfigurationProperties
+import org.springframework.stereotype.Service
+
+@Service
+@ConfigurationProperties(prefix = "migration")
+class MigrationSettingService {
+    var type: String = "FILE"
+    var sourceDatabase: MutableMap<String, Any> = mutableMapOf()
+    var schema: Array<MigrationSchema> = arrayOf()
+}
+
+class MigrationSchema(
+    val table: String,
+    val history: String? = null,
+    val group: String? = null,
+    val attribute: MutableMap<String, Any>
+)