|
|
@@ -1,3 +1,5 @@
|
|
|
+@file:Suppress("UNCHECKED_CAST")
|
|
|
+
|
|
|
package com.datacomsolusindo.migration
|
|
|
|
|
|
import com.datacomsolusindo.cpx_shared_code.entity.Account
|
|
|
@@ -48,181 +50,146 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
|
|
|
|
|
|
private val logger = SimpleLogger.getLogger(this::class.java)
|
|
|
|
|
|
-// fun <T> dataToMap(
|
|
|
-// clazz: Class<T>,
|
|
|
-// fields: Map<String, String>,
|
|
|
-// unique: String,
|
|
|
-// rootFile: File,
|
|
|
-// historyFile: File? = null,
|
|
|
-// groupFile: File? = null
|
|
|
-// ): List<MutableMap<String, Any?>> {
|
|
|
-// logger.info("prepare data migration class ${clazz.simpleName}")
|
|
|
-// val process = measureTimedValue {
|
|
|
-// val historyData = historyFile?.let { readQueryDataToMap(it) }
|
|
|
-// val groupData = groupFile?.let { readQueryDataToMap(it) }
|
|
|
-// val rootData = readQueryDataToMap(rootFile)
|
|
|
-// val mapRootData = rootData.mapIndexed { index, map ->
|
|
|
-// val data: MutableMap<String, Any?> = mutableMapOf()
|
|
|
-// val fieldRoots = fields.toList().filterNot { f -> f.second.contains(".") }
|
|
|
-// val joinRoots = fields.toList().filter { f -> f.second.contains(".") }
|
|
|
-//
|
|
|
-// fieldRoots.forEach { f ->
|
|
|
-// data[f.first] = map[f.second]
|
|
|
-// }
|
|
|
-// joinRoots.sortedByDescending { it.second }.forEach { f ->
|
|
|
-// data[f.first] = when {
|
|
|
-// f.second.startsWith("history.") ->
|
|
|
-// getValueAnotherFile(fields, historyData, unique, data, f.first, f.second, "history")
|
|
|
-//
|
|
|
-// f.second.toString().startsWith("group.") ->
|
|
|
-// getValueAnotherFile(fields, groupData, unique, data, f.first, f.second, "group")
|
|
|
-//
|
|
|
-// else -> map[f.second]
|
|
|
-// }
|
|
|
-// }
|
|
|
-// data
|
|
|
-// }
|
|
|
-// mapRootData
|
|
|
-// }
|
|
|
-// logger.info("finish prepare data [${process.value.size}] migration class ${clazz.simpleName} takes time ${process.duration.inWholeMilliseconds}ms")
|
|
|
-// return process.value
|
|
|
-// }
|
|
|
-//
|
|
|
-// private fun getValueAnotherFile(
|
|
|
-// fields: Map<String, String>,
|
|
|
-// dataFile: List<Map<String, Any?>>?,
|
|
|
-// unique: String,
|
|
|
-// rootData: Map<String, Any?>,
|
|
|
-// keyRoot: String,
|
|
|
-// valRoot: String,
|
|
|
-// key: String
|
|
|
-// ): Any? {
|
|
|
-// val uniqueField = dataFile?.firstOrNull()?.let {
|
|
|
-// if (it.any { a -> a.key == fields[unique] }) unique else "id"
|
|
|
-// } ?: "id"
|
|
|
-// val fieldUnique = fields[uniqueField]!!.split(".").last()
|
|
|
-//
|
|
|
-// val value = dataFile?.firstOrNull { f ->
|
|
|
-// f[fieldUnique].toString() == rootData[uniqueField].toString()
|
|
|
-// }?.get(valRoot.replace("${key}.", ""))
|
|
|
-//
|
|
|
-// return if (keyRoot.contains("_")) value?.let { id ->
|
|
|
-// dataFile.firstOrNull { f ->
|
|
|
-// f[fields["id"]!!.split(".").last()].toString() == id.toString()
|
|
|
-// }?.get(fields["code"]) ?: value
|
|
|
-// } else value
|
|
|
-// }
|
|
|
-//
|
|
|
-// private fun readQueryDataToMap(file: File): List<Map<String, Any?>> {
|
|
|
-// val map = measureTimedValue {
|
|
|
-// file.bufferedReader(StandardCharsets.UTF_8)
|
|
|
-// .readLines().filter {
|
|
|
-// it.startsWith("INSERT")
|
|
|
-// }.mapNotNull {
|
|
|
-// try {
|
|
|
-// insertSqlToMap(it)
|
|
|
-// } catch (_: Exception) {
|
|
|
-// null
|
|
|
-// }
|
|
|
-// }
|
|
|
-// }
|
|
|
-// logger.info("migration read query data ${map.value.size} takes time ${map.duration.inWholeMilliseconds}")
|
|
|
-// return map.value
|
|
|
-// }
|
|
|
-
|
|
|
- fun <T> dataToMap(
|
|
|
+ private fun <T> dataToMap(
|
|
|
clazz: Class<T>,
|
|
|
fields: Map<String, String>,
|
|
|
- unique: String,
|
|
|
rootFile: File,
|
|
|
historyFile: File? = null,
|
|
|
groupFile: File? = null
|
|
|
- ): List<MutableMap<String, Any?>> {
|
|
|
-
|
|
|
- logger.info("prepare data migration class ${clazz.simpleName}")
|
|
|
-
|
|
|
- val process = measureTimedValue {
|
|
|
+ ) {
|
|
|
+ processData(
|
|
|
+ clazz,
|
|
|
+ fields,
|
|
|
+ rootData = readQueryDataToMap(rootFile) as MutableList<MutableMap<String, Any?>>,
|
|
|
+ historyData = historyFile?.let { readQueryDataToMap(it) as MutableList<MutableMap<String, Any?>> },
|
|
|
+ groupData = groupFile?.let { readQueryDataToMap(it) as MutableList<MutableMap<String, Any?>> }
|
|
|
+ )
|
|
|
+ }
|
|
|
|
|
|
- // --- Read Files ---
|
|
|
- val historyData = historyFile?.let { readQueryDataToMap(it) }
|
|
|
- val groupData = groupFile?.let { readQueryDataToMap(it) }
|
|
|
- val rootData = readQueryDataToMap(rootFile)
|
|
|
+ private fun <T> dataToMapWithDataSource(
|
|
|
+ clazz: Class<T>,
|
|
|
+ fields: Map<String, String>,
|
|
|
+ rootData: MutableList<MutableMap<String, Any?>>,
|
|
|
+ historyData: MutableList<MutableMap<String, Any?>>? = null,
|
|
|
+ groupData: MutableList<MutableMap<String, Any?>>? = null
|
|
|
+ ) {
|
|
|
+ processData(
|
|
|
+ clazz,
|
|
|
+ fields,
|
|
|
+ rootData,
|
|
|
+ historyData,
|
|
|
+ groupData
|
|
|
+ )
|
|
|
+ }
|
|
|
|
|
|
- // --- Pre-calc field names (avoid split(".") repeatedly) ---
|
|
|
- val fieldMapping = fields.mapValues { it.value.substringAfterLast(".") }
|
|
|
- val uniqueField = fieldMapping[unique] ?: "id"
|
|
|
- val uniqueFieldId = fieldMapping["id"]
|
|
|
+ private fun <T> processData(
|
|
|
+ clazz: Class<T>,
|
|
|
+ fields: Map<String, String>,
|
|
|
+ rootData: MutableList<MutableMap<String, Any?>>,
|
|
|
+ historyData: MutableList<MutableMap<String, Any?>>? = null,
|
|
|
+ groupData: MutableList<MutableMap<String, Any?>>? = null
|
|
|
+ ) {
|
|
|
+ val fieldMapping = fields.mapValues { it.value.substringAfterLast(".") }
|
|
|
+ val uniqueField = fieldMapping["code"] ?: "id"
|
|
|
+ val uniqueFieldId = fieldMapping["id"]
|
|
|
+
|
|
|
+ val historyIndex = buildHistoryIndex(historyData, uniqueField, uniqueFieldId)
|
|
|
+ val groupIndex = buildGroupIndex(groupData, uniqueFieldId ?: uniqueField)
|
|
|
+
|
|
|
+ val fieldRoots = fields.filterValues { !it.contains(".") }
|
|
|
+ val joinRoots = fields.filterValues { it.contains(".") }
|
|
|
+
|
|
|
+ val chunkData = rootData.chunked(1000)
|
|
|
+ logger.info("data migration class ${clazz.simpleName} chunk data ${chunkData.size}")
|
|
|
+ chunkData.forEachIndexed { _, data ->
|
|
|
+ val dataMap = data.map { row ->
|
|
|
+ buildRow(
|
|
|
+ row,
|
|
|
+ fieldRoots,
|
|
|
+ joinRoots,
|
|
|
+ historyIndex,
|
|
|
+ groupIndex,
|
|
|
+ uniqueField,
|
|
|
+ uniqueFieldId
|
|
|
+ )
|
|
|
+ }.map { postProcessPassword(it) }
|
|
|
+ queueInsertData.put((clazz as Class<out BaseEntity>) to dataMap)
|
|
|
+ }
|
|
|
+ }
|
|
|
|
|
|
- // --- Create Indexes (O(n)) ---
|
|
|
- val historyIndex = historyData?.groupBy { it[uniqueField] }?.mapValues { (_, items) ->
|
|
|
+ private fun buildHistoryIndex(
|
|
|
+ historyData: List<Map<String, Any?>>?,
|
|
|
+ uniqueField: String,
|
|
|
+ uniqueFieldId: String?
|
|
|
+ ): Map<Any?, Map<String, Any?>>? {
|
|
|
+ if (historyData == null || uniqueFieldId == null) return null
|
|
|
+ return historyData
|
|
|
+ .groupBy { it[uniqueField] }
|
|
|
+ .mapValues { (_, items) ->
|
|
|
items.maxByOrNull {
|
|
|
- it[uniqueFieldId!!.removePrefix("history.")].toString().toInt()
|
|
|
- }
|
|
|
+ it[uniqueFieldId.removePrefix("history.")]
|
|
|
+ ?.toString()
|
|
|
+ ?.toIntOrNull() ?: 0
|
|
|
+ } as Map<String, Any?>
|
|
|
}
|
|
|
- //?.associateBy { it[uniqueField]?.toString() }
|
|
|
- val groupIndex = groupData?.associateBy { it[uniqueFieldId ?: uniqueField]?.toString() }
|
|
|
-
|
|
|
- val fieldRoots = fields.filter { !it.value.contains(".") }
|
|
|
- val joinRoots = fields.filter { it.value.contains(".") }
|
|
|
- .toList().sortedByDescending { it.second }.toMap()
|
|
|
+ }
|
|
|
|
|
|
- rootData.mapIndexed { index, row ->
|
|
|
- val data = mutableMapOf<String, Any?>()
|
|
|
+ private fun buildGroupIndex(groupData: List<Map<String, Any?>>?, key: String): Map<String?, Map<String, Any?>>? {
|
|
|
+ return groupData?.associateBy { it[key]?.toString() }
|
|
|
+ }
|
|
|
|
|
|
- // --- Direct Fields ---
|
|
|
- fieldRoots.forEach { (targetKey, sourceKey) ->
|
|
|
- data[targetKey] = row[sourceKey]
|
|
|
+ private fun buildRow(
|
|
|
+ row: Map<String, Any?>,
|
|
|
+ fieldRoots: Map<String, String>,
|
|
|
+ joinRoots: Map<String, String>,
|
|
|
+ historyIndex: Map<Any?, Map<String, Any?>>?,
|
|
|
+ groupIndex: Map<String?, Map<String, Any?>>?,
|
|
|
+ uniqueField: String,
|
|
|
+ uniqueFieldId: String?
|
|
|
+ ): MutableMap<String, Any?> {
|
|
|
+ val data = mutableMapOf<String, Any?>()
|
|
|
+ // Direct fields
|
|
|
+ fieldRoots.forEach { (target, source) -> data[target] = row[source] }
|
|
|
+ // Join fields
|
|
|
+ joinRoots.forEach { (target, sourceFull) ->
|
|
|
+
|
|
|
+ val value = when {
|
|
|
+ sourceFull.startsWith("history.") -> {
|
|
|
+ val key = sourceFull.removePrefix("history.")
|
|
|
+ historyIndex
|
|
|
+ ?.get(row[uniqueField])
|
|
|
+ ?.get(key.substringAfterLast("."))
|
|
|
}
|
|
|
|
|
|
- // --- Join Fields ---
|
|
|
- joinRoots.forEach { (targetKey, sourceKeyFull) ->
|
|
|
-
|
|
|
- val value = when {
|
|
|
- sourceKeyFull.startsWith("history.") -> {
|
|
|
- val sourceKey = sourceKeyFull.removePrefix("history.")
|
|
|
- historyIndex
|
|
|
- ?.get(row[uniqueField]?.toString())
|
|
|
- ?.get(sourceKey.substringAfterLast("."))
|
|
|
- }
|
|
|
-
|
|
|
- sourceKeyFull.startsWith("group.") -> {
|
|
|
- val sourceKey = sourceKeyFull.removePrefix("group.")
|
|
|
- val idKey = historyIndex
|
|
|
- ?.get(row[uniqueField]?.toString())
|
|
|
- ?.get(uniqueFieldId) ?: row[uniqueFieldId]
|
|
|
- groupIndex
|
|
|
- ?.get(idKey.toString())
|
|
|
- ?.get(sourceKey.substringAfterLast("."))
|
|
|
- }
|
|
|
-
|
|
|
- else -> row[sourceKeyFull]
|
|
|
- }
|
|
|
-
|
|
|
- data[targetKey] = value
|
|
|
+ sourceFull.startsWith("group.") -> {
|
|
|
+ val key = sourceFull.removePrefix("group.")
|
|
|
+ val idKey = historyIndex
|
|
|
+ ?.get(row[uniqueField])
|
|
|
+ ?.get(uniqueFieldId)
|
|
|
+ ?: row[uniqueFieldId]
|
|
|
+ groupIndex
|
|
|
+ ?.get(idKey?.toString())
|
|
|
+ ?.get(key.substringAfterLast("."))
|
|
|
}
|
|
|
- data
|
|
|
+
|
|
|
+ else -> row[sourceFull]
|
|
|
}
|
|
|
+ data[target] = value
|
|
|
}
|
|
|
+ return data
|
|
|
+ }
|
|
|
|
|
|
- val value = process.value.map { m ->
|
|
|
- val dt = m as MutableMap<String, Any?>
|
|
|
- dt["password"]?.toString()?.let {
|
|
|
- dt["password"] = if (it.isBlank()) ""
|
|
|
- else cpDecrypt.decrypt(it)?.let { p ->
|
|
|
- tempPassword[p] ?: run {
|
|
|
- val pass = passwordEncoder.encode(p)
|
|
|
- tempPassword[p] = pass
|
|
|
- pass
|
|
|
- }
|
|
|
- } ?: ""
|
|
|
- }
|
|
|
- dt
|
|
|
+ private fun postProcessPassword(data: MutableMap<String, Any?>): MutableMap<String, Any?> {
|
|
|
+ val raw = data["password"]?.toString() ?: return data
|
|
|
+ data["password"] = when {
|
|
|
+ raw.isBlank() -> ""
|
|
|
+ else -> cpDecrypt.decrypt(raw)?.let { plain ->
|
|
|
+ tempPassword[plain] ?: passwordEncoder.encode(plain).also {
|
|
|
+ tempPassword[plain] = it
|
|
|
+ }
|
|
|
+ } ?: ""
|
|
|
}
|
|
|
- logger.info(
|
|
|
- "finish prepare data [${process.value.size}] migration class ${clazz.simpleName} " +
|
|
|
- "takes time ${process.duration.inWholeMilliseconds}ms"
|
|
|
- )
|
|
|
- return value
|
|
|
+ return data
|
|
|
}
|
|
|
|
|
|
private fun readQueryDataToMap(file: File): List<Map<String, Any?>> {
|
|
|
@@ -309,16 +276,22 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
|
|
|
}
|
|
|
|
|
|
private val failed: MutableList<Any> = mutableListOf()
|
|
|
+
|
|
|
fun <T : BaseEntity> execute(
|
|
|
clazz: Class<T>,
|
|
|
fields: Map<String, String>,
|
|
|
rootFile: File,
|
|
|
historyFile: File?,
|
|
|
groupFile: File?
|
|
|
- ): List<MutableMap<String, Any?>> {
|
|
|
- val data = dataToMap(clazz, fields, "code", rootFile, historyFile, groupFile)
|
|
|
- return data
|
|
|
- }
|
|
|
+ ) = dataToMap(clazz, fields, rootFile, historyFile, groupFile)
|
|
|
+
|
|
|
+ fun <T : BaseEntity> executeWithDataSource(
|
|
|
+ clazz: Class<T>,
|
|
|
+ fields: Map<String, String>,
|
|
|
+ rootData: MutableList<MutableMap<String, Any?>>,
|
|
|
+ historyData: MutableList<MutableMap<String, Any?>>?,
|
|
|
+ groupData: MutableList<MutableMap<String, Any?>>?
|
|
|
+ ) = dataToMapWithDataSource(clazz, fields, rootData, historyData, groupData)
|
|
|
|
|
|
fun clazzEntity(migrationTarget: String): Class<out BaseEntity>? {
|
|
|
return when (migrationTarget) {
|
|
|
@@ -351,8 +324,8 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
|
|
|
val phoneUserExtension = finalizer["extension"]
|
|
|
|
|
|
// budget
|
|
|
- val budgetAnnual = finalizer["budget.maxCost"]?.toString()?.toDoubleOrNull()
|
|
|
- val warningAnnual = finalizer["budget.warnCost"]?.toString()?.toDoubleOrNull()
|
|
|
+ val budgetAnnual = (finalizer["budget.maxCost"]?.toString() ?: finalizer["budget__maxCost"]?.toString())?.toDoubleOrNull()
|
|
|
+ val warningAnnual = (finalizer["budget.warnCost"]?.toString() ?: finalizer["budget__warnCost"]?.toString())?.toDoubleOrNull()
|
|
|
val budgetMaxCost = budgetAnnual?.let { max ->
|
|
|
List(12) { max }.joinToString(";")
|
|
|
}
|
|
|
@@ -365,7 +338,7 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
|
|
|
// phoneUserPbx
|
|
|
val phoneUserPbxIds: MutableList<Any> = mutableListOf()
|
|
|
val data = if (clazz.simpleName == "PhoneUser") {
|
|
|
- finalizer["pbx.list"]?.toString()?.let {
|
|
|
+ (finalizer["pbx.list"]?.toString() ?: finalizer["pbx__list"]?.toString())?.let {
|
|
|
it.split(";").forEach { fi ->
|
|
|
findUidByCode(Pbx::class.java, fi)?.let { id -> phoneUserPbxIds.add(id) }
|
|
|
}
|
|
|
@@ -378,11 +351,14 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
|
|
|
val finalMap = finalizer.filterNot { fi ->
|
|
|
listOf(
|
|
|
"pbx.list",
|
|
|
+ "pbx__list",
|
|
|
"pbx_id",
|
|
|
"extension",
|
|
|
"pin",
|
|
|
"budget.maxCost",
|
|
|
+ "budget__maxCost",
|
|
|
"budget.warnCost",
|
|
|
+ "budget__warnCost",
|
|
|
"maxCost",
|
|
|
"warnCost"
|
|
|
).any { a -> a == fi.key }
|
|
|
@@ -393,7 +369,7 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
|
|
|
}
|
|
|
}]"
|
|
|
if (phoneUserPbxIds.isEmpty()) {
|
|
|
- finalMap["expiredDate"] = LocalDate.now().atStartOfDay()
|
|
|
+ finalMap["expiredDate"] = finalMap["expiredDate"] ?: LocalDate.now().atStartOfDay()
|
|
|
}
|
|
|
queryNativeService.insertDataWithNativeQuery(clazz, finalMap)
|
|
|
} else {
|
|
|
@@ -503,17 +479,6 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-// mapFinalize["password"]?.toString()?.let {
|
|
|
-// mapFinalize["password"] = if (it.isBlank()) ""
|
|
|
-// else cpDecrypt.decrypt(it)?.let { p ->
|
|
|
-// tempPassword[p] ?: run {
|
|
|
-// val pass = passwordEncoder.encode(p)
|
|
|
-// tempPassword[p] = pass
|
|
|
-// pass
|
|
|
-// }
|
|
|
-// } ?: ""
|
|
|
-// }
|
|
|
-
|
|
|
if (className == "transaction") {
|
|
|
val to = mapFinalize["extTransferTo"]?.toString() ?: ""
|
|
|
val from = mapFinalize["extTransferFrom"]?.toString() ?: ""
|
|
|
@@ -594,13 +559,15 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
|
|
|
}
|
|
|
|
|
|
private fun <T : BaseEntity> findUidByCode(clazz: Class<T>, value: Any): String? {
|
|
|
- val tmpData = temporaryData[clazz.simpleName] ?: run {
|
|
|
- val data = apiService.findListAll(clazz)
|
|
|
- .associateBy { it["code"]?.toString() ?: it["id"]!!.toString() }
|
|
|
- temporaryData[clazz.simpleName] = data
|
|
|
- data
|
|
|
+ return tempDataParent["${clazz.simpleName}_$value"] ?: run {
|
|
|
+ val tmpData = temporaryData[clazz.simpleName] ?: run {
|
|
|
+ val data = apiService.findListAll(clazz)
|
|
|
+ .associateBy { it["code"]?.toString() ?: it["id"]!!.toString() }
|
|
|
+ temporaryData[clazz.simpleName] = data
|
|
|
+ data
|
|
|
+ }
|
|
|
+ tmpData[value.toString()]?.get("uid")?.toString()
|
|
|
}
|
|
|
- return tmpData[value.toString()]?.get("uid")?.toString() ?: tempDataParent["${clazz.simpleName}_$value"]
|
|
|
}
|
|
|
|
|
|
private fun findPinPhonePbx(phoneUserCode: String): Pair<String, String>? {
|
masarifyuli