|
|
@@ -130,7 +130,102 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
|
|
|
// return map.value
|
|
|
// }
|
|
|
|
|
|
- fun <T> dataToMap(
|
|
|
+// fun <T> dataToMap(
|
|
|
+// clazz: Class<T>,
|
|
|
+// fields: Map<String, String>,
|
|
|
+// unique: String,
|
|
|
+// rootFile: File,
|
|
|
+// historyFile: File? = null,
|
|
|
+// groupFile: File? = null
|
|
|
+// ): List<MutableMap<String, Any?>> {
|
|
|
+//
|
|
|
+// logger.info("prepare data migration class ${clazz.simpleName}")
|
|
|
+//
|
|
|
+// val process = measureTimedValue {
|
|
|
+//
|
|
|
+// // --- Read Files ---
|
|
|
+// val historyData = historyFile?.let { readQueryDataToMap(it) }
|
|
|
+// val groupData = groupFile?.let { readQueryDataToMap(it) }
|
|
|
+// val rootData = readQueryDataToMap(rootFile)
|
|
|
+//
|
|
|
+// // --- Pre-calc field names (avoid split(".") repeatedly) ---
|
|
|
+// val fieldMapping = fields.mapValues { it.value.substringAfterLast(".") }
|
|
|
+// val uniqueField = fieldMapping[unique] ?: "id"
|
|
|
+// val uniqueFieldId = fieldMapping["id"]
|
|
|
+//
|
|
|
+// // --- Create Indexes (O(n)) ---
|
|
|
+// val historyIndex = historyData?.groupBy { it[uniqueField] }?.mapValues { (_, items) ->
|
|
|
+// items.maxByOrNull {
|
|
|
+// it[uniqueFieldId!!.removePrefix("history.")].toString().toInt()
|
|
|
+// }
|
|
|
+// }
|
|
|
+// //?.associateBy { it[uniqueField]?.toString() }
|
|
|
+// val groupIndex = groupData?.associateBy { it[uniqueFieldId ?: uniqueField]?.toString() }
|
|
|
+//
|
|
|
+// val fieldRoots = fields.filter { !it.value.contains(".") }
|
|
|
+// val joinRoots = fields.filter { it.value.contains(".") }
|
|
|
+// .toList().sortedByDescending { it.second }.toMap()
|
|
|
+//
|
|
|
+// rootData.mapIndexed { index, row ->
|
|
|
+// val data = mutableMapOf<String, Any?>()
|
|
|
+//
|
|
|
+// // --- Direct Fields ---
|
|
|
+// fieldRoots.forEach { (targetKey, sourceKey) ->
|
|
|
+// data[targetKey] = row[sourceKey]
|
|
|
+// }
|
|
|
+//
|
|
|
+// // --- Join Fields ---
|
|
|
+// joinRoots.forEach { (targetKey, sourceKeyFull) ->
|
|
|
+//
|
|
|
+// val value = when {
|
|
|
+// sourceKeyFull.startsWith("history.") -> {
|
|
|
+// val sourceKey = sourceKeyFull.removePrefix("history.")
|
|
|
+// historyIndex
|
|
|
+// ?.get(row[uniqueField]?.toString())
|
|
|
+// ?.get(sourceKey.substringAfterLast("."))
|
|
|
+// }
|
|
|
+//
|
|
|
+// sourceKeyFull.startsWith("group.") -> {
|
|
|
+// val sourceKey = sourceKeyFull.removePrefix("group.")
|
|
|
+// val idKey = historyIndex
|
|
|
+// ?.get(row[uniqueField]?.toString())
|
|
|
+// ?.get(uniqueFieldId) ?: row[uniqueFieldId]
|
|
|
+// groupIndex
|
|
|
+// ?.get(idKey.toString())
|
|
|
+// ?.get(sourceKey.substringAfterLast("."))
|
|
|
+// }
|
|
|
+//
|
|
|
+// else -> row[sourceKeyFull]
|
|
|
+// }
|
|
|
+//
|
|
|
+// data[targetKey] = value
|
|
|
+// }
|
|
|
+// data
|
|
|
+// }
|
|
|
+// }
|
|
|
+//
|
|
|
+// val value = process.value.map { m ->
|
|
|
+// val dt = m as MutableMap<String, Any?>
|
|
|
+// dt["password"]?.toString()?.let {
|
|
|
+// dt["password"] = if (it.isBlank()) ""
|
|
|
+// else cpDecrypt.decrypt(it)?.let { p ->
|
|
|
+// tempPassword[p] ?: run {
|
|
|
+// val pass = passwordEncoder.encode(p)
|
|
|
+// tempPassword[p] = pass
|
|
|
+// pass
|
|
|
+// }
|
|
|
+// } ?: ""
|
|
|
+// }
|
|
|
+// dt
|
|
|
+// }
|
|
|
+// logger.info(
|
|
|
+// "finish prepare data [${process.value.size}] migration class ${clazz.simpleName} " +
|
|
|
+// "takes time ${process.duration.inWholeMilliseconds}ms"
|
|
|
+// )
|
|
|
+// return value
|
|
|
+// }
|
|
|
+
|
|
|
+ private fun <T> dataToMap(
|
|
|
clazz: Class<T>,
|
|
|
fields: Map<String, String>,
|
|
|
unique: String,
|
|
|
@@ -138,91 +233,149 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
|
|
|
historyFile: File? = null,
|
|
|
groupFile: File? = null
|
|
|
): List<MutableMap<String, Any?>> {
|
|
|
+ return processData(
|
|
|
+ clazz,
|
|
|
+ fields,
|
|
|
+ unique,
|
|
|
+ rootData = readQueryDataToMap(rootFile) as MutableList<MutableMap<String, Any?>>,
|
|
|
+ historyData = historyFile?.let { readQueryDataToMap(it) as MutableList<MutableMap<String, Any?>> },
|
|
|
+ groupData = groupFile?.let { readQueryDataToMap(it) as MutableList<MutableMap<String, Any?>> }
|
|
|
+ )
|
|
|
+ }
|
|
|
|
|
|
- logger.info("prepare data migration class ${clazz.simpleName}")
|
|
|
+ private fun <T> dataToMapWithDataSource(
|
|
|
+ clazz: Class<T>,
|
|
|
+ fields: Map<String, String>,
|
|
|
+ unique: String,
|
|
|
+ rootData: MutableList<MutableMap<String, Any?>>,
|
|
|
+ historyData: MutableList<MutableMap<String, Any?>>? = null,
|
|
|
+ groupData: MutableList<MutableMap<String, Any?>>? = null
|
|
|
+ ): List<MutableMap<String, Any?>> {
|
|
|
+ return processData(
|
|
|
+ clazz,
|
|
|
+ fields,
|
|
|
+ unique,
|
|
|
+ rootData,
|
|
|
+ historyData,
|
|
|
+ groupData
|
|
|
+ )
|
|
|
+ }
|
|
|
|
|
|
- val process = measureTimedValue {
|
|
|
+ private fun <T> processData(
|
|
|
+ clazz: Class<T>,
|
|
|
+ fields: Map<String, String>,
|
|
|
+ unique: String,
|
|
|
+ rootData: MutableList<MutableMap<String, Any?>>,
|
|
|
+ historyData: MutableList<MutableMap<String, Any?>>? = null,
|
|
|
+ groupData: MutableList<MutableMap<String, Any?>>? = null
|
|
|
+ ): List<MutableMap<String, Any?>> {
|
|
|
|
|
|
- // --- Read Files ---
|
|
|
- val historyData = historyFile?.let { readQueryDataToMap(it) }
|
|
|
- val groupData = groupFile?.let { readQueryDataToMap(it) }
|
|
|
- val rootData = readQueryDataToMap(rootFile)
|
|
|
+ logger.info("prepare data migration class ${clazz.simpleName}")
|
|
|
|
|
|
- // --- Pre-calc field names (avoid split(".") repeatedly) ---
|
|
|
+ val process = measureTimedValue {
|
|
|
val fieldMapping = fields.mapValues { it.value.substringAfterLast(".") }
|
|
|
val uniqueField = fieldMapping[unique] ?: "id"
|
|
|
val uniqueFieldId = fieldMapping["id"]
|
|
|
|
|
|
- // --- Create Indexes (O(n)) ---
|
|
|
- val historyIndex = historyData?.groupBy { it[uniqueField] }?.mapValues { (_, items) ->
|
|
|
- items.maxByOrNull {
|
|
|
- it[uniqueFieldId!!.removePrefix("history.")].toString().toInt()
|
|
|
- }
|
|
|
+ val historyIndex = buildHistoryIndex(historyData, uniqueField, uniqueFieldId)
|
|
|
+ val groupIndex = buildGroupIndex(groupData, uniqueFieldId ?: uniqueField)
|
|
|
+
|
|
|
+ val fieldRoots = fields.filterValues { !it.contains(".") }
|
|
|
+ val joinRoots = fields.filterValues { it.contains(".") }
|
|
|
+
|
|
|
+ rootData.map { row ->
|
|
|
+ buildRow(
|
|
|
+ row,
|
|
|
+ fieldRoots,
|
|
|
+ joinRoots,
|
|
|
+ historyIndex,
|
|
|
+ groupIndex,
|
|
|
+ uniqueField,
|
|
|
+ uniqueFieldId
|
|
|
+ )
|
|
|
}
|
|
|
- //?.associateBy { it[uniqueField]?.toString() }
|
|
|
- val groupIndex = groupData?.associateBy { it[uniqueFieldId ?: uniqueField]?.toString() }
|
|
|
-
|
|
|
- val fieldRoots = fields.filter { !it.value.contains(".") }
|
|
|
- val joinRoots = fields.filter { it.value.contains(".") }
|
|
|
- .toList().sortedByDescending { it.second }.toMap()
|
|
|
-
|
|
|
- rootData.mapIndexed { index, row ->
|
|
|
- val data = mutableMapOf<String, Any?>()
|
|
|
+ }
|
|
|
|
|
|
- // --- Direct Fields ---
|
|
|
- fieldRoots.forEach { (targetKey, sourceKey) ->
|
|
|
- data[targetKey] = row[sourceKey]
|
|
|
- }
|
|
|
+ logger.info(
|
|
|
+ "finish prepare data [${process.value.size}] migration class ${clazz.simpleName} " +
|
|
|
+ "takes time ${process.duration.inWholeMilliseconds}ms"
|
|
|
+ )
|
|
|
|
|
|
- // --- Join Fields ---
|
|
|
- joinRoots.forEach { (targetKey, sourceKeyFull) ->
|
|
|
+ return process.value.map { postProcessPassword(it) }
|
|
|
+ }
|
|
|
|
|
|
- val value = when {
|
|
|
- sourceKeyFull.startsWith("history.") -> {
|
|
|
- val sourceKey = sourceKeyFull.removePrefix("history.")
|
|
|
- historyIndex
|
|
|
- ?.get(row[uniqueField]?.toString())
|
|
|
- ?.get(sourceKey.substringAfterLast("."))
|
|
|
- }
|
|
|
+ private fun buildHistoryIndex(
|
|
|
+ historyData: List<Map<String, Any?>>?,
|
|
|
+ uniqueField: String,
|
|
|
+ uniqueFieldId: String?
|
|
|
+ ): Map<Any?, Map<String, Any?>>? {
|
|
|
+ if (historyData == null || uniqueFieldId == null) return null
|
|
|
+ return historyData
|
|
|
+ .groupBy { it[uniqueField] }
|
|
|
+ .mapValues { (_, items) ->
|
|
|
+ items.maxByOrNull {
|
|
|
+ it[uniqueFieldId.removePrefix("history.")]
|
|
|
+ ?.toString()
|
|
|
+ ?.toIntOrNull() ?: 0
|
|
|
+ } as Map<String, Any?>
|
|
|
+ }
|
|
|
+ }
|
|
|
|
|
|
- sourceKeyFull.startsWith("group.") -> {
|
|
|
- val sourceKey = sourceKeyFull.removePrefix("group.")
|
|
|
- val idKey = historyIndex
|
|
|
- ?.get(row[uniqueField]?.toString())
|
|
|
- ?.get(uniqueFieldId) ?: row[uniqueFieldId]
|
|
|
- groupIndex
|
|
|
- ?.get(idKey.toString())
|
|
|
- ?.get(sourceKey.substringAfterLast("."))
|
|
|
- }
|
|
|
+ private fun buildGroupIndex(groupData: List<Map<String, Any?>>?, key: String): Map<String?, Map<String, Any?>>? {
|
|
|
+ return groupData?.associateBy { it[key]?.toString() }
|
|
|
+ }
|
|
|
|
|
|
- else -> row[sourceKeyFull]
|
|
|
- }
|
|
|
+ private fun buildRow(
|
|
|
+ row: Map<String, Any?>,
|
|
|
+ fieldRoots: Map<String, String>,
|
|
|
+ joinRoots: Map<String, String>,
|
|
|
+ historyIndex: Map<Any?, Map<String, Any?>>?,
|
|
|
+ groupIndex: Map<String?, Map<String, Any?>>?,
|
|
|
+ uniqueField: String,
|
|
|
+ uniqueFieldId: String?
|
|
|
+ ): MutableMap<String, Any?> {
|
|
|
+ val data = mutableMapOf<String, Any?>()
|
|
|
+ // Direct fields
|
|
|
+ fieldRoots.forEach { (target, source) -> data[target] = row[source] }
|
|
|
+ // Join fields
|
|
|
+ joinRoots.forEach { (target, sourceFull) ->
|
|
|
+
|
|
|
+ val value = when {
|
|
|
+ sourceFull.startsWith("history.") -> {
|
|
|
+ val key = sourceFull.removePrefix("history.")
|
|
|
+ historyIndex
|
|
|
+ ?.get(row[uniqueField])
|
|
|
+ ?.get(key.substringAfterLast("."))
|
|
|
+ }
|
|
|
|
|
|
- data[targetKey] = value
|
|
|
+ sourceFull.startsWith("group.") -> {
|
|
|
+ val key = sourceFull.removePrefix("group.")
|
|
|
+ val idKey = historyIndex
|
|
|
+ ?.get(row[uniqueField])
|
|
|
+ ?.get(uniqueFieldId)
|
|
|
+ ?: row[uniqueFieldId]
|
|
|
+ groupIndex
|
|
|
+ ?.get(idKey?.toString())
|
|
|
+ ?.get(key.substringAfterLast("."))
|
|
|
}
|
|
|
- data
|
|
|
+ else -> row[sourceFull]
|
|
|
}
|
|
|
+ data[target] = value
|
|
|
}
|
|
|
+ return data
|
|
|
+ }
|
|
|
|
|
|
- val value = process.value.map { m ->
|
|
|
- val dt = m as MutableMap<String, Any?>
|
|
|
- dt["password"]?.toString()?.let {
|
|
|
- dt["password"] = if (it.isBlank()) ""
|
|
|
- else cpDecrypt.decrypt(it)?.let { p ->
|
|
|
- tempPassword[p] ?: run {
|
|
|
- val pass = passwordEncoder.encode(p)
|
|
|
- tempPassword[p] = pass
|
|
|
- pass
|
|
|
- }
|
|
|
- } ?: ""
|
|
|
- }
|
|
|
- dt
|
|
|
+ private fun postProcessPassword(data: MutableMap<String, Any?>): MutableMap<String, Any?> {
|
|
|
+ val raw = data["password"]?.toString() ?: return data
|
|
|
+ data["password"] = when {
|
|
|
+ raw.isBlank() -> ""
|
|
|
+ else -> cpDecrypt.decrypt(raw)?.let { plain ->
|
|
|
+ tempPassword[plain] ?: passwordEncoder.encode(plain).also {
|
|
|
+ tempPassword[plain] = it
|
|
|
+ }
|
|
|
+ } ?: ""
|
|
|
}
|
|
|
- logger.info(
|
|
|
- "finish prepare data [${process.value.size}] migration class ${clazz.simpleName} " +
|
|
|
- "takes time ${process.duration.inWholeMilliseconds}ms"
|
|
|
- )
|
|
|
- return value
|
|
|
+ return data
|
|
|
}
|
|
|
|
|
|
private fun readQueryDataToMap(file: File): List<Map<String, Any?>> {
|
|
|
@@ -309,6 +462,7 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
|
|
|
}
|
|
|
|
|
|
private val failed: MutableList<Any> = mutableListOf()
|
|
|
+
|
|
|
fun <T : BaseEntity> execute(
|
|
|
clazz: Class<T>,
|
|
|
fields: Map<String, String>,
|
|
|
@@ -320,6 +474,17 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
|
|
|
return data
|
|
|
}
|
|
|
|
|
|
+ fun <T : BaseEntity> executeWithDataSource(
|
|
|
+ clazz: Class<T>,
|
|
|
+ fields: Map<String, String>,
|
|
|
+ rootData: MutableList<MutableMap<String, Any?>>,
|
|
|
+ historyData: MutableList<MutableMap<String, Any?>>?,
|
|
|
+ groupData: MutableList<MutableMap<String, Any?>>?
|
|
|
+ ): List<MutableMap<String, Any?>> {
|
|
|
+ val data = dataToMapWithDataSource(clazz, fields, "code", rootData, historyData, groupData)
|
|
|
+ return data
|
|
|
+ }
|
|
|
+
|
|
|
fun clazzEntity(migrationTarget: String): Class<out BaseEntity>? {
|
|
|
return when (migrationTarget) {
|
|
|
"organization" -> Organization::class.java
|
|
|
@@ -503,17 +668,6 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-// mapFinalize["password"]?.toString()?.let {
|
|
|
-// mapFinalize["password"] = if (it.isBlank()) ""
|
|
|
-// else cpDecrypt.decrypt(it)?.let { p ->
|
|
|
-// tempPassword[p] ?: run {
|
|
|
-// val pass = passwordEncoder.encode(p)
|
|
|
-// tempPassword[p] = pass
|
|
|
-// pass
|
|
|
-// }
|
|
|
-// } ?: ""
|
|
|
-// }
|
|
|
-
|
|
|
if (className == "transaction") {
|
|
|
val to = mapFinalize["extTransferTo"]?.toString() ?: ""
|
|
|
val from = mapFinalize["extTransferFrom"]?.toString() ?: ""
|
|
|
@@ -594,13 +748,15 @@ class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeServi
|
|
|
}
|
|
|
|
|
|
private fun <T : BaseEntity> findUidByCode(clazz: Class<T>, value: Any): String? {
|
|
|
- val tmpData = temporaryData[clazz.simpleName] ?: run {
|
|
|
- val data = apiService.findListAll(clazz)
|
|
|
- .associateBy { it["code"]?.toString() ?: it["id"]!!.toString() }
|
|
|
- temporaryData[clazz.simpleName] = data
|
|
|
- data
|
|
|
+ return tempDataParent["${clazz.simpleName}_$value"] ?: run {
|
|
|
+ val tmpData = temporaryData[clazz.simpleName] ?: run {
|
|
|
+ val data = apiService.findListAll(clazz)
|
|
|
+ .associateBy { it["code"]?.toString() ?: it["id"]!!.toString() }
|
|
|
+ temporaryData[clazz.simpleName] = data
|
|
|
+ data
|
|
|
+ }
|
|
|
+ tmpData[value.toString()]?.get("uid")?.toString()
|
|
|
}
|
|
|
- return tmpData[value.toString()]?.get("uid")?.toString() ?: tempDataParent["${clazz.simpleName}_$value"]
|
|
|
}
|
|
|
|
|
|
private fun findPinPhonePbx(phoneUserCode: String): Pair<String, String>? {
|
