MigrationEntity.kt 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554
  1. package com.datacomsolusindo.migration
  2. import com.datacomsolusindo.cpx_shared_code.entity.Account
  3. import com.datacomsolusindo.cpx_shared_code.entity.Area
  4. import com.datacomsolusindo.cpx_shared_code.entity.BaseEntity
  5. import com.datacomsolusindo.cpx_shared_code.entity.Budget
  6. import com.datacomsolusindo.cpx_shared_code.entity.CdrModifier
  7. import com.datacomsolusindo.cpx_shared_code.entity.Corcos
  8. import com.datacomsolusindo.cpx_shared_code.entity.CostCenter
  9. import com.datacomsolusindo.cpx_shared_code.entity.Organization
  10. import com.datacomsolusindo.cpx_shared_code.entity.Pbx
  11. import com.datacomsolusindo.cpx_shared_code.entity.PhoneUser
  12. import com.datacomsolusindo.cpx_shared_code.entity.PhoneUserPbx
  13. import com.datacomsolusindo.cpx_shared_code.entity.Provider
  14. import com.datacomsolusindo.cpx_shared_code.entity.Rate
  15. import com.datacomsolusindo.cpx_shared_code.entity.Rights
  16. import com.datacomsolusindo.cpx_shared_code.entity.Transaction
  17. import com.datacomsolusindo.cpx_shared_code.entity.Trunk
  18. import com.datacomsolusindo.cpx_shared_code.entity.WebUser
  19. import com.datacomsolusindo.cpx_shared_code.service.ApiService
  20. import com.datacomsolusindo.cpx_shared_code.service.CpDecrypt
  21. import com.datacomsolusindo.cpx_shared_code.utility.*
  22. import io.azam.ulidj.ULID
  23. import io.github.semutkecil.simplecriteria.FilterData
  24. import org.springframework.beans.factory.annotation.Autowired
  25. import org.springframework.context.annotation.Bean
  26. import org.springframework.context.annotation.Configuration
  27. import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder
  28. import org.springframework.security.crypto.factory.PasswordEncoderFactories
  29. import org.springframework.security.crypto.password.PasswordEncoder
  30. import org.springframework.stereotype.Service
  31. import org.springframework.transaction.annotation.Transactional
  32. import java.io.File
  33. import java.nio.charset.StandardCharsets
  34. import java.util.UUID
  35. import kotlin.time.measureTime
  36. import kotlin.time.measureTimedValue
  37. @Service
  38. class MigrationEntity(val passwordEncoder: PasswordEncoder, val queryNativeService: QueryNativeService) {
  39. @Autowired
  40. lateinit var apiService: ApiService
  41. @Autowired
  42. lateinit var cpDecrypt: CpDecrypt
  43. private val logger = SimpleLogger.getLogger(this::class.java)
  44. fun <T> dataToMap(
  45. clazz: Class<T>,
  46. fields: Map<String, String>,
  47. unique: String,
  48. rootFile: File,
  49. historyFile: File? = null,
  50. groupFile: File? = null
  51. ): List<MutableMap<String, Any?>> {
  52. logger.info("prepare data migration class ${clazz.simpleName}")
  53. val process = measureTimedValue {
  54. val historyData = historyFile?.let { readQueryDataToMap(it) }
  55. val groupData = groupFile?.let { readQueryDataToMap(it) }
  56. val rootData = readQueryDataToMap(rootFile)
  57. val mapRootData = rootData.map { map ->
  58. val data: MutableMap<String, Any?> = mutableMapOf()
  59. val fieldRoots = fields.toList().filterNot { f -> f.second.contains(".") }
  60. val joinRoots = fields.toList().filter { f -> f.second.contains(".") }
  61. fieldRoots.forEach { f ->
  62. data[f.first] = map[f.second]
  63. }
  64. joinRoots.sortedByDescending { it.second }.forEach { f ->
  65. data[f.first] = when {
  66. f.second.startsWith("history.") ->
  67. getValueAnotherFile(fields, historyData, unique, data, f.first, f.second, "history")
  68. f.second.toString().startsWith("group.") ->
  69. getValueAnotherFile(fields, groupData, unique, data, f.first, f.second, "group")
  70. else -> map[f.second]
  71. }
  72. }
  73. // fields.forEach { (t, u) ->
  74. // data[t] = when {
  75. // u.startsWith("history.") ->
  76. // getValueAnotherFile(fields, historyData, unique, data, t, u, "history")
  77. //
  78. // u.startsWith("group.") -> getValueAnotherFile(fields, groupData, unique, data, t, u, "group")
  79. // else -> map[u]
  80. // }
  81. // }
  82. data
  83. }
  84. mapRootData
  85. }
  86. logger.info("finish prepare data [${process.value.size}] migration class ${clazz.simpleName} takes time ${process.duration.inWholeMilliseconds}ms")
  87. return process.value
  88. }
  89. private fun getValueAnotherFile(
  90. fields: Map<String, String>,
  91. dataFile: List<Map<String, Any?>>?,
  92. unique: String,
  93. rootData: Map<String, Any?>,
  94. keyRoot: String,
  95. valRoot: String,
  96. key: String
  97. ): Any? {
  98. val uniqueField = dataFile?.firstOrNull()?.let {
  99. if (it.any { a -> a.key == fields[unique] }) unique else "id"
  100. } ?: "id"
  101. val fieldUnique = fields[uniqueField]!!.split(".").last()
  102. val value = dataFile?.firstOrNull { f ->
  103. f[fieldUnique].toString() == rootData[uniqueField].toString()
  104. }?.get(valRoot.replace("${key}.", ""))
  105. return if (keyRoot.contains("_")) value?.let { id ->
  106. dataFile.firstOrNull { f ->
  107. f[fields["id"]!!.split(".").last()].toString() == id.toString()
  108. }?.get(fields["code"]) ?: value
  109. } else value
  110. }
  111. private fun readQueryDataToMap(file: File): List<Map<String, Any?>> {
  112. val map = measureTimedValue {
  113. file.bufferedReader(StandardCharsets.UTF_8)
  114. .readLines().filter {
  115. it.startsWith("INSERT")
  116. }.mapNotNull {
  117. try {
  118. insertSqlToMap(it)
  119. } catch (_: Exception) {
  120. null
  121. }
  122. }
  123. }
  124. logger.info("migration read query data ${map.value.size} takes time ${map.duration.inWholeMilliseconds}")
  125. return map.value
  126. }
  127. private fun insertSqlToMap(sql: String): Map<String, Any?> {
  128. val splitSql = sql.split(") VALUES (")
  129. val fieldColumn = splitSql[0].split("(").last().split(",")
  130. .map { it.trim().removeSurrounding("[", "]") }
  131. val valueColumn = splitValues(splitSql[1].dropLast(1))
  132. .map { parseValue(it.trim()) }
  133. val map: MutableMap<String, Any?> = mutableMapOf()
  134. fieldColumn.mapIndexed { index, key ->
  135. map[key] = valueColumn[index]
  136. }
  137. return map
  138. }
  139. private fun splitValues(input: String): List<String> {
  140. val result = mutableListOf<String>()
  141. var depth = 0
  142. var current = StringBuilder()
  143. for (c in input) {
  144. when (c) {
  145. '(' -> depth++
  146. ')' -> depth--
  147. ',' -> {
  148. if (depth == 0) {
  149. result.add(current.toString())
  150. current = StringBuilder()
  151. continue
  152. }
  153. }
  154. }
  155. current.append(c)
  156. }
  157. result.add(current.toString())
  158. return result
  159. }
  160. private fun parseValue(raw: String): Any? {
  161. val value = raw.trim()
  162. if (value.equals("NULL", true)) return null
  163. // CAST(x AS ...)
  164. if (value.startsWith("CAST", true)) {
  165. val inner = Regex("CAST\\((.*)\\s+AS", RegexOption.IGNORE_CASE)
  166. .find(value)?.groupValues?.get(1)?.trim()
  167. return parseValue(inner ?: value)
  168. }
  169. // N'string'
  170. if (value.startsWith("N'") && value.endsWith("'")) {
  171. return value.substring(2, value.length - 1)
  172. }
  173. // 'string'
  174. if (value.startsWith("'") && value.endsWith("'")) {
  175. return value.substring(1, value.length - 1)
  176. }
  177. // number
  178. return value.toLongOrNull()
  179. ?: value.toDoubleOrNull()
  180. ?: value
  181. }
  182. private val failed: MutableList<Any> = mutableListOf()
  183. fun <T : BaseEntity> execute(
  184. clazz: Class<T>,
  185. fields: Map<String, String>,
  186. rootFile: File,
  187. historyFile: File?,
  188. groupFile: File?
  189. ): List<MutableMap<String, Any?>> {
  190. val data = dataToMap(clazz, fields, "code", rootFile, historyFile, groupFile)
  191. return data
  192. // val process = measureTime { insertData(clazz, data) }
  193. // logger.info(
  194. // "finished process migration ${clazz.simpleName} " +
  195. // "data ${data.size} " +
  196. // "success ${data.size - failed.size} " +
  197. // "failed ${failed.size} " +
  198. // "takes time ${process.inWholeMilliseconds}ms"
  199. // )
  200. }
  201. fun clazzEntity(migrationTarget: String): Class<out BaseEntity>? {
  202. return when (migrationTarget) {
  203. "organization" -> Organization::class.java
  204. "costCenter" -> CostCenter::class.java
  205. "pbx" -> Pbx::class.java
  206. "trunk" -> Trunk::class.java
  207. "callTransaction" -> Transaction::class.java
  208. "account" -> Account::class.java
  209. "area" -> Area::class.java
  210. "cdrModifier" -> CdrModifier::class.java
  211. "phoneUser" -> PhoneUser::class.java
  212. "provider" -> Provider::class.java
  213. "rate" -> Rate::class.java
  214. "webUser" -> WebUser::class.java
  215. "corcos" -> Corcos::class.java
  216. else -> null
  217. }
  218. }
  219. fun <T : BaseEntity> insertData(clazz: Class<T>, dataMap: List<MutableMap<String, Any?>>): MutableList<Any> {
  220. failed.clear()
  221. val sortingData = if (dataMap.any { it.any { a -> a.key == "structure" } }) {
  222. dataMap.sortedBy { f -> f["structure"].toString().length }
  223. } else dataMap
  224. sortingData.forEach { map ->
  225. try {
  226. val finalizer = finalizeMap(clazz.simpleName.camelCase(), map)
  227. val phoneUserPin = finalizer["pin"]
  228. val phoneUserExtension = finalizer["extension"]
  229. // budget
  230. val budgetAnnual = finalizer["budget.maxCost"]?.toString()?.toDoubleOrNull()
  231. val warningAnnual = finalizer["budget.warnCost"]?.toString()?.toDoubleOrNull()
  232. val budgetMaxCost = budgetAnnual?.let {
  233. val monthBudget = it / 12.0
  234. List(12) { monthBudget }.joinToString(";")
  235. }
  236. val budgetWarnCost = warningAnnual?.let {
  237. val monthBudget = budgetAnnual?.let { b -> b / 12.0 } ?: 0.0
  238. val warnPercent = ((it / 12.0) / monthBudget) * 100
  239. List(12) { "%.2f".format(warnPercent) }.joinToString(";")
  240. }
  241. // phoneUserPbx
  242. val phoneUserPbxIds: MutableList<Any> = mutableListOf()
  243. val data = if (clazz.simpleName == "PhoneUser") {
  244. finalizer["pbx.list"]?.toString()?.let {
  245. it.split(";").forEach { fi ->
  246. findId(Pbx::class.java, fi)?.let { id -> phoneUserPbxIds.add(id) }
  247. }
  248. }
  249. finalizer["pbx_id"]?.toString()?.let {
  250. if (it.isNotBlank()) {
  251. phoneUserPbxIds.add(it)
  252. }
  253. }
  254. val finalMap = finalizer.filterNot { fi ->
  255. listOf(
  256. "pbx.list",
  257. "pbx_id",
  258. "extension",
  259. "pin",
  260. "budget.maxCost",
  261. "budget.warnCost",
  262. "maxCost",
  263. "warnCost"
  264. ).any { a -> a == fi.key }
  265. } as MutableMap<String, Any?>
  266. queryNativeService.insertDataWithNativeQuery(clazz, finalMap)
  267. // apiService.create(clazz, finalMap)
  268. } else {
  269. // apiService.create(clazz, finalizer)
  270. queryNativeService.insertDataWithNativeQuery(clazz, finalizer)
  271. }
  272. // create budget
  273. if (BudgetUserType.entries.any { a -> a.name.snakeToCamel() == clazz.simpleName.camelCase() }) {
  274. // val id = clazz.collectAllField().findId().value(data)
  275. // val res = apiService.findById(clazz, id!!, listOf("uid"))
  276. // res?.get("uid")?.toString()?.let { uid ->
  277. // apiService.create(
  278. // Budget::class.java,
  279. // mutableMapOf(
  280. // "userType" to BudgetUserType.valueOf(
  281. // clazz.simpleName.camelCase().camelToSnake().uppercase()
  282. // ),
  283. // "userUid" to data,
  284. // "type" to "FLAT"
  285. // ),
  286. // )
  287. queryNativeService.insertDataWithNativeQuery(
  288. Budget::class.java, mutableMapOf(
  289. "userType" to BudgetUserType.valueOf(
  290. clazz.simpleName.camelCase().camelToSnake().uppercase()
  291. ).ordinal,
  292. "userUid" to data,
  293. "type" to BudgetType.FLAT.ordinal,
  294. "annualCost" to budgetAnnual,
  295. "accumulate" to 0,
  296. "maxCost" to (budgetMaxCost ?: "0;0;0;0;0;0;0;0;0;0;0;0"),
  297. "warnCostPercentage" to (budgetWarnCost ?: "0;0;0;0;0;0;0;0;0;0;0;0"),
  298. "tempCost" to "0;0;0;0;0;0;0;0;0;0;0;0"
  299. )
  300. )
  301. // create phoneUserPbx
  302. if (phoneUserPbxIds.isNotEmpty()) {
  303. phoneUserPbxIds.forEach { pbxId ->
  304. queryNativeService.insertDataWithNativeQuery(
  305. PhoneUserPbx::class.java, mutableMapOf(
  306. "pin" to phoneUserPin,
  307. "extension" to phoneUserExtension,
  308. "pbx_id" to pbxId,
  309. "phoneUser_id" to data,
  310. )
  311. )
  312. // apiService.create(
  313. // PhoneUserPbx::class.java, mutableMapOf(
  314. // "pin" to phoneUserPin,
  315. // "extension" to phoneUserExtension,
  316. // "pbx_id" to pbxId,
  317. // "phoneUser_id" to id,
  318. // )
  319. // )
  320. }
  321. }
  322. // }
  323. }
  324. } catch (e: Exception) {
  325. failed.add(map)
  326. // logger.error("failed insert data migration", e)
  327. }
  328. }
  329. return failed
  330. }
  331. private fun finalizeMap(className: String, map: MutableMap<String, Any?>): MutableMap<String, Any?> {
  332. val mapFinalize: MutableMap<String, Any?> = mutableMapOf()
  333. map["code"]?.toString()?.let {
  334. map["code"] = it.ifBlank { UUID.randomUUID().toString().take(6) }
  335. }
  336. map["name"]?.toString()?.let {
  337. map["name"] = it.ifBlank { "Auto ${(0..99999).toString().padStart(5, '0')}" }
  338. }
  339. map["emailOnOverBudget"]?.let {
  340. map["emailOnOverBudget"] = it.toString() == "1"
  341. }
  342. map["direction"]?.toString()?.let {
  343. map["direction"] = if (className == "transaction") {
  344. it.split("").mapNotNull { m ->
  345. when (m.trim()) {
  346. "C" -> Direction.INCOMING.ordinal //"C"
  347. "G" -> Direction.OUTGOING.ordinal //"G"
  348. "I" -> Direction.INTERNAL.ordinal //"I"
  349. else -> null
  350. }
  351. }.joinToString(";")
  352. } else {
  353. it.split("").mapNotNull { m ->
  354. when (m.trim()) {
  355. "C" -> "C"
  356. "G" -> "G"
  357. "I" -> "I"
  358. else -> null
  359. }
  360. }.joinToString(";")
  361. }
  362. }
  363. map.filterNot { it.key == "id" || it.key == "structure" }.forEach { (t, u) ->
  364. if (t.contains("_")) {
  365. val isParent = t.startsWith("parent")
  366. val value = if (t == "pbx_id") {
  367. findId(Pbx::class.java, u ?: "PBX01")
  368. } else u?.toString()?.let { code ->
  369. val clazzEntity = clazzEntity(if (isParent) className else t.split("_")[0])
  370. clazzEntity?.let { findId(it, code) }
  371. }
  372. if (value != null) {
  373. mapFinalize[t] = value
  374. }
  375. } else {
  376. mapFinalize[t] = u
  377. }
  378. }
  379. mapFinalize["pin"]?.toString()?.let {
  380. mapFinalize["pin"] = if (it.isBlank()) null
  381. else cpDecrypt.decrypt(it)?.let { p -> ToolAes.encrypt(p) }
  382. }
  383. mapFinalize["password"]?.toString()?.let {
  384. mapFinalize["password"] = if (it.isBlank()) ""
  385. else cpDecrypt.decrypt(it)?.let { p -> passwordEncoder.encode(p) } ?: ""
  386. }
  387. if (className == "transaction") {
  388. val to = mapFinalize["extTransferTo"]?.toString() ?: ""
  389. val from = mapFinalize["extTransferFrom"]?.toString() ?: ""
  390. mapFinalize["transferType"] = when {
  391. to.isBlank() && from.isBlank() -> TransferType.DIRECT.ordinal
  392. to.isNotBlank() && from.isBlank() -> TransferType.TRANSFER_TO.ordinal
  393. to.isBlank() && from.isNotBlank() -> TransferType.TRANSFER_FROM.ordinal
  394. else -> TransferType.TRANSFER_FROM_AND_TRANSFER_TO.ordinal
  395. }
  396. }
  397. if (className == "webUser") {
  398. defaultProfile?.let { mapFinalize["profile_id"] = it }
  399. }
  400. if (className == "corcos") {
  401. mapFinalize["name"] = "Corcos ${map["name"]}"
  402. }
  403. return mapFinalize.mapValues { v -> v.value?.toString() } as MutableMap<String, Any?>
  404. }
  405. private val defaultProfile: Any? by lazy {
  406. apiService.findListPage(
  407. Rights::class.java,
  408. listOf("uid"),
  409. FilterData.filter("name", FilterData.FILTEROP.EQ, "Default")
  410. ).firstOrNull()?.get("uid")
  411. }
  412. private fun toSnakeCase(input: String): String {
  413. return input
  414. .replace(Regex("([a-z0-9])([A-Z])"), "$1_$2")
  415. .lowercase()
  416. }
  417. private fun <T : BaseEntity> findId(clazz: Class<T>, value: Any): String? {
  418. return try {
  419. val query = "SELECT uid FROM ${toSnakeCase(clazz.simpleName)} WHERE code = :code "
  420. val id = apiService.em.createNativeQuery(query, String::class.java)
  421. .setParameter("code", value)
  422. .singleResult as String
  423. id
  424. } catch (_: Exception) {
  425. null
  426. }
  427. }
  428. }
  429. fun String.camelToSnake(): String =
  430. replace(Regex("([a-z0-9])([A-Z])"), "$1_$2")
  431. .lowercase()
  432. fun String.camelCase(): String {
  433. return when {
  434. this.contains("_") || this.contains("-") || this.contains(" ") -> {
  435. this
  436. .lowercase()
  437. .split("_", "-", " ")
  438. .filter { it.isNotBlank() }
  439. .mapIndexed { index, word ->
  440. if (index == 0) word
  441. else word.replaceFirstChar { it.uppercase() }
  442. }
  443. .joinToString("")
  444. }
  445. else -> {
  446. this.replaceFirstChar { it.lowercase() }
  447. }
  448. }
  449. }
  450. fun String.snakeToCamel(): String {
  451. return this.lowercase()
  452. .split("_")
  453. .mapIndexed { index, s ->
  454. if (index == 0) s else s.replaceFirstChar { it.uppercase() }
  455. }
  456. .joinToString("")
  457. }
  458. @Configuration
  459. class SecurityConfig {
  460. @Bean
  461. fun passwordEncoder(): PasswordEncoder = PasswordEncoderFactories.createDelegatingPasswordEncoder()
  462. }
  463. @Service
  464. @Transactional
  465. class QueryNativeService(val apiService: ApiService) {
  466. fun <T> insertDataWithNativeQuery(clazz: Class<T>, map: MutableMap<String, Any?>): String? {
  467. val uid = ULID.random()
  468. val fields = mutableListOf("uid")
  469. map.keys.forEach {
  470. val t = it.replace("_id", "_uid")
  471. fields.add(t)
  472. }
  473. val structure = map["parent_id"]?.toString()?.let {
  474. fields.add("structure")
  475. "${EntityUtility(apiService, Organization::class.java).parentStructure(it)}|$uid"
  476. }
  477. val tableName = when (clazz.simpleName.lowercase()) {
  478. "phoneuserpbx" -> "phoneuserpbx"
  479. "transaction" -> {
  480. fields.remove("uid")
  481. "calltransaction"
  482. }
  483. else -> clazz.simpleName.camelToSnake().lowercase()
  484. }
  485. val query = "INSERT INTO $tableName (${fields.joinToString() { it.camelToSnake() }}) " +
  486. "VALUES (${fields.joinToString() { ":$it" }})"
  487. val sqlNative = apiService.em.createNativeQuery(query)
  488. if (clazz.simpleName.lowercase() != "transaction") {
  489. sqlNative.setParameter("uid", uid)
  490. }
  491. structure?.let { sqlNative.setParameter("structure", structure) }
  492. map.forEach { (t, u) -> sqlNative.setParameter(t.replace("_id", "_uid"), u) }
  493. sqlNative.executeUpdate()
  494. return uid
  495. }
  496. }