From ad2e30a8ec5c224ef1bae7e8671c7dbfa424a528 Mon Sep 17 00:00:00 2001 From: Oleksandr Maistruk Date: Mon, 23 Sep 2024 14:14:00 +0300 Subject: [PATCH] [WLA-8515] - translations validator --- .../betterme/localizer/LocalizerExtension.kt | 2 + .../betterme/localizer/LocalizerPlugin.kt | 2 + .../localizer/TranslationsDownloaderTask.kt | 29 +++++-- .../localizer/TranslationsUploaderTask.kt | 10 ++- .../betterme/localizer/core/SlackNotifier.kt | 37 +++++++++ .../localizer/core/TranslationsLoader.kt | 16 +++- .../core/TranslationsLoaderFactory.kt | 12 ++- .../localizer/core/TranslationsValidator.kt | 83 +++++++++++++++++++ .../betterme/localizer/core/XmlParser.kt | 29 +++++++ .../localizer/data/models/ApiParams.kt | 3 +- 10 files changed, 207 insertions(+), 16 deletions(-) create mode 100644 src/main/kotlin/world/betterme/localizer/core/SlackNotifier.kt create mode 100644 src/main/kotlin/world/betterme/localizer/core/TranslationsValidator.kt create mode 100644 src/main/kotlin/world/betterme/localizer/core/XmlParser.kt diff --git a/src/main/kotlin/world/betterme/localizer/LocalizerExtension.kt b/src/main/kotlin/world/betterme/localizer/LocalizerExtension.kt index 98d91e5..d52d98d 100644 --- a/src/main/kotlin/world/betterme/localizer/LocalizerExtension.kt +++ b/src/main/kotlin/world/betterme/localizer/LocalizerExtension.kt @@ -14,4 +14,6 @@ open class LocalizerExtension(project: Project) { val filters: ListProperty = project.objects.listProperty(String::class.java) val tags: ListProperty = project.objects.listProperty(String::class.java) val supportRegions: Property = project.objects.property(String::class.java) + val slackWebHook: Property = project.objects.property(String::class.java) + val validateTranslations: Property = project.objects.property(String::class.java) } \ No newline at end of file diff --git a/src/main/kotlin/world/betterme/localizer/LocalizerPlugin.kt b/src/main/kotlin/world/betterme/localizer/LocalizerPlugin.kt index 1f0d4e1..9d98bf7 100644 --- a/src/main/kotlin/world/betterme/localizer/LocalizerPlugin.kt +++ b/src/main/kotlin/world/betterme/localizer/LocalizerPlugin.kt @@ -28,6 +28,8 @@ open class LocalizerPlugin : Plugin { it.filters?.addAll(extension.filters) it.tags?.addAll(extension.tags) it.supportRegions?.set(supportRegions) + it.slackWebHook.set(extension.slackWebHook) + it.validateTranslations.set(extension.validateTranslations) } } } diff --git a/src/main/kotlin/world/betterme/localizer/TranslationsDownloaderTask.kt b/src/main/kotlin/world/betterme/localizer/TranslationsDownloaderTask.kt index b16d859..bbd6961 100644 --- a/src/main/kotlin/world/betterme/localizer/TranslationsDownloaderTask.kt +++ b/src/main/kotlin/world/betterme/localizer/TranslationsDownloaderTask.kt @@ -1,32 +1,44 @@ package world.betterme.localizer -import org.gradle.api.tasks.Input -import org.gradle.api.tasks.Optional -import org.gradle.api.tasks.TaskAction import org.gradle.api.DefaultTask import org.gradle.api.provider.ListProperty import org.gradle.api.provider.Property -import world.betterme.localizer.data.models.ApiParams +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.Optional +import org.gradle.api.tasks.TaskAction import world.betterme.localizer.core.TranslationsLoaderFactory +import world.betterme.localizer.data.models.ApiParams open class TranslationsDownloaderTask : DefaultTask() { @Input val apiToken: Property = project.objects.property(String::class.java) + @Input val projectId: Property = project.objects.property(String::class.java) + @Input val resourcesPath: Property = project.objects.property(String::class.java) + @get:Input @get:Optional val supportRegions: Property? = project.objects.property(String::class.java) + @get:Input @get:Optional val filters: ListProperty? = project.objects.listProperty(String::class.java) + @get:Input @get:Optional val tags: ListProperty? = project.objects.listProperty(String::class.java) + @Input + val slackWebHook: Property = project.objects.property(String::class.java) + + @get:Input + @get:Optional + val validateTranslations: Property = project.objects.property(String::class.java) + init { description = "Downloads translations for all locales supported in this project" group = "translations" @@ -34,13 +46,18 @@ open class TranslationsDownloaderTask : DefaultTask() { @TaskAction fun downloadTranslations() { - val apiParams = ApiParams(apiToken = apiToken.get(), projectId = projectId.get()) + val apiParams = ApiParams( + apiToken = apiToken.get(), + projectId = projectId.get(), + slackWebHook = slackWebHook.get() + ) val translationsLoader = TranslationsLoaderFactory.create(apiParams) translationsLoader.downloadLocalizedStrings( resFolderPath = resourcesPath.get(), filters = filters?.get() ?: emptyList(), tags = tags?.get() ?: emptyList(), - supportRegions = supportRegions?.get()?.toBoolean() ?: false + supportRegions = supportRegions?.get()?.toBoolean() ?: false, + validateTranslations = validateTranslations.get().toBoolean(), ) } } diff --git a/src/main/kotlin/world/betterme/localizer/TranslationsUploaderTask.kt b/src/main/kotlin/world/betterme/localizer/TranslationsUploaderTask.kt index f69b2bb..5a773db 100644 --- a/src/main/kotlin/world/betterme/localizer/TranslationsUploaderTask.kt +++ b/src/main/kotlin/world/betterme/localizer/TranslationsUploaderTask.kt @@ -1,12 +1,12 @@ package world.betterme.localizer import org.gradle.api.DefaultTask +import org.gradle.api.provider.Property import org.gradle.api.tasks.Input import org.gradle.api.tasks.Optional import org.gradle.api.tasks.TaskAction -import org.gradle.api.provider.Property -import world.betterme.localizer.data.models.ApiParams import world.betterme.localizer.core.TranslationsLoaderFactory +import world.betterme.localizer.data.models.ApiParams open class TranslationsUploaderTask : DefaultTask() { @@ -34,7 +34,11 @@ open class TranslationsUploaderTask : DefaultTask() { @Suppress("unused") @TaskAction fun uploadTranslations() { - val apiParams = ApiParams(apiToken = apiToken.get(), projectId = projectId.get()) + val apiParams = ApiParams( + apiToken = apiToken.get(), + projectId = projectId.get(), + slackWebHook = "" // no-op + ) val translationsLoader = TranslationsLoaderFactory.create(apiParams) translationsLoader.uploadTermsAndTranslations( resFolderPath = resourcesPath.get(), diff --git a/src/main/kotlin/world/betterme/localizer/core/SlackNotifier.kt b/src/main/kotlin/world/betterme/localizer/core/SlackNotifier.kt new file mode 100644 index 0000000..2d86fc1 --- /dev/null +++ b/src/main/kotlin/world/betterme/localizer/core/SlackNotifier.kt @@ -0,0 +1,37 @@ +package world.betterme.localizer.core + +import java.io.OutputStream +import java.net.HttpURLConnection +import java.net.URL + +class SlackNotifier(private val webHookUrl: String) { + + fun sendSlackMessage(message: String) { + val url = URL(webHookUrl) + val connection = url.openConnection() as HttpURLConnection + + try { + connection.requestMethod = "POST" + connection.doOutput = true + connection.setRequestProperty("Content-Type", "application/json") + + val jsonPayload = """{"text": "$message"}""" + + connection.outputStream.use { outputStream: OutputStream -> + outputStream.write(jsonPayload.toByteArray(Charsets.UTF_8)) + outputStream.flush() + } + + val responseCode = connection.responseCode + if (responseCode == HttpURLConnection.HTTP_OK) { + println("Slack message sent successfully!") + } else { + println("Failed to send Slack message. Response code: $responseCode") + } + } catch (e: Exception) { + println("Error sending Slack message: ${e.message}") + } finally { + connection.disconnect() + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/world/betterme/localizer/core/TranslationsLoader.kt b/src/main/kotlin/world/betterme/localizer/core/TranslationsLoader.kt index a8bfe15..7d64c28 100644 --- a/src/main/kotlin/world/betterme/localizer/core/TranslationsLoader.kt +++ b/src/main/kotlin/world/betterme/localizer/core/TranslationsLoader.kt @@ -18,7 +18,8 @@ interface TranslationsLoader { resFolderPath: String, filters: List, tags: List, - supportRegions: Boolean + supportRegions: Boolean, + validateTranslations: Boolean, ) /** @@ -40,14 +41,16 @@ interface TranslationsLoader { internal class TranslationsLoaderImpl( private val apiParams: ApiParams, private val restStore: TranslationsRestStore, - private val localStore: TranslationsLocalStore + private val localStore: TranslationsLocalStore, + private val translationsValidator: TranslationsValidator, ) : TranslationsLoader { override fun downloadLocalizedStrings( resFolderPath: String, filters: List, tags: List, - supportRegions: Boolean + supportRegions: Boolean, + validateTranslations: Boolean, ) { val availableLocales = restStore.getAvailableLanguages(apiParams) println("Retrieved list of locales available for this project: $availableLocales") @@ -55,9 +58,16 @@ internal class TranslationsLoaderImpl( urls.forEach { (locale, url) -> println("Starting translations for exportLocale $locale download with filters [$filters] and tags [$tags]") val fileContent = restStore.loadTranslationsContent(url) + if (validateTranslations) { + translationsValidator.appendTranslationContent(locale, fileContent) + } println("Saving translations for exportLocale $locale") localStore.saveToFile(resFolderPath, fileContent, locale, supportRegions) } + + if (validateTranslations) { + translationsValidator.validateAll() + } } override fun uploadTermsAndTranslations( diff --git a/src/main/kotlin/world/betterme/localizer/core/TranslationsLoaderFactory.kt b/src/main/kotlin/world/betterme/localizer/core/TranslationsLoaderFactory.kt index bd56c37..e97ae2c 100644 --- a/src/main/kotlin/world/betterme/localizer/core/TranslationsLoaderFactory.kt +++ b/src/main/kotlin/world/betterme/localizer/core/TranslationsLoaderFactory.kt @@ -1,12 +1,12 @@ package world.betterme.localizer.core +import com.google.gson.Gson +import okhttp3.OkHttpClient import world.betterme.localizer.data.TranslationsLocalStore import world.betterme.localizer.data.TranslationsLocalStoreImpl import world.betterme.localizer.data.TranslationsRestStore import world.betterme.localizer.data.TranslationsRestStoreImpl import world.betterme.localizer.data.models.ApiParams -import com.google.gson.Gson -import okhttp3.OkHttpClient import java.util.concurrent.TimeUnit class TranslationsLoaderFactory { @@ -18,10 +18,16 @@ class TranslationsLoaderFactory { */ @JvmStatic fun create(apiParams: ApiParams): TranslationsLoader { + val parser = XmlParser() + val notifier = SlackNotifier(apiParams.slackWebHook) return TranslationsLoaderImpl( apiParams = apiParams, restStore = translationsRestStore(), - localStore = translationsLocalStore() + localStore = translationsLocalStore(), + translationsValidator = TranslationsValidator( + parser, + notifier + ) ) } diff --git a/src/main/kotlin/world/betterme/localizer/core/TranslationsValidator.kt b/src/main/kotlin/world/betterme/localizer/core/TranslationsValidator.kt new file mode 100644 index 0000000..6d89310 --- /dev/null +++ b/src/main/kotlin/world/betterme/localizer/core/TranslationsValidator.kt @@ -0,0 +1,83 @@ +package world.betterme.localizer.core + +import java.util.Locale + +typealias Resources = Map + +class TranslationsValidator( + private val xmlParser: XmlParser, + private val notifier: SlackNotifier +) { + private val translations = mutableMapOf() + private val validationErrors = mutableListOf() + + fun appendTranslationContent(locale: String, translationContent: String) { + val parsedTranslationContent = xmlParser.parseXmlStrings(translationContent) + translations[Locale.forLanguageTag(locale)] = parsedTranslationContent + } + + fun validateAll() { + println("Starting validation placeholders") + val referenceLocale = Locale.ENGLISH + val referenceTranslation = translations[referenceLocale] + ?: throw IllegalArgumentException("Reference locale $referenceLocale is missing.") + + translations.forEach { (locale, translationStrings) -> + translationStrings.forEach { (key, localizedText) -> + val referenceText = referenceTranslation[key] + if (referenceText != null) { + validatePlaceholders(referenceText, localizedText, locale, key) + } + } + } + + if (validationErrors.isNotEmpty()) { + val report = validationErrors.joinToString(separator = "\n") + notifier.sendSlackMessage("Translation validation issues:\n$report") + } else { + println("All translations validated successfully!") + } + } + + private fun validatePlaceholders( + referenceText: String, + localizedText: String, + locale: Locale, + key: String + ) { + val referencePlaceholders = extractPlaceholdersFromText(referenceText) + val localizedPlaceholders = extractPlaceholdersFromText(localizedText) + + // Validate placeholder count + if (referencePlaceholders.size != localizedPlaceholders.size) { + validationErrors.add( + "Placeholder count mismatch for key '$key' in locale '$locale'. " + + "Expected ${referencePlaceholders.size}, found ${localizedPlaceholders.size}." + ) + } + + // Validate placeholder types + referencePlaceholders.forEachIndexed { index, referencePlaceholder -> + val localizedPlaceholder = localizedPlaceholders.getOrNull(index) + if (localizedPlaceholder == null || referencePlaceholder != localizedPlaceholder) { + validationErrors.add( + "Placeholder type mismatch for key '$key' in locale '$locale' at position $index. " + + "Expected $referencePlaceholder, found $localizedPlaceholder." + ) + } + } + } + + /** + * Extracts placeholders like %1$s, %1$d from the text. + */ + private fun extractPlaceholdersFromText(text: String): List { + val placeholderRegex = """%\d+\$[sd]""".toRegex() + return placeholderRegex.findAll(text).map { it.value }.toList() + } +} + + + + + diff --git a/src/main/kotlin/world/betterme/localizer/core/XmlParser.kt b/src/main/kotlin/world/betterme/localizer/core/XmlParser.kt new file mode 100644 index 0000000..48a6833 --- /dev/null +++ b/src/main/kotlin/world/betterme/localizer/core/XmlParser.kt @@ -0,0 +1,29 @@ +package world.betterme.localizer.core + +import org.w3c.dom.Document +import javax.xml.parsers.DocumentBuilderFactory + + +class XmlParser { + fun parseXmlStrings(content: String): Resources { + val translations = mutableMapOf() + + val factory = DocumentBuilderFactory.newInstance() + val builder = factory.newDocumentBuilder() + val document: Document = builder.parse(content.byteInputStream()) + document.documentElement.normalize() + + val nodeList = document.getElementsByTagName("string") + + for (i in 0 until nodeList.length) { + val node = nodeList.item(i) + val name = node.attributes.getNamedItem("name").nodeValue + val textContent = node.textContent + + translations[name] = textContent + } + return translations + } + +} + diff --git a/src/main/kotlin/world/betterme/localizer/data/models/ApiParams.kt b/src/main/kotlin/world/betterme/localizer/data/models/ApiParams.kt index 8302a21..a2490c1 100644 --- a/src/main/kotlin/world/betterme/localizer/data/models/ApiParams.kt +++ b/src/main/kotlin/world/betterme/localizer/data/models/ApiParams.kt @@ -2,5 +2,6 @@ package world.betterme.localizer.data.models data class ApiParams( val apiToken: String, - val projectId: String + val projectId: String, + val slackWebHook: String, ) \ No newline at end of file