diff options
Diffstat (limited to 'MLEB/Translate/src/Synchronization')
9 files changed, 1310 insertions, 203 deletions
diff --git a/MLEB/Translate/src/Synchronization/ClearGroupSyncCacheMaintenanceScript.php b/MLEB/Translate/src/Synchronization/ClearGroupSyncCacheMaintenanceScript.php new file mode 100644 index 00000000..d5cf1568 --- /dev/null +++ b/MLEB/Translate/src/Synchronization/ClearGroupSyncCacheMaintenanceScript.php @@ -0,0 +1,83 @@ +<?php +declare( strict_types = 1 ); + +namespace MediaWiki\Extension\Translate\Synchronization; + +use MediaWiki\Extension\Translate\Services; +use MediaWiki\Extension\Translate\Utilities\BaseMaintenanceScript; +use MediaWiki\MediaWikiServices; + +/** + * Clear the contents of the group synchronization cache + * @author Abijeet Patro + * @license GPL-2.0-or-later + * @since 2021.01 + */ +class ClearGroupSyncCacheMaintenanceScript extends BaseMaintenanceScript { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Clear the contents of the group synchronization cache for a single or all groups' ); + + $this->addOption( + 'group', + '(optional) Group Id being cleared', + self::OPTIONAL, + self::HAS_ARG + ); + $this->addOption( + 'all', + '(optional) Clear all groups' + ); + + $this->requireExtension( 'Translate' ); + } + + public function execute() { + $config = MediaWikiServices::getInstance()->getMainConfig(); + + if ( !$config->get( 'TranslateGroupSynchronizationCache' ) ) { + $this->fatalError( 'GroupSynchronizationCache is not enabled' ); + } + + $this->validateParamsAndArgs(); + $groupId = $this->getOption( 'group' ); + $all = $this->hasOption( 'all' ); + $groupSyncCache = Services::getInstance()->getGroupSynchronizationCache(); + + if ( $groupId ) { + $this->clearGroupFromSync( $groupSyncCache, $groupId ); + $this->output( "Ended synchronization for group: $groupId\n" ); + } elseif ( $all ) { + // Remove all groups + $groupsInSync = $groupSyncCache->getGroupsInSync(); + $this->output( 'Found ' . count( $groupsInSync ) . " groups in sync.\n" ); + foreach ( $groupsInSync as $groupId ) { + $this->clearGroupFromSync( $groupSyncCache, $groupId ); + $this->output( "Ended synchronization for group: $groupId\n" ); + } + } + } + + public function validateParamsAndArgs() { + parent::validateParamsAndArgs(); + + $group = $this->getOption( 'group' ); + $all = $this->hasOption( 'all' ); + + if ( $all && $group !== null ) { + $this->fatalError( 'The "all" and "group" options cannot be used together.' ); + } + + if ( !$all && $group === null ) { + $this->fatalError( 'One of "all" OR "group" options must be specified.' ); + } + } + + private function clearGroupFromSync( GroupSynchronizationCache $groupSyncCache, string $groupId ): void { + if ( !$groupSyncCache->isGroupBeingProcessed( $groupId ) ) { + $this->fatalError( "$groupId is currently not being processed" ); + } + + $groupSyncCache->forceEndSync( $groupId ); + } +} diff --git a/MLEB/Translate/src/Synchronization/CompleteExternalTranslationMaintenanceScript.php b/MLEB/Translate/src/Synchronization/CompleteExternalTranslationMaintenanceScript.php index 0c4d5aec..eb8ce627 100644 --- a/MLEB/Translate/src/Synchronization/CompleteExternalTranslationMaintenanceScript.php +++ b/MLEB/Translate/src/Synchronization/CompleteExternalTranslationMaintenanceScript.php @@ -2,12 +2,12 @@ declare( strict_types = 1 ); -namespace MediaWiki\Extensions\Translate\Synchronization; +namespace MediaWiki\Extension\Translate\Synchronization; use Maintenance; -use MediaWiki\Extensions\Translate\Services; +use MediaWiki\Extension\Translate\Services; use MediaWiki\Logger\LoggerFactory; -use MessageIndex; +use MediaWiki\MediaWikiServices; /** * @author Abijeet Patro @@ -25,6 +25,12 @@ class CompleteExternalTranslationMaintenanceScript extends Maintenance { } public function execute() { + $config = MediaWikiServices::getInstance()->getMainConfig(); + + if ( !$config->get( 'TranslateGroupSynchronizationCache' ) ) { + $this->fatalError( 'GroupSynchronizationCache is not enabled' ); + } + $logger = LoggerFactory::getInstance( 'Translate.GroupSynchronization' ); $groupSyncCache = Services::getInstance()->getGroupSynchronizationCache(); $groupsInSync = $groupSyncCache->getGroupsInSync(); @@ -47,20 +53,24 @@ class CompleteExternalTranslationMaintenanceScript extends Maintenance { } if ( $groupResponse->hasTimedOut() ) { - $remainingMessageKeys = $groupResponse->getRemainingMessages(); + $remainingMessages = $groupResponse->getRemainingMessages(); $logger->warning( 'MessageUpdateJobs timed out for group - {groupId}; ' . 'Messages - {messages}; ' . 'Jobs remaining - {jobRemaining}', [ 'groupId' => $groupId , - 'jobRemaining' => count( $remainingMessageKeys ), - 'messages' => implode( ', ', $remainingMessageKeys ) + 'jobRemaining' => count( $remainingMessages ), + 'messages' => implode( ', ', array_keys( $remainingMessages ) ) ] ); - wfLogWarning( 'MessageUpdateJob timed out for group - ' . $groupId ); - $groupSyncCache->endSync( $groupId ); + $count = count( $remainingMessages ); + wfLogWarning( "MessageUpdateJob timed out for group $groupId with $count message(s) remaining" ); + $groupSyncCache->forceEndSync( $groupId ); + + $groupSyncCache->addGroupErrors( $groupResponse ); + } else { $groupsInProgress[] = $groupId; } @@ -68,8 +78,7 @@ class CompleteExternalTranslationMaintenanceScript extends Maintenance { if ( !$groupsInProgress ) { // No groups in progress. - $logger->info( 'All message groups are now in sync. Starting MessageIndex rebuild' ); - MessageIndex::singleton()->rebuild(); + $logger->info( 'All message groups are now in sync.' ); } $logger->info( @@ -81,3 +90,8 @@ class CompleteExternalTranslationMaintenanceScript extends Maintenance { ); } } + +class_alias( + CompleteExternalTranslationMaintenanceScript::class, + '\MediaWiki\Extensions\Translate\CompleteExternalTranslationMaintenanceScript' +); diff --git a/MLEB/Translate/src/Synchronization/DisplayGroupSynchronizationInfo.php b/MLEB/Translate/src/Synchronization/DisplayGroupSynchronizationInfo.php new file mode 100644 index 00000000..3eea143e --- /dev/null +++ b/MLEB/Translate/src/Synchronization/DisplayGroupSynchronizationInfo.php @@ -0,0 +1,261 @@ +<?php +declare( strict_types = 1 ); + +namespace MediaWiki\Extension\Translate\Synchronization; + +use Html; +use Language; +use MediaWiki\Linker\LinkRenderer; +use MessageLocalizer; +use Title; + +/** + * Display Group synchronization related information + * @author Abijeet Patro + * @license GPL-2.0-or-later + * @since 2021.02 + */ +class DisplayGroupSynchronizationInfo { + /** @var MessageLocalizer */ + private $localizer; + /** @var LinkRenderer */ + private $linkRenderer; + + public function __construct( MessageLocalizer $localizer, LinkRenderer $linkRenderer ) { + $this->localizer = $localizer; + $this->linkRenderer = $linkRenderer; + } + + /** @param string[] $groupsInSync */ + public function getGroupsInSyncHtml( array $groupsInSync, string $wrapperClass ): string { + sort( $groupsInSync ); + + if ( !$groupsInSync ) { + return Html::rawElement( + 'p', + [ 'class' => $wrapperClass ], + $this->localizer->msg( 'translate-smg-no-groups-in-sync' )->escaped() + . $this->addGroupSyncHelp( $wrapperClass ) + ); + } + + $htmlGroupItems = []; + foreach ( $groupsInSync as $groupId ) { + $htmlGroupItems[] = Html::element( 'li', [], $groupId ); + } + + return $this->getGroupSyncInfoHtml( + $wrapperClass, + 'translate-smg-groups-in-sync', + 'translate-smg-groups-in-sync-list', + Html::rawElement( 'ul', [], implode( '', $htmlGroupItems ) ), + $this->addGroupSyncHelp( $wrapperClass ) + ); + } + + public function getHtmlForGroupsWithError( + GroupSynchronizationCache $groupSynchronizationCache, + string $wrapperClass, + Language $currentLang + ): string { + $groupsWithErrors = $groupSynchronizationCache->getGroupsWithErrors(); + if ( !$groupsWithErrors ) { + return ''; + } + + $htmlGroupItems = []; + foreach ( $groupsWithErrors as $groupId ) { + $groupErrorResponse = $groupSynchronizationCache->getGroupErrorInfo( $groupId ); + $htmlGroupItems[] = $this->getHtmlForGroupErrors( $groupErrorResponse, $currentLang, $wrapperClass ); + } + + return $this->getGroupSyncInfoHtml( + $wrapperClass . ' js-group-sync-groups-with-error', + 'translate-smg-groups-with-error-title', + 'translate-smg-groups-with-error-desc', + implode( '', $htmlGroupItems ) + ); + } + + private function addGroupSyncHelp( string $wrapperClass ): string { + return Html::element( + 'a', + [ + 'href' => 'https://www.mediawiki.org/wiki/Special:MyLanguage/Help:Extension:Translate/' . + 'Group_management#Strong_synchronization', + 'target' => '_blank', + 'class' => "{$wrapperClass}__help", + ], + '[' . $this->localizer->msg( 'translate-smg-strong-sync-help' )->text() . ']' + ); + } + + private function getGroupSyncInfoHtml( + string $className, + string $summaryMsgKey, + string $descriptionMsgKey, + string $htmlContent, + string $preHtmlContent = null + ): string { + $output = Html::openElement( 'div', [ 'class' => $className ] ); + if ( $preHtmlContent ) { + $output .= $preHtmlContent; + } + + $output .= Html::openElement( 'details' ); + $output .= Html::element( 'summary', [], $this->localizer->msg( $summaryMsgKey )->text() ); + $output .= Html::element( 'p', [], $this->localizer->msg( $descriptionMsgKey )->text() ); + $output .= $htmlContent; + $output .= Html::closeElement( 'details' ); + $output .= Html::closeElement( 'div' ); + + return $output; + } + + private function getHtmlForGroupErrors( + GroupSynchronizationResponse $groupErrorResponse, + Language $language, + string $wrapperClass + ): string { + $groupId = $groupErrorResponse->getGroupId(); + $output = Html::openElement( + 'details', + [ 'class' => "{$wrapperClass}__group_errors js-group-sync-group-errors" ] + ); + + $groupResolveAction = Html::linkButton( + $this->localizer->msg( 'translate-smg-group-action-resolve' )->text(), + [ + 'class' => "{$wrapperClass}__resolve-action js-group-sync-group-resolve", + 'href' => '#', + 'data-group-id' => $groupId, + ] + ); + + $output .= Html::rawElement( + 'summary', + [], + $groupId . ' ' . + Html::rawElement( + 'span', + [ 'class' => "{$wrapperClass}__sync-actions" ], + $this->localizer->msg( 'parentheses' ) + ->params( $groupResolveAction )->text() + + ) + ); + + $errorMessages = $groupErrorResponse->getRemainingMessages(); + + $output .= Html::openElement( 'ol' ); + foreach ( $errorMessages as $message ) { + $output .= Html::rawElement( + 'li', + [ 'class' => "{$wrapperClass}__message-error js-group-sync-message-error" ], + $this->getErrorMessageHtml( $groupId, $message, $language, $wrapperClass ) + ); + } + $output .= Html::closeElement( 'ol' ); + + $output .= Html::closeElement( 'details' ); + + return $output; + } + + private function getErrorMessageHtml( + string $groupId, + MessageUpdateParameter $message, + Language $language, + string $wrapperClass + ): string { + $messageTitle = Title::newFromText( $message->getPageName() ); + $actions = []; + if ( $messageTitle->exists() ) { + $output = $this->linkRenderer->makeLink( $messageTitle, $message->getPageName() ); + $actions[] = $this->linkRenderer->makeLink( + $messageTitle, + $this->localizer->msg( 'translate-smg-group-message-action-history' )->text(), + [], + [ 'action' => 'history' ] + ); + } else { + $output = $this->linkRenderer->makeBrokenLink( $messageTitle, $message->getPageName() ); + } + + $actions[] = Html::linkButton( + $this->localizer->msg( 'translate-smg-group-action-resolve' )->text(), + [ + 'class' => "{$wrapperClass}__resolve-action js-group-sync-message-resolve", + 'href' => '#', + 'data-group-id' => $groupId, + 'data-msg-title' => $message->getPageName(), + ] + ); + + $output .= ' ' . Html::rawElement( + 'span', + [ 'class' => "{$wrapperClass}__sync-actions" ], + $this->localizer->msg( 'parentheses' ) + ->params( $language->pipeList( $actions ) )->text() + ); + + $output .= $this->getMessageInfoHtml( $message, $language ); + + return $output; + } + + private function getMessageInfoHtml( MessageUpdateParameter $message, Language $language ): string { + $output = Html::openElement( 'dl' ); + + $tags = []; + if ( $message->isFuzzy() ) { + $tags[] = $this->localizer->msg( 'translate-smg-group-message-tag-outdated' )->text(); + } + + if ( $message->isRename() ) { + $tags[] = $this->localizer->msg( 'translate-smg-group-message-tag-rename' )->text(); + } + + if ( $tags ) { + $output .= $this->getMessagePropHtml( + $this->localizer->msg( 'translate-smg-group-message-tag-label' ) + ->numParams( count( $tags ) )->text(), + implode( $this->localizer->msg( 'pipe-separator' )->text(), $tags ) + ); + } + + $output .= $this->getMessagePropHtml( + $this->localizer->msg( 'translate-smg-group-message-message-content' )->text(), + $message->getContent() + ); + + if ( $message->isRename() ) { + $output .= $this->getMessagePropHtml( + $this->localizer->msg( 'translate-smg-group-message-message-target' )->text(), + $message->getTargetValue() + ); + + $output .= $this->getMessagePropHtml( + $this->localizer->msg( 'translate-smg-group-message-message-replacement' )->text(), + $message->getReplacementValue() + ); + + if ( $message->getOtherLangs() ) { + $output .= $this->getMessagePropHtml( + $this->localizer->msg( 'translate-smg-group-message-message-other-langs' )->text(), + implode( + $this->localizer->msg( 'comma-separator' )->text(), + $message->getOtherLangs() + ) + ); + } + } + + $output .= Html::closeElement( 'dl' ); + return $output; + } + + private function getMessagePropHtml( string $label, string $value ): string { + return Html::element( 'dt', [], $label ) . Html::element( 'dd', [], $value ); + } +} diff --git a/MLEB/Translate/src/Synchronization/ExportTranslationsMaintenanceScript.php b/MLEB/Translate/src/Synchronization/ExportTranslationsMaintenanceScript.php new file mode 100644 index 00000000..39f7e1e8 --- /dev/null +++ b/MLEB/Translate/src/Synchronization/ExportTranslationsMaintenanceScript.php @@ -0,0 +1,391 @@ +<?php + +namespace MediaWiki\Extension\Translate\Synchronization; + +use FileBasedMessageGroup; +use GettextFFS; +use MediaWiki\Extension\Translate\Utilities\BaseMaintenanceScript; +use MediaWiki\Logger\LoggerFactory; +use MessageGroup; +use MessageGroups; +use MessageGroupStats; +use MessageHandle; +use Title; +use TranslateUtils; + +/** + * Script to export translations of message groups to files. + * + * @author Niklas Laxström + * @author Siebrand Mazeland + * @copyright Copyright © 2008-2013, Niklas Laxström, Siebrand Mazeland + * @license GPL-2.0-or-later + */ +class ExportTranslationsMaintenanceScript extends BaseMaintenanceScript { + /// The translation file should be deleted if it exists + private const ACTION_DELETE = 'delete'; + /// The translation file should be created or updated + private const ACTION_CREATE = 'create'; + /// The translation file should be updated if exists, but not created as a new + private const ACTION_UPDATE = 'update'; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Export translations to files.' ); + + $this->addOption( + 'group', + 'Comma separated list of message group IDs (supports * wildcard) to export', + self::REQUIRED, + self::HAS_ARG + ); + $this->addOption( + 'lang', + 'Comma separated list of language codes or *', + self::REQUIRED, + self::HAS_ARG + ); + $this->addOption( + 'target', + 'Target directory for exported files', + self::REQUIRED, + self::HAS_ARG + ); + $this->addOption( + 'skip', + '(optional) Languages to skip, comma separated list', + self::OPTIONAL, + self::HAS_ARG + ); + $this->addOption( + 'skipgroup', + '(optional) Comma separated list of message group IDs (supports * wildcard) to not export', + self::OPTIONAL, + self::HAS_ARG + ); + $this->addOption( + 'threshold', + '(optional) Threshold for translation completion percentage that must be exceeded for initial export', + self::OPTIONAL, + self::HAS_ARG + ); + $this->addOption( + 'removal-threshold', + '(optional) Threshold for translation completion percentage that must be exceeded to keep the file', + self::OPTIONAL, + self::HAS_ARG + ); + $this->addOption( + 'hours', + '(optional) Only export languages with changes in the last given number of hours', + self::OPTIONAL, + self::HAS_ARG + ); + $this->addOption( + 'no-fuzzy', + '(optional) Do not include any messages marked as fuzzy/outdated' + ); + $this->addOption( + 'offline-gettext-format', + '(optional) Export languages in offline Gettext format. Give a file pattern with ' + . '%GROUPID% and %CODE%. Empty pattern defaults to %GROUPID%/%CODE%.po.', + self::OPTIONAL, + self::HAS_ARG + ); + + $this->requireExtension( 'Translate' ); + } + + public function execute() { + $logger = LoggerFactory::getInstance( 'Translate.GroupSynchronization' ); + $groupPattern = $this->getOption( 'group' ) ?? ''; + $groupSkipPattern = $this->getOption( 'skipgroup' ) ?? ''; + + $logger->info( + 'Starting exports for groups {groups}', + [ 'groups' => $groupPattern ] + ); + $exportStartTime = microtime( true ); + + $target = $this->getOption( 'target' ); + if ( !is_writable( $target ) ) { + $this->fatalError( "Target directory is not writable ($target)." ); + } + + $exportThreshold = $this->getOption( 'threshold' ); + $removalThreshold = $this->getOption( 'removal-threshold' ); + $noFuzzy = $this->hasOption( 'no-fuzzy' ); + + $reqLangs = TranslateUtils::parseLanguageCodes( $this->getOption( 'lang' ) ); + if ( $this->hasOption( 'skip' ) ) { + $skipLangs = array_map( 'trim', explode( ',', $this->getOption( 'skip' ) ) ); + $reqLangs = array_diff( $reqLangs, $skipLangs ); + } + + $forOffline = $this->hasOption( 'offline-gettext-format' ); + $offlineTargetPattern = $this->getOption( 'offline-gettext-format' ) ?: "%GROUPID%/%CODE%.po"; + + $groups = $this->getMessageGroups( $groupPattern, $groupSkipPattern, $forOffline ); + if ( $groups === [] ) { + $this->fatalError( 'EE1: No valid message groups identified.' ); + } + + $changeFilter = null; + if ( $this->hasOption( 'hours' ) ) { + $changeFilter = $this->getRecentlyChangedItems( + (int)$this->getOption( 'hours' ), + $this->getNamespacesForGroups( $groups ) + ); + } + + foreach ( $groups as $groupId => $group ) { + // No changes to this group at all + if ( is_array( $changeFilter ) && !isset( $changeFilter[$groupId] ) ) { + $this->output( "No recent changes to $groupId.\n" ); + continue; + } + + if ( $exportThreshold || $removalThreshold ) { + $logger->info( 'Calculating stats for group {groupId}', [ 'groupId' => $groupId ] ); + $tStartTime = microtime( true ); + + $languageExportActions = $this->getLanguageExportActions( + $groupId, + $reqLangs, + (int)$exportThreshold, + (int)$removalThreshold + ); + + $tEndTime = microtime( true ); + $logger->info( + 'Finished calculating stats for group {groupId}. Time: {duration} secs', + [ + 'groupId' => $groupId, + 'duration' => round( $tEndTime - $tStartTime, 3 ), + ] + ); + } else { + // Convert list to an associate array + $languageExportActions = array_fill_keys( $reqLangs, self::ACTION_CREATE ); + } + + if ( $languageExportActions === [] ) { + continue; + } + + $this->output( "Exporting group $groupId\n" ); + $logger->info( 'Exporting group {groupId}', [ 'groupId' => $groupId ] ); + + /** @var FileBasedMessageGroup $fileBasedGroup */ + if ( $forOffline ) { + $fileBasedGroup = FileBasedMessageGroup::newFromMessageGroup( $group, $offlineTargetPattern ); + $ffs = new GettextFFS( $fileBasedGroup ); + $ffs->setOfflineMode( true ); + } else { + $fileBasedGroup = $group; + $ffs = $group->getFFS(); + } + + $ffs->setWritePath( $target ); + $sourceLanguage = $group->getSourceLanguage(); + $collection = $group->initCollection( $sourceLanguage ); + + $inclusionList = $group->getTranslatableLanguages(); + + $langExportTimes = [ + 'collection' => 0, + 'ffs' => 0, + ]; + + $languagesExportedCount = 0; + + $langStartTime = microtime( true ); + foreach ( $languageExportActions as $lang => $action ) { + // Do not export languages that are excluded (or not included). + // Also check that inclusion list is not null, which means that all + // languages are allowed for translation and export. + if ( is_array( $inclusionList ) && !isset( $inclusionList[$lang] ) ) { + continue; + } + + // Skip languages not present in recent changes + if ( is_array( $changeFilter ) && !isset( $changeFilter[$groupId][$lang] ) ) { + continue; + } + + $targetFilePath = $target . '/' . $fileBasedGroup->getTargetFilename( $lang ); + if ( $action === self::ACTION_DELETE ) { + // phpcs:ignore Generic.PHP.NoSilencedErrors.Discouraged + @$ok = unlink( $targetFilePath ); + if ( $ok ) { + $logger->info( "Removed $targetFilePath due to removal threshold" ); + } + continue; + } elseif ( $action === self::ACTION_UPDATE && !file_exists( $targetFilePath ) ) { + // Language is under export threshold, do not export yet + $logger->info( "Not creating $targetFilePath due to export threshold" ); + continue; + } + + $startTime = microtime( true ); + $collection->resetForNewLanguage( $lang ); + $collection->loadTranslations(); + // Don't export ignored, unless it is the source language + // or message documentation + global $wgTranslateDocumentationLanguageCode; + if ( $lang !== $wgTranslateDocumentationLanguageCode + && $lang !== $sourceLanguage + ) { + $collection->filter( 'ignored' ); + } + + if ( $noFuzzy ) { + $collection->filter( 'fuzzy' ); + } + + $languagesExportedCount++; + + $endTime = microtime( true ); + $langExportTimes['collection'] += ( $endTime - $startTime ); + + $startTime = microtime( true ); + $ffs->write( $collection ); + $endTime = microtime( true ); + $langExportTimes['ffs'] += ( $endTime - $startTime ); + } + $langEndTime = microtime( true ); + + $logger->info( + 'Done exporting {count} languages for group {groupId}. Time taken {duration} secs.', + [ + 'count' => $languagesExportedCount, + 'groupId' => $groupId, + 'duration' => round( $langEndTime - $langStartTime, 3 ), + ] + ); + + foreach ( $langExportTimes as $type => $time ) { + $logger->info( + 'Time taken by "{type}" for group {groupId} – {duration} secs.', + [ + 'groupId' => $groupId, + 'type' => $type, + 'duration' => round( $time, 3 ), + ] + ); + } + } + + $exportEndTime = microtime( true ); + $logger->info( + 'Finished export process for groups {groups}. Time: {duration} secs.', + [ + 'groups' => $groupPattern, + 'duration' => round( $exportEndTime - $exportStartTime, 3 ), + ] + ); + } + + /** @return MessageGroup[] */ + private function getMessageGroups( + string $groupPattern, + string $excludePattern, + bool $forOffline + ): array { + $groupIds = MessageGroups::expandWildcards( explode( ',', trim( $groupPattern ) ) ); + $groups = MessageGroups::getGroupsById( $groupIds ); + + foreach ( $groups as $groupId => $group ) { + if ( $group->isMeta() ) { + $this->output( "Skipping meta message group $groupId.\n" ); + unset( $groups[$groupId] ); + continue; + } + + if ( !$forOffline && !$group instanceof FileBasedMessageGroup ) { + $this->output( "EE2: Unexportable message group $groupId.\n" ); + unset( $groups[$groupId] ); + } + } + + $skipIds = MessageGroups::expandWildcards( explode( ',', trim( $excludePattern ) ) ); + foreach ( $skipIds as $groupId ) { + if ( isset( $groups[$groupId] ) ) { + unset( $groups[$groupId] ); + $this->output( "Group $groupId is in skipgroup.\n" ); + } + } + + return $groups; + } + + /** + * @param int $hours + * @param int[] $namespaces + * @return array[] + */ + private function getRecentlyChangedItems( int $hours, array $namespaces ): array { + $bots = true; + $changeFilter = []; + $rows = TranslateUtils::translationChanges( $hours, $bots, $namespaces ); + foreach ( $rows as $row ) { + $title = Title::makeTitle( $row->rc_namespace, $row->rc_title ); + $handle = new MessageHandle( $title ); + $code = $handle->getCode(); + if ( !$code ) { + continue; + } + $groupIds = $handle->getGroupIds(); + foreach ( $groupIds as $groupId ) { + $changeFilter[$groupId][$code] = true; + } + } + + return $changeFilter; + } + + /** + * @param MessageGroup[] $groups + * @return int[] + */ + private function getNamespacesForGroups( array $groups ): array { + $namespaces = []; + foreach ( $groups as $group ) { + $namespaces[$group->getNamespace()] = true; + } + + return array_keys( $namespaces ); + } + + private function getLanguageExportActions( + string $groupId, + array $requestedLanguages, + int $exportThreshold = 0, + int $removalThreshold = 0 + ): array { + $stats = MessageGroupStats::forGroup( $groupId ); + + $languages = []; + + foreach ( $requestedLanguages as $code ) { + // Statistics unavailable. This should only happen if unknown language code requested. + if ( !isset( $stats[$code] ) ) { + continue; + } + + $total = $stats[$code][MessageGroupStats::TOTAL]; + $translated = $stats[$code][MessageGroupStats::TRANSLATED]; + $percentage = $total === 0 ? 0 : $translated / $total * 100; + + if ( $percentage === 0 || $percentage < $removalThreshold ) { + $languages[$code] = self::ACTION_DELETE; + } elseif ( $percentage > $exportThreshold ) { + $languages[$code] = self::ACTION_CREATE; + } else { + $languages[$code] = self::ACTION_UPDATE; + } + } + + return $languages; + } +} diff --git a/MLEB/Translate/src/Synchronization/GroupSynchronizationCache.php b/MLEB/Translate/src/Synchronization/GroupSynchronizationCache.php index 34b0c3f4..64ea5ad0 100644 --- a/MLEB/Translate/src/Synchronization/GroupSynchronizationCache.php +++ b/MLEB/Translate/src/Synchronization/GroupSynchronizationCache.php @@ -1,34 +1,52 @@ <?php declare( strict_types = 1 ); -namespace MediaWiki\Extensions\Translate\Synchronization; +namespace MediaWiki\Extension\Translate\Synchronization; -use BagOStuff; use DateTime; +use InvalidArgumentException; +use LogicException; +use MediaWiki\Extension\Translate\Cache\PersistentCache; +use MediaWiki\Extension\Translate\Cache\PersistentCacheEntry; +use RuntimeException; /** * Message group synchronization cache. Handles storage of data in the cache - * to track which groups are currently being synchronized + * to track which groups are currently being synchronized. + * Stores: + * + * 1. Groups in sync: + * - Key: {hash($groupId)}_$groupId + * - Value: $groupId + * - Tag: See GroupSynchronizationCache::getGroupsTag() + * - Exptime: Set when startSyncTimer is called + * + * 2. Message under each group being modified: + * - Key: {hash($groupId_$messageKey)}_$messageKey + * - Value: MessageUpdateParameter + * - Tag: gsc_$groupId + * - Exptime: none + * * @author Abijeet Patro * @license GPL-2.0-or-later * @since 2020.06 */ class GroupSynchronizationCache { - private const CACHE_PREFIX = 'translate-msg-group-sync'; - - private const OP_ADD = 'add'; - - private const OP_DEL = 'remove'; - - /** @var BagOStuff */ + /** @var PersistentCache */ private $cache; - /** @var int */ - private $timeout; + private $timeoutSeconds; + + /** @var string Cache tag used for groups */ + private const GROUP_LIST_TAG = 'gsc_%group_in_sync%'; + /** @var string Cache tag used for tracking groups that have errors */ + private const GROUP_ERROR_TAG = 'gsc_%group_with_error%'; - public function __construct( BagOStuff $cache, int $timeout = 600 ) { + // TODO: Decide timeout based on monitoring. Also check if it needs to be configurable + // based on the number of messages in the group. + public function __construct( PersistentCache $cache, int $timeoutSeconds = 2400 ) { $this->cache = $cache; - $this->timeout = $timeout; + $this->timeoutSeconds = $timeoutSeconds; } /** @@ -36,118 +54,130 @@ class GroupSynchronizationCache { * @return string[] */ public function getGroupsInSync(): array { - $groupsCacheKey = $this->getGroupsKey(); - $groupsInSync = $this->cache->get( $groupsCacheKey ); + $groupsInSyncEntries = $this->cache->getByTag( self::GROUP_LIST_TAG ); + /** @var string[] */ + $groups = []; + foreach ( $groupsInSyncEntries as $entry ) { + $groups[] = $entry->value(); + } - return $groupsInSync === false ? [] : $groupsInSync; + return $groups; } - /** Start the synchronization process for a group with the given groupId */ - public function startSync( string $groupId ): void { - $this->cache->set( $this->getSyncTimeKey( $groupId ), ( new DateTime() )->getTimestamp() ); - $this->cache->set( $this->getGroupKey( $groupId ), [] ); - - $this->modifyGroupsInSync( $groupId, self::OP_ADD ); + /** Start synchronization process for a group and starts the expiry time */ + public function markGroupForSync( string $groupId ): void { + $expTime = $this->getExpireTime(); + $this->cache->set( + new PersistentCacheEntry( + $this->getGroupKey( $groupId ), + $groupId, + $expTime, + self::GROUP_LIST_TAG + ) + ); } - public function getSyncStartTime( string $groupId ): ?int { - $timestamp = $this->cache->get( $this->getSyncTimeKey( $groupId ) ); - if ( $timestamp === false ) { - return null; - } - - return (int)$timestamp; + public function getSyncEndTime( string $groupId ): ?int { + $cacheEntry = $this->cache->get( $this->getGroupKey( $groupId ) ); + return $cacheEntry ? $cacheEntry[0]->exptime() : null; } - /** - * End synchronization for a group. Removes the sync time, deletes the group key, and - * removes the groupId from groups in sync list - */ + /** End synchronization for a group. Deletes the group key */ public function endSync( string $groupId ): void { - // Remove all the messages for the group - $groupKey = $this->getGroupKey( $groupId ); - $groupMessageKeys = $this->cache->get( $groupKey ); - $this->removeMessages( ...$groupMessageKeys ); + if ( $this->cache->hasEntryWithTag( $this->getGroupTag( $groupId ) ) ) { + throw new InvalidArgumentException( + 'Cannot end synchronization for a group that still has messages to be processed.' + ); + } - // Remove the group message list + $groupKey = $this->getGroupKey( $groupId ); $this->cache->delete( $groupKey ); + } - // Delete the group sync start time - $this->cache->delete( $this->getSyncTimeKey( $groupId ) ); - - // Remove the group from groups in sync list - $this->modifyGroupsInSync( $groupId, self::OP_DEL ); + /** End synchronization for a group. Deletes the group key and messages */ + public function forceEndSync( string $groupId ): void { + $this->cache->deleteEntriesWithTag( $this->getGroupTag( $groupId ) ); + $this->endSync( $groupId ); } - /** Add multiple messages from a group to the cache */ + /** Add messages for a group to the cache */ public function addMessages( string $groupId, MessageUpdateParameter ...$messageParams ): void { $messagesToAdd = []; + $groupTag = $this->getGroupTag( $groupId ); foreach ( $messageParams as $messageParam ) { - $messagesToAdd[ $this->getMessageTitleKey( $messageParam->getPageName() ) ] = - $messageParam; + $titleKey = $this->getMessageKeys( $groupId, $messageParam->getPageName() )[0]; + $messagesToAdd[] = new PersistentCacheEntry( + $titleKey, + $messageParam, + null, + $groupTag + ); } - $this->cache->setMulti( $messagesToAdd ); - $this->modifyGroupMessagesInSync( $groupId, $messageParams, self::OP_ADD ); + $this->cache->set( ...$messagesToAdd ); } /** Check if the group is in synchronization */ public function isGroupBeingProcessed( string $groupId ): bool { - $groupMessages = $this->cache->get( $this->getGroupKey( $groupId ) ); - return $groupMessages !== false; + $groupEntry = $this->cache->get( $this->getGroupKey( $groupId ) ); + return $groupEntry !== []; } /** - * Return messages keys belonging to group Id currently in synchronization. + * Return all messages in a group * @param string $groupId - * @return string[] - */ - public function getGroupMessageKeys( string $groupId ): array { - $groupMessages = $this->cache->get( $this->getGroupKey( $groupId ) ); - if ( $groupMessages === false ) { - return []; - } - - return $groupMessages; - } - - /** - * Return values for multiple messages from the cache. - * @param string ...$messageKeys * @return MessageUpdateParameter[] Returns a key value pair, with the key being the - * messageKey and value being MessageUpdateParameter or null if the key is not available - * in the cache. + * messageKey and value being MessageUpdateParameter */ - public function getMessages( string ...$messageKeys ): array { - $messageCacheKeys = []; - foreach ( $messageKeys as $messageKey ) { - $messageCacheKeys[] = $this->getMessageTitleKey( $messageKey ); - } - - $messageParams = $this->cache->getMulti( $messageCacheKeys ); + public function getGroupMessages( string $groupId ): array { + $messageEntries = $this->cache->getByTag( $this->getGroupTag( $groupId ) ); $allMessageParams = []; - foreach ( $messageCacheKeys as $index => $messageCacheKey ) { - $allMessageParams[$messageKeys[$index]] = $messageParams[$messageCacheKey] ?? null; + foreach ( $messageEntries as $entry ) { + $message = $entry->value(); + if ( $message instanceof MessageUpdateParameter ) { + $allMessageParams[$message->getPageName()] = $message; + } else { + // Should not happen, but handle primarily to keep phan happy. + throw $this->invalidArgument( $message, MessageUpdateParameter::class ); + } } return $allMessageParams; } - /** - * Update the group cache with the latest information with the status of message - * update jobs, then check if the group has timed out and returns the latest information - */ + /** Check if a message is being processed */ + public function isMessageBeingProcessed( string $groupId, string $messageKey ): bool { + $messageCacheKey = $this->getMessageKeys( $groupId, $messageKey ); + return $this->cache->has( $messageCacheKey[0] ); + } + + /** Get the current synchronization status of the group. Does not perform any updates. */ public function getSynchronizationStatus( string $groupId ): GroupSynchronizationResponse { - $this->syncGroup( $groupId ); - $syncStartTime = $this->getSyncStartTime( $groupId ); - if ( !$syncStartTime ) { - // Processing is done + if ( !$this->isGroupBeingProcessed( $groupId ) ) { + // Group is currently not being processed. + throw new LogicException( + 'Sync requested for a group currently not being processed. Check if ' . + 'group is being processed by calling isGroupBeingProcessed() first' + ); + } + + $remainingMessages = $this->getGroupMessages( $groupId ); + + // No messages are present + if ( !$remainingMessages ) { return new GroupSynchronizationResponse( $groupId, [], false ); } - $hasTimedOut = $this->hasGroupTimedOut( $syncStartTime ); - $remainingMessages = $this->getGroupMessageKeys( $groupId ); + $syncExpTime = $this->getSyncEndTime( $groupId ); + if ( $syncExpTime === null ) { + // This should not happen + throw new RuntimeException( + "Unexpected condition. Group: $groupId; Messages present, but group key not found." + ); + } + + $hasTimedOut = $this->hasGroupTimedOut( $syncExpTime ); return new GroupSynchronizationResponse( $groupId, @@ -156,125 +186,224 @@ class GroupSynchronizationCache { ); } - /** - * Remove messages from the cache. Removes the message keys, but DOES NOT the update group - * message key list. - */ - public function removeMessages( string ...$messageKeys ): void { - $messageCacheKeys = []; - foreach ( $messageKeys as $key ) { - $messageCacheKeys[] = $this->getMessageTitleKey( $key ); - } + /** Remove messages from the cache. */ + public function removeMessages( string $groupId, string ...$messageKeys ): void { + $messageCacheKeys = $this->getMessageKeys( $groupId, ...$messageKeys ); - $this->cache->deleteMulti( $messageCacheKeys ); + $this->cache->delete( ...$messageCacheKeys ); } - /** - * Check messages keys that are still present in the cache and update the list of keys - * in the message group. - */ - private function syncGroup( string $groupId ): void { - $groupCacheKey = $this->getGroupKey( $groupId ); - $groupMessages = $this->cache->get( $groupCacheKey ); - if ( $groupMessages === false ) { - return; + public function addGroupErrors( GroupSynchronizationResponse $response ): void { + $groupId = $response->getGroupId(); + $remainingMessages = $response->getRemainingMessages(); + + if ( !$remainingMessages ) { + throw new LogicException( 'Cannot add a group without any remaining messages to the errors list' ); } - $messageCacheKeys = []; - foreach ( $groupMessages as $messageKey ) { - $messageCacheKeys[] = $this->getMessageTitleKey( $messageKey ); + $groupMessageErrorTag = $this->getGroupMessageErrorTag( $groupId ); + + $entriesToSave = []; + foreach ( $remainingMessages as $messageParam ) { + $titleErrorKey = $this->getMessageErrorKey( $groupId, $messageParam->getPageName() )[0]; + $entriesToSave[] = new PersistentCacheEntry( + $titleErrorKey, + $messageParam, + null, + $groupMessageErrorTag + ); } - $messageParams = $this->cache->getMulti( $messageCacheKeys ); + $this->cache->set( ...$entriesToSave ); + + $groupErrorKey = $this->getGroupErrorKey( $groupId ); - // No keys are present, delete the message and mark the group as synced - if ( !$messageParams ) { - $this->endSync( $groupId ); + // Check if the group already has errors + $groupInfo = $this->cache->get( $groupErrorKey ); + if ( $groupInfo ) { return; } - // Make a list of remaining jobs that are running. - $remainingJobTitle = []; - foreach ( $messageCacheKeys as $index => $messageCacheKey ) { - if ( isset( $messageParams[$messageCacheKey] ) ) { - $groupMessageTitle = $groupMessages[$index]; - $remainingJobTitle[] = $groupMessageTitle; + // Group did not have an error previously, add it now. When adding, + // remove the remaining messages from the GroupSynchronizationResponse to + // avoid the value in the cache becoming too big. The remaining messages + // are stored as separate items in the cache. + $trimmedGroupSyncResponse = new GroupSynchronizationResponse( + $groupId, + [], + $response->hasTimedOut() + ); + + $entriesToSave[] = new PersistentCacheEntry( + $groupErrorKey, + $trimmedGroupSyncResponse, + null, + self::GROUP_ERROR_TAG + ); + + $this->cache->set( ...$entriesToSave ); + } + + /** + * Return the groups that have errors + * @return string[] + */ + public function getGroupsWithErrors(): array { + $groupsInSyncEntries = $this->cache->getByTag( self::GROUP_ERROR_TAG ); + /** @var string[] */ + $groupIds = []; + foreach ( $groupsInSyncEntries as $entry ) { + $groupResponse = $entry->value(); + if ( $groupResponse instanceof GroupSynchronizationResponse ) { + $groupIds[] = $groupResponse->getGroupId(); + } else { + // Should not happen, but handle primarily to keep phan happy. + throw $this->invalidArgument( $groupResponse, GroupSynchronizationResponse::class ); } } - // Set the group cache with the remaining job title. - $this->cache->set( $groupCacheKey, $remainingJobTitle ); + return $groupIds; } - private function hasGroupTimedOut( int $syncStartTime ): bool { - $secondsSinceSyncStart = ( new DateTime() )->getTimestamp() - $syncStartTime; - return $secondsSinceSyncStart > $this->timeout; + /** Fetch information about a particular group that has errors including messages that failed */ + public function getGroupErrorInfo( string $groupId ): GroupSynchronizationResponse { + $groupMessageErrorTag = $this->getGroupMessageErrorTag( $groupId ); + $groupMessageEntries = $this->cache->getByTag( $groupMessageErrorTag ); + + $groupErrorKey = $this->getGroupErrorKey( $groupId ); + $groupResponseEntry = $this->cache->get( $groupErrorKey ); + $groupResponse = $groupResponseEntry[0] ? $groupResponseEntry[0]->value() : null; + if ( $groupResponse ) { + if ( !$groupResponse instanceof GroupSynchronizationResponse ) { + // Should not happen, but handle primarily to keep phan happy. + throw $this->invalidArgument( $groupResponse, GroupSynchronizationResponse::class ); + } + } else { + throw new LogicException( 'Requested to fetch errors for a group that has no errors.' ); + } + + $messageParams = []; + foreach ( $groupMessageEntries as $messageEntries ) { + $messageParam = $messageEntries->value(); + if ( $messageParam instanceof MessageUpdateParameter ) { + $messageParams[] = $messageParam; + } else { + // Should not happen, but handle primarily to keep phan happy. + throw $this->invalidArgument( $messageParam, MessageUpdateParameter::class ); + } + } + + return new GroupSynchronizationResponse( + $groupId, + $messageParams, + $groupResponse->hasTimedOut() + ); } - private function modifyGroupsInSync( string $groupId, string $op ): void { - $groupsCacheKey = $this->getGroupsKey(); - $this->cache->lock( $groupsCacheKey ); + /** Marks all messages in a group and the group itself as resolved */ + public function markGroupAsResolved( string $groupId ): GroupSynchronizationResponse { + $groupSyncResponse = $this->getGroupErrorInfo( $groupId ); + $errorMessages = $groupSyncResponse->getRemainingMessages(); - $groupsInSync = $this->getGroupsInSync(); - if ( $groupsInSync === [] && $op === self::OP_DEL ) { - return; + $errorMessageKeys = []; + foreach ( $errorMessages as $message ) { + $errorMessageKeys[] = $this->getMessageErrorKey( $groupId, $message->getPageName() )[0]; } - $this->modifyArray( $groupsInSync, $groupId, $op ); - - $this->cache->set( $groupsCacheKey, $groupsInSync ); - $this->cache->unlock( $groupsCacheKey ); + $this->cache->delete( ...$errorMessageKeys ); + return $this->syncGroupErrors( $groupId ); } - private function modifyGroupMessagesInSync( - string $groupId, array $messageParams, string $op - ): void { - $groupCacheKey = $this->getGroupKey( $groupId ); + /** Marks errors for a message as resolved */ + public function markMessageAsResolved( string $groupId, string $messagePageName ): void { + $messageErrorKey = $this->getMessageErrorKey( $groupId, $messagePageName )[0]; + $messageInCache = $this->cache->get( $messageErrorKey ); + if ( !$messageInCache ) { + throw new InvalidArgumentException( + 'Message does not appear to have synchronization errors' + ); + } - $this->cache->lock( $groupCacheKey ); + $this->cache->delete( $messageErrorKey ); + } - $groupMessages = $this->getGroupMessageKeys( $groupId ); - if ( $groupMessages === [] && $op === self::OP_DEL ) { - return; + /** Checks if group has unresolved error messages. If not clears the group from error list */ + public function syncGroupErrors( string $groupId ): GroupSynchronizationResponse { + $groupSyncResponse = $this->getGroupErrorInfo( $groupId ); + if ( $groupSyncResponse->getRemainingMessages() ) { + return $groupSyncResponse; } - /** @var MessageUpdateParameter $messageParam */ - foreach ( $messageParams as $messageParam ) { - $messageTitle = $messageParam->getPageName(); - $this->modifyArray( $groupMessages, $messageTitle, $op ); - } + // No remaining messages left, remove group from errors list. + $groupErrorKey = $this->getGroupErrorKey( $groupId ); + $this->cache->delete( $groupErrorKey ); - $this->cache->set( $groupCacheKey, $groupMessages ); - $this->cache->unlock( $groupCacheKey ); + return $groupSyncResponse; } - private function modifyArray( - array &$toModify, string $needle, string $op - ): void { - $needleIndex = array_search( $needle, $toModify ); - if ( $op === self::OP_ADD && $needleIndex === false ) { - $toModify[] = $needle; - } elseif ( $op === self::OP_DEL && $needleIndex !== false ) { - array_splice( $toModify, $needleIndex, 1 ); - } + private function hasGroupTimedOut( int $syncExpTime ): bool { + return ( new DateTime() )->getTimestamp() > $syncExpTime; } - // Cache keys related functions start here. + private function getExpireTime(): int { + $currentTime = ( new DateTime() )->getTimestamp(); + $expTime = ( new DateTime() ) + ->setTimestamp( $currentTime + $this->timeoutSeconds ) + ->getTimestamp(); - private function getGroupsKey(): string { - return $this->cache->makeKey( self::CACHE_PREFIX ); + return $expTime; } - private function getSyncTimeKey( string $groupId ): string { - return $this->cache->makeKey( self::CACHE_PREFIX, $groupId, 'time' ); + private function invalidArgument( $value, string $expectedType ): RuntimeException { + $valueType = $value ? get_class( $value ) : gettype( $value ); + return new RuntimeException( "Expected $expectedType, got $valueType" ); + } + + // Cache keys / tag related functions start here. + + private function getGroupTag( string $groupId ): string { + return 'gsc_' . $groupId; } private function getGroupKey( string $groupId ): string { - return $this->cache->makeKey( self::CACHE_PREFIX, 'group', $groupId ); + $hash = substr( hash( 'sha256', $groupId ), 0, 40 ); + return substr( "{$hash}_$groupId", 0, 255 ); } - private function getMessageTitleKey( string $title ): string { - return $this->cache->makeKey( self::CACHE_PREFIX, 'msg-title', $title ); + /** @return string[] */ + private function getMessageKeys( string $groupId, string ...$messages ): array { + $messageKeys = []; + foreach ( $messages as $message ) { + $key = $groupId . '_' . $message; + $hash = substr( hash( 'sha256', $key ), 0, 40 ); + $finalKey = substr( $hash . '_' . $key, 0, 255 ); + $messageKeys[] = $finalKey; + } + + return $messageKeys; + } + + private function getGroupErrorKey( string $groupId ): string { + $hash = substr( hash( 'sha256', $groupId ), 0, 40 ); + return substr( "{$hash}_gsc_error_$groupId", 0, 255 ); + } + + /** @return string[] */ + private function getMessageErrorKey( string $groupId, string ...$messages ): array { + $messageKeys = []; + foreach ( $messages as $message ) { + $key = $groupId . '_' . $message; + $hash = substr( hash( 'sha256', $key ), 0, 40 ); + $finalKey = substr( $hash . '_gsc_error_' . $key, 0, 255 ); + $messageKeys[] = $finalKey; + } + + return $messageKeys; } + private function getGroupMessageErrorTag( string $groupId ): string { + return "gsc_%error%_$groupId"; + } } + +class_alias( GroupSynchronizationCache::class, '\MediaWiki\Extensions\Translate\GroupSynchronizationCache' ); diff --git a/MLEB/Translate/src/Synchronization/GroupSynchronizationResponse.php b/MLEB/Translate/src/Synchronization/GroupSynchronizationResponse.php index 83a58ef6..069be720 100644 --- a/MLEB/Translate/src/Synchronization/GroupSynchronizationResponse.php +++ b/MLEB/Translate/src/Synchronization/GroupSynchronizationResponse.php @@ -2,7 +2,11 @@ declare( strict_types = 1 ); -namespace MediaWiki\Extensions\Translate\Synchronization; +namespace MediaWiki\Extension\Translate\Synchronization; + +use JsonSerializable; +use MediaWiki\Extension\Translate\Utilities\Json\JsonUnserializable; +use MediaWiki\Extension\Translate\Utilities\Json\JsonUnserializableTrait; /** * Class encapsulating the response returned by the GroupSynchronizationCache @@ -11,30 +15,31 @@ namespace MediaWiki\Extensions\Translate\Synchronization; * @license GPL-2.0-or-later * @since 2020.06 */ -class GroupSynchronizationResponse { - /** @var array */ - private $remainingMessageKeys; +class GroupSynchronizationResponse implements JsonSerializable, JsonUnserializable { + use JsonUnserializableTrait; + /** @var MessageUpdateParameter[] */ + private $remainingMessages; /** @var string */ private $groupId; - /** @var bool */ private $timeout; public function __construct( - string $groupId, array $remainingMessageKeys, bool $hasTimedOut + string $groupId, array $remainingMessages, bool $hasTimedOut ) { $this->groupId = $groupId; - $this->remainingMessageKeys = $remainingMessageKeys; + $this->remainingMessages = $remainingMessages; $this->timeout = $hasTimedOut; } public function isDone(): bool { - return $this->remainingMessageKeys === []; + return $this->remainingMessages === []; } + /** @return MessageUpdateParameter[] */ public function getRemainingMessages(): array { - return $this->remainingMessageKeys; + return $this->remainingMessages; } public function getGroupId(): string { @@ -44,4 +49,19 @@ class GroupSynchronizationResponse { public function hasTimedOut(): bool { return $this->timeout; } + + /** @return mixed[] */ + protected function toJsonArray(): array { + return get_object_vars( $this ); + } + + public static function newFromJsonArray( array $params ) { + return new self( + $params['groupId'], + $params['remainingMessages'], + $params['timeout'] + ); + } } + +class_alias( GroupSynchronizationResponse::class, '\MediaWiki\Extensions\Translate\GroupSynchronizationResponse' ); diff --git a/MLEB/Translate/src/Synchronization/ManageGroupSynchronizationCacheActionApi.php b/MLEB/Translate/src/Synchronization/ManageGroupSynchronizationCacheActionApi.php new file mode 100644 index 00000000..9c6127bf --- /dev/null +++ b/MLEB/Translate/src/Synchronization/ManageGroupSynchronizationCacheActionApi.php @@ -0,0 +1,114 @@ +<?php +declare( strict_types = 1 ); + +namespace MediaWiki\Extension\Translate\Synchronization; + +use ApiBase; +use ApiMain; +use Exception; +use FormatJson; +use MediaWiki\Logger\LoggerFactory; +use MessageGroups; + +/** + * Api module for managing group synchronization cache + * @ingroup API TranslateAPI + * @since 2021.03 + * @author Abijeet Patro + * @license GPL-2.0-or-later + */ +class ManageGroupSynchronizationCacheActionApi extends ApiBase { + private const RIGHT = 'translate-manage'; + private const VALID_OPS = [ 'resolveMessage', 'resolveGroup' ]; + /** @var GroupSynchronizationCache */ + private $groupSyncCache; + + public function __construct( ApiMain $mainModule, $moduleName, GroupSynchronizationCache $groupSyncCache ) { + parent::__construct( $mainModule, $moduleName ); + $this->groupSyncCache = $groupSyncCache; + } + + public function execute() { + $this->checkUserRightsAny( self::RIGHT ); + + $params = $this->extractRequestParams(); + $operation = $params['operation']; + $groupId = $params['group']; + $titleStr = $params['title'] ?? null; + + $group = MessageGroups::getGroup( $groupId ); + if ( $group === null ) { + return $this->dieWithError( 'apierror-translate-invalidgroup', 'invalidgroup' ); + } + + try { + if ( $operation === 'resolveMessage' ) { + if ( $titleStr === null ) { + return $this->dieWithError( [ 'apierror-missingparam', 'title' ] ); + } + $this->markAsResolved( $groupId, $titleStr ); + } elseif ( $operation === 'resolveGroup' ) { + $this->markAsResolved( $groupId ); + } + } catch ( Exception $e ) { + $data = [ + 'requestParams' => $params, + 'exceptionMessage' => $e->getMessage() + ]; + + LoggerFactory::getInstance( 'Translate.GroupSynchronization' )->error( + "Error while running: ManageGroupSynchronizationCacheActionApi::execute. Details: \n" . + FormatJson::encode( $data, true ) + ); + + $this->dieWithError( + [ + 'apierror-translate-operation-error', + wfEscapeWikiText( $e->getMessage() ) + ] + ); + } + } + + private function markAsResolved( string $groupId, ?string $messageTitle = null ): void { + if ( $messageTitle === null ) { + $currentGroupStatus = $this->groupSyncCache->markGroupAsResolved( $groupId ); + } else { + $this->groupSyncCache->markMessageAsResolved( $groupId, $messageTitle ); + $currentGroupStatus = $this->groupSyncCache->syncGroupErrors( $groupId ); + } + + $this->getResult()->addValue( null, $this->getModuleName(), [ + 'success' => 1, + 'data' => [ + 'groupRemainingMessageCount' => count( $currentGroupStatus->getRemainingMessages() ) + ] + ] ); + } + + protected function getAllowedParams() { + return [ + 'operation' => [ + ApiBase::PARAM_TYPE => self::VALID_OPS, + ApiBase::PARAM_ISMULTI => false, + ApiBase::PARAM_REQUIRED => true, + ], + 'title' => [ + ApiBase::PARAM_TYPE => 'string', + ApiBase::PARAM_REQUIRED => false + ], + 'group' => [ + ApiBase::PARAM_TYPE => 'string', + ApiBase::PARAM_REQUIRED => true + ] + ]; + } + + public function isInternal() { + return true; + } + + public function needsToken() { + return 'csrf'; + } +} diff --git a/MLEB/Translate/src/Synchronization/MessageUpdateParameter.php b/MLEB/Translate/src/Synchronization/MessageUpdateParameter.php index 21109c85..6874caef 100644 --- a/MLEB/Translate/src/Synchronization/MessageUpdateParameter.php +++ b/MLEB/Translate/src/Synchronization/MessageUpdateParameter.php @@ -1,11 +1,12 @@ <?php declare( strict_types = 1 ); -namespace MediaWiki\Extensions\Translate\Synchronization; +namespace MediaWiki\Extension\Translate\Synchronization; -use FormatJson; +use JsonSerializable; +use MediaWiki\Extension\Translate\Utilities\Json\JsonUnserializable; +use MediaWiki\Extension\Translate\Utilities\Json\JsonUnserializableTrait; use MessageUpdateJob; -use Serializable; /** * Store params for MessageUpdateJob. @@ -13,26 +14,22 @@ use Serializable; * @license GPL-2.0-or-later * @since 2020.06 */ -class MessageUpdateParameter implements Serializable { +class MessageUpdateParameter implements JsonSerializable, JsonUnserializable { + use JsonUnserializableTrait; + /** @var string */ private $pageName; - /** @var bool */ private $rename; - /** @var bool */ private $fuzzy; - /** @var string */ private $content; - /** @var string */ private $target; - /** @var string */ private $replacement; - - /** @var array */ + /** @var array|null */ private $otherLangs; public function __construct( array $params ) { @@ -63,18 +60,17 @@ class MessageUpdateParameter implements Serializable { return $this->fuzzy; } - public function getOtherLangs(): array { + public function getOtherLangs(): ?array { return $this->otherLangs; } - public function serialize(): string { - $return = FormatJson::encode( get_object_vars( $this ), false, FormatJson::ALL_OK ); - return $return; + public static function newFromJsonArray( array $params ) { + return new self( $params ); } - public function unserialize( $deserialized ) { - $params = FormatJson::decode( $deserialized, true ); - $this->assignPropsFromArray( $params ); + /** @return mixed[] */ + protected function toJsonArray(): array { + return get_object_vars( $this ); } private function assignPropsFromArray( array $params ) { @@ -100,3 +96,5 @@ class MessageUpdateParameter implements Serializable { return new self( $jobParams ); } } + +class_alias( MessageUpdateParameter::class, '\MediaWiki\Extensions\Translate\MessageUpdateParameter' ); diff --git a/MLEB/Translate/src/Synchronization/QueryGroupSyncCacheMaintenanceScript.php b/MLEB/Translate/src/Synchronization/QueryGroupSyncCacheMaintenanceScript.php new file mode 100644 index 00000000..101ecab7 --- /dev/null +++ b/MLEB/Translate/src/Synchronization/QueryGroupSyncCacheMaintenanceScript.php @@ -0,0 +1,97 @@ +<?php +declare( strict_types = 1 ); + +namespace MediaWiki\Extension\Translate\Synchronization; + +use MediaWiki\Extension\Translate\Services; +use MediaWiki\Extension\Translate\Utilities\BaseMaintenanceScript; +use MediaWiki\MediaWikiServices; + +/** + * Query information in the group synchronization cache. + * @author Abijeet Patro + * @license GPL-2.0-or-later + * @since 2021.01 + */ +class QueryGroupSyncCacheMaintenanceScript extends BaseMaintenanceScript { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Query the contents of the group synchronization cache' ); + + $this->addOption( + 'group', + '(optional) Group Id being queried', + self::OPTIONAL, + self::HAS_ARG + ); + + $this->requireExtension( 'Translate' ); + } + + public function execute() { + $config = MediaWikiServices::getInstance()->getMainConfig(); + + if ( !$config->get( 'TranslateGroupSynchronizationCache' ) ) { + $this->fatalError( 'GroupSynchronizationCache is not enabled' ); + } + + $groupSyncCache = Services::getInstance()->getGroupSynchronizationCache(); + + $groupId = $this->getOption( 'group' ); + if ( $groupId ) { + $groupMessages = $groupSyncCache->getGroupMessages( $groupId ); + $this->displayGroupMessages( $groupId, $groupMessages ); + } else { + $groups = $groupSyncCache->getGroupsInSync(); + $this->displayGroups( $groups ); + } + } + + private function displayGroups( array $groupIds ): void { + if ( !$groupIds ) { + $this->output( "No groups found in synchronization\n" ); + return; + } + + $this->output( "Groups found in sync:\n" ); + foreach ( $groupIds as $groupId ) { + $this->output( "\t- $groupId\n" ); + } + } + + /** + * @param string $groupId + * @param MessageUpdateParameter[] $groupMessages + */ + private function displayGroupMessages( string $groupId, array $groupMessages ): void { + if ( !$groupMessages ) { + $this->output( "No messages found for group $groupId\n" ); + return; + } + + $this->output( "Messages in group $groupId:\n" ); + foreach ( $groupMessages as $message ) { + $this->displayMessageDetails( $message ); + } + } + + private function displayMessageDetails( MessageUpdateParameter $messageParam ): void { + $tags = []; + if ( $messageParam->isRename() ) { + $tags[] = 'rename'; + } + + if ( $messageParam->isFuzzy() ) { + $tags[] = 'fuzzy'; + } + + $otherLangs = $messageParam->getOtherLangs() ?: [ 'N/A' ]; + $this->output( "\t- Title: " . $messageParam->getPageName() . "\n" ); + $this->output( "\t Tags: " . ( $tags ? implode( ', ', $tags ) : 'N/A' ) . "\n" ); + if ( $messageParam->isRename() ) { + $this->output( "\t Target: " . $messageParam->getTargetValue() . "\n" ); + $this->output( "\t Replacement: " . $messageParam->getReplacementValue() . "\n" ); + $this->output( "\t Other languages: " . ( implode( ', ', $otherLangs ) ) . "\n" ); + } + } +} |