diff options
author | Yaco <franco@reevo.org> | 2020-06-04 11:01:00 -0300 |
---|---|---|
committer | Yaco <franco@reevo.org> | 2020-06-04 11:01:00 -0300 |
commit | fc7369835258467bf97eb64f184b93691f9a9fd5 (patch) | |
tree | daabd60089d2dd76d9f5fb416b005fbe159c799d /www/wiki/includes/cache |
first commit
Diffstat (limited to 'www/wiki/includes/cache')
19 files changed, 6136 insertions, 0 deletions
diff --git a/www/wiki/includes/cache/BacklinkCache.php b/www/wiki/includes/cache/BacklinkCache.php new file mode 100644 index 00000000..48809d07 --- /dev/null +++ b/www/wiki/includes/cache/BacklinkCache.php @@ -0,0 +1,578 @@ +<?php +/** + * Class for fetching backlink lists, approximate backlink counts and + * partitions. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @author Tim Starling + * @copyright © 2009, Tim Starling, Domas Mituzas + * @copyright © 2010, Max Sem + * @copyright © 2011, Antoine Musso + */ + +use Wikimedia\Rdbms\ResultWrapper; +use Wikimedia\Rdbms\FakeResultWrapper; +use Wikimedia\Rdbms\IDatabase; +use MediaWiki\MediaWikiServices; + +/** + * Class for fetching backlink lists, approximate backlink counts and + * partitions. This is a shared cache. + * + * Instances of this class should typically be fetched with the method + * $title->getBacklinkCache(). + * + * Ideally you should only get your backlinks from here when you think + * there is some advantage in caching them. Otherwise it's just a waste + * of memory. + * + * Introduced by r47317 + */ +class BacklinkCache { + /** @var BacklinkCache */ + protected static $instance; + + /** + * Multi dimensions array representing batches. Keys are: + * > (string) links table name + * > (int) batch size + * > 'numRows' : Number of rows for this link table + * > 'batches' : [ $start, $end ] + * + * @see BacklinkCache::partitionResult() + * + * Cleared with BacklinkCache::clear() + * @var array[] + */ + protected $partitionCache = []; + + /** + * Contains the whole links from a database result. + * This is raw data that will be partitioned in $partitionCache + * + * Initialized with BacklinkCache::getLinks() + * Cleared with BacklinkCache::clear() + * @var ResultWrapper[] + */ + protected $fullResultCache = []; + + /** + * @var WANObjectCache + */ + protected $wanCache; + + /** + * Local copy of a database object. + * + * Accessor: BacklinkCache::getDB() + * Mutator : BacklinkCache::setDB() + * Cleared with BacklinkCache::clear() + */ + protected $db; + + /** + * Local copy of a Title object + */ + protected $title; + + const CACHE_EXPIRY = 3600; + + /** + * Create a new BacklinkCache + * + * @param Title $title : Title object to create a backlink cache for + */ + public function __construct( Title $title ) { + $this->title = $title; + $this->wanCache = MediaWikiServices::getInstance()->getMainWANObjectCache(); + } + + /** + * Create a new BacklinkCache or reuse any existing one. + * Currently, only one cache instance can exist; callers that + * need multiple backlink cache objects should keep them in scope. + * + * @param Title $title Title object to get a backlink cache for + * @return BacklinkCache + */ + public static function get( Title $title ) { + if ( !self::$instance || !self::$instance->title->equals( $title ) ) { + self::$instance = new self( $title ); + } + return self::$instance; + } + + /** + * Serialization handler, diasallows to serialize the database to prevent + * failures after this class is deserialized from cache with dead DB + * connection. + * + * @return array + */ + function __sleep() { + return [ 'partitionCache', 'fullResultCache', 'title' ]; + } + + /** + * Clear locally stored data and database object. Invalidate data in memcache. + */ + public function clear() { + $this->partitionCache = []; + $this->fullResultCache = []; + $this->wanCache->touchCheckKey( $this->makeCheckKey() ); + unset( $this->db ); + } + + /** + * Set the Database object to use + * + * @param IDatabase $db + */ + public function setDB( $db ) { + $this->db = $db; + } + + /** + * Get the replica DB connection to the database + * When non existing, will initialize the connection. + * @return IDatabase + */ + protected function getDB() { + if ( !isset( $this->db ) ) { + $this->db = wfGetDB( DB_REPLICA ); + } + + return $this->db; + } + + /** + * Get the backlinks for a given table. Cached in process memory only. + * @param string $table + * @param int|bool $startId + * @param int|bool $endId + * @param int $max + * @return TitleArrayFromResult + */ + public function getLinks( $table, $startId = false, $endId = false, $max = INF ) { + return TitleArray::newFromResult( $this->queryLinks( $table, $startId, $endId, $max ) ); + } + + /** + * Get the backlinks for a given table. Cached in process memory only. + * @param string $table + * @param int|bool $startId + * @param int|bool $endId + * @param int $max + * @param string $select 'all' or 'ids' + * @return ResultWrapper + */ + protected function queryLinks( $table, $startId, $endId, $max, $select = 'all' ) { + $fromField = $this->getPrefix( $table ) . '_from'; + + if ( !$startId && !$endId && is_infinite( $max ) + && isset( $this->fullResultCache[$table] ) + ) { + wfDebug( __METHOD__ . ": got results from cache\n" ); + $res = $this->fullResultCache[$table]; + } else { + wfDebug( __METHOD__ . ": got results from DB\n" ); + $conds = $this->getConditions( $table ); + // Use the from field in the condition rather than the joined page_id, + // because databases are stupid and don't necessarily propagate indexes. + if ( $startId ) { + $conds[] = "$fromField >= " . intval( $startId ); + } + if ( $endId ) { + $conds[] = "$fromField <= " . intval( $endId ); + } + $options = [ 'ORDER BY' => $fromField ]; + if ( is_finite( $max ) && $max > 0 ) { + $options['LIMIT'] = $max; + } + + if ( $select === 'ids' ) { + // Just select from the backlink table and ignore the page JOIN + $res = $this->getDB()->select( + $table, + [ $this->getPrefix( $table ) . '_from AS page_id' ], + array_filter( $conds, function ( $clause ) { // kind of janky + return !preg_match( '/(\b|=)page_id(\b|=)/', $clause ); + } ), + __METHOD__, + $options + ); + } else { + // Select from the backlink table and JOIN with page title information + $res = $this->getDB()->select( + [ $table, 'page' ], + [ 'page_namespace', 'page_title', 'page_id' ], + $conds, + __METHOD__, + array_merge( [ 'STRAIGHT_JOIN' ], $options ) + ); + } + + if ( $select === 'all' && !$startId && !$endId && $res->numRows() < $max ) { + // The full results fit within the limit, so cache them + $this->fullResultCache[$table] = $res; + } else { + wfDebug( __METHOD__ . ": results from DB were uncacheable\n" ); + } + } + + return $res; + } + + /** + * Get the field name prefix for a given table + * @param string $table + * @throws MWException + * @return null|string + */ + protected function getPrefix( $table ) { + static $prefixes = [ + 'pagelinks' => 'pl', + 'imagelinks' => 'il', + 'categorylinks' => 'cl', + 'templatelinks' => 'tl', + 'redirect' => 'rd', + ]; + + if ( isset( $prefixes[$table] ) ) { + return $prefixes[$table]; + } else { + $prefix = null; + Hooks::run( 'BacklinkCacheGetPrefix', [ $table, &$prefix ] ); + if ( $prefix ) { + return $prefix; + } else { + throw new MWException( "Invalid table \"$table\" in " . __CLASS__ ); + } + } + } + + /** + * Get the SQL condition array for selecting backlinks, with a join + * on the page table. + * @param string $table + * @throws MWException + * @return array|null + */ + protected function getConditions( $table ) { + $prefix = $this->getPrefix( $table ); + + switch ( $table ) { + case 'pagelinks': + case 'templatelinks': + $conds = [ + "{$prefix}_namespace" => $this->title->getNamespace(), + "{$prefix}_title" => $this->title->getDBkey(), + "page_id={$prefix}_from" + ]; + break; + case 'redirect': + $conds = [ + "{$prefix}_namespace" => $this->title->getNamespace(), + "{$prefix}_title" => $this->title->getDBkey(), + $this->getDB()->makeList( [ + "{$prefix}_interwiki" => '', + "{$prefix}_interwiki IS NULL", + ], LIST_OR ), + "page_id={$prefix}_from" + ]; + break; + case 'imagelinks': + case 'categorylinks': + $conds = [ + "{$prefix}_to" => $this->title->getDBkey(), + "page_id={$prefix}_from" + ]; + break; + default: + $conds = null; + Hooks::run( 'BacklinkCacheGetConditions', [ $table, $this->title, &$conds ] ); + if ( !$conds ) { + throw new MWException( "Invalid table \"$table\" in " . __CLASS__ ); + } + } + + return $conds; + } + + /** + * Check if there are any backlinks + * @param string $table + * @return bool + */ + public function hasLinks( $table ) { + return ( $this->getNumLinks( $table, 1 ) > 0 ); + } + + /** + * Get the approximate number of backlinks + * @param string $table + * @param int $max Only count up to this many backlinks + * @return int + */ + public function getNumLinks( $table, $max = INF ) { + global $wgUpdateRowsPerJob; + + // 1) try partition cache ... + if ( isset( $this->partitionCache[$table] ) ) { + $entry = reset( $this->partitionCache[$table] ); + + return min( $max, $entry['numRows'] ); + } + + // 2) ... then try full result cache ... + if ( isset( $this->fullResultCache[$table] ) ) { + return min( $max, $this->fullResultCache[$table]->numRows() ); + } + + $memcKey = $this->wanCache->makeKey( + 'numbacklinks', + md5( $this->title->getPrefixedDBkey() ), + $table + ); + + // 3) ... fallback to memcached ... + $curTTL = INF; + $count = $this->wanCache->get( + $memcKey, + $curTTL, + [ + $this->makeCheckKey() + ] + ); + if ( $count && ( $curTTL > 0 ) ) { + return min( $max, $count ); + } + + // 4) fetch from the database ... + if ( is_infinite( $max ) ) { // no limit at all + // Use partition() since it will batch the query and skip the JOIN. + // Use $wgUpdateRowsPerJob just to encourage cache reuse for jobs. + $this->partition( $table, $wgUpdateRowsPerJob ); // updates $this->partitionCache + return $this->partitionCache[$table][$wgUpdateRowsPerJob]['numRows']; + } else { // probably some sane limit + // Fetch the full title info, since the caller will likely need it next + $count = $this->getLinks( $table, false, false, $max )->count(); + if ( $count < $max ) { // full count + $this->wanCache->set( $memcKey, $count, self::CACHE_EXPIRY ); + } + } + + return min( $max, $count ); + } + + /** + * Partition the backlinks into batches. + * Returns an array giving the start and end of each range. The first + * batch has a start of false, and the last batch has an end of false. + * + * @param string $table The links table name + * @param int $batchSize + * @return array + */ + public function partition( $table, $batchSize ) { + // 1) try partition cache ... + if ( isset( $this->partitionCache[$table][$batchSize] ) ) { + wfDebug( __METHOD__ . ": got from partition cache\n" ); + + return $this->partitionCache[$table][$batchSize]['batches']; + } + + $this->partitionCache[$table][$batchSize] = false; + $cacheEntry =& $this->partitionCache[$table][$batchSize]; + + // 2) ... then try full result cache ... + if ( isset( $this->fullResultCache[$table] ) ) { + $cacheEntry = $this->partitionResult( $this->fullResultCache[$table], $batchSize ); + wfDebug( __METHOD__ . ": got from full result cache\n" ); + + return $cacheEntry['batches']; + } + + $memcKey = $this->wanCache->makeKey( + 'backlinks', + md5( $this->title->getPrefixedDBkey() ), + $table, + $batchSize + ); + + // 3) ... fallback to memcached ... + $curTTL = 0; + $memcValue = $this->wanCache->get( + $memcKey, + $curTTL, + [ + $this->makeCheckKey() + ] + ); + if ( is_array( $memcValue ) && ( $curTTL > 0 ) ) { + $cacheEntry = $memcValue; + wfDebug( __METHOD__ . ": got from memcached $memcKey\n" ); + + return $cacheEntry['batches']; + } + + // 4) ... finally fetch from the slow database :( + $cacheEntry = [ 'numRows' => 0, 'batches' => [] ]; // final result + // Do the selects in batches to avoid client-side OOMs (T45452). + // Use a LIMIT that plays well with $batchSize to keep equal sized partitions. + $selectSize = max( $batchSize, 200000 - ( 200000 % $batchSize ) ); + $start = false; + do { + $res = $this->queryLinks( $table, $start, false, $selectSize, 'ids' ); + $partitions = $this->partitionResult( $res, $batchSize, false ); + // Merge the link count and range partitions for this chunk + $cacheEntry['numRows'] += $partitions['numRows']; + $cacheEntry['batches'] = array_merge( $cacheEntry['batches'], $partitions['batches'] ); + if ( count( $partitions['batches'] ) ) { + list( , $lEnd ) = end( $partitions['batches'] ); + $start = $lEnd + 1; // pick up after this inclusive range + } + } while ( $partitions['numRows'] >= $selectSize ); + // Make sure the first range has start=false and the last one has end=false + if ( count( $cacheEntry['batches'] ) ) { + $cacheEntry['batches'][0][0] = false; + $cacheEntry['batches'][count( $cacheEntry['batches'] ) - 1][1] = false; + } + + // Save partitions to memcached + $this->wanCache->set( $memcKey, $cacheEntry, self::CACHE_EXPIRY ); + + // Save backlink count to memcached + $memcKey = $this->wanCache->makeKey( + 'numbacklinks', + md5( $this->title->getPrefixedDBkey() ), + $table + ); + $this->wanCache->set( $memcKey, $cacheEntry['numRows'], self::CACHE_EXPIRY ); + + wfDebug( __METHOD__ . ": got from database\n" ); + + return $cacheEntry['batches']; + } + + /** + * Partition a DB result with backlinks in it into batches + * @param ResultWrapper $res Database result + * @param int $batchSize + * @param bool $isComplete Whether $res includes all the backlinks + * @throws MWException + * @return array + */ + protected function partitionResult( $res, $batchSize, $isComplete = true ) { + $batches = []; + $numRows = $res->numRows(); + $numBatches = ceil( $numRows / $batchSize ); + + for ( $i = 0; $i < $numBatches; $i++ ) { + if ( $i == 0 && $isComplete ) { + $start = false; + } else { + $rowNum = $i * $batchSize; + $res->seek( $rowNum ); + $row = $res->fetchObject(); + $start = (int)$row->page_id; + } + + if ( $i == ( $numBatches - 1 ) && $isComplete ) { + $end = false; + } else { + $rowNum = min( $numRows - 1, ( $i + 1 ) * $batchSize - 1 ); + $res->seek( $rowNum ); + $row = $res->fetchObject(); + $end = (int)$row->page_id; + } + + # Sanity check order + if ( $start && $end && $start > $end ) { + throw new MWException( __METHOD__ . ': Internal error: query result out of order' ); + } + + $batches[] = [ $start, $end ]; + } + + return [ 'numRows' => $numRows, 'batches' => $batches ]; + } + + /** + * Get a Title iterator for cascade-protected template/file use backlinks + * + * @return TitleArray + * @since 1.25 + */ + public function getCascadeProtectedLinks() { + $dbr = $this->getDB(); + + // @todo: use UNION without breaking tests that use temp tables + $resSets = []; + $resSets[] = $dbr->select( + [ 'templatelinks', 'page_restrictions', 'page' ], + [ 'page_namespace', 'page_title', 'page_id' ], + [ + 'tl_namespace' => $this->title->getNamespace(), + 'tl_title' => $this->title->getDBkey(), + 'tl_from = pr_page', + 'pr_cascade' => 1, + 'page_id = tl_from' + ], + __METHOD__, + [ 'DISTINCT' ] + ); + if ( $this->title->getNamespace() == NS_FILE ) { + $resSets[] = $dbr->select( + [ 'imagelinks', 'page_restrictions', 'page' ], + [ 'page_namespace', 'page_title', 'page_id' ], + [ + 'il_to' => $this->title->getDBkey(), + 'il_from = pr_page', + 'pr_cascade' => 1, + 'page_id = il_from' + ], + __METHOD__, + [ 'DISTINCT' ] + ); + } + + // Combine and de-duplicate the results + $mergedRes = []; + foreach ( $resSets as $res ) { + foreach ( $res as $row ) { + $mergedRes[$row->page_id] = $row; + } + } + + return TitleArray::newFromResult( + new FakeResultWrapper( array_values( $mergedRes ) ) ); + } + + /** + * Returns check key for the backlinks cache for a particular title + * + * @return String + */ + private function makeCheckKey() { + return $this->wanCache->makeKey( + 'backlinks', + md5( $this->title->getPrefixedDBkey() ) + ); + } +} diff --git a/www/wiki/includes/cache/CacheDependency.php b/www/wiki/includes/cache/CacheDependency.php new file mode 100644 index 00000000..4ff10047 --- /dev/null +++ b/www/wiki/includes/cache/CacheDependency.php @@ -0,0 +1,293 @@ +<?php +/** + * Data caching with dependencies. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Cache + */ +use MediaWiki\MediaWikiServices; + +/** + * This class stores an arbitrary value along with its dependencies. + * Users should typically only use DependencyWrapper::getValueFromCache(), + * rather than instantiating one of these objects directly. + * @ingroup Cache + */ +class DependencyWrapper { + private $value; + /** @var CacheDependency[] */ + private $deps; + + /** + * @param mixed $value The user-supplied value + * @param CacheDependency|CacheDependency[] $deps A dependency or dependency + * array. All dependencies must be objects implementing CacheDependency. + */ + function __construct( $value = false, $deps = [] ) { + $this->value = $value; + + if ( !is_array( $deps ) ) { + $deps = [ $deps ]; + } + + $this->deps = $deps; + } + + /** + * Returns true if any of the dependencies have expired + * + * @return bool + */ + function isExpired() { + foreach ( $this->deps as $dep ) { + if ( $dep->isExpired() ) { + return true; + } + } + + return false; + } + + /** + * Initialise dependency values in preparation for storing. This must be + * called before serialization. + */ + function initialiseDeps() { + foreach ( $this->deps as $dep ) { + $dep->loadDependencyValues(); + } + } + + /** + * Get the user-defined value + * @return bool|mixed + */ + function getValue() { + return $this->value; + } + + /** + * Store the wrapper to a cache + * + * @param BagOStuff $cache + * @param string $key + * @param int $expiry + */ + function storeToCache( $cache, $key, $expiry = 0 ) { + $this->initialiseDeps(); + $cache->set( $key, $this, $expiry ); + } + + /** + * Attempt to get a value from the cache. If the value is expired or missing, + * it will be generated with the callback function (if present), and the newly + * calculated value will be stored to the cache in a wrapper. + * + * @param BagOStuff $cache + * @param string $key The cache key + * @param int $expiry The expiry timestamp or interval in seconds + * @param bool|callable $callback The callback for generating the value, or false + * @param array $callbackParams The function parameters for the callback + * @param array $deps The dependencies to store on a cache miss. Note: these + * are not the dependencies used on a cache hit! Cache hits use the stored + * dependency array. + * + * @return mixed The value, or null if it was not present in the cache and no + * callback was defined. + */ + static function getValueFromCache( $cache, $key, $expiry = 0, $callback = false, + $callbackParams = [], $deps = [] + ) { + $obj = $cache->get( $key ); + + if ( is_object( $obj ) && $obj instanceof DependencyWrapper && !$obj->isExpired() ) { + $value = $obj->value; + } elseif ( $callback ) { + $value = call_user_func_array( $callback, $callbackParams ); + # Cache the newly-generated value + $wrapper = new DependencyWrapper( $value, $deps ); + $wrapper->storeToCache( $cache, $key, $expiry ); + } else { + $value = null; + } + + return $value; + } +} + +/** + * @ingroup Cache + */ +abstract class CacheDependency { + /** + * Returns true if the dependency is expired, false otherwise + */ + abstract function isExpired(); + + /** + * Hook to perform any expensive pre-serialize loading of dependency values. + */ + function loadDependencyValues() { + } +} + +/** + * @ingroup Cache + */ +class FileDependency extends CacheDependency { + private $filename; + private $timestamp; + + /** + * Create a file dependency + * + * @param string $filename The name of the file, preferably fully qualified + * @param null|bool|int $timestamp The unix last modified timestamp, or false if the + * file does not exist. If omitted, the timestamp will be loaded from + * the file. + * + * A dependency on a nonexistent file will be triggered when the file is + * created. A dependency on an existing file will be triggered when the + * file is changed. + */ + function __construct( $filename, $timestamp = null ) { + $this->filename = $filename; + $this->timestamp = $timestamp; + } + + /** + * @return array + */ + function __sleep() { + $this->loadDependencyValues(); + + return [ 'filename', 'timestamp' ]; + } + + function loadDependencyValues() { + if ( is_null( $this->timestamp ) ) { + Wikimedia\suppressWarnings(); + # Dependency on a non-existent file stores "false" + # This is a valid concept! + $this->timestamp = filemtime( $this->filename ); + Wikimedia\restoreWarnings(); + } + } + + /** + * @return bool + */ + function isExpired() { + Wikimedia\suppressWarnings(); + $lastmod = filemtime( $this->filename ); + Wikimedia\restoreWarnings(); + if ( $lastmod === false ) { + if ( $this->timestamp === false ) { + # Still nonexistent + return false; + } else { + # Deleted + wfDebug( "Dependency triggered: {$this->filename} deleted.\n" ); + + return true; + } + } else { + if ( $lastmod > $this->timestamp ) { + # Modified or created + wfDebug( "Dependency triggered: {$this->filename} changed.\n" ); + + return true; + } else { + # Not modified + return false; + } + } + } +} + +/** + * @ingroup Cache + */ +class GlobalDependency extends CacheDependency { + private $name; + private $value; + + function __construct( $name ) { + $this->name = $name; + $this->value = $GLOBALS[$name]; + } + + /** + * @return bool + */ + function isExpired() { + if ( !isset( $GLOBALS[$this->name] ) ) { + return true; + } + + return $GLOBALS[$this->name] != $this->value; + } +} + +/** + * @ingroup Cache + */ +class MainConfigDependency extends CacheDependency { + private $name; + private $value; + + function __construct( $name ) { + $this->name = $name; + $this->value = $this->getConfig()->get( $this->name ); + } + + private function getConfig() { + return MediaWikiServices::getInstance()->getMainConfig(); + } + + /** + * @return bool + */ + function isExpired() { + if ( !$this->getConfig()->has( $this->name ) ) { + return true; + } + + return $this->getConfig()->get( $this->name ) != $this->value; + } +} + +/** + * @ingroup Cache + */ +class ConstantDependency extends CacheDependency { + private $name; + private $value; + + function __construct( $name ) { + $this->name = $name; + $this->value = constant( $name ); + } + + /** + * @return bool + */ + function isExpired() { + return constant( $this->name ) != $this->value; + } +} diff --git a/www/wiki/includes/cache/CacheHelper.php b/www/wiki/includes/cache/CacheHelper.php new file mode 100644 index 00000000..e77e2515 --- /dev/null +++ b/www/wiki/includes/cache/CacheHelper.php @@ -0,0 +1,388 @@ +<?php +/** + * Cache of various elements in a single cache entry. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @license GNU GPL v2 or later + * @author Jeroen De Dauw < jeroendedauw@gmail.com > + */ + +/** + * Interface for all classes implementing CacheHelper functionality. + * + * @since 1.20 + */ +interface ICacheHelper { + /** + * Sets if the cache should be enabled or not. + * + * @since 1.20 + * @param bool $cacheEnabled + */ + function setCacheEnabled( $cacheEnabled ); + + /** + * Initializes the caching. + * Should be called before the first time anything is added via addCachedHTML. + * + * @since 1.20 + * + * @param int|null $cacheExpiry Sets the cache expiry, either ttl in seconds or unix timestamp. + * @param bool|null $cacheEnabled Sets if the cache should be enabled or not. + */ + function startCache( $cacheExpiry = null, $cacheEnabled = null ); + + /** + * Get a cached value if available or compute it if not and then cache it if possible. + * The provided $computeFunction is only called when the computation needs to happen + * and should return a result value. $args are arguments that will be passed to the + * compute function when called. + * + * @since 1.20 + * + * @param callable $computeFunction + * @param array|mixed $args + * @param string|null $key + * + * @return mixed + */ + function getCachedValue( $computeFunction, $args = [], $key = null ); + + /** + * Saves the HTML to the cache in case it got recomputed. + * Should be called after the last time anything is added via addCachedHTML. + * + * @since 1.20 + */ + function saveCache(); + + /** + * Sets the time to live for the cache, in seconds or a unix timestamp + * indicating the point of expiry... + * + * @since 1.20 + * + * @param int $cacheExpiry + */ + function setExpiry( $cacheExpiry ); +} + +use MediaWiki\MediaWikiServices; + +/** + * Helper class for caching various elements in a single cache entry. + * + * To get a cached value or compute it, use getCachedValue like this: + * $this->getCachedValue( $callback ); + * + * To add HTML that should be cached, use addCachedHTML like this: + * $this->addCachedHTML( $callback ); + * + * The callback function is only called when needed, so do all your expensive + * computations here. This function should returns the HTML to be cached. + * It should not add anything to the PageOutput object! + * + * Before the first addCachedHTML call, you should call $this->startCache(); + * After adding the last HTML that should be cached, call $this->saveCache(); + * + * @since 1.20 + */ +class CacheHelper implements ICacheHelper { + /** + * The time to live for the cache, in seconds or a unix timestamp indicating the point of expiry. + * + * @since 1.20 + * @var int + */ + protected $cacheExpiry = 3600; + + /** + * List of HTML chunks to be cached (if !hasCached) or that where cached (of hasCached). + * If not cached already, then the newly computed chunks are added here, + * if it as cached already, chunks are removed from this list as they are needed. + * + * @since 1.20 + * @var array + */ + protected $cachedChunks; + + /** + * Indicates if the to be cached content was already cached. + * Null if this information is not available yet. + * + * @since 1.20 + * @var bool|null + */ + protected $hasCached = null; + + /** + * If the cache is enabled or not. + * + * @since 1.20 + * @var bool + */ + protected $cacheEnabled = true; + + /** + * Function that gets called when initialization is done. + * + * @since 1.20 + * @var callable + */ + protected $onInitHandler = false; + + /** + * Elements to build a cache key with. + * + * @since 1.20 + * @var array + */ + protected $cacheKey = []; + + /** + * Sets if the cache should be enabled or not. + * + * @since 1.20 + * @param bool $cacheEnabled + */ + public function setCacheEnabled( $cacheEnabled ) { + $this->cacheEnabled = $cacheEnabled; + } + + /** + * Initializes the caching. + * Should be called before the first time anything is added via addCachedHTML. + * + * @since 1.20 + * + * @param int|null $cacheExpiry Sets the cache expiry, either ttl in seconds or unix timestamp. + * @param bool|null $cacheEnabled Sets if the cache should be enabled or not. + */ + public function startCache( $cacheExpiry = null, $cacheEnabled = null ) { + if ( is_null( $this->hasCached ) ) { + if ( !is_null( $cacheExpiry ) ) { + $this->cacheExpiry = $cacheExpiry; + } + + if ( !is_null( $cacheEnabled ) ) { + $this->setCacheEnabled( $cacheEnabled ); + } + + $this->initCaching(); + } + } + + /** + * Returns a message that notifies the user he/she is looking at + * a cached version of the page, including a refresh link. + * + * @since 1.20 + * + * @param IContextSource $context + * @param bool $includePurgeLink + * + * @return string + */ + public function getCachedNotice( IContextSource $context, $includePurgeLink = true ) { + if ( $this->cacheExpiry < 86400 * 3650 ) { + $message = $context->msg( + 'cachedspecial-viewing-cached-ttl', + $context->getLanguage()->formatDuration( $this->cacheExpiry ) + )->escaped(); + } else { + $message = $context->msg( + 'cachedspecial-viewing-cached-ts' + )->escaped(); + } + + if ( $includePurgeLink ) { + $refreshArgs = $context->getRequest()->getQueryValues(); + unset( $refreshArgs['title'] ); + $refreshArgs['action'] = 'purge'; + + $subPage = $context->getTitle()->getFullText(); + $subPage = explode( '/', $subPage, 2 ); + $subPage = count( $subPage ) > 1 ? $subPage[1] : false; + + $message .= ' ' . MediaWikiServices::getInstance()->getLinkRenderer()->makeLink( + $context->getTitle( $subPage ), + $context->msg( 'cachedspecial-refresh-now' )->text(), + [], + $refreshArgs + ); + } + + return $message; + } + + /** + * Initializes the caching if not already done so. + * Should be called before any of the caching functionality is used. + * + * @since 1.20 + */ + protected function initCaching() { + if ( $this->cacheEnabled && is_null( $this->hasCached ) ) { + $cachedChunks = wfGetCache( CACHE_ANYTHING )->get( $this->getCacheKeyString() ); + + $this->hasCached = is_array( $cachedChunks ); + $this->cachedChunks = $this->hasCached ? $cachedChunks : []; + + if ( $this->onInitHandler !== false ) { + call_user_func( $this->onInitHandler, $this->hasCached ); + } + } + } + + /** + * Get a cached value if available or compute it if not and then cache it if possible. + * The provided $computeFunction is only called when the computation needs to happen + * and should return a result value. $args are arguments that will be passed to the + * compute function when called. + * + * @since 1.20 + * + * @param callable $computeFunction + * @param array|mixed $args + * @param string|null $key + * + * @return mixed + */ + public function getCachedValue( $computeFunction, $args = [], $key = null ) { + $this->initCaching(); + + if ( $this->cacheEnabled && $this->hasCached ) { + $value = null; + + if ( is_null( $key ) ) { + $itemKey = array_keys( array_slice( $this->cachedChunks, 0, 1 ) ); + $itemKey = array_shift( $itemKey ); + + if ( !is_int( $itemKey ) ) { + wfWarn( "Attempted to get item with non-numeric key while " . + "the next item in the queue has a key ($itemKey) in " . __METHOD__ ); + } elseif ( is_null( $itemKey ) ) { + wfWarn( "Attempted to get an item while the queue is empty in " . __METHOD__ ); + } else { + $value = array_shift( $this->cachedChunks ); + } + } else { + if ( array_key_exists( $key, $this->cachedChunks ) ) { + $value = $this->cachedChunks[$key]; + unset( $this->cachedChunks[$key] ); + } else { + wfWarn( "There is no item with key '$key' in this->cachedChunks in " . __METHOD__ ); + } + } + } else { + if ( !is_array( $args ) ) { + $args = [ $args ]; + } + + $value = call_user_func_array( $computeFunction, $args ); + + if ( $this->cacheEnabled ) { + if ( is_null( $key ) ) { + $this->cachedChunks[] = $value; + } else { + $this->cachedChunks[$key] = $value; + } + } + } + + return $value; + } + + /** + * Saves the HTML to the cache in case it got recomputed. + * Should be called after the last time anything is added via addCachedHTML. + * + * @since 1.20 + */ + public function saveCache() { + if ( $this->cacheEnabled && $this->hasCached === false && !empty( $this->cachedChunks ) ) { + wfGetCache( CACHE_ANYTHING )->set( + $this->getCacheKeyString(), + $this->cachedChunks, + $this->cacheExpiry + ); + } + } + + /** + * Sets the time to live for the cache, in seconds or a unix timestamp + * indicating the point of expiry... + * + * @since 1.20 + * + * @param int $cacheExpiry + */ + public function setExpiry( $cacheExpiry ) { + $this->cacheExpiry = $cacheExpiry; + } + + /** + * Returns the cache key to use to cache this page's HTML output. + * Is constructed from the special page name and language code. + * + * @since 1.20 + * + * @return string + * @throws MWException + */ + protected function getCacheKeyString() { + if ( $this->cacheKey === [] ) { + throw new MWException( 'No cache key set, so cannot obtain or save the CacheHelper values.' ); + } + + return call_user_func_array( 'wfMemcKey', $this->cacheKey ); + } + + /** + * Sets the cache key that should be used. + * + * @since 1.20 + * + * @param array $cacheKey + */ + public function setCacheKey( array $cacheKey ) { + $this->cacheKey = $cacheKey; + } + + /** + * Rebuild the content, even if it's already cached. + * This effectively has the same effect as purging the cache, + * since it will be overridden with the new value on the next request. + * + * @since 1.20 + */ + public function rebuildOnDemand() { + $this->hasCached = false; + } + + /** + * Sets a function that gets called when initialization of the cache is done. + * + * @since 1.20 + * + * @param callable $handlerFunction + */ + public function setOnInitializedHandler( $handlerFunction ) { + $this->onInitHandler = $handlerFunction; + } +} diff --git a/www/wiki/includes/cache/FileCacheBase.php b/www/wiki/includes/cache/FileCacheBase.php new file mode 100644 index 00000000..ce5a019b --- /dev/null +++ b/www/wiki/includes/cache/FileCacheBase.php @@ -0,0 +1,278 @@ +<?php +/** + * Data storage in the file system. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Cache + */ + +/** + * Base class for data storage in the file system. + * + * @ingroup Cache + */ +abstract class FileCacheBase { + protected $mKey; + protected $mType = 'object'; + protected $mExt = 'cache'; + protected $mFilePath; + protected $mUseGzip; + /* lazy loaded */ + protected $mCached; + + /* @todo configurable? */ + const MISS_FACTOR = 15; // log 1 every MISS_FACTOR cache misses + const MISS_TTL_SEC = 3600; // how many seconds ago is "recent" + + protected function __construct() { + global $wgUseGzip; + + $this->mUseGzip = (bool)$wgUseGzip; + } + + /** + * Get the base file cache directory + * @return string + */ + final protected function baseCacheDirectory() { + global $wgFileCacheDirectory; + + return $wgFileCacheDirectory; + } + + /** + * Get the base cache directory (not specific to this file) + * @return string + */ + abstract protected function cacheDirectory(); + + /** + * Get the path to the cache file + * @return string + */ + protected function cachePath() { + if ( $this->mFilePath !== null ) { + return $this->mFilePath; + } + + $dir = $this->cacheDirectory(); + # Build directories (methods include the trailing "/") + $subDirs = $this->typeSubdirectory() . $this->hashSubdirectory(); + # Avoid extension confusion + $key = str_replace( '.', '%2E', urlencode( $this->mKey ) ); + # Build the full file path + $this->mFilePath = "{$dir}/{$subDirs}{$key}.{$this->mExt}"; + if ( $this->useGzip() ) { + $this->mFilePath .= '.gz'; + } + + return $this->mFilePath; + } + + /** + * Check if the cache file exists + * @return bool + */ + public function isCached() { + if ( $this->mCached === null ) { + $this->mCached = file_exists( $this->cachePath() ); + } + + return $this->mCached; + } + + /** + * Get the last-modified timestamp of the cache file + * @return string|bool TS_MW timestamp + */ + public function cacheTimestamp() { + $timestamp = filemtime( $this->cachePath() ); + + return ( $timestamp !== false ) + ? wfTimestamp( TS_MW, $timestamp ) + : false; + } + + /** + * Check if up to date cache file exists + * @param string $timestamp MW_TS timestamp + * + * @return bool + */ + public function isCacheGood( $timestamp = '' ) { + global $wgCacheEpoch; + + if ( !$this->isCached() ) { + return false; + } + + $cachetime = $this->cacheTimestamp(); + $good = ( $timestamp <= $cachetime && $wgCacheEpoch <= $cachetime ); + wfDebug( __METHOD__ . + ": cachetime $cachetime, touched '{$timestamp}' epoch {$wgCacheEpoch}, good $good\n" ); + + return $good; + } + + /** + * Check if the cache is gzipped + * @return bool + */ + protected function useGzip() { + return $this->mUseGzip; + } + + /** + * Get the uncompressed text from the cache + * @return string + */ + public function fetchText() { + if ( $this->useGzip() ) { + $fh = gzopen( $this->cachePath(), 'rb' ); + + return stream_get_contents( $fh ); + } else { + return file_get_contents( $this->cachePath() ); + } + } + + /** + * Save and compress text to the cache + * @param string $text + * @return string|false Compressed text + */ + public function saveText( $text ) { + if ( $this->useGzip() ) { + $text = gzencode( $text ); + } + + $this->checkCacheDirs(); // build parent dir + if ( !file_put_contents( $this->cachePath(), $text, LOCK_EX ) ) { + wfDebug( __METHOD__ . "() failed saving " . $this->cachePath() . "\n" ); + $this->mCached = null; + + return false; + } + + $this->mCached = true; + + return $text; + } + + /** + * Clear the cache for this page + * @return void + */ + public function clearCache() { + Wikimedia\suppressWarnings(); + unlink( $this->cachePath() ); + Wikimedia\restoreWarnings(); + $this->mCached = false; + } + + /** + * Create parent directors of $this->cachePath() + * @return void + */ + protected function checkCacheDirs() { + wfMkdirParents( dirname( $this->cachePath() ), null, __METHOD__ ); + } + + /** + * Get the cache type subdirectory (with trailing slash) + * An extending class could use that method to alter the type -> directory + * mapping. @see HTMLFileCache::typeSubdirectory() for an example. + * + * @return string + */ + protected function typeSubdirectory() { + return $this->mType . '/'; + } + + /** + * Return relative multi-level hash subdirectory (with trailing slash) + * or the empty string if not $wgFileCacheDepth + * @return string + */ + protected function hashSubdirectory() { + global $wgFileCacheDepth; + + $subdir = ''; + if ( $wgFileCacheDepth > 0 ) { + $hash = md5( $this->mKey ); + for ( $i = 1; $i <= $wgFileCacheDepth; $i++ ) { + $subdir .= substr( $hash, 0, $i ) . '/'; + } + } + + return $subdir; + } + + /** + * Roughly increments the cache misses in the last hour by unique visitors + * @param WebRequest $request + * @return void + */ + public function incrMissesRecent( WebRequest $request ) { + if ( mt_rand( 0, self::MISS_FACTOR - 1 ) == 0 ) { + $cache = ObjectCache::getLocalClusterInstance(); + # Get a large IP range that should include the user even if that + # person's IP address changes + $ip = $request->getIP(); + if ( !IP::isValid( $ip ) ) { + return; + } + $ip = IP::isIPv6( $ip ) + ? IP::sanitizeRange( "$ip/32" ) + : IP::sanitizeRange( "$ip/16" ); + + # Bail out if a request already came from this range... + $key = $cache->makeKey( static::class, 'attempt', $this->mType, $this->mKey, $ip ); + if ( $cache->get( $key ) ) { + return; // possibly the same user + } + $cache->set( $key, 1, self::MISS_TTL_SEC ); + + # Increment the number of cache misses... + $key = $this->cacheMissKey( $cache ); + if ( $cache->get( $key ) === false ) { + $cache->set( $key, 1, self::MISS_TTL_SEC ); + } else { + $cache->incr( $key ); + } + } + } + + /** + * Roughly gets the cache misses in the last hour by unique visitors + * @return int + */ + public function getMissesRecent() { + $cache = ObjectCache::getLocalClusterInstance(); + + return self::MISS_FACTOR * $cache->get( $this->cacheMissKey( $cache ) ); + } + + /** + * @param BagOStuff $cache Instance that the key will be used with + * @return string + */ + protected function cacheMissKey( BagOStuff $cache ) { + return $cache->makeKey( static::class, 'misses', $this->mType, $this->mKey ); + } +} diff --git a/www/wiki/includes/cache/GenderCache.php b/www/wiki/includes/cache/GenderCache.php new file mode 100644 index 00000000..099a986f --- /dev/null +++ b/www/wiki/includes/cache/GenderCache.php @@ -0,0 +1,188 @@ +<?php +/** + * Caches user genders when needed to use correct namespace aliases. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @author Niklas Laxström + * @ingroup Cache + */ +use MediaWiki\MediaWikiServices; + +/** + * Caches user genders when needed to use correct namespace aliases. + * + * @since 1.18 + */ +class GenderCache { + protected $cache = []; + protected $default; + protected $misses = 0; + protected $missLimit = 1000; + + /** + * @deprecated in 1.28 see MediaWikiServices::getInstance()->getGenderCache() + * @return GenderCache + */ + public static function singleton() { + return MediaWikiServices::getInstance()->getGenderCache(); + } + + /** + * Returns the default gender option in this wiki. + * @return string + */ + protected function getDefault() { + if ( $this->default === null ) { + $this->default = User::getDefaultOption( 'gender' ); + } + + return $this->default; + } + + /** + * Returns the gender for given username. + * @param string|User $username + * @param string $caller The calling method + * @return string + */ + public function getGenderOf( $username, $caller = '' ) { + global $wgUser; + + if ( $username instanceof User ) { + $username = $username->getName(); + } + + $username = self::normalizeUsername( $username ); + if ( !isset( $this->cache[$username] ) ) { + if ( $this->misses >= $this->missLimit && $wgUser->getName() !== $username ) { + if ( $this->misses === $this->missLimit ) { + $this->misses++; + wfDebug( __METHOD__ . ": too many misses, returning default onwards\n" ); + } + + return $this->getDefault(); + } else { + $this->misses++; + $this->doQuery( $username, $caller ); + } + } + + /* Undefined if there is a valid username which for some reason doesn't + * exist in the database. + */ + return isset( $this->cache[$username] ) ? $this->cache[$username] : $this->getDefault(); + } + + /** + * Wrapper for doQuery that processes raw LinkBatch data. + * + * @param array $data + * @param string $caller + */ + public function doLinkBatch( $data, $caller = '' ) { + $users = []; + foreach ( $data as $ns => $pagenames ) { + if ( !MWNamespace::hasGenderDistinction( $ns ) ) { + continue; + } + foreach ( array_keys( $pagenames ) as $username ) { + $users[$username] = true; + } + } + + $this->doQuery( array_keys( $users ), $caller ); + } + + /** + * Wrapper for doQuery that processes a title or string array. + * + * @since 1.20 + * @param array $titles Array of Title objects or strings + * @param string $caller The calling method + */ + public function doTitlesArray( $titles, $caller = '' ) { + $users = []; + foreach ( $titles as $title ) { + $titleObj = is_string( $title ) ? Title::newFromText( $title ) : $title; + if ( !$titleObj ) { + continue; + } + if ( !MWNamespace::hasGenderDistinction( $titleObj->getNamespace() ) ) { + continue; + } + $users[] = $titleObj->getText(); + } + + $this->doQuery( $users, $caller ); + } + + /** + * Preloads genders for given list of users. + * @param array|string $users Usernames + * @param string $caller The calling method + */ + public function doQuery( $users, $caller = '' ) { + $default = $this->getDefault(); + + $usersToCheck = []; + foreach ( (array)$users as $value ) { + $name = self::normalizeUsername( $value ); + // Skip users whose gender setting we already know + if ( !isset( $this->cache[$name] ) ) { + // For existing users, this value will be overwritten by the correct value + $this->cache[$name] = $default; + // query only for valid names, which can be in the database + if ( User::isValidUserName( $name ) ) { + $usersToCheck[] = $name; + } + } + } + + if ( count( $usersToCheck ) === 0 ) { + return; + } + + $dbr = wfGetDB( DB_REPLICA ); + $table = [ 'user', 'user_properties' ]; + $fields = [ 'user_name', 'up_value' ]; + $conds = [ 'user_name' => $usersToCheck ]; + $joins = [ 'user_properties' => + [ 'LEFT JOIN', [ 'user_id = up_user', 'up_property' => 'gender' ] ] ]; + + $comment = __METHOD__; + if ( strval( $caller ) !== '' ) { + $comment .= "/$caller"; + } + $res = $dbr->select( $table, $fields, $conds, $comment, [], $joins ); + + foreach ( $res as $row ) { + $this->cache[$row->user_name] = $row->up_value ? $row->up_value : $default; + } + } + + private static function normalizeUsername( $username ) { + // Strip off subpages + $indexSlash = strpos( $username, '/' ); + if ( $indexSlash !== false ) { + $username = substr( $username, 0, $indexSlash ); + } + + // normalize underscore/spaces + return strtr( $username, '_', ' ' ); + } +} diff --git a/www/wiki/includes/cache/HTMLFileCache.php b/www/wiki/includes/cache/HTMLFileCache.php new file mode 100644 index 00000000..7ae2ee0e --- /dev/null +++ b/www/wiki/includes/cache/HTMLFileCache.php @@ -0,0 +1,246 @@ +<?php +/** + * Page view caching in the file system. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Cache + */ + +use MediaWiki\MediaWikiServices; + +/** + * Page view caching in the file system. + * The only cacheable actions are "view" and "history". Also special pages + * will not be cached. + * + * @ingroup Cache + */ +class HTMLFileCache extends FileCacheBase { + const MODE_NORMAL = 0; // normal cache mode + const MODE_OUTAGE = 1; // fallback cache for DB outages + const MODE_REBUILD = 2; // background cache rebuild mode + + /** + * @param Title|string $title Title object or prefixed DB key string + * @param string $action + * @throws MWException + */ + public function __construct( $title, $action ) { + parent::__construct(); + + $allowedTypes = self::cacheablePageActions(); + if ( !in_array( $action, $allowedTypes ) ) { + throw new MWException( 'Invalid file cache type given.' ); + } + $this->mKey = ( $title instanceof Title ) + ? $title->getPrefixedDBkey() + : (string)$title; + $this->mType = (string)$action; + $this->mExt = 'html'; + } + + /** + * Cacheable actions + * @return array + */ + protected static function cacheablePageActions() { + return [ 'view', 'history' ]; + } + + /** + * Get the base file cache directory + * @return string + */ + protected function cacheDirectory() { + return $this->baseCacheDirectory(); // no subdir for b/c with old cache files + } + + /** + * Get the cache type subdirectory (with the trailing slash) or the empty string + * Alter the type -> directory mapping to put action=view cache at the root. + * + * @return string + */ + protected function typeSubdirectory() { + if ( $this->mType === 'view' ) { + return ''; // b/c to not skip existing cache + } else { + return $this->mType . '/'; + } + } + + /** + * Check if pages can be cached for this request/user + * @param IContextSource $context + * @param int $mode One of the HTMLFileCache::MODE_* constants (since 1.28) + * @return bool + */ + public static function useFileCache( IContextSource $context, $mode = self::MODE_NORMAL ) { + $config = MediaWikiServices::getInstance()->getMainConfig(); + + if ( !$config->get( 'UseFileCache' ) && $mode !== self::MODE_REBUILD ) { + return false; + } elseif ( $config->get( 'DebugToolbar' ) ) { + wfDebug( "HTML file cache skipped. \$wgDebugToolbar on\n" ); + + return false; + } + + // Get all query values + $queryVals = $context->getRequest()->getValues(); + foreach ( $queryVals as $query => $val ) { + if ( $query === 'title' || $query === 'curid' ) { + continue; // note: curid sets title + // Normal page view in query form can have action=view. + } elseif ( $query === 'action' && in_array( $val, self::cacheablePageActions() ) ) { + continue; + // Below are header setting params + } elseif ( $query === 'maxage' || $query === 'smaxage' ) { + continue; + } + + return false; + } + + $user = $context->getUser(); + // Check for non-standard user language; this covers uselang, + // and extensions for auto-detecting user language. + $ulang = $context->getLanguage(); + + // Check that there are no other sources of variation + if ( $user->getId() || $ulang->getCode() !== $config->get( 'LanguageCode' ) ) { + return false; + } + + if ( $mode === self::MODE_NORMAL ) { + if ( $user->getNewtalk() ) { + return false; + } + } + + // Allow extensions to disable caching + return Hooks::run( 'HTMLFileCache::useFileCache', [ $context ] ); + } + + /** + * Read from cache to context output + * @param IContextSource $context + * @param int $mode One of the HTMLFileCache::MODE_* constants + * @return void + */ + public function loadFromFileCache( IContextSource $context, $mode = self::MODE_NORMAL ) { + global $wgContLang; + $config = MediaWikiServices::getInstance()->getMainConfig(); + + wfDebug( __METHOD__ . "()\n" ); + $filename = $this->cachePath(); + + if ( $mode === self::MODE_OUTAGE ) { + // Avoid DB errors for queries in sendCacheControl() + $context->getTitle()->resetArticleID( 0 ); + } + + $context->getOutput()->sendCacheControl(); + header( "Content-Type: {$config->get( 'MimeType' )}; charset=UTF-8" ); + header( "Content-Language: {$wgContLang->getHtmlCode()}" ); + if ( $this->useGzip() ) { + if ( wfClientAcceptsGzip() ) { + header( 'Content-Encoding: gzip' ); + readfile( $filename ); + } else { + /* Send uncompressed */ + wfDebug( __METHOD__ . " uncompressing cache file and sending it\n" ); + readgzfile( $filename ); + } + } else { + readfile( $filename ); + } + + $context->getOutput()->disable(); // tell $wgOut that output is taken care of + } + + /** + * Save this cache object with the given text. + * Use this as an ob_start() handler. + * + * Normally this is only registed as a handler if $wgUseFileCache is on. + * If can be explicitly called by rebuildFileCache.php when it takes over + * handling file caching itself, disabling any automatic handling the the + * process. + * + * @param string $text + * @return string|bool The annotated $text or false on error + */ + public function saveToFileCache( $text ) { + if ( strlen( $text ) < 512 ) { + // Disabled or empty/broken output (OOM and PHP errors) + return $text; + } + + wfDebug( __METHOD__ . "()\n", 'private' ); + + $now = wfTimestampNow(); + if ( $this->useGzip() ) { + $text = str_replace( + '</html>', '<!-- Cached/compressed ' . $now . " -->\n</html>", $text ); + } else { + $text = str_replace( + '</html>', '<!-- Cached ' . $now . " -->\n</html>", $text ); + } + + // Store text to FS... + $compressed = $this->saveText( $text ); + if ( $compressed === false ) { + return $text; // error + } + + // gzip output to buffer as needed and set headers... + if ( $this->useGzip() ) { + // @todo Ugly wfClientAcceptsGzip() function - use context! + if ( wfClientAcceptsGzip() ) { + header( 'Content-Encoding: gzip' ); + + return $compressed; + } else { + return $text; + } + } else { + return $text; + } + } + + /** + * Clear the file caches for a page for all actions + * @param Title $title + * @return bool Whether $wgUseFileCache is enabled + */ + public static function clearFileCache( Title $title ) { + $config = MediaWikiServices::getInstance()->getMainConfig(); + + if ( !$config->get( 'UseFileCache' ) ) { + return false; + } + + foreach ( self::cacheablePageActions() as $type ) { + $fc = new self( $title, $type ); + $fc->clearCache(); + } + + return true; + } +} diff --git a/www/wiki/includes/cache/LinkBatch.php b/www/wiki/includes/cache/LinkBatch.php new file mode 100644 index 00000000..30d105b2 --- /dev/null +++ b/www/wiki/includes/cache/LinkBatch.php @@ -0,0 +1,246 @@ +<?php +/** + * Batch query to determine page existence. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Cache + */ +use MediaWiki\Linker\LinkTarget; +use MediaWiki\MediaWikiServices; +use Wikimedia\Rdbms\ResultWrapper; +use Wikimedia\Rdbms\IDatabase; + +/** + * Class representing a list of titles + * The execute() method checks them all for existence and adds them to a LinkCache object + * + * @ingroup Cache + */ +class LinkBatch { + /** + * 2-d array, first index namespace, second index dbkey, value arbitrary + */ + public $data = []; + + /** + * For debugging which method is using this class. + */ + protected $caller; + + /** + * @param Traversable|LinkTarget[] $arr Initial items to be added to the batch + */ + public function __construct( $arr = [] ) { + foreach ( $arr as $item ) { + $this->addObj( $item ); + } + } + + /** + * Use ->setCaller( __METHOD__ ) to indicate which code is using this + * class. Only used in debugging output. + * @since 1.17 + * + * @param string $caller + */ + public function setCaller( $caller ) { + $this->caller = $caller; + } + + /** + * @param LinkTarget $linkTarget + */ + public function addObj( $linkTarget ) { + if ( is_object( $linkTarget ) ) { + $this->add( $linkTarget->getNamespace(), $linkTarget->getDBkey() ); + } else { + wfDebug( "Warning: LinkBatch::addObj got invalid LinkTarget object\n" ); + } + } + + /** + * @param int $ns + * @param string $dbkey + */ + public function add( $ns, $dbkey ) { + if ( $ns < 0 || $dbkey === '' ) { + return; // T137083 + } + if ( !array_key_exists( $ns, $this->data ) ) { + $this->data[$ns] = []; + } + + $this->data[$ns][strtr( $dbkey, ' ', '_' )] = 1; + } + + /** + * Set the link list to a given 2-d array + * First key is the namespace, second is the DB key, value arbitrary + * + * @param array $array + */ + public function setArray( $array ) { + $this->data = $array; + } + + /** + * Returns true if no pages have been added, false otherwise. + * + * @return bool + */ + public function isEmpty() { + return $this->getSize() == 0; + } + + /** + * Returns the size of the batch. + * + * @return int + */ + public function getSize() { + return count( $this->data ); + } + + /** + * Do the query and add the results to the LinkCache object + * + * @return array Mapping PDBK to ID + */ + public function execute() { + $linkCache = MediaWikiServices::getInstance()->getLinkCache(); + + return $this->executeInto( $linkCache ); + } + + /** + * Do the query and add the results to a given LinkCache object + * Return an array mapping PDBK to ID + * + * @param LinkCache &$cache + * @return array Remaining IDs + */ + protected function executeInto( &$cache ) { + $res = $this->doQuery(); + $this->doGenderQuery(); + $ids = $this->addResultToCache( $cache, $res ); + + return $ids; + } + + /** + * Add a ResultWrapper containing IDs and titles to a LinkCache object. + * As normal, titles will go into the static Title cache field. + * This function *also* stores extra fields of the title used for link + * parsing to avoid extra DB queries. + * + * @param LinkCache $cache + * @param ResultWrapper $res + * @return array Array of remaining titles + */ + public function addResultToCache( $cache, $res ) { + if ( !$res ) { + return []; + } + + $titleFormatter = MediaWikiServices::getInstance()->getTitleFormatter(); + // For each returned entry, add it to the list of good links, and remove it from $remaining + + $ids = []; + $remaining = $this->data; + foreach ( $res as $row ) { + $title = new TitleValue( (int)$row->page_namespace, $row->page_title ); + $cache->addGoodLinkObjFromRow( $title, $row ); + $pdbk = $titleFormatter->getPrefixedDBkey( $title ); + $ids[$pdbk] = $row->page_id; + unset( $remaining[$row->page_namespace][$row->page_title] ); + } + + // The remaining links in $data are bad links, register them as such + foreach ( $remaining as $ns => $dbkeys ) { + foreach ( $dbkeys as $dbkey => $unused ) { + $title = new TitleValue( (int)$ns, (string)$dbkey ); + $cache->addBadLinkObj( $title ); + $pdbk = $titleFormatter->getPrefixedDBkey( $title ); + $ids[$pdbk] = 0; + } + } + + return $ids; + } + + /** + * Perform the existence test query, return a ResultWrapper with page_id fields + * @return bool|ResultWrapper + */ + public function doQuery() { + if ( $this->isEmpty() ) { + return false; + } + + // This is similar to LinkHolderArray::replaceInternal + $dbr = wfGetDB( DB_REPLICA ); + $table = 'page'; + $fields = array_merge( + LinkCache::getSelectFields(), + [ 'page_namespace', 'page_title' ] + ); + + $conds = $this->constructSet( 'page', $dbr ); + + // Do query + $caller = __METHOD__; + if ( strval( $this->caller ) !== '' ) { + $caller .= " (for {$this->caller})"; + } + $res = $dbr->select( $table, $fields, $conds, $caller ); + + return $res; + } + + /** + * Do (and cache) {{GENDER:...}} information for userpages in this LinkBatch + * + * @return bool Whether the query was successful + */ + public function doGenderQuery() { + if ( $this->isEmpty() ) { + return false; + } + + global $wgContLang; + if ( !$wgContLang->needsGenderDistinction() ) { + return false; + } + + $genderCache = MediaWikiServices::getInstance()->getGenderCache(); + $genderCache->doLinkBatch( $this->data, $this->caller ); + + return true; + } + + /** + * Construct a WHERE clause which will match all the given titles. + * + * @param string $prefix The appropriate table's field name prefix ('page', 'pl', etc) + * @param IDatabase $db DB object to use + * @return string|bool String with SQL where clause fragment, or false if no items. + */ + public function constructSet( $prefix, $db ) { + return $db->makeWhereFrom2d( $this->data, "{$prefix}_namespace", "{$prefix}_title" ); + } +} diff --git a/www/wiki/includes/cache/LinkCache.php b/www/wiki/includes/cache/LinkCache.php new file mode 100644 index 00000000..2d088952 --- /dev/null +++ b/www/wiki/includes/cache/LinkCache.php @@ -0,0 +1,338 @@ +<?php +/** + * Page existence cache. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Cache + */ + +use Wikimedia\Rdbms\Database; +use Wikimedia\Rdbms\IDatabase; +use MediaWiki\Linker\LinkTarget; +use MediaWiki\MediaWikiServices; + +/** + * Cache for article titles (prefixed DB keys) and ids linked from one source + * + * @ingroup Cache + */ +class LinkCache { + /** @var HashBagOStuff */ + private $mGoodLinks; + /** @var HashBagOStuff */ + private $mBadLinks; + /** @var WANObjectCache */ + private $wanCache; + + /** @var bool */ + private $mForUpdate = false; + + /** @var TitleFormatter */ + private $titleFormatter; + + /** + * How many Titles to store. There are two caches, so the amount actually + * stored in memory can be up to twice this. + */ + const MAX_SIZE = 10000; + + public function __construct( TitleFormatter $titleFormatter, WANObjectCache $cache ) { + $this->mGoodLinks = new HashBagOStuff( [ 'maxKeys' => self::MAX_SIZE ] ); + $this->mBadLinks = new HashBagOStuff( [ 'maxKeys' => self::MAX_SIZE ] ); + $this->wanCache = $cache; + $this->titleFormatter = $titleFormatter; + } + + /** + * Get an instance of this class. + * + * @return LinkCache + * @deprecated since 1.28, use MediaWikiServices instead + */ + public static function singleton() { + return MediaWikiServices::getInstance()->getLinkCache(); + } + + /** + * General accessor to get/set whether the master DB should be used + * + * This used to also set the FOR UPDATE option (locking the rows read + * in order to avoid link table inconsistency), which was later removed + * for performance on wikis with a high edit rate. + * + * @param bool $update + * @return bool + */ + public function forUpdate( $update = null ) { + return wfSetVar( $this->mForUpdate, $update ); + } + + /** + * @param string $title Prefixed DB key + * @return int Page ID or zero + */ + public function getGoodLinkID( $title ) { + $info = $this->mGoodLinks->get( $title ); + if ( !$info ) { + return 0; + } + return $info['id']; + } + + /** + * Get a field of a title object from cache. + * If this link is not a cached good title, it will return NULL. + * @param LinkTarget $target + * @param string $field ('length','redirect','revision','model') + * @return string|int|null + */ + public function getGoodLinkFieldObj( LinkTarget $target, $field ) { + $dbkey = $this->titleFormatter->getPrefixedDBkey( $target ); + $info = $this->mGoodLinks->get( $dbkey ); + if ( !$info ) { + return null; + } + return $info[$field]; + } + + /** + * @param string $title Prefixed DB key + * @return bool + */ + public function isBadLink( $title ) { + // Use get() to ensure it records as used for LRU. + return $this->mBadLinks->get( $title ) !== false; + } + + /** + * Add a link for the title to the link cache + * + * @param int $id Page's ID + * @param LinkTarget $target + * @param int $len Text's length + * @param int $redir Whether the page is a redirect + * @param int $revision Latest revision's ID + * @param string|null $model Latest revision's content model ID + * @param string|null $lang Language code of the page, if not the content language + */ + public function addGoodLinkObj( $id, LinkTarget $target, $len = -1, $redir = null, + $revision = 0, $model = null, $lang = null + ) { + $dbkey = $this->titleFormatter->getPrefixedDBkey( $target ); + $this->mGoodLinks->set( $dbkey, [ + 'id' => (int)$id, + 'length' => (int)$len, + 'redirect' => (int)$redir, + 'revision' => (int)$revision, + 'model' => $model ? (string)$model : null, + 'lang' => $lang ? (string)$lang : null, + ] ); + } + + /** + * Same as above with better interface. + * @since 1.19 + * @param LinkTarget $target + * @param stdClass $row Object which has the fields page_id, page_is_redirect, + * page_latest and page_content_model + */ + public function addGoodLinkObjFromRow( LinkTarget $target, $row ) { + $dbkey = $this->titleFormatter->getPrefixedDBkey( $target ); + $this->mGoodLinks->set( $dbkey, [ + 'id' => intval( $row->page_id ), + 'length' => intval( $row->page_len ), + 'redirect' => intval( $row->page_is_redirect ), + 'revision' => intval( $row->page_latest ), + 'model' => !empty( $row->page_content_model ) ? strval( $row->page_content_model ) : null, + 'lang' => !empty( $row->page_lang ) ? strval( $row->page_lang ) : null, + ] ); + } + + /** + * @param LinkTarget $target + */ + public function addBadLinkObj( LinkTarget $target ) { + $dbkey = $this->titleFormatter->getPrefixedDBkey( $target ); + if ( !$this->isBadLink( $dbkey ) ) { + $this->mBadLinks->set( $dbkey, 1 ); + } + } + + /** + * @param string $title Prefixed DB key + */ + public function clearBadLink( $title ) { + $this->mBadLinks->delete( $title ); + } + + /** + * @param LinkTarget $target + */ + public function clearLink( LinkTarget $target ) { + $dbkey = $this->titleFormatter->getPrefixedDBkey( $target ); + $this->mBadLinks->delete( $dbkey ); + $this->mGoodLinks->delete( $dbkey ); + } + + /** + * Add a title to the link cache, return the page_id or zero if non-existent + * + * @deprecated since 1.27, unused + * @param string $title Prefixed DB key + * @return int Page ID or zero + */ + public function addLink( $title ) { + $nt = Title::newFromDBkey( $title ); + if ( !$nt ) { + return 0; + } + return $this->addLinkObj( $nt ); + } + + /** + * Fields that LinkCache needs to select + * + * @since 1.28 + * @return array + */ + public static function getSelectFields() { + global $wgContentHandlerUseDB, $wgPageLanguageUseDB; + + $fields = [ 'page_id', 'page_len', 'page_is_redirect', 'page_latest' ]; + if ( $wgContentHandlerUseDB ) { + $fields[] = 'page_content_model'; + } + if ( $wgPageLanguageUseDB ) { + $fields[] = 'page_lang'; + } + + return $fields; + } + + /** + * Add a title to the link cache, return the page_id or zero if non-existent + * + * @param LinkTarget $nt LinkTarget object to add + * @return int Page ID or zero + */ + public function addLinkObj( LinkTarget $nt ) { + $key = $this->titleFormatter->getPrefixedDBkey( $nt ); + if ( $this->isBadLink( $key ) || $nt->isExternal() + || $nt->inNamespace( NS_SPECIAL ) + ) { + return 0; + } + $id = $this->getGoodLinkID( $key ); + if ( $id != 0 ) { + return $id; + } + + if ( $key === '' ) { + return 0; + } + + // Cache template/file pages as they are less often viewed but heavily used + if ( $this->mForUpdate ) { + $row = $this->fetchPageRow( wfGetDB( DB_MASTER ), $nt ); + } elseif ( $this->isCacheable( $nt ) ) { + // These pages are often transcluded heavily, so cache them + $cache = $this->wanCache; + $row = $cache->getWithSetCallback( + $cache->makeKey( 'page', $nt->getNamespace(), sha1( $nt->getDBkey() ) ), + $cache::TTL_DAY, + function ( $curValue, &$ttl, array &$setOpts ) use ( $cache, $nt ) { + $dbr = wfGetDB( DB_REPLICA ); + $setOpts += Database::getCacheSetOptions( $dbr ); + + $row = $this->fetchPageRow( $dbr, $nt ); + $mtime = $row ? wfTimestamp( TS_UNIX, $row->page_touched ) : false; + $ttl = $cache->adaptiveTTL( $mtime, $ttl ); + + return $row; + } + ); + } else { + $row = $this->fetchPageRow( wfGetDB( DB_REPLICA ), $nt ); + } + + if ( $row ) { + $this->addGoodLinkObjFromRow( $nt, $row ); + $id = intval( $row->page_id ); + } else { + $this->addBadLinkObj( $nt ); + $id = 0; + } + + return $id; + } + + /** + * @param WANObjectCache $cache + * @param TitleValue $t + * @return string[] + * @since 1.28 + */ + public function getMutableCacheKeys( WANObjectCache $cache, TitleValue $t ) { + if ( $this->isCacheable( $t ) ) { + return [ $cache->makeKey( 'page', $t->getNamespace(), sha1( $t->getDBkey() ) ) ]; + } + + return []; + } + + private function isCacheable( LinkTarget $title ) { + return ( $title->inNamespace( NS_TEMPLATE ) || $title->inNamespace( NS_FILE ) ); + } + + private function fetchPageRow( IDatabase $db, LinkTarget $nt ) { + $fields = self::getSelectFields(); + if ( $this->isCacheable( $nt ) ) { + $fields[] = 'page_touched'; + } + + return $db->selectRow( + 'page', + $fields, + [ 'page_namespace' => $nt->getNamespace(), 'page_title' => $nt->getDBkey() ], + __METHOD__ + ); + } + + /** + * Purge the link cache for a title + * + * @param LinkTarget $title + * @since 1.28 + */ + public function invalidateTitle( LinkTarget $title ) { + if ( $this->isCacheable( $title ) ) { + $cache = ObjectCache::getMainWANInstance(); + $cache->delete( + $cache->makeKey( 'page', $title->getNamespace(), sha1( $title->getDBkey() ) ) + ); + } + } + + /** + * Clears cache + */ + public function clear() { + $this->mGoodLinks->clear(); + $this->mBadLinks->clear(); + } +} diff --git a/www/wiki/includes/cache/MessageBlobStore.php b/www/wiki/includes/cache/MessageBlobStore.php new file mode 100644 index 00000000..b262eab6 --- /dev/null +++ b/www/wiki/includes/cache/MessageBlobStore.php @@ -0,0 +1,245 @@ +<?php +/** + * Message blobs storage used by ResourceLoader. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @author Roan Kattouw + * @author Trevor Parscal + * @author Timo Tijhof + */ + +use Psr\Log\LoggerAwareInterface; +use Psr\Log\LoggerInterface; +use Psr\Log\NullLogger; +use Wikimedia\Rdbms\Database; + +/** + * This class generates message blobs for use by ResourceLoader modules. + * + * A message blob is a JSON object containing the interface messages for a certain module in + * a certain language. + */ +class MessageBlobStore implements LoggerAwareInterface { + + /* @var ResourceLoader|null */ + private $resourceloader; + + /** + * @var LoggerInterface + */ + protected $logger; + + /** + * @var WANObjectCache + */ + protected $wanCache; + + /** + * @param ResourceLoader $rl + * @param LoggerInterface $logger + */ + public function __construct( ResourceLoader $rl = null, LoggerInterface $logger = null ) { + $this->resourceloader = $rl; + $this->logger = $logger ?: new NullLogger(); + $this->wanCache = ObjectCache::getMainWANInstance(); + } + + /** + * @since 1.27 + * @param LoggerInterface $logger + */ + public function setLogger( LoggerInterface $logger ) { + $this->logger = $logger; + } + + /** + * Get the message blob for a module + * + * @since 1.27 + * @param ResourceLoaderModule $module + * @param string $lang Language code + * @return string JSON + */ + public function getBlob( ResourceLoaderModule $module, $lang ) { + $blobs = $this->getBlobs( [ $module->getName() => $module ], $lang ); + return $blobs[$module->getName()]; + } + + /** + * Get the message blobs for a set of modules + * + * @since 1.27 + * @param ResourceLoaderModule[] $modules Array of module objects keyed by name + * @param string $lang Language code + * @return array An array mapping module names to message blobs + */ + public function getBlobs( array $modules, $lang ) { + // Each cache key for a message blob by module name and language code also has a generic + // check key without language code. This is used to invalidate any and all language subkeys + // that exist for a module from the updateMessage() method. + $cache = $this->wanCache; + $checkKeys = [ + // Global check key, see clear() + $cache->makeKey( __CLASS__ ) + ]; + $cacheKeys = []; + foreach ( $modules as $name => $module ) { + $cacheKey = $this->makeCacheKey( $module, $lang ); + $cacheKeys[$name] = $cacheKey; + // Per-module check key, see updateMessage() + $checkKeys[$cacheKey][] = $cache->makeKey( __CLASS__, $name ); + } + $curTTLs = []; + $result = $cache->getMulti( array_values( $cacheKeys ), $curTTLs, $checkKeys ); + + $blobs = []; + foreach ( $modules as $name => $module ) { + $key = $cacheKeys[$name]; + if ( !isset( $result[$key] ) || $curTTLs[$key] === null || $curTTLs[$key] < 0 ) { + $blobs[$name] = $this->recacheMessageBlob( $key, $module, $lang ); + } else { + // Use unexpired cache + $blobs[$name] = $result[$key]; + } + } + return $blobs; + } + + /** + * @deprecated since 1.27 Use getBlobs() instead + * @return array + */ + public function get( ResourceLoader $resourceLoader, $modules, $lang ) { + return $this->getBlobs( $modules, $lang ); + } + + /** + * @since 1.27 + * @param ResourceLoaderModule $module + * @param string $lang + * @return string Cache key + */ + private function makeCacheKey( ResourceLoaderModule $module, $lang ) { + $messages = array_values( array_unique( $module->getMessages() ) ); + sort( $messages ); + return $this->wanCache->makeKey( __CLASS__, $module->getName(), $lang, + md5( json_encode( $messages ) ) + ); + } + + /** + * @since 1.27 + * @param string $cacheKey + * @param ResourceLoaderModule $module + * @param string $lang + * @return string JSON blob + */ + protected function recacheMessageBlob( $cacheKey, ResourceLoaderModule $module, $lang ) { + $blob = $this->generateMessageBlob( $module, $lang ); + $cache = $this->wanCache; + $cache->set( $cacheKey, $blob, + // Add part of a day to TTL to avoid all modules expiring at once + $cache::TTL_WEEK + mt_rand( 0, $cache::TTL_DAY ), + Database::getCacheSetOptions( wfGetDB( DB_REPLICA ) ) + ); + return $blob; + } + + /** + * Invalidate cache keys for modules using this message key. + * Called by MessageCache when a message has changed. + * + * @param string $key Message key + */ + public function updateMessage( $key ) { + $moduleNames = $this->getResourceLoader()->getModulesByMessage( $key ); + foreach ( $moduleNames as $moduleName ) { + // Uses a holdoff to account for database replica DB lag (for MessageCache) + $this->wanCache->touchCheckKey( $this->wanCache->makeKey( __CLASS__, $moduleName ) ); + } + } + + /** + * Invalidate cache keys for all known modules. + * Called by LocalisationCache after cache is regenerated. + */ + public function clear() { + $cache = $this->wanCache; + // Disable holdoff because this invalidates all modules and also not needed since + // LocalisationCache is stored outside the database and doesn't have lag. + $cache->touchCheckKey( $cache->makeKey( __CLASS__ ), $cache::HOLDOFF_NONE ); + } + + /** + * @since 1.27 + * @return ResourceLoader + */ + protected function getResourceLoader() { + // Back-compat: This class supports instantiation without a ResourceLoader object. + // Lazy-initialise this property because most callers don't need it. + if ( $this->resourceloader === null ) { + $this->logger->warning( __CLASS__ . ' created without a ResourceLoader instance' ); + $this->resourceloader = new ResourceLoader(); + } + return $this->resourceloader; + } + + /** + * @since 1.27 + * @param string $key Message key + * @param string $lang Language code + * @return string + */ + protected function fetchMessage( $key, $lang ) { + $message = wfMessage( $key )->inLanguage( $lang ); + $value = $message->plain(); + if ( !$message->exists() ) { + $this->logger->warning( 'Failed to find {messageKey} ({lang})', [ + 'messageKey' => $key, + 'lang' => $lang, + ] ); + } + return $value; + } + + /** + * Generate the message blob for a given module in a given language. + * + * @param ResourceLoaderModule $module + * @param string $lang Language code + * @return string JSON blob + */ + private function generateMessageBlob( ResourceLoaderModule $module, $lang ) { + $messages = []; + foreach ( $module->getMessages() as $key ) { + $messages[$key] = $this->fetchMessage( $key, $lang ); + } + + $json = FormatJson::encode( (object)$messages ); + // @codeCoverageIgnoreStart + if ( $json === false ) { + $this->logger->warning( 'Failed to encode message blob for {module} ({lang})', [ + 'module' => $module->getName(), + 'lang' => $lang, + ] ); + $json = '{}'; + } + // codeCoverageIgnoreEnd + return $json; + } +} diff --git a/www/wiki/includes/cache/MessageCache.php b/www/wiki/includes/cache/MessageCache.php new file mode 100644 index 00000000..71fcd8bd --- /dev/null +++ b/www/wiki/includes/cache/MessageCache.php @@ -0,0 +1,1318 @@ +<?php +/** + * Localisation messages cache. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Cache + */ +use MediaWiki\MediaWikiServices; +use Wikimedia\ScopedCallback; +use MediaWiki\Logger\LoggerFactory; +use Wikimedia\Rdbms\Database; + +/** + * MediaWiki message cache structure version. + * Bump this whenever the message cache format has changed. + */ +define( 'MSG_CACHE_VERSION', 2 ); + +/** + * Message cache + * Performs various MediaWiki namespace-related functions + * @ingroup Cache + */ +class MessageCache { + const FOR_UPDATE = 1; // force message reload + + /** How long to wait for memcached locks */ + const WAIT_SEC = 15; + /** How long memcached locks last */ + const LOCK_TTL = 30; + + /** + * Process local cache of loaded messages that are defined in + * MediaWiki namespace. First array level is a language code, + * second level is message key and the values are either message + * content prefixed with space, or !NONEXISTENT for negative + * caching. + * @var array $mCache + */ + protected $mCache; + + /** + * @var bool[] Map of (language code => boolean) + */ + protected $mCacheVolatile = []; + + /** + * Should mean that database cannot be used, but check + * @var bool $mDisable + */ + protected $mDisable; + + /** + * Lifetime for cache, used by object caching. + * Set on construction, see __construct(). + */ + protected $mExpiry; + + /** + * Message cache has its own parser which it uses to transform messages + * @var ParserOptions + */ + protected $mParserOptions; + /** @var Parser */ + protected $mParser; + + /** + * Variable for tracking which variables are already loaded + * @var array $mLoadedLanguages + */ + protected $mLoadedLanguages = []; + + /** + * @var bool $mInParser + */ + protected $mInParser = false; + + /** @var WANObjectCache */ + protected $wanCache; + /** @var BagOStuff */ + protected $clusterCache; + /** @var BagOStuff */ + protected $srvCache; + + /** + * Singleton instance + * + * @var MessageCache $instance + */ + private static $instance; + + /** + * Get the signleton instance of this class + * + * @since 1.18 + * @return MessageCache + */ + public static function singleton() { + if ( self::$instance === null ) { + global $wgUseDatabaseMessages, $wgMsgCacheExpiry, $wgUseLocalMessageCache; + self::$instance = new self( + MediaWikiServices::getInstance()->getMainWANObjectCache(), + wfGetMessageCacheStorage(), + $wgUseLocalMessageCache + ? MediaWikiServices::getInstance()->getLocalServerObjectCache() + : new EmptyBagOStuff(), + $wgUseDatabaseMessages, + $wgMsgCacheExpiry + ); + } + + return self::$instance; + } + + /** + * Destroy the singleton instance + * + * @since 1.18 + */ + public static function destroyInstance() { + self::$instance = null; + } + + /** + * Normalize message key input + * + * @param string $key Input message key to be normalized + * @return string Normalized message key + */ + public static function normalizeKey( $key ) { + global $wgContLang; + + $lckey = strtr( $key, ' ', '_' ); + if ( ord( $lckey ) < 128 ) { + $lckey[0] = strtolower( $lckey[0] ); + } else { + $lckey = $wgContLang->lcfirst( $lckey ); + } + + return $lckey; + } + + /** + * @param WANObjectCache $wanCache + * @param BagOStuff $clusterCache + * @param BagOStuff $serverCache + * @param bool $useDB Whether to look for message overrides (e.g. MediaWiki: pages) + * @param int $expiry Lifetime for cache. @see $mExpiry. + */ + public function __construct( + WANObjectCache $wanCache, + BagOStuff $clusterCache, + BagOStuff $serverCache, + $useDB, + $expiry + ) { + $this->wanCache = $wanCache; + $this->clusterCache = $clusterCache; + $this->srvCache = $serverCache; + + $this->mDisable = !$useDB; + $this->mExpiry = $expiry; + } + + /** + * ParserOptions is lazy initialised. + * + * @return ParserOptions + */ + function getParserOptions() { + global $wgUser; + + if ( !$this->mParserOptions ) { + if ( !$wgUser->isSafeToLoad() ) { + // $wgUser isn't unstubbable yet, so don't try to get a + // ParserOptions for it. And don't cache this ParserOptions + // either. + $po = ParserOptions::newFromAnon(); + $po->setAllowUnsafeRawHtml( false ); + return $po; + } + + $this->mParserOptions = new ParserOptions; + // Messages may take parameters that could come + // from malicious sources. As a precaution, disable + // the <html> parser tag when parsing messages. + $this->mParserOptions->setAllowUnsafeRawHtml( false ); + } + + return $this->mParserOptions; + } + + /** + * Try to load the cache from APC. + * + * @param string $code Optional language code, see documenation of load(). + * @return array|bool The cache array, or false if not in cache. + */ + protected function getLocalCache( $code ) { + $cacheKey = $this->srvCache->makeKey( __CLASS__, $code ); + + return $this->srvCache->get( $cacheKey ); + } + + /** + * Save the cache to APC. + * + * @param string $code + * @param array $cache The cache array + */ + protected function saveToLocalCache( $code, $cache ) { + $cacheKey = $this->srvCache->makeKey( __CLASS__, $code ); + $this->srvCache->set( $cacheKey, $cache ); + } + + /** + * Loads messages from caches or from database in this order: + * (1) local message cache (if $wgUseLocalMessageCache is enabled) + * (2) memcached + * (3) from the database. + * + * When succesfully loading from (2) or (3), all higher level caches are + * updated for the newest version. + * + * Nothing is loaded if member variable mDisable is true, either manually + * set by calling code or if message loading fails (is this possible?). + * + * Returns true if cache is already populated or it was succesfully populated, + * or false if populating empty cache fails. Also returns true if MessageCache + * is disabled. + * + * @param string $code Language to which load messages + * @param int $mode Use MessageCache::FOR_UPDATE to skip process cache [optional] + * @throws MWException + * @return bool + */ + protected function load( $code, $mode = null ) { + if ( !is_string( $code ) ) { + throw new InvalidArgumentException( "Missing language code" ); + } + + # Don't do double loading... + if ( isset( $this->mLoadedLanguages[$code] ) && $mode != self::FOR_UPDATE ) { + return true; + } + + # 8 lines of code just to say (once) that message cache is disabled + if ( $this->mDisable ) { + static $shownDisabled = false; + if ( !$shownDisabled ) { + wfDebug( __METHOD__ . ": disabled\n" ); + $shownDisabled = true; + } + + return true; + } + + # Loading code starts + $success = false; # Keep track of success + $staleCache = false; # a cache array with expired data, or false if none has been loaded + $where = []; # Debug info, delayed to avoid spamming debug log too much + + # Hash of the contents is stored in memcache, to detect if data-center cache + # or local cache goes out of date (e.g. due to replace() on some other server) + list( $hash, $hashVolatile ) = $this->getValidationHash( $code ); + $this->mCacheVolatile[$code] = $hashVolatile; + + # Try the local cache and check against the cluster hash key... + $cache = $this->getLocalCache( $code ); + if ( !$cache ) { + $where[] = 'local cache is empty'; + } elseif ( !isset( $cache['HASH'] ) || $cache['HASH'] !== $hash ) { + $where[] = 'local cache has the wrong hash'; + $staleCache = $cache; + } elseif ( $this->isCacheExpired( $cache ) ) { + $where[] = 'local cache is expired'; + $staleCache = $cache; + } elseif ( $hashVolatile ) { + $where[] = 'local cache validation key is expired/volatile'; + $staleCache = $cache; + } else { + $where[] = 'got from local cache'; + $success = true; + $this->mCache[$code] = $cache; + } + + if ( !$success ) { + $cacheKey = $this->clusterCache->makeKey( 'messages', $code ); + # Try the global cache. If it is empty, try to acquire a lock. If + # the lock can't be acquired, wait for the other thread to finish + # and then try the global cache a second time. + for ( $failedAttempts = 0; $failedAttempts <= 1; $failedAttempts++ ) { + if ( $hashVolatile && $staleCache ) { + # Do not bother fetching the whole cache blob to avoid I/O. + # Instead, just try to get the non-blocking $statusKey lock + # below, and use the local stale value if it was not acquired. + $where[] = 'global cache is presumed expired'; + } else { + $cache = $this->clusterCache->get( $cacheKey ); + if ( !$cache ) { + $where[] = 'global cache is empty'; + } elseif ( $this->isCacheExpired( $cache ) ) { + $where[] = 'global cache is expired'; + $staleCache = $cache; + } elseif ( $hashVolatile ) { + # DB results are replica DB lag prone until the holdoff TTL passes. + # By then, updates should be reflected in loadFromDBWithLock(). + # One thread renerates the cache while others use old values. + $where[] = 'global cache is expired/volatile'; + $staleCache = $cache; + } else { + $where[] = 'got from global cache'; + $this->mCache[$code] = $cache; + $this->saveToCaches( $cache, 'local-only', $code ); + $success = true; + } + } + + if ( $success ) { + # Done, no need to retry + break; + } + + # We need to call loadFromDB. Limit the concurrency to one process. + # This prevents the site from going down when the cache expires. + # Note that the DB slam protection lock here is non-blocking. + $loadStatus = $this->loadFromDBWithLock( $code, $where, $mode ); + if ( $loadStatus === true ) { + $success = true; + break; + } elseif ( $staleCache ) { + # Use the stale cache while some other thread constructs the new one + $where[] = 'using stale cache'; + $this->mCache[$code] = $staleCache; + $success = true; + break; + } elseif ( $failedAttempts > 0 ) { + # Already blocked once, so avoid another lock/unlock cycle. + # This case will typically be hit if memcached is down, or if + # loadFromDB() takes longer than LOCK_WAIT. + $where[] = "could not acquire status key."; + break; + } elseif ( $loadStatus === 'cantacquire' ) { + # Wait for the other thread to finish, then retry. Normally, + # the memcached get() will then yeild the other thread's result. + $where[] = 'waited for other thread to complete'; + $this->getReentrantScopedLock( $cacheKey ); + } else { + # Disable cache; $loadStatus is 'disabled' + break; + } + } + } + + if ( !$success ) { + $where[] = 'loading FAILED - cache is disabled'; + $this->mDisable = true; + $this->mCache = false; + wfDebugLog( 'MessageCacheError', __METHOD__ . ": Failed to load $code\n" ); + # This used to throw an exception, but that led to nasty side effects like + # the whole wiki being instantly down if the memcached server died + } else { + # All good, just record the success + $this->mLoadedLanguages[$code] = true; + } + + $info = implode( ', ', $where ); + wfDebugLog( 'MessageCache', __METHOD__ . ": Loading $code... $info\n" ); + + return $success; + } + + /** + * @param string $code + * @param array &$where List of wfDebug() comments + * @param int $mode Use MessageCache::FOR_UPDATE to use DB_MASTER + * @return bool|string True on success or one of ("cantacquire", "disabled") + */ + protected function loadFromDBWithLock( $code, array &$where, $mode = null ) { + # If cache updates on all levels fail, give up on message overrides. + # This is to avoid easy site outages; see $saveSuccess comments below. + $statusKey = $this->clusterCache->makeKey( 'messages', $code, 'status' ); + $status = $this->clusterCache->get( $statusKey ); + if ( $status === 'error' ) { + $where[] = "could not load; method is still globally disabled"; + return 'disabled'; + } + + # Now let's regenerate + $where[] = 'loading from database'; + + # Lock the cache to prevent conflicting writes. + # This lock is non-blocking so stale cache can quickly be used. + # Note that load() will call a blocking getReentrantScopedLock() + # after this if it really need to wait for any current thread. + $cacheKey = $this->clusterCache->makeKey( 'messages', $code ); + $scopedLock = $this->getReentrantScopedLock( $cacheKey, 0 ); + if ( !$scopedLock ) { + $where[] = 'could not acquire main lock'; + return 'cantacquire'; + } + + $cache = $this->loadFromDB( $code, $mode ); + $this->mCache[$code] = $cache; + $saveSuccess = $this->saveToCaches( $cache, 'all', $code ); + + if ( !$saveSuccess ) { + /** + * Cache save has failed. + * + * There are two main scenarios where this could be a problem: + * - The cache is more than the maximum size (typically 1MB compressed). + * - Memcached has no space remaining in the relevant slab class. This is + * unlikely with recent versions of memcached. + * + * Either way, if there is a local cache, nothing bad will happen. If there + * is no local cache, disabling the message cache for all requests avoids + * incurring a loadFromDB() overhead on every request, and thus saves the + * wiki from complete downtime under moderate traffic conditions. + */ + if ( $this->srvCache instanceof EmptyBagOStuff ) { + $this->clusterCache->set( $statusKey, 'error', 60 * 5 ); + $where[] = 'could not save cache, disabled globally for 5 minutes'; + } else { + $where[] = "could not save global cache"; + } + } + + return true; + } + + /** + * Loads cacheable messages from the database. Messages bigger than + * $wgMaxMsgCacheEntrySize are assigned a special value, and are loaded + * on-demand from the database later. + * + * @param string $code Language code + * @param int $mode Use MessageCache::FOR_UPDATE to skip process cache + * @return array Loaded messages for storing in caches + */ + protected function loadFromDB( $code, $mode = null ) { + global $wgMaxMsgCacheEntrySize, $wgLanguageCode, $wgAdaptiveMessageCache; + + // (T164666) The query here performs really poorly on WMF's + // contributions replicas. We don't have a way to say "any group except + // contributions", so for the moment let's specify 'api'. + // @todo: Get rid of this hack. + $dbr = wfGetDB( ( $mode == self::FOR_UPDATE ) ? DB_MASTER : DB_REPLICA, 'api' ); + + $cache = []; + + # Common conditions + $conds = [ + 'page_is_redirect' => 0, + 'page_namespace' => NS_MEDIAWIKI, + ]; + + $mostused = []; + if ( $wgAdaptiveMessageCache && $code !== $wgLanguageCode ) { + if ( !isset( $this->mCache[$wgLanguageCode] ) ) { + $this->load( $wgLanguageCode ); + } + $mostused = array_keys( $this->mCache[$wgLanguageCode] ); + foreach ( $mostused as $key => $value ) { + $mostused[$key] = "$value/$code"; + } + } + + if ( count( $mostused ) ) { + $conds['page_title'] = $mostused; + } elseif ( $code !== $wgLanguageCode ) { + $conds[] = 'page_title' . $dbr->buildLike( $dbr->anyString(), '/', $code ); + } else { + # Effectively disallows use of '/' character in NS_MEDIAWIKI for uses + # other than language code. + $conds[] = 'page_title NOT' . + $dbr->buildLike( $dbr->anyString(), '/', $dbr->anyString() ); + } + + # Conditions to fetch oversized pages to ignore them + $bigConds = $conds; + $bigConds[] = 'page_len > ' . intval( $wgMaxMsgCacheEntrySize ); + + # Load titles for all oversized pages in the MediaWiki namespace + $res = $dbr->select( + 'page', + [ 'page_title', 'page_latest' ], + $bigConds, + __METHOD__ . "($code)-big" + ); + foreach ( $res as $row ) { + $cache[$row->page_title] = '!TOO BIG'; + // At least include revision ID so page changes are reflected in the hash + $cache['EXCESSIVE'][$row->page_title] = $row->page_latest; + } + + # Conditions to load the remaining pages with their contents + $smallConds = $conds; + $smallConds[] = 'page_len <= ' . intval( $wgMaxMsgCacheEntrySize ); + + $res = $dbr->select( + [ 'page', 'revision', 'text' ], + [ 'page_title', 'old_id', 'old_text', 'old_flags' ], + $smallConds, + __METHOD__ . "($code)-small", + [], + [ + 'revision' => [ 'JOIN', 'page_latest=rev_id' ], + 'text' => [ 'JOIN', 'rev_text_id=old_id' ], + ] + ); + + foreach ( $res as $row ) { + $text = Revision::getRevisionText( $row ); + if ( $text === false ) { + // Failed to fetch data; possible ES errors? + // Store a marker to fetch on-demand as a workaround... + // TODO Use a differnt marker + $entry = '!TOO BIG'; + wfDebugLog( + 'MessageCache', + __METHOD__ + . ": failed to load message page text for {$row->page_title} ($code)" + ); + } else { + $entry = ' ' . $text; + } + $cache[$row->page_title] = $entry; + } + + $cache['VERSION'] = MSG_CACHE_VERSION; + ksort( $cache ); + + # Hash for validating local cache (APC). No need to take into account + # messages larger than $wgMaxMsgCacheEntrySize, since those are only + # stored and fetched from memcache. + $cache['HASH'] = md5( serialize( $cache ) ); + $cache['EXPIRY'] = wfTimestamp( TS_MW, time() + $this->mExpiry ); + + return $cache; + } + + /** + * Updates cache as necessary when message page is changed + * + * @param string $title Message cache key with initial uppercase letter + * @param string|bool $text New contents of the page (false if deleted) + */ + public function replace( $title, $text ) { + global $wgLanguageCode; + + if ( $this->mDisable ) { + return; + } + + list( $msg, $code ) = $this->figureMessage( $title ); + if ( strpos( $title, '/' ) !== false && $code === $wgLanguageCode ) { + // Content language overrides do not use the /<code> suffix + return; + } + + // (a) Update the process cache with the new message text + if ( $text === false ) { + // Page deleted + $this->mCache[$code][$title] = '!NONEXISTENT'; + } else { + // Ignore $wgMaxMsgCacheEntrySize so the process cache is up to date + $this->mCache[$code][$title] = ' ' . $text; + } + + // (b) Update the shared caches in a deferred update with a fresh DB snapshot + DeferredUpdates::addCallableUpdate( + function () use ( $title, $msg, $code ) { + global $wgContLang, $wgMaxMsgCacheEntrySize; + // Allow one caller at a time to avoid race conditions + $scopedLock = $this->getReentrantScopedLock( + $this->clusterCache->makeKey( 'messages', $code ) + ); + if ( !$scopedLock ) { + LoggerFactory::getInstance( 'MessageCache' )->error( + __METHOD__ . ': could not acquire lock to update {title} ({code})', + [ 'title' => $title, 'code' => $code ] ); + return; + } + // Load the messages from the master DB to avoid race conditions + $cache = $this->loadFromDB( $code, self::FOR_UPDATE ); + $this->mCache[$code] = $cache; + // Load the process cache values and set the per-title cache keys + $page = WikiPage::factory( Title::makeTitle( NS_MEDIAWIKI, $title ) ); + $page->loadPageData( $page::READ_LATEST ); + $text = $this->getMessageTextFromContent( $page->getContent() ); + // Check if an individual cache key should exist and update cache accordingly + if ( is_string( $text ) && strlen( $text ) > $wgMaxMsgCacheEntrySize ) { + $titleKey = $this->bigMessageCacheKey( $this->mCache[$code]['HASH'], $title ); + $this->wanCache->set( $titleKey, ' ' . $text, $this->mExpiry ); + } + // Mark this cache as definitely being "latest" (non-volatile) so + // load() calls do try to refresh the cache with replica DB data + $this->mCache[$code]['LATEST'] = time(); + // Pre-emptively update the local datacenter cache so things like edit filter and + // blacklist changes are reflected immediately; these often use MediaWiki: pages. + // The datacenter handling replace() calls should be the same one handling edits + // as they require HTTP POST. + $this->saveToCaches( $this->mCache[$code], 'all', $code ); + // Release the lock now that the cache is saved + ScopedCallback::consume( $scopedLock ); + + // Relay the purge. Touching this check key expires cache contents + // and local cache (APC) validation hash across all datacenters. + $this->wanCache->touchCheckKey( $this->getCheckKey( $code ) ); + + // Purge the message in the message blob store + $resourceloader = RequestContext::getMain()->getOutput()->getResourceLoader(); + $blobStore = $resourceloader->getMessageBlobStore(); + $blobStore->updateMessage( $wgContLang->lcfirst( $msg ) ); + + Hooks::run( 'MessageCacheReplace', [ $title, $text ] ); + }, + DeferredUpdates::PRESEND + ); + } + + /** + * Is the given cache array expired due to time passing or a version change? + * + * @param array $cache + * @return bool + */ + protected function isCacheExpired( $cache ) { + if ( !isset( $cache['VERSION'] ) || !isset( $cache['EXPIRY'] ) ) { + return true; + } + if ( $cache['VERSION'] != MSG_CACHE_VERSION ) { + return true; + } + if ( wfTimestampNow() >= $cache['EXPIRY'] ) { + return true; + } + + return false; + } + + /** + * Shortcut to update caches. + * + * @param array $cache Cached messages with a version. + * @param string $dest Either "local-only" to save to local caches only + * or "all" to save to all caches. + * @param string|bool $code Language code (default: false) + * @return bool + */ + protected function saveToCaches( array $cache, $dest, $code = false ) { + if ( $dest === 'all' ) { + $cacheKey = $this->clusterCache->makeKey( 'messages', $code ); + $success = $this->clusterCache->set( $cacheKey, $cache ); + $this->setValidationHash( $code, $cache ); + } else { + $success = true; + } + + $this->saveToLocalCache( $code, $cache ); + + return $success; + } + + /** + * Get the md5 used to validate the local APC cache + * + * @param string $code + * @return array (hash or false, bool expiry/volatility status) + */ + protected function getValidationHash( $code ) { + $curTTL = null; + $value = $this->wanCache->get( + $this->wanCache->makeKey( 'messages', $code, 'hash', 'v1' ), + $curTTL, + [ $this->getCheckKey( $code ) ] + ); + + if ( $value ) { + $hash = $value['hash']; + if ( ( time() - $value['latest'] ) < WANObjectCache::TTL_MINUTE ) { + // Cache was recently updated via replace() and should be up-to-date. + // That method is only called in the primary datacenter and uses FOR_UPDATE. + // Also, it is unlikely that the current datacenter is *now* secondary one. + $expired = false; + } else { + // See if the "check" key was bumped after the hash was generated + $expired = ( $curTTL < 0 ); + } + } else { + // No hash found at all; cache must regenerate to be safe + $hash = false; + $expired = true; + } + + return [ $hash, $expired ]; + } + + /** + * Set the md5 used to validate the local disk cache + * + * If $cache has a 'LATEST' UNIX timestamp key, then the hash will not + * be treated as "volatile" by getValidationHash() for the next few seconds. + * This is triggered when $cache is generated using FOR_UPDATE mode. + * + * @param string $code + * @param array $cache Cached messages with a version + */ + protected function setValidationHash( $code, array $cache ) { + $this->wanCache->set( + $this->wanCache->makeKey( 'messages', $code, 'hash', 'v1' ), + [ + 'hash' => $cache['HASH'], + 'latest' => isset( $cache['LATEST'] ) ? $cache['LATEST'] : 0 + ], + WANObjectCache::TTL_INDEFINITE + ); + } + + /** + * @param string $key A language message cache key that stores blobs + * @param int $timeout Wait timeout in seconds + * @return null|ScopedCallback + */ + protected function getReentrantScopedLock( $key, $timeout = self::WAIT_SEC ) { + return $this->clusterCache->getScopedLock( $key, $timeout, self::LOCK_TTL, __METHOD__ ); + } + + /** + * Get a message from either the content language or the user language. + * + * First, assemble a list of languages to attempt getting the message from. This + * chain begins with the requested language and its fallbacks and then continues with + * the content language and its fallbacks. For each language in the chain, the following + * process will occur (in this order): + * 1. If a language-specific override, i.e., [[MW:msg/lang]], is available, use that. + * Note: for the content language, there is no /lang subpage. + * 2. Fetch from the static CDB cache. + * 3. If available, check the database for fallback language overrides. + * + * This process provides a number of guarantees. When changing this code, make sure all + * of these guarantees are preserved. + * * If the requested language is *not* the content language, then the CDB cache for that + * specific language will take precedence over the root database page ([[MW:msg]]). + * * Fallbacks will be just that: fallbacks. A fallback language will never be reached if + * the message is available *anywhere* in the language for which it is a fallback. + * + * @param string $key The message key + * @param bool $useDB If true, look for the message in the DB, false + * to use only the compiled l10n cache. + * @param bool|string|object $langcode Code of the language to get the message for. + * - If string and a valid code, will create a standard language object + * - If string but not a valid code, will create a basic language object + * - If boolean and false, create object from the current users language + * - If boolean and true, create object from the wikis content language + * - If language object, use it as given + * @param bool $isFullKey Specifies whether $key is a two part key "msg/lang". + * + * @throws MWException When given an invalid key + * @return string|bool False if the message doesn't exist, otherwise the + * message (which can be empty) + */ + function get( $key, $useDB = true, $langcode = true, $isFullKey = false ) { + if ( is_int( $key ) ) { + // Fix numerical strings that somehow become ints + // on their way here + $key = (string)$key; + } elseif ( !is_string( $key ) ) { + throw new MWException( 'Non-string key given' ); + } elseif ( $key === '' ) { + // Shortcut: the empty key is always missing + return false; + } + + // For full keys, get the language code from the key + $pos = strrpos( $key, '/' ); + if ( $isFullKey && $pos !== false ) { + $langcode = substr( $key, $pos + 1 ); + $key = substr( $key, 0, $pos ); + } + + // Normalise title-case input (with some inlining) + $lckey = self::normalizeKey( $key ); + + Hooks::run( 'MessageCache::get', [ &$lckey ] ); + + // Loop through each language in the fallback list until we find something useful + $lang = wfGetLangObj( $langcode ); + $message = $this->getMessageFromFallbackChain( + $lang, + $lckey, + !$this->mDisable && $useDB + ); + + // If we still have no message, maybe the key was in fact a full key so try that + if ( $message === false ) { + $parts = explode( '/', $lckey ); + // We may get calls for things that are http-urls from sidebar + // Let's not load nonexistent languages for those + // They usually have more than one slash. + if ( count( $parts ) == 2 && $parts[1] !== '' ) { + $message = Language::getMessageFor( $parts[0], $parts[1] ); + if ( $message === null ) { + $message = false; + } + } + } + + // Post-processing if the message exists + if ( $message !== false ) { + // Fix whitespace + $message = str_replace( + [ + # Fix for trailing whitespace, removed by textarea + ' ', + # Fix for NBSP, converted to space by firefox + ' ', + ' ', + '­' + ], + [ + ' ', + "\xc2\xa0", + "\xc2\xa0", + "\xc2\xad" + ], + $message + ); + } + + return $message; + } + + /** + * Given a language, try and fetch messages from that language. + * + * Will also consider fallbacks of that language, the site language, and fallbacks for + * the site language. + * + * @see MessageCache::get + * @param Language|StubObject $lang Preferred language + * @param string $lckey Lowercase key for the message (as for localisation cache) + * @param bool $useDB Whether to include messages from the wiki database + * @return string|bool The message, or false if not found + */ + protected function getMessageFromFallbackChain( $lang, $lckey, $useDB ) { + global $wgContLang; + + $alreadyTried = []; + + // First try the requested language. + $message = $this->getMessageForLang( $lang, $lckey, $useDB, $alreadyTried ); + if ( $message !== false ) { + return $message; + } + + // Now try checking the site language. + $message = $this->getMessageForLang( $wgContLang, $lckey, $useDB, $alreadyTried ); + return $message; + } + + /** + * Given a language, try and fetch messages from that language and its fallbacks. + * + * @see MessageCache::get + * @param Language|StubObject $lang Preferred language + * @param string $lckey Lowercase key for the message (as for localisation cache) + * @param bool $useDB Whether to include messages from the wiki database + * @param bool[] $alreadyTried Contains true for each language that has been tried already + * @return string|bool The message, or false if not found + */ + private function getMessageForLang( $lang, $lckey, $useDB, &$alreadyTried ) { + global $wgContLang; + + $langcode = $lang->getCode(); + + // Try checking the database for the requested language + if ( $useDB ) { + $uckey = $wgContLang->ucfirst( $lckey ); + + if ( !isset( $alreadyTried[$langcode] ) ) { + $message = $this->getMsgFromNamespace( + $this->getMessagePageName( $langcode, $uckey ), + $langcode + ); + + if ( $message !== false ) { + return $message; + } + $alreadyTried[$langcode] = true; + } + } else { + $uckey = null; + } + + // Check the CDB cache + $message = $lang->getMessage( $lckey ); + if ( $message !== null ) { + return $message; + } + + // Try checking the database for all of the fallback languages + if ( $useDB ) { + $fallbackChain = Language::getFallbacksFor( $langcode ); + + foreach ( $fallbackChain as $code ) { + if ( isset( $alreadyTried[$code] ) ) { + continue; + } + + $message = $this->getMsgFromNamespace( + $this->getMessagePageName( $code, $uckey ), $code ); + + if ( $message !== false ) { + return $message; + } + $alreadyTried[$code] = true; + } + } + + return false; + } + + /** + * Get the message page name for a given language + * + * @param string $langcode + * @param string $uckey Uppercase key for the message + * @return string The page name + */ + private function getMessagePageName( $langcode, $uckey ) { + global $wgLanguageCode; + + if ( $langcode === $wgLanguageCode ) { + // Messages created in the content language will not have the /lang extension + return $uckey; + } else { + return "$uckey/$langcode"; + } + } + + /** + * Get a message from the MediaWiki namespace, with caching. The key must + * first be converted to two-part lang/msg form if necessary. + * + * Unlike self::get(), this function doesn't resolve fallback chains, and + * some callers require this behavior. LanguageConverter::parseCachedTable() + * and self::get() are some examples in core. + * + * @param string $title Message cache key with initial uppercase letter + * @param string $code Code denoting the language to try + * @return string|bool The message, or false if it does not exist or on error + */ + public function getMsgFromNamespace( $title, $code ) { + $this->load( $code ); + + if ( isset( $this->mCache[$code][$title] ) ) { + $entry = $this->mCache[$code][$title]; + if ( substr( $entry, 0, 1 ) === ' ' ) { + // The message exists and is not '!TOO BIG' + return (string)substr( $entry, 1 ); + } elseif ( $entry === '!NONEXISTENT' ) { + return false; + } + // Fall through and try invididual message cache below + } else { + // XXX: This is not cached in process cache, should it? + $message = false; + Hooks::run( 'MessagesPreLoad', [ $title, &$message, $code ] ); + if ( $message !== false ) { + return $message; + } + + return false; + } + + // Individual message cache key + $titleKey = $this->bigMessageCacheKey( $this->mCache[$code]['HASH'], $title ); + + if ( $this->mCacheVolatile[$code] ) { + $entry = false; + // Make sure that individual keys respect the WAN cache holdoff period too + LoggerFactory::getInstance( 'MessageCache' )->debug( + __METHOD__ . ': loading volatile key \'{titleKey}\'', + [ 'titleKey' => $titleKey, 'code' => $code ] ); + } else { + // Try the individual message cache + $entry = $this->wanCache->get( $titleKey ); + } + + if ( $entry !== false ) { + if ( substr( $entry, 0, 1 ) === ' ' ) { + $this->mCache[$code][$title] = $entry; + // The message exists, so make sure a string is returned + return (string)substr( $entry, 1 ); + } elseif ( $entry === '!NONEXISTENT' ) { + $this->mCache[$code][$title] = '!NONEXISTENT'; + + return false; + } else { + // Corrupt/obsolete entry, delete it + $this->wanCache->delete( $titleKey ); + } + } + + // Try loading the message from the database + $dbr = wfGetDB( DB_REPLICA ); + $cacheOpts = Database::getCacheSetOptions( $dbr ); + // Use newKnownCurrent() to avoid querying revision/user tables + $titleObj = Title::makeTitle( NS_MEDIAWIKI, $title ); + if ( $titleObj->getLatestRevID() ) { + $revision = Revision::newKnownCurrent( + $dbr, + $titleObj + ); + } else { + $revision = false; + } + + if ( $revision ) { + $content = $revision->getContent(); + if ( $content ) { + $message = $this->getMessageTextFromContent( $content ); + if ( is_string( $message ) ) { + $this->mCache[$code][$title] = ' ' . $message; + $this->wanCache->set( $titleKey, ' ' . $message, $this->mExpiry, $cacheOpts ); + } + } else { + // A possibly temporary loading failure + LoggerFactory::getInstance( 'MessageCache' )->warning( + __METHOD__ . ': failed to load message page text for \'{titleKey}\'', + [ 'titleKey' => $titleKey, 'code' => $code ] ); + $message = null; // no negative caching + } + } else { + $message = false; // negative caching + } + + if ( $message === false ) { + // Negative caching in case a "too big" message is no longer available (deleted) + $this->mCache[$code][$title] = '!NONEXISTENT'; + $this->wanCache->set( $titleKey, '!NONEXISTENT', $this->mExpiry, $cacheOpts ); + } + + return $message; + } + + /** + * @param string $message + * @param bool $interface + * @param Language $language + * @param Title $title + * @return string + */ + public function transform( $message, $interface = false, $language = null, $title = null ) { + // Avoid creating parser if nothing to transform + if ( strpos( $message, '{{' ) === false ) { + return $message; + } + + if ( $this->mInParser ) { + return $message; + } + + $parser = $this->getParser(); + if ( $parser ) { + $popts = $this->getParserOptions(); + $popts->setInterfaceMessage( $interface ); + $popts->setTargetLanguage( $language ); + + $userlang = $popts->setUserLang( $language ); + $this->mInParser = true; + $message = $parser->transformMsg( $message, $popts, $title ); + $this->mInParser = false; + $popts->setUserLang( $userlang ); + } + + return $message; + } + + /** + * @return Parser + */ + public function getParser() { + global $wgParser, $wgParserConf; + + if ( !$this->mParser && isset( $wgParser ) ) { + # Do some initialisation so that we don't have to do it twice + $wgParser->firstCallInit(); + # Clone it and store it + $class = $wgParserConf['class']; + if ( $class == ParserDiffTest::class ) { + # Uncloneable + $this->mParser = new $class( $wgParserConf ); + } else { + $this->mParser = clone $wgParser; + } + } + + return $this->mParser; + } + + /** + * @param string $text + * @param Title $title + * @param bool $linestart Whether or not this is at the start of a line + * @param bool $interface Whether this is an interface message + * @param Language|string $language Language code + * @return ParserOutput|string + */ + public function parse( $text, $title = null, $linestart = true, + $interface = false, $language = null + ) { + global $wgTitle; + + if ( $this->mInParser ) { + return htmlspecialchars( $text ); + } + + $parser = $this->getParser(); + $popts = $this->getParserOptions(); + $popts->setInterfaceMessage( $interface ); + + if ( is_string( $language ) ) { + $language = Language::factory( $language ); + } + $popts->setTargetLanguage( $language ); + + if ( !$title || !$title instanceof Title ) { + wfDebugLog( 'GlobalTitleFail', __METHOD__ . ' called by ' . + wfGetAllCallers( 6 ) . ' with no title set.' ); + $title = $wgTitle; + } + // Sometimes $wgTitle isn't set either... + if ( !$title ) { + # It's not uncommon having a null $wgTitle in scripts. See r80898 + # Create a ghost title in such case + $title = Title::makeTitle( NS_SPECIAL, 'Badtitle/title not set in ' . __METHOD__ ); + } + + $this->mInParser = true; + $res = $parser->parse( $text, $title, $popts, $linestart ); + $this->mInParser = false; + + return $res; + } + + public function disable() { + $this->mDisable = true; + } + + public function enable() { + $this->mDisable = false; + } + + /** + * Whether DB/cache usage is disabled for determining messages + * + * If so, this typically indicates either: + * - a) load() failed to find a cached copy nor query the DB + * - b) we are in a special context or error mode that cannot use the DB + * If the DB is ignored, any derived HTML output or cached objects may be wrong. + * To avoid long-term cache pollution, TTLs can be adjusted accordingly. + * + * @return bool + * @since 1.27 + */ + public function isDisabled() { + return $this->mDisable; + } + + /** + * Clear all stored messages in global and local cache + * + * Mainly used after a mass rebuild + */ + function clear() { + $langs = Language::fetchLanguageNames( null, 'mw' ); + foreach ( array_keys( $langs ) as $code ) { + $this->wanCache->touchCheckKey( $this->getCheckKey( $code ) ); + } + + $this->mLoadedLanguages = []; + } + + /** + * @param string $key + * @return array + */ + public function figureMessage( $key ) { + global $wgLanguageCode; + + $pieces = explode( '/', $key ); + if ( count( $pieces ) < 2 ) { + return [ $key, $wgLanguageCode ]; + } + + $lang = array_pop( $pieces ); + if ( !Language::fetchLanguageName( $lang, null, 'mw' ) ) { + return [ $key, $wgLanguageCode ]; + } + + $message = implode( '/', $pieces ); + + return [ $message, $lang ]; + } + + /** + * Get all message keys stored in the message cache for a given language. + * If $code is the content language code, this will return all message keys + * for which MediaWiki:msgkey exists. If $code is another language code, this + * will ONLY return message keys for which MediaWiki:msgkey/$code exists. + * @param string $code Language code + * @return array Array of message keys (strings) + */ + public function getAllMessageKeys( $code ) { + global $wgContLang; + + $this->load( $code ); + if ( !isset( $this->mCache[$code] ) ) { + // Apparently load() failed + return null; + } + // Remove administrative keys + $cache = $this->mCache[$code]; + unset( $cache['VERSION'] ); + unset( $cache['EXPIRY'] ); + unset( $cache['EXCESSIVE'] ); + // Remove any !NONEXISTENT keys + $cache = array_diff( $cache, [ '!NONEXISTENT' ] ); + + // Keys may appear with a capital first letter. lcfirst them. + return array_map( [ $wgContLang, 'lcfirst' ], array_keys( $cache ) ); + } + + /** + * Purge message caches when a MediaWiki: page is created, updated, or deleted + * + * @param Title $title Message page title + * @param Content|null $content New content for edit/create, null on deletion + * @since 1.29 + */ + public function updateMessageOverride( Title $title, Content $content = null ) { + global $wgContLang; + + $msgText = $this->getMessageTextFromContent( $content ); + if ( $msgText === null ) { + $msgText = false; // treat as not existing + } + + $this->replace( $title->getDBkey(), $msgText ); + + if ( $wgContLang->hasVariants() ) { + $wgContLang->updateConversionTable( $title ); + } + } + + /** + * @param string $code Language code + * @return string WAN cache key usable as a "check key" against language page edits + */ + public function getCheckKey( $code ) { + return $this->wanCache->makeKey( 'messages', $code ); + } + + /** + * @param Content|null $content Content or null if the message page does not exist + * @return string|bool|null Returns false if $content is null and null on error + */ + private function getMessageTextFromContent( Content $content = null ) { + // @TODO: could skip pseudo-messages like js/css here, based on content model + if ( $content ) { + // Message page exists... + // XXX: Is this the right way to turn a Content object into a message? + // NOTE: $content is typically either WikitextContent, JavaScriptContent or + // CssContent. MessageContent is *not* used for storing messages, it's + // only used for wrapping them when needed. + $msgText = $content->getWikitextForTransclusion(); + if ( $msgText === false || $msgText === null ) { + // This might be due to some kind of misconfiguration... + $msgText = null; + LoggerFactory::getInstance( 'MessageCache' )->warning( + __METHOD__ . ": message content doesn't provide wikitext " + . "(content model: " . $content->getModel() . ")" ); + } + } else { + // Message page does not exist... + $msgText = false; + } + + return $msgText; + } + + /** + * @param string $hash Hash for this version of the entire key/value overrides map + * @param string $title Message cache key with initial uppercase letter + * @return string + */ + private function bigMessageCacheKey( $hash, $title ) { + return $this->wanCache->makeKey( 'messages-big', $hash, $title ); + } +} diff --git a/www/wiki/includes/cache/ResourceFileCache.php b/www/wiki/includes/cache/ResourceFileCache.php new file mode 100644 index 00000000..326d0659 --- /dev/null +++ b/www/wiki/includes/cache/ResourceFileCache.php @@ -0,0 +1,117 @@ +<?php +/** + * ResourceLoader request result caching in the file system. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Cache + */ + +/** + * ResourceLoader request result caching in the file system. + * + * @ingroup Cache + */ +class ResourceFileCache extends FileCacheBase { + protected $mCacheWorthy; + + /* @todo configurable? */ + const MISS_THRESHOLD = 360; // 6/min * 60 min + + /** + * Construct an ResourceFileCache from a context + * @param ResourceLoaderContext $context + * @return ResourceFileCache + */ + public static function newFromContext( ResourceLoaderContext $context ) { + $cache = new self(); + + if ( $context->getImage() ) { + $cache->mType = 'image'; + } elseif ( $context->getOnly() === 'styles' ) { + $cache->mType = 'css'; + } else { + $cache->mType = 'js'; + } + $modules = array_unique( $context->getModules() ); // remove duplicates + sort( $modules ); // normalize the order (permutation => combination) + $cache->mKey = sha1( $context->getHash() . implode( '|', $modules ) ); + if ( count( $modules ) == 1 ) { + $cache->mCacheWorthy = true; // won't take up much space + } + + return $cache; + } + + /** + * Check if an RL request can be cached. + * Caller is responsible for checking if any modules are private. + * @param ResourceLoaderContext $context + * @return bool + */ + public static function useFileCache( ResourceLoaderContext $context ) { + global $wgUseFileCache, $wgDefaultSkin, $wgLanguageCode; + if ( !$wgUseFileCache ) { + return false; + } + // Get all query values + $queryVals = $context->getRequest()->getValues(); + foreach ( $queryVals as $query => $val ) { + if ( in_array( $query, [ 'modules', 'image', 'variant', 'version', '*' ] ) ) { + // Use file cache regardless of the value of this parameter + continue; // note: &* added as IE fix + } elseif ( $query === 'skin' && $val === $wgDefaultSkin ) { + continue; + } elseif ( $query === 'lang' && $val === $wgLanguageCode ) { + continue; + } elseif ( $query === 'only' && in_array( $val, [ 'styles', 'scripts' ] ) ) { + continue; + } elseif ( $query === 'debug' && $val === 'false' ) { + continue; + } elseif ( $query === 'format' && $val === 'rasterized' ) { + continue; + } + + return false; + } + + return true; // cacheable + } + + /** + * Get the base file cache directory + * @return string + */ + protected function cacheDirectory() { + return $this->baseCacheDirectory() . '/resources'; + } + + /** + * Item has many recent cache misses + * @return bool + */ + public function isCacheWorthy() { + if ( $this->mCacheWorthy === null ) { + $this->mCacheWorthy = ( + $this->isCached() || // even stale cache indicates it was cache worthy + $this->getMissesRecent() >= self::MISS_THRESHOLD // many misses + ); + } + + return $this->mCacheWorthy; + } +} diff --git a/www/wiki/includes/cache/UserCache.php b/www/wiki/includes/cache/UserCache.php new file mode 100644 index 00000000..cb685712 --- /dev/null +++ b/www/wiki/includes/cache/UserCache.php @@ -0,0 +1,162 @@ +<?php +/** + * Caches current user names and other info based on user IDs. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Cache + */ + +/** + * @since 1.20 + */ +class UserCache { + protected $cache = []; // (uid => property => value) + protected $typesCached = []; // (uid => cache type => 1) + + /** + * @return UserCache + */ + public static function singleton() { + static $instance = null; + if ( $instance === null ) { + $instance = new self(); + } + + return $instance; + } + + protected function __construct() { + } + + /** + * Get a property of a user based on their user ID + * + * @param int $userId User ID + * @param string $prop User property + * @return mixed|bool The property or false if the user does not exist + */ + public function getProp( $userId, $prop ) { + if ( !isset( $this->cache[$userId][$prop] ) ) { + wfDebug( __METHOD__ . ": querying DB for prop '$prop' for user ID '$userId'.\n" ); + $this->doQuery( [ $userId ] ); // cache miss + } + + return isset( $this->cache[$userId][$prop] ) + ? $this->cache[$userId][$prop] + : false; // user does not exist? + } + + /** + * Get the name of a user or return $ip if the user ID is 0 + * + * @param int $userId + * @param string $ip + * @return string + * @since 1.22 + */ + public function getUserName( $userId, $ip ) { + return $userId > 0 ? $this->getProp( $userId, 'name' ) : $ip; + } + + /** + * Preloads user names for given list of users. + * @param array $userIds List of user IDs + * @param array $options Option flags; include 'userpage' and 'usertalk' + * @param string $caller The calling method + */ + public function doQuery( array $userIds, $options = [], $caller = '' ) { + global $wgActorTableSchemaMigrationStage; + + $usersToCheck = []; + $usersToQuery = []; + + $userIds = array_unique( $userIds ); + + foreach ( $userIds as $userId ) { + $userId = (int)$userId; + if ( $userId <= 0 ) { + continue; // skip anons + } + if ( isset( $this->cache[$userId]['name'] ) ) { + $usersToCheck[$userId] = $this->cache[$userId]['name']; // already have name + } else { + $usersToQuery[] = $userId; // we need to get the name + } + } + + // Lookup basic info for users not yet loaded... + if ( count( $usersToQuery ) ) { + $dbr = wfGetDB( DB_REPLICA ); + $tables = [ 'user' ]; + $conds = [ 'user_id' => $usersToQuery ]; + $fields = [ 'user_name', 'user_real_name', 'user_registration', 'user_id' ]; + $joinConds = []; + + if ( $wgActorTableSchemaMigrationStage > MIGRATION_OLD ) { + $tables[] = 'actor'; + $fields[] = 'actor_id'; + $joinConds['actor'] = [ + $wgActorTableSchemaMigrationStage === MIGRATION_NEW ? 'JOIN' : 'LEFT JOIN', + [ 'actor_user = user_id' ] + ]; + } + + $comment = __METHOD__; + if ( strval( $caller ) !== '' ) { + $comment .= "/$caller"; + } + + $res = $dbr->select( $tables, $fields, $conds, $comment, [], $joinConds ); + foreach ( $res as $row ) { // load each user into cache + $userId = (int)$row->user_id; + $this->cache[$userId]['name'] = $row->user_name; + $this->cache[$userId]['real_name'] = $row->user_real_name; + $this->cache[$userId]['registration'] = $row->user_registration; + if ( $wgActorTableSchemaMigrationStage > MIGRATION_OLD ) { + $this->cache[$userId]['actor'] = $row->actor_id; + } + $usersToCheck[$userId] = $row->user_name; + } + } + + $lb = new LinkBatch(); + foreach ( $usersToCheck as $userId => $name ) { + if ( $this->queryNeeded( $userId, 'userpage', $options ) ) { + $lb->add( NS_USER, $name ); + $this->typesCached[$userId]['userpage'] = 1; + } + if ( $this->queryNeeded( $userId, 'usertalk', $options ) ) { + $lb->add( NS_USER_TALK, $name ); + $this->typesCached[$userId]['usertalk'] = 1; + } + } + $lb->execute(); + } + + /** + * Check if a cache type is in $options and was not loaded for this user + * + * @param int $uid User ID + * @param string $type Cache type + * @param array $options Requested cache types + * @return bool + */ + protected function queryNeeded( $uid, $type, array $options ) { + return ( in_array( $type, $options ) && !isset( $this->typesCached[$uid][$type] ) ); + } +} diff --git a/www/wiki/includes/cache/localisation/LCStore.php b/www/wiki/includes/cache/localisation/LCStore.php new file mode 100644 index 00000000..cb1e2612 --- /dev/null +++ b/www/wiki/includes/cache/localisation/LCStore.php @@ -0,0 +1,66 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +/** + * Interface for the persistence layer of LocalisationCache. + * + * The persistence layer is two-level hierarchical cache. The first level + * is the language, the second level is the item or subitem. + * + * Since the data for a whole language is rebuilt in one operation, it needs + * to have a fast and atomic method for deleting or replacing all of the + * current data for a given language. The interface reflects this bulk update + * operation. Callers writing to the cache must first call startWrite(), then + * will call set() a couple of thousand times, then will call finishWrite() + * to commit the operation. When finishWrite() is called, the cache is + * expected to delete all data previously stored for that language. + * + * The values stored are PHP variables suitable for serialize(). Implementations + * of LCStore are responsible for serializing and unserializing. + */ +interface LCStore { + + /** + * Get a value. + * @param string $code Language code + * @param string $key Cache key + */ + function get( $code, $key ); + + /** + * Start a write transaction. + * @param string $code Language code + */ + function startWrite( $code ); + + /** + * Finish a write transaction. + */ + function finishWrite(); + + /** + * Set a key to a given value. startWrite() must be called before this + * is called, and finishWrite() must be called afterwards. + * @param string $key + * @param mixed $value + */ + function set( $key, $value ); + +} diff --git a/www/wiki/includes/cache/localisation/LCStoreCDB.php b/www/wiki/includes/cache/localisation/LCStoreCDB.php new file mode 100644 index 00000000..78a4863f --- /dev/null +++ b/www/wiki/includes/cache/localisation/LCStoreCDB.php @@ -0,0 +1,144 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ +use Cdb\Exception; +use Cdb\Reader; +use Cdb\Writer; + +/** + * LCStore implementation which stores data as a collection of CDB files in the + * directory given by $wgCacheDirectory. If $wgCacheDirectory is not set, this + * will throw an exception. + * + * Profiling indicates that on Linux, this implementation outperforms MySQL if + * the directory is on a local filesystem and there is ample kernel cache + * space. The performance advantage is greater when the DBA extension is + * available than it is with the PHP port. + * + * See Cdb.php and https://cr.yp.to/cdb.html + */ +class LCStoreCDB implements LCStore { + + /** @var Reader[] */ + private $readers; + + /** @var Writer */ + private $writer; + + /** @var string Current language code */ + private $currentLang; + + /** @var bool|string Cache directory. False if not set */ + private $directory; + + function __construct( $conf = [] ) { + global $wgCacheDirectory; + + if ( isset( $conf['directory'] ) ) { + $this->directory = $conf['directory']; + } else { + $this->directory = $wgCacheDirectory; + } + } + + public function get( $code, $key ) { + if ( !isset( $this->readers[$code] ) ) { + $fileName = $this->getFileName( $code ); + + $this->readers[$code] = false; + if ( file_exists( $fileName ) ) { + try { + $this->readers[$code] = Reader::open( $fileName ); + } catch ( Exception $e ) { + wfDebug( __METHOD__ . ": unable to open cdb file for reading\n" ); + } + } + } + + if ( !$this->readers[$code] ) { + return null; + } else { + $value = false; + try { + $value = $this->readers[$code]->get( $key ); + } catch ( Exception $e ) { + wfDebug( __METHOD__ . ": \Cdb\Exception caught, error message was " + . $e->getMessage() . "\n" ); + } + if ( $value === false ) { + return null; + } + + return unserialize( $value ); + } + } + + public function startWrite( $code ) { + if ( !file_exists( $this->directory ) ) { + if ( !wfMkdirParents( $this->directory, null, __METHOD__ ) ) { + throw new MWException( "Unable to create the localisation store " . + "directory \"{$this->directory}\"" ); + } + } + + // Close reader to stop permission errors on write + if ( !empty( $this->readers[$code] ) ) { + $this->readers[$code]->close(); + } + + try { + $this->writer = Writer::open( $this->getFileName( $code ) ); + } catch ( Exception $e ) { + throw new MWException( $e->getMessage() ); + } + $this->currentLang = $code; + } + + public function finishWrite() { + // Close the writer + try { + $this->writer->close(); + } catch ( Exception $e ) { + throw new MWException( $e->getMessage() ); + } + $this->writer = null; + unset( $this->readers[$this->currentLang] ); + $this->currentLang = null; + } + + public function set( $key, $value ) { + if ( is_null( $this->writer ) ) { + throw new MWException( __CLASS__ . ': must call startWrite() before calling set()' ); + } + try { + $this->writer->set( $key, serialize( $value ) ); + } catch ( Exception $e ) { + throw new MWException( $e->getMessage() ); + } + } + + protected function getFileName( $code ) { + if ( strval( $code ) === '' || strpos( $code, '/' ) !== false ) { + throw new MWException( __METHOD__ . ": Invalid language \"$code\"" ); + } + + return "{$this->directory}/l10n_cache-$code.cdb"; + } + +} diff --git a/www/wiki/includes/cache/localisation/LCStoreDB.php b/www/wiki/includes/cache/localisation/LCStoreDB.php new file mode 100644 index 00000000..c57145c0 --- /dev/null +++ b/www/wiki/includes/cache/localisation/LCStoreDB.php @@ -0,0 +1,117 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +use Wikimedia\Rdbms\IDatabase; +use Wikimedia\Rdbms\DBQueryError; + +/** + * LCStore implementation which uses the standard DB functions to store data. + * This will work on any MediaWiki installation. + */ +class LCStoreDB implements LCStore { + + /** @var string */ + private $currentLang; + /** @var bool */ + private $writesDone = false; + /** @var IDatabase */ + private $dbw; + /** @var array */ + private $batch = []; + /** @var bool */ + private $readOnly = false; + + public function get( $code, $key ) { + if ( $this->writesDone && $this->dbw ) { + $db = $this->dbw; // see the changes in finishWrite() + } else { + $db = wfGetDB( DB_REPLICA ); + } + + $value = $db->selectField( + 'l10n_cache', + 'lc_value', + [ 'lc_lang' => $code, 'lc_key' => $key ], + __METHOD__ + ); + + return ( $value !== false ) ? unserialize( $db->decodeBlob( $value ) ) : null; + } + + public function startWrite( $code ) { + if ( $this->readOnly ) { + return; + } elseif ( !$code ) { + throw new MWException( __METHOD__ . ": Invalid language \"$code\"" ); + } + + $this->dbw = wfGetDB( DB_MASTER ); + $this->readOnly = $this->dbw->isReadOnly(); + + $this->currentLang = $code; + $this->batch = []; + } + + public function finishWrite() { + if ( $this->readOnly ) { + return; + } elseif ( is_null( $this->currentLang ) ) { + throw new MWException( __CLASS__ . ': must call startWrite() before finishWrite()' ); + } + + $this->dbw->startAtomic( __METHOD__ ); + try { + $this->dbw->delete( + 'l10n_cache', + [ 'lc_lang' => $this->currentLang ], + __METHOD__ + ); + foreach ( array_chunk( $this->batch, 500 ) as $rows ) { + $this->dbw->insert( 'l10n_cache', $rows, __METHOD__ ); + } + $this->writesDone = true; + } catch ( DBQueryError $e ) { + if ( $this->dbw->wasReadOnlyError() ) { + $this->readOnly = true; // just avoid site down time + } else { + throw $e; + } + } + $this->dbw->endAtomic( __METHOD__ ); + + $this->currentLang = null; + $this->batch = []; + } + + public function set( $key, $value ) { + if ( $this->readOnly ) { + return; + } elseif ( is_null( $this->currentLang ) ) { + throw new MWException( __CLASS__ . ': must call startWrite() before set()' ); + } + + $this->batch[] = [ + 'lc_lang' => $this->currentLang, + 'lc_key' => $key, + 'lc_value' => $this->dbw->encodeBlob( serialize( $value ) ) + ]; + } + +} diff --git a/www/wiki/includes/cache/localisation/LCStoreNull.php b/www/wiki/includes/cache/localisation/LCStoreNull.php new file mode 100644 index 00000000..62f88ebf --- /dev/null +++ b/www/wiki/includes/cache/localisation/LCStoreNull.php @@ -0,0 +1,39 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +/** + * Null store backend, used to avoid DB errors during install + */ +class LCStoreNull implements LCStore { + + public function get( $code, $key ) { + return null; + } + + public function startWrite( $code ) { + } + + public function finishWrite() { + } + + public function set( $key, $value ) { + } + +} diff --git a/www/wiki/includes/cache/localisation/LCStoreStaticArray.php b/www/wiki/includes/cache/localisation/LCStoreStaticArray.php new file mode 100644 index 00000000..602c0ac4 --- /dev/null +++ b/www/wiki/includes/cache/localisation/LCStoreStaticArray.php @@ -0,0 +1,140 @@ +<?php +/** + * Localisation cache storage based on PHP files and static arrays. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +/** + * @since 1.26 + */ +class LCStoreStaticArray implements LCStore { + /** @var string|null Current language code. */ + private $currentLang = null; + + /** @var array Localisation data. */ + private $data = []; + + /** @var string File name. */ + private $fname = null; + + /** @var string Directory for cache files. */ + private $directory; + + public function __construct( $conf = [] ) { + global $wgCacheDirectory; + + if ( isset( $conf['directory'] ) ) { + $this->directory = $conf['directory']; + } else { + $this->directory = $wgCacheDirectory; + } + } + + public function startWrite( $code ) { + $this->currentLang = $code; + $this->fname = $this->directory . '/' . $code . '.l10n.php'; + $this->data[$code] = []; + if ( file_exists( $this->fname ) ) { + $this->data[$code] = require $this->fname; + } + } + + public function set( $key, $value ) { + $this->data[$this->currentLang][$key] = self::encode( $value ); + } + + /** + * Encodes a value into an array format + * + * @param mixed $value + * @return array + * @throws RuntimeException + */ + public static function encode( $value ) { + if ( is_scalar( $value ) || $value === null ) { + // [V]alue + return [ 'v', $value ]; + } + if ( is_object( $value ) ) { + // [S]erialized + return [ 's', serialize( $value ) ]; + } + if ( is_array( $value ) ) { + // [A]rray + return [ 'a', array_map( function ( $v ) { + return LCStoreStaticArray::encode( $v ); + }, $value ) ]; + } + + throw new RuntimeException( 'Cannot encode ' . var_export( $value, true ) ); + } + + /** + * Decode something that was encoded with encode + * + * @param array $encoded + * @return array|mixed + * @throws RuntimeException + */ + public static function decode( array $encoded ) { + $type = $encoded[0]; + $data = $encoded[1]; + + switch ( $type ) { + case 'v': + return $data; + case 's': + return unserialize( $data ); + case 'a': + return array_map( function ( $v ) { + return LCStoreStaticArray::decode( $v ); + }, $data ); + default: + throw new RuntimeException( + 'Unable to decode ' . var_export( $encoded, true ) ); + } + } + + public function finishWrite() { + file_put_contents( + $this->fname, + "<?php\n" . + "// Generated by LCStoreStaticArray.php -- do not edit!\n" . + "return " . + var_export( $this->data[$this->currentLang], true ) . ';' + ); + $this->currentLang = null; + $this->fname = null; + } + + public function get( $code, $key ) { + if ( !array_key_exists( $code, $this->data ) ) { + $fname = $this->directory . '/' . $code . '.l10n.php'; + if ( !file_exists( $fname ) ) { + return null; + } + $this->data[$code] = require $fname; + } + $data = $this->data[$code]; + if ( array_key_exists( $key, $data ) ) { + return self::decode( $data[$key] ); + } + return null; + } +} diff --git a/www/wiki/includes/cache/localisation/LocalisationCache.php b/www/wiki/includes/cache/localisation/LocalisationCache.php new file mode 100644 index 00000000..e7b95548 --- /dev/null +++ b/www/wiki/includes/cache/localisation/LocalisationCache.php @@ -0,0 +1,1107 @@ +<?php +/** + * Cache of the contents of localisation files. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +use CLDRPluralRuleParser\Evaluator; +use CLDRPluralRuleParser\Error as CLDRPluralRuleError; +use MediaWiki\MediaWikiServices; + +/** + * Class for caching the contents of localisation files, Messages*.php + * and *.i18n.php. + * + * An instance of this class is available using Language::getLocalisationCache(). + * + * The values retrieved from here are merged, containing items from extension + * files, core messages files and the language fallback sequence (e.g. zh-cn -> + * zh-hans -> en ). Some common errors are corrected, for example namespace + * names with spaces instead of underscores, but heavyweight processing, such + * as grammatical transformation, is done by the caller. + */ +class LocalisationCache { + const VERSION = 4; + + /** Configuration associative array */ + private $conf; + + /** + * True if recaching should only be done on an explicit call to recache(). + * Setting this reduces the overhead of cache freshness checking, which + * requires doing a stat() for every extension i18n file. + */ + private $manualRecache = false; + + /** + * True to treat all files as expired until they are regenerated by this object. + */ + private $forceRecache = false; + + /** + * The cache data. 3-d array, where the first key is the language code, + * the second key is the item key e.g. 'messages', and the third key is + * an item specific subkey index. Some items are not arrays and so for those + * items, there are no subkeys. + */ + protected $data = []; + + /** + * The persistent store object. An instance of LCStore. + * + * @var LCStore + */ + private $store; + + /** + * A 2-d associative array, code/key, where presence indicates that the item + * is loaded. Value arbitrary. + * + * For split items, if set, this indicates that all of the subitems have been + * loaded. + */ + private $loadedItems = []; + + /** + * A 3-d associative array, code/key/subkey, where presence indicates that + * the subitem is loaded. Only used for the split items, i.e. messages. + */ + private $loadedSubitems = []; + + /** + * An array where presence of a key indicates that that language has been + * initialised. Initialisation includes checking for cache expiry and doing + * any necessary updates. + */ + private $initialisedLangs = []; + + /** + * An array mapping non-existent pseudo-languages to fallback languages. This + * is filled by initShallowFallback() when data is requested from a language + * that lacks a Messages*.php file. + */ + private $shallowFallbacks = []; + + /** + * An array where the keys are codes that have been recached by this instance. + */ + private $recachedLangs = []; + + /** + * All item keys + */ + static public $allKeys = [ + 'fallback', 'namespaceNames', 'bookstoreList', + 'magicWords', 'messages', 'rtl', 'capitalizeAllNouns', 'digitTransformTable', + 'separatorTransformTable', 'minimumGroupingDigits', + 'fallback8bitEncoding', 'linkPrefixExtension', + 'linkTrail', 'linkPrefixCharset', 'namespaceAliases', + 'dateFormats', 'datePreferences', 'datePreferenceMigrationMap', + 'defaultDateFormat', 'extraUserToggles', 'specialPageAliases', + 'imageFiles', 'preloadedMessages', 'namespaceGenderAliases', + 'digitGroupingPattern', 'pluralRules', 'pluralRuleTypes', 'compiledPluralRules', + ]; + + /** + * Keys for items which consist of associative arrays, which may be merged + * by a fallback sequence. + */ + static public $mergeableMapKeys = [ 'messages', 'namespaceNames', + 'namespaceAliases', 'dateFormats', 'imageFiles', 'preloadedMessages' + ]; + + /** + * Keys for items which are a numbered array. + */ + static public $mergeableListKeys = [ 'extraUserToggles' ]; + + /** + * Keys for items which contain an array of arrays of equivalent aliases + * for each subitem. The aliases may be merged by a fallback sequence. + */ + static public $mergeableAliasListKeys = [ 'specialPageAliases' ]; + + /** + * Keys for items which contain an associative array, and may be merged if + * the primary value contains the special array key "inherit". That array + * key is removed after the first merge. + */ + static public $optionalMergeKeys = [ 'bookstoreList' ]; + + /** + * Keys for items that are formatted like $magicWords + */ + static public $magicWordKeys = [ 'magicWords' ]; + + /** + * Keys for items where the subitems are stored in the backend separately. + */ + static public $splitKeys = [ 'messages' ]; + + /** + * Keys which are loaded automatically by initLanguage() + */ + static public $preloadedKeys = [ 'dateFormats', 'namespaceNames' ]; + + /** + * Associative array of cached plural rules. The key is the language code, + * the value is an array of plural rules for that language. + */ + private $pluralRules = null; + + /** + * Associative array of cached plural rule types. The key is the language + * code, the value is an array of plural rule types for that language. For + * example, $pluralRuleTypes['ar'] = ['zero', 'one', 'two', 'few', 'many']. + * The index for each rule type matches the index for the rule in + * $pluralRules, thus allowing correlation between the two. The reason we + * don't just use the type names as the keys in $pluralRules is because + * Language::convertPlural applies the rules based on numeric order (or + * explicit numeric parameter), not based on the name of the rule type. For + * example, {{plural:count|wordform1|wordform2|wordform3}}, rather than + * {{plural:count|one=wordform1|two=wordform2|many=wordform3}}. + */ + private $pluralRuleTypes = null; + + private $mergeableKeys = null; + + /** + * For constructor parameters, see the documentation in DefaultSettings.php + * for $wgLocalisationCacheConf. + * + * @param array $conf + * @throws MWException + */ + function __construct( $conf ) { + global $wgCacheDirectory; + + $this->conf = $conf; + $storeConf = []; + if ( !empty( $conf['storeClass'] ) ) { + $storeClass = $conf['storeClass']; + } else { + switch ( $conf['store'] ) { + case 'files': + case 'file': + $storeClass = LCStoreCDB::class; + break; + case 'db': + $storeClass = LCStoreDB::class; + break; + case 'array': + $storeClass = LCStoreStaticArray::class; + break; + case 'detect': + if ( !empty( $conf['storeDirectory'] ) ) { + $storeClass = LCStoreCDB::class; + } elseif ( $wgCacheDirectory ) { + $storeConf['directory'] = $wgCacheDirectory; + $storeClass = LCStoreCDB::class; + } else { + $storeClass = LCStoreDB::class; + } + break; + default: + throw new MWException( + 'Please set $wgLocalisationCacheConf[\'store\'] to something sensible.' + ); + } + } + + wfDebugLog( 'caches', static::class . ": using store $storeClass" ); + if ( !empty( $conf['storeDirectory'] ) ) { + $storeConf['directory'] = $conf['storeDirectory']; + } + + $this->store = new $storeClass( $storeConf ); + foreach ( [ 'manualRecache', 'forceRecache' ] as $var ) { + if ( isset( $conf[$var] ) ) { + $this->$var = $conf[$var]; + } + } + } + + /** + * Returns true if the given key is mergeable, that is, if it is an associative + * array which can be merged through a fallback sequence. + * @param string $key + * @return bool + */ + public function isMergeableKey( $key ) { + if ( $this->mergeableKeys === null ) { + $this->mergeableKeys = array_flip( array_merge( + self::$mergeableMapKeys, + self::$mergeableListKeys, + self::$mergeableAliasListKeys, + self::$optionalMergeKeys, + self::$magicWordKeys + ) ); + } + + return isset( $this->mergeableKeys[$key] ); + } + + /** + * Get a cache item. + * + * Warning: this may be slow for split items (messages), since it will + * need to fetch all of the subitems from the cache individually. + * @param string $code + * @param string $key + * @return mixed + */ + public function getItem( $code, $key ) { + if ( !isset( $this->loadedItems[$code][$key] ) ) { + $this->loadItem( $code, $key ); + } + + if ( $key === 'fallback' && isset( $this->shallowFallbacks[$code] ) ) { + return $this->shallowFallbacks[$code]; + } + + return $this->data[$code][$key]; + } + + /** + * Get a subitem, for instance a single message for a given language. + * @param string $code + * @param string $key + * @param string $subkey + * @return mixed|null + */ + public function getSubitem( $code, $key, $subkey ) { + if ( !isset( $this->loadedSubitems[$code][$key][$subkey] ) && + !isset( $this->loadedItems[$code][$key] ) + ) { + $this->loadSubitem( $code, $key, $subkey ); + } + + if ( isset( $this->data[$code][$key][$subkey] ) ) { + return $this->data[$code][$key][$subkey]; + } else { + return null; + } + } + + /** + * Get the list of subitem keys for a given item. + * + * This is faster than array_keys($lc->getItem(...)) for the items listed in + * self::$splitKeys. + * + * Will return null if the item is not found, or false if the item is not an + * array. + * @param string $code + * @param string $key + * @return bool|null|string|string[] + */ + public function getSubitemList( $code, $key ) { + if ( in_array( $key, self::$splitKeys ) ) { + return $this->getSubitem( $code, 'list', $key ); + } else { + $item = $this->getItem( $code, $key ); + if ( is_array( $item ) ) { + return array_keys( $item ); + } else { + return false; + } + } + } + + /** + * Load an item into the cache. + * @param string $code + * @param string $key + */ + protected function loadItem( $code, $key ) { + if ( !isset( $this->initialisedLangs[$code] ) ) { + $this->initLanguage( $code ); + } + + // Check to see if initLanguage() loaded it for us + if ( isset( $this->loadedItems[$code][$key] ) ) { + return; + } + + if ( isset( $this->shallowFallbacks[$code] ) ) { + $this->loadItem( $this->shallowFallbacks[$code], $key ); + + return; + } + + if ( in_array( $key, self::$splitKeys ) ) { + $subkeyList = $this->getSubitem( $code, 'list', $key ); + foreach ( $subkeyList as $subkey ) { + if ( isset( $this->data[$code][$key][$subkey] ) ) { + continue; + } + $this->data[$code][$key][$subkey] = $this->getSubitem( $code, $key, $subkey ); + } + } else { + $this->data[$code][$key] = $this->store->get( $code, $key ); + } + + $this->loadedItems[$code][$key] = true; + } + + /** + * Load a subitem into the cache + * @param string $code + * @param string $key + * @param string $subkey + */ + protected function loadSubitem( $code, $key, $subkey ) { + if ( !in_array( $key, self::$splitKeys ) ) { + $this->loadItem( $code, $key ); + + return; + } + + if ( !isset( $this->initialisedLangs[$code] ) ) { + $this->initLanguage( $code ); + } + + // Check to see if initLanguage() loaded it for us + if ( isset( $this->loadedItems[$code][$key] ) || + isset( $this->loadedSubitems[$code][$key][$subkey] ) + ) { + return; + } + + if ( isset( $this->shallowFallbacks[$code] ) ) { + $this->loadSubitem( $this->shallowFallbacks[$code], $key, $subkey ); + + return; + } + + $value = $this->store->get( $code, "$key:$subkey" ); + $this->data[$code][$key][$subkey] = $value; + $this->loadedSubitems[$code][$key][$subkey] = true; + } + + /** + * Returns true if the cache identified by $code is missing or expired. + * + * @param string $code + * + * @return bool + */ + public function isExpired( $code ) { + if ( $this->forceRecache && !isset( $this->recachedLangs[$code] ) ) { + wfDebug( __METHOD__ . "($code): forced reload\n" ); + + return true; + } + + $deps = $this->store->get( $code, 'deps' ); + $keys = $this->store->get( $code, 'list' ); + $preload = $this->store->get( $code, 'preload' ); + // Different keys may expire separately for some stores + if ( $deps === null || $keys === null || $preload === null ) { + wfDebug( __METHOD__ . "($code): cache missing, need to make one\n" ); + + return true; + } + + foreach ( $deps as $dep ) { + // Because we're unserializing stuff from cache, we + // could receive objects of classes that don't exist + // anymore (e.g. uninstalled extensions) + // When this happens, always expire the cache + if ( !$dep instanceof CacheDependency || $dep->isExpired() ) { + wfDebug( __METHOD__ . "($code): cache for $code expired due to " . + get_class( $dep ) . "\n" ); + + return true; + } + } + + return false; + } + + /** + * Initialise a language in this object. Rebuild the cache if necessary. + * @param string $code + * @throws MWException + */ + protected function initLanguage( $code ) { + if ( isset( $this->initialisedLangs[$code] ) ) { + return; + } + + $this->initialisedLangs[$code] = true; + + # If the code is of the wrong form for a Messages*.php file, do a shallow fallback + if ( !Language::isValidBuiltInCode( $code ) ) { + $this->initShallowFallback( $code, 'en' ); + + return; + } + + # Recache the data if necessary + if ( !$this->manualRecache && $this->isExpired( $code ) ) { + if ( Language::isSupportedLanguage( $code ) ) { + $this->recache( $code ); + } elseif ( $code === 'en' ) { + throw new MWException( 'MessagesEn.php is missing.' ); + } else { + $this->initShallowFallback( $code, 'en' ); + } + + return; + } + + # Preload some stuff + $preload = $this->getItem( $code, 'preload' ); + if ( $preload === null ) { + if ( $this->manualRecache ) { + // No Messages*.php file. Do shallow fallback to en. + if ( $code === 'en' ) { + throw new MWException( 'No localisation cache found for English. ' . + 'Please run maintenance/rebuildLocalisationCache.php.' ); + } + $this->initShallowFallback( $code, 'en' ); + + return; + } else { + throw new MWException( 'Invalid or missing localisation cache.' ); + } + } + $this->data[$code] = $preload; + foreach ( $preload as $key => $item ) { + if ( in_array( $key, self::$splitKeys ) ) { + foreach ( $item as $subkey => $subitem ) { + $this->loadedSubitems[$code][$key][$subkey] = true; + } + } else { + $this->loadedItems[$code][$key] = true; + } + } + } + + /** + * Create a fallback from one language to another, without creating a + * complete persistent cache. + * @param string $primaryCode + * @param string $fallbackCode + */ + public function initShallowFallback( $primaryCode, $fallbackCode ) { + $this->data[$primaryCode] =& $this->data[$fallbackCode]; + $this->loadedItems[$primaryCode] =& $this->loadedItems[$fallbackCode]; + $this->loadedSubitems[$primaryCode] =& $this->loadedSubitems[$fallbackCode]; + $this->shallowFallbacks[$primaryCode] = $fallbackCode; + } + + /** + * Read a PHP file containing localisation data. + * @param string $_fileName + * @param string $_fileType + * @throws MWException + * @return array + */ + protected function readPHPFile( $_fileName, $_fileType ) { + // Disable APC caching + Wikimedia\suppressWarnings(); + $_apcEnabled = ini_set( 'apc.cache_by_default', '0' ); + Wikimedia\restoreWarnings(); + + include $_fileName; + + Wikimedia\suppressWarnings(); + ini_set( 'apc.cache_by_default', $_apcEnabled ); + Wikimedia\restoreWarnings(); + + $data = []; + if ( $_fileType == 'core' || $_fileType == 'extension' ) { + foreach ( self::$allKeys as $key ) { + // Not all keys are set in language files, so + // check they exist first + if ( isset( $$key ) ) { + $data[$key] = $$key; + } + } + } elseif ( $_fileType == 'aliases' ) { + if ( isset( $aliases ) ) { + /** @suppress PhanUndeclaredVariable */ + $data['aliases'] = $aliases; + } + } else { + throw new MWException( __METHOD__ . ": Invalid file type: $_fileType" ); + } + + return $data; + } + + /** + * Read a JSON file containing localisation messages. + * @param string $fileName Name of file to read + * @throws MWException If there is a syntax error in the JSON file + * @return array Array with a 'messages' key, or empty array if the file doesn't exist + */ + public function readJSONFile( $fileName ) { + if ( !is_readable( $fileName ) ) { + return []; + } + + $json = file_get_contents( $fileName ); + if ( $json === false ) { + return []; + } + + $data = FormatJson::decode( $json, true ); + if ( $data === null ) { + throw new MWException( __METHOD__ . ": Invalid JSON file: $fileName" ); + } + + // Remove keys starting with '@', they're reserved for metadata and non-message data + foreach ( $data as $key => $unused ) { + if ( $key === '' || $key[0] === '@' ) { + unset( $data[$key] ); + } + } + + // The JSON format only supports messages, none of the other variables, so wrap the data + return [ 'messages' => $data ]; + } + + /** + * Get the compiled plural rules for a given language from the XML files. + * @since 1.20 + * @param string $code + * @return array|null + */ + public function getCompiledPluralRules( $code ) { + $rules = $this->getPluralRules( $code ); + if ( $rules === null ) { + return null; + } + try { + $compiledRules = Evaluator::compile( $rules ); + } catch ( CLDRPluralRuleError $e ) { + wfDebugLog( 'l10n', $e->getMessage() ); + + return []; + } + + return $compiledRules; + } + + /** + * Get the plural rules for a given language from the XML files. + * Cached. + * @since 1.20 + * @param string $code + * @return array|null + */ + public function getPluralRules( $code ) { + if ( $this->pluralRules === null ) { + $this->loadPluralFiles(); + } + if ( !isset( $this->pluralRules[$code] ) ) { + return null; + } else { + return $this->pluralRules[$code]; + } + } + + /** + * Get the plural rule types for a given language from the XML files. + * Cached. + * @since 1.22 + * @param string $code + * @return array|null + */ + public function getPluralRuleTypes( $code ) { + if ( $this->pluralRuleTypes === null ) { + $this->loadPluralFiles(); + } + if ( !isset( $this->pluralRuleTypes[$code] ) ) { + return null; + } else { + return $this->pluralRuleTypes[$code]; + } + } + + /** + * Load the plural XML files. + */ + protected function loadPluralFiles() { + global $IP; + $cldrPlural = "$IP/languages/data/plurals.xml"; + $mwPlural = "$IP/languages/data/plurals-mediawiki.xml"; + // Load CLDR plural rules + $this->loadPluralFile( $cldrPlural ); + if ( file_exists( $mwPlural ) ) { + // Override or extend + $this->loadPluralFile( $mwPlural ); + } + } + + /** + * Load a plural XML file with the given filename, compile the relevant + * rules, and save the compiled rules in a process-local cache. + * + * @param string $fileName + * @throws MWException + */ + protected function loadPluralFile( $fileName ) { + // Use file_get_contents instead of DOMDocument::load (T58439) + $xml = file_get_contents( $fileName ); + if ( !$xml ) { + throw new MWException( "Unable to read plurals file $fileName" ); + } + $doc = new DOMDocument; + $doc->loadXML( $xml ); + $rulesets = $doc->getElementsByTagName( "pluralRules" ); + foreach ( $rulesets as $ruleset ) { + $codes = $ruleset->getAttribute( 'locales' ); + $rules = []; + $ruleTypes = []; + $ruleElements = $ruleset->getElementsByTagName( "pluralRule" ); + foreach ( $ruleElements as $elt ) { + $ruleType = $elt->getAttribute( 'count' ); + if ( $ruleType === 'other' ) { + // Don't record "other" rules, which have an empty condition + continue; + } + $rules[] = $elt->nodeValue; + $ruleTypes[] = $ruleType; + } + foreach ( explode( ' ', $codes ) as $code ) { + $this->pluralRules[$code] = $rules; + $this->pluralRuleTypes[$code] = $ruleTypes; + } + } + } + + /** + * Read the data from the source files for a given language, and register + * the relevant dependencies in the $deps array. If the localisation + * exists, the data array is returned, otherwise false is returned. + * + * @param string $code + * @param array &$deps + * @return array + */ + protected function readSourceFilesAndRegisterDeps( $code, &$deps ) { + global $IP; + + // This reads in the PHP i18n file with non-messages l10n data + $fileName = Language::getMessagesFileName( $code ); + if ( !file_exists( $fileName ) ) { + $data = []; + } else { + $deps[] = new FileDependency( $fileName ); + $data = $this->readPHPFile( $fileName, 'core' ); + } + + # Load CLDR plural rules for JavaScript + $data['pluralRules'] = $this->getPluralRules( $code ); + # And for PHP + $data['compiledPluralRules'] = $this->getCompiledPluralRules( $code ); + # Load plural rule types + $data['pluralRuleTypes'] = $this->getPluralRuleTypes( $code ); + + $deps['plurals'] = new FileDependency( "$IP/languages/data/plurals.xml" ); + $deps['plurals-mw'] = new FileDependency( "$IP/languages/data/plurals-mediawiki.xml" ); + + return $data; + } + + /** + * Merge two localisation values, a primary and a fallback, overwriting the + * primary value in place. + * @param string $key + * @param mixed &$value + * @param mixed $fallbackValue + */ + protected function mergeItem( $key, &$value, $fallbackValue ) { + if ( !is_null( $value ) ) { + if ( !is_null( $fallbackValue ) ) { + if ( in_array( $key, self::$mergeableMapKeys ) ) { + $value = $value + $fallbackValue; + } elseif ( in_array( $key, self::$mergeableListKeys ) ) { + $value = array_unique( array_merge( $fallbackValue, $value ) ); + } elseif ( in_array( $key, self::$mergeableAliasListKeys ) ) { + $value = array_merge_recursive( $value, $fallbackValue ); + } elseif ( in_array( $key, self::$optionalMergeKeys ) ) { + if ( !empty( $value['inherit'] ) ) { + $value = array_merge( $fallbackValue, $value ); + } + + if ( isset( $value['inherit'] ) ) { + unset( $value['inherit'] ); + } + } elseif ( in_array( $key, self::$magicWordKeys ) ) { + $this->mergeMagicWords( $value, $fallbackValue ); + } + } + } else { + $value = $fallbackValue; + } + } + + /** + * @param mixed &$value + * @param mixed $fallbackValue + */ + protected function mergeMagicWords( &$value, $fallbackValue ) { + foreach ( $fallbackValue as $magicName => $fallbackInfo ) { + if ( !isset( $value[$magicName] ) ) { + $value[$magicName] = $fallbackInfo; + } else { + $oldSynonyms = array_slice( $fallbackInfo, 1 ); + $newSynonyms = array_slice( $value[$magicName], 1 ); + $synonyms = array_values( array_unique( array_merge( + $newSynonyms, $oldSynonyms ) ) ); + $value[$magicName] = array_merge( [ $fallbackInfo[0] ], $synonyms ); + } + } + } + + /** + * Given an array mapping language code to localisation value, such as is + * found in extension *.i18n.php files, iterate through a fallback sequence + * to merge the given data with an existing primary value. + * + * Returns true if any data from the extension array was used, false + * otherwise. + * @param array $codeSequence + * @param string $key + * @param mixed &$value + * @param mixed $fallbackValue + * @return bool + */ + protected function mergeExtensionItem( $codeSequence, $key, &$value, $fallbackValue ) { + $used = false; + foreach ( $codeSequence as $code ) { + if ( isset( $fallbackValue[$code] ) ) { + $this->mergeItem( $key, $value, $fallbackValue[$code] ); + $used = true; + } + } + + return $used; + } + + /** + * Gets the combined list of messages dirs from + * core and extensions + * + * @since 1.25 + * @return array + */ + public function getMessagesDirs() { + global $IP; + + $config = MediaWikiServices::getInstance()->getMainConfig(); + $messagesDirs = $config->get( 'MessagesDirs' ); + return [ + 'core' => "$IP/languages/i18n", + 'api' => "$IP/includes/api/i18n", + 'oojs-ui' => "$IP/resources/lib/oojs-ui/i18n", + ] + $messagesDirs; + } + + /** + * Load localisation data for a given language for both core and extensions + * and save it to the persistent cache store and the process cache + * @param string $code + * @throws MWException + */ + public function recache( $code ) { + global $wgExtensionMessagesFiles; + + if ( !$code ) { + throw new MWException( "Invalid language code requested" ); + } + $this->recachedLangs[$code] = true; + + # Initial values + $initialData = array_fill_keys( self::$allKeys, null ); + $coreData = $initialData; + $deps = []; + + # Load the primary localisation from the source file + $data = $this->readSourceFilesAndRegisterDeps( $code, $deps ); + if ( $data === false ) { + wfDebug( __METHOD__ . ": no localisation file for $code, using fallback to en\n" ); + $coreData['fallback'] = 'en'; + } else { + wfDebug( __METHOD__ . ": got localisation for $code from source\n" ); + + # Merge primary localisation + foreach ( $data as $key => $value ) { + $this->mergeItem( $key, $coreData[$key], $value ); + } + } + + # Fill in the fallback if it's not there already + if ( is_null( $coreData['fallback'] ) ) { + $coreData['fallback'] = $code === 'en' ? false : 'en'; + } + if ( $coreData['fallback'] === false ) { + $coreData['fallbackSequence'] = []; + } else { + $coreData['fallbackSequence'] = array_map( 'trim', explode( ',', $coreData['fallback'] ) ); + $len = count( $coreData['fallbackSequence'] ); + + # Ensure that the sequence ends at en + if ( $coreData['fallbackSequence'][$len - 1] !== 'en' ) { + $coreData['fallbackSequence'][] = 'en'; + } + } + + $codeSequence = array_merge( [ $code ], $coreData['fallbackSequence'] ); + $messageDirs = $this->getMessagesDirs(); + + # Load non-JSON localisation data for extensions + $extensionData = array_fill_keys( $codeSequence, $initialData ); + foreach ( $wgExtensionMessagesFiles as $extension => $fileName ) { + if ( isset( $messageDirs[$extension] ) ) { + # This extension has JSON message data; skip the PHP shim + continue; + } + + $data = $this->readPHPFile( $fileName, 'extension' ); + $used = false; + + foreach ( $data as $key => $item ) { + foreach ( $codeSequence as $csCode ) { + if ( isset( $item[$csCode] ) ) { + $this->mergeItem( $key, $extensionData[$csCode][$key], $item[$csCode] ); + $used = true; + } + } + } + + if ( $used ) { + $deps[] = new FileDependency( $fileName ); + } + } + + # Load the localisation data for each fallback, then merge it into the full array + $allData = $initialData; + foreach ( $codeSequence as $csCode ) { + $csData = $initialData; + + # Load core messages and the extension localisations. + foreach ( $messageDirs as $dirs ) { + foreach ( (array)$dirs as $dir ) { + $fileName = "$dir/$csCode.json"; + $data = $this->readJSONFile( $fileName ); + + foreach ( $data as $key => $item ) { + $this->mergeItem( $key, $csData[$key], $item ); + } + + $deps[] = new FileDependency( $fileName ); + } + } + + # Merge non-JSON extension data + if ( isset( $extensionData[$csCode] ) ) { + foreach ( $extensionData[$csCode] as $key => $item ) { + $this->mergeItem( $key, $csData[$key], $item ); + } + } + + if ( $csCode === $code ) { + # Merge core data into extension data + foreach ( $coreData as $key => $item ) { + $this->mergeItem( $key, $csData[$key], $item ); + } + } else { + # Load the secondary localisation from the source file to + # avoid infinite cycles on cyclic fallbacks + $fbData = $this->readSourceFilesAndRegisterDeps( $csCode, $deps ); + if ( $fbData !== false ) { + # Only merge the keys that make sense to merge + foreach ( self::$allKeys as $key ) { + if ( !isset( $fbData[$key] ) ) { + continue; + } + + if ( is_null( $coreData[$key] ) || $this->isMergeableKey( $key ) ) { + $this->mergeItem( $key, $csData[$key], $fbData[$key] ); + } + } + } + } + + # Allow extensions an opportunity to adjust the data for this + # fallback + Hooks::run( 'LocalisationCacheRecacheFallback', [ $this, $csCode, &$csData ] ); + + # Merge the data for this fallback into the final array + if ( $csCode === $code ) { + $allData = $csData; + } else { + foreach ( self::$allKeys as $key ) { + if ( !isset( $csData[$key] ) ) { + continue; + } + + if ( is_null( $allData[$key] ) || $this->isMergeableKey( $key ) ) { + $this->mergeItem( $key, $allData[$key], $csData[$key] ); + } + } + } + } + + # Add cache dependencies for any referenced globals + $deps['wgExtensionMessagesFiles'] = new GlobalDependency( 'wgExtensionMessagesFiles' ); + // The 'MessagesDirs' config setting is used in LocalisationCache::getMessagesDirs(). + // We use the key 'wgMessagesDirs' for historical reasons. + $deps['wgMessagesDirs'] = new MainConfigDependency( 'MessagesDirs' ); + $deps['version'] = new ConstantDependency( 'LocalisationCache::VERSION' ); + + # Add dependencies to the cache entry + $allData['deps'] = $deps; + + # Replace spaces with underscores in namespace names + $allData['namespaceNames'] = str_replace( ' ', '_', $allData['namespaceNames'] ); + + # And do the same for special page aliases. $page is an array. + foreach ( $allData['specialPageAliases'] as &$page ) { + $page = str_replace( ' ', '_', $page ); + } + # Decouple the reference to prevent accidental damage + unset( $page ); + + # If there were no plural rules, return an empty array + if ( $allData['pluralRules'] === null ) { + $allData['pluralRules'] = []; + } + if ( $allData['compiledPluralRules'] === null ) { + $allData['compiledPluralRules'] = []; + } + # If there were no plural rule types, return an empty array + if ( $allData['pluralRuleTypes'] === null ) { + $allData['pluralRuleTypes'] = []; + } + + # Set the list keys + $allData['list'] = []; + foreach ( self::$splitKeys as $key ) { + $allData['list'][$key] = array_keys( $allData[$key] ); + } + # Run hooks + $purgeBlobs = true; + Hooks::run( 'LocalisationCacheRecache', [ $this, $code, &$allData, &$purgeBlobs ] ); + + if ( is_null( $allData['namespaceNames'] ) ) { + throw new MWException( __METHOD__ . ': Localisation data failed sanity check! ' . + 'Check that your languages/messages/MessagesEn.php file is intact.' ); + } + + # Set the preload key + $allData['preload'] = $this->buildPreload( $allData ); + + # Save to the process cache and register the items loaded + $this->data[$code] = $allData; + foreach ( $allData as $key => $item ) { + $this->loadedItems[$code][$key] = true; + } + + # Save to the persistent cache + $this->store->startWrite( $code ); + foreach ( $allData as $key => $value ) { + if ( in_array( $key, self::$splitKeys ) ) { + foreach ( $value as $subkey => $subvalue ) { + $this->store->set( "$key:$subkey", $subvalue ); + } + } else { + $this->store->set( $key, $value ); + } + } + $this->store->finishWrite(); + + # Clear out the MessageBlobStore + # HACK: If using a null (i.e. disabled) storage backend, we + # can't write to the MessageBlobStore either + if ( $purgeBlobs && !$this->store instanceof LCStoreNull ) { + $blobStore = new MessageBlobStore(); + $blobStore->clear(); + } + } + + /** + * Build the preload item from the given pre-cache data. + * + * The preload item will be loaded automatically, improving performance + * for the commonly-requested items it contains. + * @param array $data + * @return array + */ + protected function buildPreload( $data ) { + $preload = [ 'messages' => [] ]; + foreach ( self::$preloadedKeys as $key ) { + $preload[$key] = $data[$key]; + } + + foreach ( $data['preloadedMessages'] as $subkey ) { + if ( isset( $data['messages'][$subkey] ) ) { + $subitem = $data['messages'][$subkey]; + } else { + $subitem = null; + } + $preload['messages'][$subkey] = $subitem; + } + + return $preload; + } + + /** + * Unload the data for a given language from the object cache. + * Reduces memory usage. + * @param string $code + */ + public function unload( $code ) { + unset( $this->data[$code] ); + unset( $this->loadedItems[$code] ); + unset( $this->loadedSubitems[$code] ); + unset( $this->initialisedLangs[$code] ); + unset( $this->shallowFallbacks[$code] ); + + foreach ( $this->shallowFallbacks as $shallowCode => $fbCode ) { + if ( $fbCode === $code ) { + $this->unload( $shallowCode ); + } + } + } + + /** + * Unload all data + */ + public function unloadAll() { + foreach ( $this->initialisedLangs as $lang => $unused ) { + $this->unload( $lang ); + } + } + + /** + * Disable the storage backend + */ + public function disableBackend() { + $this->store = new LCStoreNull; + $this->manualRecache = false; + } + +} diff --git a/www/wiki/includes/cache/localisation/LocalisationCacheBulkLoad.php b/www/wiki/includes/cache/localisation/LocalisationCacheBulkLoad.php new file mode 100644 index 00000000..30c7d375 --- /dev/null +++ b/www/wiki/includes/cache/localisation/LocalisationCacheBulkLoad.php @@ -0,0 +1,126 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +/** + * A localisation cache optimised for loading large amounts of data for many + * languages. Used by rebuildLocalisationCache.php. + */ +class LocalisationCacheBulkLoad extends LocalisationCache { + + /** + * A cache of the contents of data files. + * Core files are serialized to avoid using ~1GB of RAM during a recache. + */ + private $fileCache = []; + + /** + * Most recently used languages. Uses the linked-list aspect of PHP hashtables + * to keep the most recently used language codes at the end of the array, and + * the language codes that are ready to be deleted at the beginning. + */ + private $mruLangs = []; + + /** + * Maximum number of languages that may be loaded into $this->data + */ + private $maxLoadedLangs = 10; + + /** + * @param string $fileName + * @param string $fileType + * @return array|mixed + */ + protected function readPHPFile( $fileName, $fileType ) { + $serialize = $fileType === 'core'; + if ( !isset( $this->fileCache[$fileName][$fileType] ) ) { + $data = parent::readPHPFile( $fileName, $fileType ); + + if ( $serialize ) { + $encData = serialize( $data ); + } else { + $encData = $data; + } + + $this->fileCache[$fileName][$fileType] = $encData; + + return $data; + } elseif ( $serialize ) { + return unserialize( $this->fileCache[$fileName][$fileType] ); + } else { + return $this->fileCache[$fileName][$fileType]; + } + } + + /** + * @param string $code + * @param string $key + * @return mixed + */ + public function getItem( $code, $key ) { + unset( $this->mruLangs[$code] ); + $this->mruLangs[$code] = true; + + return parent::getItem( $code, $key ); + } + + /** + * @param string $code + * @param string $key + * @param string $subkey + * @return mixed + */ + public function getSubitem( $code, $key, $subkey ) { + unset( $this->mruLangs[$code] ); + $this->mruLangs[$code] = true; + + return parent::getSubitem( $code, $key, $subkey ); + } + + /** + * @param string $code + */ + public function recache( $code ) { + parent::recache( $code ); + unset( $this->mruLangs[$code] ); + $this->mruLangs[$code] = true; + $this->trimCache(); + } + + /** + * @param string $code + */ + public function unload( $code ) { + unset( $this->mruLangs[$code] ); + parent::unload( $code ); + } + + /** + * Unload cached languages until there are less than $this->maxLoadedLangs + */ + protected function trimCache() { + while ( count( $this->data ) > $this->maxLoadedLangs && count( $this->mruLangs ) ) { + reset( $this->mruLangs ); + $code = key( $this->mruLangs ); + wfDebug( __METHOD__ . ": unloading $code\n" ); + $this->unload( $code ); + } + } + +} |