diff options
author | Yaco <franco@reevo.org> | 2019-01-06 00:20:37 -0300 |
---|---|---|
committer | Yaco <franco@reevo.org> | 2019-01-06 00:20:37 -0300 |
commit | dab3fd4a501df5c3fc30b4c9fe79bfada4415958 (patch) | |
tree | 3d1971414457ff62418a69b6a95bc4b4e93ab5e9 /www/wiki/maintenance | |
parent | 71ddfdcf197d529e0964059ad7b796913908f2b3 (diff) |
grandes avances previos al primer deployment en reevo.wiki
Diffstat (limited to 'www/wiki/maintenance')
712 files changed, 71155 insertions, 0 deletions
diff --git a/www/wiki/maintenance/.htaccess b/www/wiki/maintenance/.htaccess new file mode 100644 index 00000000..3a428827 --- /dev/null +++ b/www/wiki/maintenance/.htaccess @@ -0,0 +1 @@ +Deny from all diff --git a/www/wiki/maintenance/7zip.inc b/www/wiki/maintenance/7zip.inc new file mode 100644 index 00000000..751a1311 --- /dev/null +++ b/www/wiki/maintenance/7zip.inc @@ -0,0 +1,96 @@ +<?php +/** + * 7z stream wrapper + * + * Copyright © 2005 Brion Vibber <brion@pobox.com> + * https://www.mediawiki.org/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +/** + * Stream wrapper around 7za filter program. + * Required since we can't pass an open file resource to XMLReader->open() + * which is used for the text prefetch. + * + * @ingroup Maintenance + */ +class SevenZipStream { + protected $stream; + + private function stripPath( $path ) { + $prefix = 'mediawiki.compress.7z://'; + + return substr( $path, strlen( $prefix ) ); + } + + function stream_open( $path, $mode, $options, &$opened_path ) { + if ( $mode[0] == 'r' ) { + $options = 'e -bd -so'; + } elseif ( $mode[0] == 'w' ) { + $options = 'a -bd -si'; + } else { + return false; + } + $arg = wfEscapeShellArg( $this->stripPath( $path ) ); + $command = "7za $options $arg"; + if ( !wfIsWindows() ) { + // Suppress the stupid messages on stderr + $command .= ' 2>/dev/null'; + } + $this->stream = popen( $command, $mode[0] ); // popen() doesn't like two-letter modes + return ( $this->stream !== false ); + } + + function url_stat( $path, $flags ) { + return stat( $this->stripPath( $path ) ); + } + + // This is all so lame; there should be a default class we can extend + + function stream_close() { + return fclose( $this->stream ); + } + + function stream_flush() { + return fflush( $this->stream ); + } + + function stream_read( $count ) { + return fread( $this->stream, $count ); + } + + function stream_write( $data ) { + return fwrite( $this->stream, $data ); + } + + function stream_tell() { + return ftell( $this->stream ); + } + + function stream_eof() { + return feof( $this->stream ); + } + + function stream_seek( $offset, $whence ) { + return fseek( $this->stream, $offset, $whence ); + } +} + +stream_wrapper_register( 'mediawiki.compress.7z', 'SevenZipStream' ); diff --git a/www/wiki/maintenance/CodeCleanerGlobalsPass.inc b/www/wiki/maintenance/CodeCleanerGlobalsPass.inc new file mode 100644 index 00000000..9ccf6d63 --- /dev/null +++ b/www/wiki/maintenance/CodeCleanerGlobalsPass.inc @@ -0,0 +1,51 @@ +<?php +/** + * Psy CodeCleaner to allow PHP super globals. + * + * https://github.com/bobthecow/psysh/issues/353 + * + * Copyright © 2017 Justin Hileman <justin@justinhileman.info> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * + * @author Justin Hileman <justin@justinhileman.info> + */ + +/** + * Prefix the real command with a bunch of 'global $VAR;' commands, one for each global. + * This will make the shell behave as if it was running in the global scope (almost; + * variables created in the shell won't become global if no global variable by that name + * existed before). + */ +class CodeCleanerGlobalsPass extends \Psy\CodeCleaner\CodeCleanerPass { + private static $superglobals = [ + 'GLOBALS', '_SERVER', '_ENV', '_FILES', '_COOKIE', '_POST', '_GET', '_SESSION' + ]; + + public function beforeTraverse( array $nodes ) { + $names = []; + foreach ( array_diff( array_keys( $GLOBALS ), self::$superglobals ) as $name ) { + array_push( $names, new \PhpParser\Node\Expr\Variable( $name ) ); + } + + array_unshift( $nodes, new \PhpParser\Node\Stmt\Global_( $names ) ); + + return $nodes; + } +} diff --git a/www/wiki/maintenance/Doxyfile b/www/wiki/maintenance/Doxyfile new file mode 100644 index 00000000..7e9220c7 --- /dev/null +++ b/www/wiki/maintenance/Doxyfile @@ -0,0 +1,398 @@ +# Doxyfile 1.8.6 + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for MediaWiki. +# +# Some placeholders have been added for MediaWiki usage: +# OUTPUT_DIRECTORY = {{OUTPUT_DIRECTORY}} +# CURRENT_VERSION = {{CURRENT_VERSION}} +# STRIP_FROM_PATH = {{STRIP_FROM_PATH}} +# INPUT = {{INPUT}} +# +# To generate documentation run: php mwdocgen.php --no-extensions + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- +DOXYFILE_ENCODING = UTF-8 +PROJECT_NAME = MediaWiki +PROJECT_NUMBER = {{CURRENT_VERSION}} +PROJECT_BRIEF = +PROJECT_LOGO = +OUTPUT_DIRECTORY = {{OUTPUT_DIRECTORY}} +CREATE_SUBDIRS = NO +OUTPUT_LANGUAGE = English +BRIEF_MEMBER_DESC = YES +REPEAT_BRIEF = YES +ABBREVIATE_BRIEF = "The $name class" \ + "The $name widget" \ + "The $name file" \ + is \ + provides \ + specifies \ + contains \ + represents \ + a \ + an \ + the +ALWAYS_DETAILED_SEC = NO +INLINE_INHERITED_MEMB = NO +FULL_PATH_NAMES = YES +STRIP_FROM_PATH = {{STRIP_FROM_PATH}} +STRIP_FROM_INC_PATH = +SHORT_NAMES = NO +JAVADOC_AUTOBRIEF = YES +QT_AUTOBRIEF = NO +MULTILINE_CPP_IS_BRIEF = NO +INHERIT_DOCS = YES +SEPARATE_MEMBER_PAGES = NO +TAB_SIZE = 4 +ALIASES = "type{1}=<b> \1 </b>:" \ + "types{2}=<b> \1 </b> or <b> \2 </b>:" \ + "types{3}=<b> \1 </b>, <b> \2 </b>, or <b> \3 </b>:" \ + "arrayof{2}=<b> Array </b> of \2" \ + "null=\type{Null}" \ + "boolean=\type{Boolean}" \ + "bool=\type{Boolean}" \ + "integer=\type{Integer}" \ + "int=\type{Integer}" \ + "string=\type{String}" \ + "str=\type{String}" \ + "mixed=\type{Mixed}" \ + "access=\par Access:\n" \ + "private=\access private" \ + "protected=\access protected" \ + "public=\access public" \ + "copyright=\note" \ + "license=\note" \ + "codeCoverageIgnore=" \ + "codingStandardsIgnoreStart=" \ + "group=" \ + "covers=" \ + "dataProvider=" \ + "expectedException=" \ + "expectedExceptionMessage=" +TCL_SUBST = +OPTIMIZE_OUTPUT_FOR_C = NO +OPTIMIZE_OUTPUT_JAVA = NO +OPTIMIZE_FOR_FORTRAN = NO +OPTIMIZE_OUTPUT_VHDL = NO +EXTENSION_MAPPING = +MARKDOWN_SUPPORT = YES +AUTOLINK_SUPPORT = YES +BUILTIN_STL_SUPPORT = NO +CPP_CLI_SUPPORT = NO +SIP_SUPPORT = NO +IDL_PROPERTY_SUPPORT = YES +DISTRIBUTE_GROUP_DOC = YES +SUBGROUPING = YES +INLINE_GROUPED_CLASSES = NO +INLINE_SIMPLE_STRUCTS = NO +TYPEDEF_HIDES_STRUCT = NO +LOOKUP_CACHE_SIZE = 2 +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- +EXTRACT_ALL = YES +EXTRACT_PRIVATE = YES +EXTRACT_PACKAGE = NO +EXTRACT_STATIC = YES +EXTRACT_LOCAL_CLASSES = YES +EXTRACT_LOCAL_METHODS = NO +EXTRACT_ANON_NSPACES = NO +HIDE_UNDOC_MEMBERS = NO +HIDE_UNDOC_CLASSES = NO +HIDE_FRIEND_COMPOUNDS = NO +HIDE_IN_BODY_DOCS = YES +INTERNAL_DOCS = NO +CASE_SENSE_NAMES = YES +HIDE_SCOPE_NAMES = NO +SHOW_INCLUDE_FILES = YES +SHOW_GROUPED_MEMB_INC = NO +FORCE_LOCAL_INCLUDES = NO +INLINE_INFO = YES +SORT_MEMBER_DOCS = YES +SORT_BRIEF_DOCS = YES +SORT_MEMBERS_CTORS_1ST = NO +SORT_GROUP_NAMES = NO +SORT_BY_SCOPE_NAME = NO +STRICT_PROTO_MATCHING = NO +GENERATE_TODOLIST = YES +GENERATE_TESTLIST = YES +GENERATE_BUGLIST = YES +GENERATE_DEPRECATEDLIST= YES +ENABLED_SECTIONS = +MAX_INITIALIZER_LINES = 30 +SHOW_USED_FILES = YES +SHOW_FILES = YES +SHOW_NAMESPACES = NO +FILE_VERSION_FILTER = +LAYOUT_FILE = +CITE_BIB_FILES = +#--------------------------------------------------------------------------- +# Configuration options related to warning and progress messages +#--------------------------------------------------------------------------- +QUIET = YES +WARNINGS = YES +WARN_IF_UNDOCUMENTED = YES +WARN_IF_DOC_ERROR = YES +WARN_NO_PARAMDOC = NO +WARN_FORMAT = "$file:$line: $text" +WARN_LOGFILE = +#--------------------------------------------------------------------------- +# Configuration options related to the input files +#--------------------------------------------------------------------------- +INPUT = {{INPUT}} +INPUT_ENCODING = UTF-8 +FILE_PATTERNS = *.c \ + *.cc \ + *.cxx \ + *.cpp \ + *.c++ \ + *.d \ + *.java \ + *.ii \ + *.ixx \ + *.ipp \ + *.i++ \ + *.inl \ + *.h \ + *.hh \ + *.hxx \ + *.hpp \ + *.h++ \ + *.idl \ + *.odl \ + *.cs \ + *.php \ + *.php5 \ + *.inc \ + *.m \ + *.mm \ + *.dox \ + *.py \ + *.C \ + *.CC \ + *.C++ \ + *.II \ + *.I++ \ + *.H \ + *.HH \ + *.H++ \ + *.CS \ + *.PHP \ + *.PHP5 \ + *.M \ + *.MM \ + *.PY \ + *.txt \ + README +RECURSIVE = YES +EXCLUDE = {{EXCLUDE}} +EXCLUDE_SYMLINKS = YES +EXCLUDE_PATTERNS = LocalSettings.php \ + AdminSettings.php \ + StartProfiler.php \ + .svn \ + */.git/* \ + {{EXCLUDE_PATTERNS}} +EXCLUDE_SYMBOLS = +EXAMPLE_PATH = +EXAMPLE_PATTERNS = * +EXAMPLE_RECURSIVE = NO +IMAGE_PATH = +INPUT_FILTER = "{{INPUT_FILTER}}" +FILTER_PATTERNS = +FILTER_SOURCE_FILES = NO +FILTER_SOURCE_PATTERNS = +USE_MDFILE_AS_MAINPAGE = +#--------------------------------------------------------------------------- +# Configuration options related to source browsing +#--------------------------------------------------------------------------- +SOURCE_BROWSER = YES +INLINE_SOURCES = NO +STRIP_CODE_COMMENTS = YES +REFERENCED_BY_RELATION = YES +REFERENCES_RELATION = YES +REFERENCES_LINK_SOURCE = YES +SOURCE_TOOLTIPS = YES +USE_HTAGS = NO +VERBATIM_HEADERS = YES +#--------------------------------------------------------------------------- +# Configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- +ALPHABETICAL_INDEX = NO +COLS_IN_ALPHA_INDEX = 5 +IGNORE_PREFIX = +#--------------------------------------------------------------------------- +# Configuration options related to the HTML output +#--------------------------------------------------------------------------- +GENERATE_HTML = YES +HTML_OUTPUT = html +HTML_FILE_EXTENSION = .html +HTML_HEADER = +HTML_FOOTER = +HTML_STYLESHEET = +HTML_EXTRA_STYLESHEET = +HTML_EXTRA_FILES = +HTML_COLORSTYLE_HUE = 220 +HTML_COLORSTYLE_SAT = 100 +HTML_COLORSTYLE_GAMMA = 80 +HTML_TIMESTAMP = YES +HTML_DYNAMIC_SECTIONS = NO +HTML_INDEX_NUM_ENTRIES = 100 +GENERATE_DOCSET = NO +DOCSET_FEEDNAME = "Doxygen generated docs" +DOCSET_BUNDLE_ID = org.doxygen.Project +DOCSET_PUBLISHER_ID = org.doxygen.Publisher +DOCSET_PUBLISHER_NAME = Publisher +GENERATE_HTMLHELP = NO +CHM_FILE = +HHC_LOCATION = +GENERATE_CHI = NO +CHM_INDEX_ENCODING = +BINARY_TOC = NO +TOC_EXPAND = YES +GENERATE_QHP = NO +QCH_FILE = +QHP_NAMESPACE = org.doxygen.Project +QHP_VIRTUAL_FOLDER = doc +QHP_CUST_FILTER_NAME = +QHP_CUST_FILTER_ATTRS = +QHP_SECT_FILTER_ATTRS = +QHG_LOCATION = +GENERATE_ECLIPSEHELP = NO +ECLIPSE_DOC_ID = org.doxygen.Project +DISABLE_INDEX = NO +GENERATE_TREEVIEW = YES +ENUM_VALUES_PER_LINE = 4 +TREEVIEW_WIDTH = 250 +EXT_LINKS_IN_WINDOW = NO +FORMULA_FONTSIZE = 10 +FORMULA_TRANSPARENT = YES +USE_MATHJAX = NO +MATHJAX_FORMAT = HTML-CSS +MATHJAX_RELPATH = http://www.mathjax.org/mathjax +MATHJAX_EXTENSIONS = +MATHJAX_CODEFILE = +SEARCHENGINE = YES +SERVER_BASED_SEARCH = YES +EXTERNAL_SEARCH = NO +SEARCHENGINE_URL = +SEARCHDATA_FILE = searchdata.xml +EXTERNAL_SEARCH_ID = +EXTRA_SEARCH_MAPPINGS = +#--------------------------------------------------------------------------- +# Configuration options related to the LaTeX output +#--------------------------------------------------------------------------- +GENERATE_LATEX = NO +LATEX_OUTPUT = latex +LATEX_CMD_NAME = latex +MAKEINDEX_CMD_NAME = makeindex +COMPACT_LATEX = NO +PAPER_TYPE = a4wide +EXTRA_PACKAGES = +LATEX_HEADER = +LATEX_FOOTER = +LATEX_EXTRA_FILES = +PDF_HYPERLINKS = YES +USE_PDFLATEX = YES +LATEX_BATCHMODE = NO +LATEX_HIDE_INDICES = NO +LATEX_SOURCE_CODE = NO +LATEX_BIB_STYLE = plain +#--------------------------------------------------------------------------- +# Configuration options related to the RTF output +#--------------------------------------------------------------------------- +GENERATE_RTF = NO +RTF_OUTPUT = rtf +COMPACT_RTF = NO +RTF_HYPERLINKS = NO +RTF_STYLESHEET_FILE = +RTF_EXTENSIONS_FILE = +#--------------------------------------------------------------------------- +# Configuration options related to the man page output +#--------------------------------------------------------------------------- +GENERATE_MAN = {{GENERATE_MAN}} +MAN_OUTPUT = man +MAN_EXTENSION = .3 +MAN_LINKS = NO +#--------------------------------------------------------------------------- +# Configuration options related to the XML output +#--------------------------------------------------------------------------- +GENERATE_XML = NO +XML_OUTPUT = xml +XML_PROGRAMLISTING = YES +#--------------------------------------------------------------------------- +# Configuration options related to the DOCBOOK output +#--------------------------------------------------------------------------- +GENERATE_DOCBOOK = NO +DOCBOOK_OUTPUT = docbook +#--------------------------------------------------------------------------- +# Configuration options for the AutoGen Definitions output +#--------------------------------------------------------------------------- +GENERATE_AUTOGEN_DEF = NO +#--------------------------------------------------------------------------- +# Configuration options related to the Perl module output +#--------------------------------------------------------------------------- +GENERATE_PERLMOD = NO +PERLMOD_LATEX = NO +PERLMOD_PRETTY = YES +PERLMOD_MAKEVAR_PREFIX = +#--------------------------------------------------------------------------- +# Configuration options related to the preprocessor +#--------------------------------------------------------------------------- +ENABLE_PREPROCESSING = YES +MACRO_EXPANSION = NO +EXPAND_ONLY_PREDEF = NO +SEARCH_INCLUDES = YES +INCLUDE_PATH = +INCLUDE_FILE_PATTERNS = +PREDEFINED = +EXPAND_AS_DEFINED = +SKIP_FUNCTION_MACROS = YES +#--------------------------------------------------------------------------- +# Configuration options related to external references +#--------------------------------------------------------------------------- +TAGFILES = +GENERATE_TAGFILE = {{OUTPUT_DIRECTORY}}/html/tagfile.xml +ALLEXTERNALS = NO +EXTERNAL_GROUPS = YES +EXTERNAL_PAGES = YES +PERL_PATH = /usr/bin/perl +#--------------------------------------------------------------------------- +# Configuration options related to the dot tool +#--------------------------------------------------------------------------- +CLASS_DIAGRAMS = NO +MSCGEN_PATH = +DIA_PATH = +HIDE_UNDOC_RELATIONS = YES +HAVE_DOT = {{HAVE_DOT}} +DOT_NUM_THREADS = 0 +DOT_FONTNAME = Helvetica +DOT_FONTSIZE = 10 +DOT_FONTPATH = +CLASS_GRAPH = YES +COLLABORATION_GRAPH = YES +GROUP_GRAPHS = YES +UML_LOOK = NO +UML_LIMIT_NUM_FIELDS = 10 +TEMPLATE_RELATIONS = NO +INCLUDE_GRAPH = YES +INCLUDED_BY_GRAPH = YES +CALL_GRAPH = NO +CALLER_GRAPH = NO +GRAPHICAL_HIERARCHY = YES +DIRECTORY_GRAPH = YES +DOT_IMAGE_FORMAT = png +INTERACTIVE_SVG = NO +DOT_PATH = +DOTFILE_DIRS = +MSCFILE_DIRS = +DIAFILE_DIRS = +DOT_GRAPH_MAX_NODES = 50 +MAX_DOT_GRAPH_DEPTH = 1000 +DOT_TRANSPARENT = NO +DOT_MULTI_TARGETS = YES +GENERATE_LEGEND = YES +DOT_CLEANUP = YES diff --git a/www/wiki/maintenance/Maintenance.php b/www/wiki/maintenance/Maintenance.php new file mode 100644 index 00000000..ecbbb851 --- /dev/null +++ b/www/wiki/maintenance/Maintenance.php @@ -0,0 +1,1626 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @defgroup Maintenance Maintenance + */ + +// Bail on old versions of PHP, or if composer has not been run yet to install +// dependencies. +require_once __DIR__ . '/../includes/PHPVersionCheck.php'; +wfEntryPointCheck( 'cli' ); + +use Wikimedia\Rdbms\DBReplicationWaitError; + +/** + * @defgroup MaintenanceArchive Maintenance archives + * @ingroup Maintenance + */ + +// Define this so scripts can easily find doMaintenance.php +define( 'RUN_MAINTENANCE_IF_MAIN', __DIR__ . '/doMaintenance.php' ); +define( 'DO_MAINTENANCE', RUN_MAINTENANCE_IF_MAIN ); // original name, harmless + +$maintClass = false; + +use Wikimedia\Rdbms\IDatabase; +use MediaWiki\Logger\LoggerFactory; +use MediaWiki\MediaWikiServices; +use Wikimedia\Rdbms\LBFactory; +use Wikimedia\Rdbms\IMaintainableDatabase; + +/** + * Abstract maintenance class for quickly writing and churning out + * maintenance scripts with minimal effort. All that _must_ be defined + * is the execute() method. See docs/maintenance.txt for more info + * and a quick demo of how to use it. + * + * @author Chad Horohoe <chad@anyonecanedit.org> + * @since 1.16 + * @ingroup Maintenance + */ +abstract class Maintenance { + /** + * Constants for DB access type + * @see Maintenance::getDbType() + */ + const DB_NONE = 0; + const DB_STD = 1; + const DB_ADMIN = 2; + + // Const for getStdin() + const STDIN_ALL = 'all'; + + // This is the desired params + protected $mParams = []; + + // Array of mapping short parameters to long ones + protected $mShortParamsMap = []; + + // Array of desired args + protected $mArgList = []; + + // This is the list of options that were actually passed + protected $mOptions = []; + + // This is the list of arguments that were actually passed + protected $mArgs = []; + + // Name of the script currently running + protected $mSelf; + + // Special vars for params that are always used + protected $mQuiet = false; + protected $mDbUser, $mDbPass; + + // A description of the script, children should change this via addDescription() + protected $mDescription = ''; + + // Have we already loaded our user input? + protected $mInputLoaded = false; + + /** + * Batch size. If a script supports this, they should set + * a default with setBatchSize() + * + * @var int + */ + protected $mBatchSize = null; + + // Generic options added by addDefaultParams() + private $mGenericParameters = []; + // Generic options which might or not be supported by the script + private $mDependantParameters = []; + + /** + * Used by getDB() / setDB() + * @var IMaintainableDatabase + */ + private $mDb = null; + + /** @var float UNIX timestamp */ + private $lastReplicationWait = 0.0; + + /** + * Used when creating separate schema files. + * @var resource + */ + public $fileHandle; + + /** + * Accessible via getConfig() + * + * @var Config + */ + private $config; + + /** + * @see Maintenance::requireExtension + * @var array + */ + private $requiredExtensions = []; + + /** + * Used to read the options in the order they were passed. + * Useful for option chaining (Ex. dumpBackup.php). It will + * be an empty array if the options are passed in through + * loadParamsAndArgs( $self, $opts, $args ). + * + * This is an array of arrays where + * 0 => the option and 1 => parameter value. + * + * @var array + */ + public $orderedOptions = []; + + /** + * Default constructor. Children should call this *first* if implementing + * their own constructors + */ + public function __construct() { + // Setup $IP, using MW_INSTALL_PATH if it exists + global $IP; + $IP = strval( getenv( 'MW_INSTALL_PATH' ) ) !== '' + ? getenv( 'MW_INSTALL_PATH' ) + : realpath( __DIR__ . '/..' ); + + $this->addDefaultParams(); + register_shutdown_function( [ $this, 'outputChanneled' ], false ); + } + + /** + * Should we execute the maintenance script, or just allow it to be included + * as a standalone class? It checks that the call stack only includes this + * function and "requires" (meaning was called from the file scope) + * + * @return bool + */ + public static function shouldExecute() { + global $wgCommandLineMode; + + if ( !function_exists( 'debug_backtrace' ) ) { + // If someone has a better idea... + return $wgCommandLineMode; + } + + $bt = debug_backtrace(); + $count = count( $bt ); + if ( $count < 2 ) { + return false; // sanity + } + if ( $bt[0]['class'] !== 'Maintenance' || $bt[0]['function'] !== 'shouldExecute' ) { + return false; // last call should be to this function + } + $includeFuncs = [ 'require_once', 'require', 'include', 'include_once' ]; + for ( $i = 1; $i < $count; $i++ ) { + if ( !in_array( $bt[$i]['function'], $includeFuncs ) ) { + return false; // previous calls should all be "requires" + } + } + + return true; + } + + /** + * Do the actual work. All child classes will need to implement this + */ + abstract public function execute(); + + /** + * Add a parameter to the script. Will be displayed on --help + * with the associated description + * + * @param string $name The name of the param (help, version, etc) + * @param string $description The description of the param to show on --help + * @param bool $required Is the param required? + * @param bool $withArg Is an argument required with this option? + * @param string|bool $shortName Character to use as short name + * @param bool $multiOccurrence Can this option be passed multiple times? + */ + protected function addOption( $name, $description, $required = false, + $withArg = false, $shortName = false, $multiOccurrence = false + ) { + $this->mParams[$name] = [ + 'desc' => $description, + 'require' => $required, + 'withArg' => $withArg, + 'shortName' => $shortName, + 'multiOccurrence' => $multiOccurrence + ]; + + if ( $shortName !== false ) { + $this->mShortParamsMap[$shortName] = $name; + } + } + + /** + * Checks to see if a particular param exists. + * @param string $name The name of the param + * @return bool + */ + protected function hasOption( $name ) { + return isset( $this->mOptions[$name] ); + } + + /** + * Get an option, or return the default. + * + * If the option was added to support multiple occurrences, + * this will return an array. + * + * @param string $name The name of the param + * @param mixed $default Anything you want, default null + * @return mixed + */ + protected function getOption( $name, $default = null ) { + if ( $this->hasOption( $name ) ) { + return $this->mOptions[$name]; + } else { + // Set it so we don't have to provide the default again + $this->mOptions[$name] = $default; + + return $this->mOptions[$name]; + } + } + + /** + * Add some args that are needed + * @param string $arg Name of the arg, like 'start' + * @param string $description Short description of the arg + * @param bool $required Is this required? + */ + protected function addArg( $arg, $description, $required = true ) { + $this->mArgList[] = [ + 'name' => $arg, + 'desc' => $description, + 'require' => $required + ]; + } + + /** + * Remove an option. Useful for removing options that won't be used in your script. + * @param string $name The option to remove. + */ + protected function deleteOption( $name ) { + unset( $this->mParams[$name] ); + } + + /** + * Set the description text. + * @param string $text The text of the description + */ + protected function addDescription( $text ) { + $this->mDescription = $text; + } + + /** + * Does a given argument exist? + * @param int $argId The integer value (from zero) for the arg + * @return bool + */ + protected function hasArg( $argId = 0 ) { + return isset( $this->mArgs[$argId] ); + } + + /** + * Get an argument. + * @param int $argId The integer value (from zero) for the arg + * @param mixed $default The default if it doesn't exist + * @return mixed + */ + protected function getArg( $argId = 0, $default = null ) { + return $this->hasArg( $argId ) ? $this->mArgs[$argId] : $default; + } + + /** + * Set the batch size. + * @param int $s The number of operations to do in a batch + */ + protected function setBatchSize( $s = 0 ) { + $this->mBatchSize = $s; + + // If we support $mBatchSize, show the option. + // Used to be in addDefaultParams, but in order for that to + // work, subclasses would have to call this function in the constructor + // before they called parent::__construct which is just weird + // (and really wasn't done). + if ( $this->mBatchSize ) { + $this->addOption( 'batch-size', 'Run this many operations ' . + 'per batch, default: ' . $this->mBatchSize, false, true ); + if ( isset( $this->mParams['batch-size'] ) ) { + // This seems a little ugly... + $this->mDependantParameters['batch-size'] = $this->mParams['batch-size']; + } + } + } + + /** + * Get the script's name + * @return string + */ + public function getName() { + return $this->mSelf; + } + + /** + * Return input from stdin. + * @param int $len The number of bytes to read. If null, just return the handle. + * Maintenance::STDIN_ALL returns the full length + * @return mixed + */ + protected function getStdin( $len = null ) { + if ( $len == self::STDIN_ALL ) { + return file_get_contents( 'php://stdin' ); + } + $f = fopen( 'php://stdin', 'rt' ); + if ( !$len ) { + return $f; + } + $input = fgets( $f, $len ); + fclose( $f ); + + return rtrim( $input ); + } + + /** + * @return bool + */ + public function isQuiet() { + return $this->mQuiet; + } + + /** + * Throw some output to the user. Scripts can call this with no fears, + * as we handle all --quiet stuff here + * @param string $out The text to show to the user + * @param mixed $channel Unique identifier for the channel. See function outputChanneled. + */ + protected function output( $out, $channel = null ) { + if ( $this->mQuiet ) { + return; + } + if ( $channel === null ) { + $this->cleanupChanneled(); + print $out; + } else { + $out = preg_replace( '/\n\z/', '', $out ); + $this->outputChanneled( $out, $channel ); + } + } + + /** + * Throw an error to the user. Doesn't respect --quiet, so don't use + * this for non-error output + * @param string $err The error to display + * @param int $die If > 0, go ahead and die out using this int as the code + */ + protected function error( $err, $die = 0 ) { + $this->outputChanneled( false ); + if ( PHP_SAPI == 'cli' ) { + fwrite( STDERR, $err . "\n" ); + } else { + print $err; + } + $die = intval( $die ); + if ( $die > 0 ) { + die( $die ); + } + } + + private $atLineStart = true; + private $lastChannel = null; + + /** + * Clean up channeled output. Output a newline if necessary. + */ + public function cleanupChanneled() { + if ( !$this->atLineStart ) { + print "\n"; + $this->atLineStart = true; + } + } + + /** + * Message outputter with channeled message support. Messages on the + * same channel are concatenated, but any intervening messages in another + * channel start a new line. + * @param string $msg The message without trailing newline + * @param string $channel Channel identifier or null for no + * channel. Channel comparison uses ===. + */ + public function outputChanneled( $msg, $channel = null ) { + if ( $msg === false ) { + $this->cleanupChanneled(); + + return; + } + + // End the current line if necessary + if ( !$this->atLineStart && $channel !== $this->lastChannel ) { + print "\n"; + } + + print $msg; + + $this->atLineStart = false; + if ( $channel === null ) { + // For unchanneled messages, output trailing newline immediately + print "\n"; + $this->atLineStart = true; + } + $this->lastChannel = $channel; + } + + /** + * Does the script need different DB access? By default, we give Maintenance + * scripts normal rights to the DB. Sometimes, a script needs admin rights + * access for a reason and sometimes they want no access. Subclasses should + * override and return one of the following values, as needed: + * Maintenance::DB_NONE - For no DB access at all + * Maintenance::DB_STD - For normal DB access, default + * Maintenance::DB_ADMIN - For admin DB access + * @return int + */ + public function getDbType() { + return self::DB_STD; + } + + /** + * Add the default parameters to the scripts + */ + protected function addDefaultParams() { + # Generic (non script dependant) options: + + $this->addOption( 'help', 'Display this help message', false, false, 'h' ); + $this->addOption( 'quiet', 'Whether to supress non-error output', false, false, 'q' ); + $this->addOption( 'conf', 'Location of LocalSettings.php, if not default', false, true ); + $this->addOption( 'wiki', 'For specifying the wiki ID', false, true ); + $this->addOption( 'globals', 'Output globals at the end of processing for debugging' ); + $this->addOption( + 'memory-limit', + 'Set a specific memory limit for the script, ' + . '"max" for no limit or "default" to avoid changing it' + ); + $this->addOption( 'server', "The protocol and server name to use in URLs, e.g. " . + "http://en.wikipedia.org. This is sometimes necessary because " . + "server name detection may fail in command line scripts.", false, true ); + $this->addOption( 'profiler', 'Profiler output format (usually "text")', false, true ); + + # Save generic options to display them separately in help + $this->mGenericParameters = $this->mParams; + + # Script dependant options: + + // If we support a DB, show the options + if ( $this->getDbType() > 0 ) { + $this->addOption( 'dbuser', 'The DB user to use for this script', false, true ); + $this->addOption( 'dbpass', 'The password to use for this script', false, true ); + } + + # Save additional script dependant options to display + # them separately in help + $this->mDependantParameters = array_diff_key( $this->mParams, $this->mGenericParameters ); + } + + /** + * @since 1.24 + * @return Config + */ + public function getConfig() { + if ( $this->config === null ) { + $this->config = MediaWikiServices::getInstance()->getMainConfig(); + } + + return $this->config; + } + + /** + * @since 1.24 + * @param Config $config + */ + public function setConfig( Config $config ) { + $this->config = $config; + } + + /** + * Indicate that the specified extension must be + * loaded before the script can run. + * + * This *must* be called in the constructor. + * + * @since 1.28 + * @param string $name + */ + protected function requireExtension( $name ) { + $this->requiredExtensions[] = $name; + } + + /** + * Verify that the required extensions are installed + * + * @since 1.28 + */ + public function checkRequiredExtensions() { + $registry = ExtensionRegistry::getInstance(); + $missing = []; + foreach ( $this->requiredExtensions as $name ) { + if ( !$registry->isLoaded( $name ) ) { + $missing[] = $name; + } + } + + if ( $missing ) { + $joined = implode( ', ', $missing ); + $msg = "The following extensions are required to be installed " + . "for this script to run: $joined. Please enable them and then try again."; + $this->error( $msg, 1 ); + } + } + + /** + * Set triggers like when to try to run deferred updates + * @since 1.28 + */ + public function setAgentAndTriggers() { + if ( function_exists( 'posix_getpwuid' ) ) { + $agent = posix_getpwuid( posix_geteuid() )['name']; + } else { + $agent = 'sysadmin'; + } + $agent .= '@' . wfHostname(); + + $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory(); + // Add a comment for easy SHOW PROCESSLIST interpretation + $lbFactory->setAgentName( + mb_strlen( $agent ) > 15 ? mb_substr( $agent, 0, 15 ) . '...' : $agent + ); + self::setLBFactoryTriggers( $lbFactory ); + } + + /** + * @param LBFactory $LBFactory + * @since 1.28 + */ + public static function setLBFactoryTriggers( LBFactory $LBFactory ) { + // Hook into period lag checks which often happen in long-running scripts + $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory(); + $lbFactory->setWaitForReplicationListener( + __METHOD__, + function () { + global $wgCommandLineMode; + // Check config in case of JobRunner and unit tests + if ( $wgCommandLineMode ) { + DeferredUpdates::tryOpportunisticExecute( 'run' ); + } + } + ); + // Check for other windows to run them. A script may read or do a few writes + // to the master but mostly be writing to something else, like a file store. + $lbFactory->getMainLB()->setTransactionListener( + __METHOD__, + function ( $trigger ) { + global $wgCommandLineMode; + // Check config in case of JobRunner and unit tests + if ( $wgCommandLineMode && $trigger === IDatabase::TRIGGER_COMMIT ) { + DeferredUpdates::tryOpportunisticExecute( 'run' ); + } + } + ); + } + + /** + * Run a child maintenance script. Pass all of the current arguments + * to it. + * @param string $maintClass A name of a child maintenance class + * @param string $classFile Full path of where the child is + * @return Maintenance + */ + public function runChild( $maintClass, $classFile = null ) { + // Make sure the class is loaded first + if ( !class_exists( $maintClass ) ) { + if ( $classFile ) { + require_once $classFile; + } + if ( !class_exists( $maintClass ) ) { + $this->error( "Cannot spawn child: $maintClass" ); + } + } + + /** + * @var $child Maintenance + */ + $child = new $maintClass(); + $child->loadParamsAndArgs( $this->mSelf, $this->mOptions, $this->mArgs ); + if ( !is_null( $this->mDb ) ) { + $child->setDB( $this->mDb ); + } + + return $child; + } + + /** + * Do some sanity checking and basic setup + */ + public function setup() { + global $IP, $wgCommandLineMode, $wgRequestTime; + + # Abort if called from a web server + if ( isset( $_SERVER ) && isset( $_SERVER['REQUEST_METHOD'] ) ) { + $this->error( 'This script must be run from the command line', true ); + } + + if ( $IP === null ) { + $this->error( "\$IP not set, aborting!\n" . + '(Did you forget to call parent::__construct() in your maintenance script?)', 1 ); + } + + # Make sure we can handle script parameters + if ( !defined( 'HPHP_VERSION' ) && !ini_get( 'register_argc_argv' ) ) { + $this->error( 'Cannot get command line arguments, register_argc_argv is set to false', true ); + } + + // Send PHP warnings and errors to stderr instead of stdout. + // This aids in diagnosing problems, while keeping messages + // out of redirected output. + if ( ini_get( 'display_errors' ) ) { + ini_set( 'display_errors', 'stderr' ); + } + + $this->loadParamsAndArgs(); + $this->maybeHelp(); + + # Set the memory limit + # Note we need to set it again later in cache LocalSettings changed it + $this->adjustMemoryLimit(); + + # Set max execution time to 0 (no limit). PHP.net says that + # "When running PHP from the command line the default setting is 0." + # But sometimes this doesn't seem to be the case. + ini_set( 'max_execution_time', 0 ); + + $wgRequestTime = microtime( true ); + + # Define us as being in MediaWiki + define( 'MEDIAWIKI', true ); + + $wgCommandLineMode = true; + + # Turn off output buffering if it's on + while ( ob_get_level() > 0 ) { + ob_end_flush(); + } + + $this->validateParamsAndArgs(); + } + + /** + * Normally we disable the memory_limit when running admin scripts. + * Some scripts may wish to actually set a limit, however, to avoid + * blowing up unexpectedly. We also support a --memory-limit option, + * to allow sysadmins to explicitly set one if they'd prefer to override + * defaults (or for people using Suhosin which yells at you for trying + * to disable the limits) + * @return string + */ + public function memoryLimit() { + $limit = $this->getOption( 'memory-limit', 'max' ); + $limit = trim( $limit, "\" '" ); // trim quotes in case someone misunderstood + return $limit; + } + + /** + * Adjusts PHP's memory limit to better suit our needs, if needed. + */ + protected function adjustMemoryLimit() { + $limit = $this->memoryLimit(); + if ( $limit == 'max' ) { + $limit = -1; // no memory limit + } + if ( $limit != 'default' ) { + ini_set( 'memory_limit', $limit ); + } + } + + /** + * Activate the profiler (assuming $wgProfiler is set) + */ + protected function activateProfiler() { + global $wgProfiler, $wgProfileLimit, $wgTrxProfilerLimits; + + $output = $this->getOption( 'profiler' ); + if ( !$output ) { + return; + } + + if ( is_array( $wgProfiler ) && isset( $wgProfiler['class'] ) ) { + $class = $wgProfiler['class']; + /** @var Profiler $profiler */ + $profiler = new $class( + [ 'sampling' => 1, 'output' => [ $output ] ] + + $wgProfiler + + [ 'threshold' => $wgProfileLimit ] + ); + $profiler->setTemplated( true ); + Profiler::replaceStubInstance( $profiler ); + } + + $trxProfiler = Profiler::instance()->getTransactionProfiler(); + $trxProfiler->setLogger( LoggerFactory::getInstance( 'DBPerformance' ) ); + $trxProfiler->setExpectations( $wgTrxProfilerLimits['Maintenance'], __METHOD__ ); + } + + /** + * Clear all params and arguments. + */ + public function clearParamsAndArgs() { + $this->mOptions = []; + $this->mArgs = []; + $this->mInputLoaded = false; + } + + /** + * Load params and arguments from a given array + * of command-line arguments + * + * @since 1.27 + * @param array $argv + */ + public function loadWithArgv( $argv ) { + $options = []; + $args = []; + $this->orderedOptions = []; + + # Parse arguments + for ( $arg = reset( $argv ); $arg !== false; $arg = next( $argv ) ) { + if ( $arg == '--' ) { + # End of options, remainder should be considered arguments + $arg = next( $argv ); + while ( $arg !== false ) { + $args[] = $arg; + $arg = next( $argv ); + } + break; + } elseif ( substr( $arg, 0, 2 ) == '--' ) { + # Long options + $option = substr( $arg, 2 ); + if ( isset( $this->mParams[$option] ) && $this->mParams[$option]['withArg'] ) { + $param = next( $argv ); + if ( $param === false ) { + $this->error( "\nERROR: $option parameter needs a value after it\n" ); + $this->maybeHelp( true ); + } + + $this->setParam( $options, $option, $param ); + } else { + $bits = explode( '=', $option, 2 ); + if ( count( $bits ) > 1 ) { + $option = $bits[0]; + $param = $bits[1]; + } else { + $param = 1; + } + + $this->setParam( $options, $option, $param ); + } + } elseif ( $arg == '-' ) { + # Lonely "-", often used to indicate stdin or stdout. + $args[] = $arg; + } elseif ( substr( $arg, 0, 1 ) == '-' ) { + # Short options + $argLength = strlen( $arg ); + for ( $p = 1; $p < $argLength; $p++ ) { + $option = $arg[$p]; + if ( !isset( $this->mParams[$option] ) && isset( $this->mShortParamsMap[$option] ) ) { + $option = $this->mShortParamsMap[$option]; + } + + if ( isset( $this->mParams[$option]['withArg'] ) && $this->mParams[$option]['withArg'] ) { + $param = next( $argv ); + if ( $param === false ) { + $this->error( "\nERROR: $option parameter needs a value after it\n" ); + $this->maybeHelp( true ); + } + $this->setParam( $options, $option, $param ); + } else { + $this->setParam( $options, $option, 1 ); + } + } + } else { + $args[] = $arg; + } + } + + $this->mOptions = $options; + $this->mArgs = $args; + $this->loadSpecialVars(); + $this->mInputLoaded = true; + } + + /** + * Helper function used solely by loadParamsAndArgs + * to prevent code duplication + * + * This sets the param in the options array based on + * whether or not it can be specified multiple times. + * + * @since 1.27 + * @param array $options + * @param string $option + * @param mixed $value + */ + private function setParam( &$options, $option, $value ) { + $this->orderedOptions[] = [ $option, $value ]; + + if ( isset( $this->mParams[$option] ) ) { + $multi = $this->mParams[$option]['multiOccurrence']; + } else { + $multi = false; + } + $exists = array_key_exists( $option, $options ); + if ( $multi && $exists ) { + $options[$option][] = $value; + } elseif ( $multi ) { + $options[$option] = [ $value ]; + } elseif ( !$exists ) { + $options[$option] = $value; + } else { + $this->error( "\nERROR: $option parameter given twice\n" ); + $this->maybeHelp( true ); + } + } + + /** + * Process command line arguments + * $mOptions becomes an array with keys set to the option names + * $mArgs becomes a zero-based array containing the non-option arguments + * + * @param string $self The name of the script, if any + * @param array $opts An array of options, in form of key=>value + * @param array $args An array of command line arguments + */ + public function loadParamsAndArgs( $self = null, $opts = null, $args = null ) { + # If we were given opts or args, set those and return early + if ( $self ) { + $this->mSelf = $self; + $this->mInputLoaded = true; + } + if ( $opts ) { + $this->mOptions = $opts; + $this->mInputLoaded = true; + } + if ( $args ) { + $this->mArgs = $args; + $this->mInputLoaded = true; + } + + # If we've already loaded input (either by user values or from $argv) + # skip on loading it again. The array_shift() will corrupt values if + # it's run again and again + if ( $this->mInputLoaded ) { + $this->loadSpecialVars(); + + return; + } + + global $argv; + $this->mSelf = $argv[0]; + $this->loadWithArgv( array_slice( $argv, 1 ) ); + } + + /** + * Run some validation checks on the params, etc + */ + protected function validateParamsAndArgs() { + $die = false; + # Check to make sure we've got all the required options + foreach ( $this->mParams as $opt => $info ) { + if ( $info['require'] && !$this->hasOption( $opt ) ) { + $this->error( "Param $opt required!" ); + $die = true; + } + } + # Check arg list too + foreach ( $this->mArgList as $k => $info ) { + if ( $info['require'] && !$this->hasArg( $k ) ) { + $this->error( 'Argument <' . $info['name'] . '> required!' ); + $die = true; + } + } + + if ( $die ) { + $this->maybeHelp( true ); + } + } + + /** + * Handle the special variables that are global to all scripts + */ + protected function loadSpecialVars() { + if ( $this->hasOption( 'dbuser' ) ) { + $this->mDbUser = $this->getOption( 'dbuser' ); + } + if ( $this->hasOption( 'dbpass' ) ) { + $this->mDbPass = $this->getOption( 'dbpass' ); + } + if ( $this->hasOption( 'quiet' ) ) { + $this->mQuiet = true; + } + if ( $this->hasOption( 'batch-size' ) ) { + $this->mBatchSize = intval( $this->getOption( 'batch-size' ) ); + } + } + + /** + * Maybe show the help. + * @param bool $force Whether to force the help to show, default false + */ + protected function maybeHelp( $force = false ) { + if ( !$force && !$this->hasOption( 'help' ) ) { + return; + } + + $screenWidth = 80; // TODO: Calculate this! + $tab = " "; + $descWidth = $screenWidth - ( 2 * strlen( $tab ) ); + + ksort( $this->mParams ); + $this->mQuiet = false; + + // Description ... + if ( $this->mDescription ) { + $this->output( "\n" . wordwrap( $this->mDescription, $screenWidth ) . "\n" ); + } + $output = "\nUsage: php " . basename( $this->mSelf ); + + // ... append parameters ... + if ( $this->mParams ) { + $output .= " [--" . implode( array_keys( $this->mParams ), "|--" ) . "]"; + } + + // ... and append arguments. + if ( $this->mArgList ) { + $output .= ' '; + foreach ( $this->mArgList as $k => $arg ) { + if ( $arg['require'] ) { + $output .= '<' . $arg['name'] . '>'; + } else { + $output .= '[' . $arg['name'] . ']'; + } + if ( $k < count( $this->mArgList ) - 1 ) { + $output .= ' '; + } + } + } + $this->output( "$output\n\n" ); + + # TODO abstract some repetitive code below + + // Generic parameters + $this->output( "Generic maintenance parameters:\n" ); + foreach ( $this->mGenericParameters as $par => $info ) { + if ( $info['shortName'] !== false ) { + $par .= " (-{$info['shortName']})"; + } + $this->output( + wordwrap( "$tab--$par: " . $info['desc'], $descWidth, + "\n$tab$tab" ) . "\n" + ); + } + $this->output( "\n" ); + + $scriptDependantParams = $this->mDependantParameters; + if ( count( $scriptDependantParams ) > 0 ) { + $this->output( "Script dependant parameters:\n" ); + // Parameters description + foreach ( $scriptDependantParams as $par => $info ) { + if ( $info['shortName'] !== false ) { + $par .= " (-{$info['shortName']})"; + } + $this->output( + wordwrap( "$tab--$par: " . $info['desc'], $descWidth, + "\n$tab$tab" ) . "\n" + ); + } + $this->output( "\n" ); + } + + // Script specific parameters not defined on construction by + // Maintenance::addDefaultParams() + $scriptSpecificParams = array_diff_key( + # all script parameters: + $this->mParams, + # remove the Maintenance default parameters: + $this->mGenericParameters, + $this->mDependantParameters + ); + if ( count( $scriptSpecificParams ) > 0 ) { + $this->output( "Script specific parameters:\n" ); + // Parameters description + foreach ( $scriptSpecificParams as $par => $info ) { + if ( $info['shortName'] !== false ) { + $par .= " (-{$info['shortName']})"; + } + $this->output( + wordwrap( "$tab--$par: " . $info['desc'], $descWidth, + "\n$tab$tab" ) . "\n" + ); + } + $this->output( "\n" ); + } + + // Print arguments + if ( count( $this->mArgList ) > 0 ) { + $this->output( "Arguments:\n" ); + // Arguments description + foreach ( $this->mArgList as $info ) { + $openChar = $info['require'] ? '<' : '['; + $closeChar = $info['require'] ? '>' : ']'; + $this->output( + wordwrap( "$tab$openChar" . $info['name'] . "$closeChar: " . + $info['desc'], $descWidth, "\n$tab$tab" ) . "\n" + ); + } + $this->output( "\n" ); + } + + die( 1 ); + } + + /** + * Handle some last-minute setup here. + */ + public function finalSetup() { + global $wgCommandLineMode, $wgShowSQLErrors, $wgServer; + global $wgDBadminuser, $wgDBadminpassword; + global $wgDBuser, $wgDBpassword, $wgDBservers, $wgLBFactoryConf; + + # Turn off output buffering again, it might have been turned on in the settings files + if ( ob_get_level() ) { + ob_end_flush(); + } + # Same with these + $wgCommandLineMode = true; + + # Override $wgServer + if ( $this->hasOption( 'server' ) ) { + $wgServer = $this->getOption( 'server', $wgServer ); + } + + # If these were passed, use them + if ( $this->mDbUser ) { + $wgDBadminuser = $this->mDbUser; + } + if ( $this->mDbPass ) { + $wgDBadminpassword = $this->mDbPass; + } + + if ( $this->getDbType() == self::DB_ADMIN && isset( $wgDBadminuser ) ) { + $wgDBuser = $wgDBadminuser; + $wgDBpassword = $wgDBadminpassword; + + if ( $wgDBservers ) { + /** + * @var $wgDBservers array + */ + foreach ( $wgDBservers as $i => $server ) { + $wgDBservers[$i]['user'] = $wgDBuser; + $wgDBservers[$i]['password'] = $wgDBpassword; + } + } + if ( isset( $wgLBFactoryConf['serverTemplate'] ) ) { + $wgLBFactoryConf['serverTemplate']['user'] = $wgDBuser; + $wgLBFactoryConf['serverTemplate']['password'] = $wgDBpassword; + } + MediaWikiServices::getInstance()->getDBLoadBalancerFactory()->destroy(); + } + + // Per-script profiling; useful for debugging + $this->activateProfiler(); + + $this->afterFinalSetup(); + + $wgShowSQLErrors = true; + + MediaWiki\suppressWarnings(); + set_time_limit( 0 ); + MediaWiki\restoreWarnings(); + + $this->adjustMemoryLimit(); + } + + /** + * Execute a callback function at the end of initialisation + */ + protected function afterFinalSetup() { + if ( defined( 'MW_CMDLINE_CALLBACK' ) ) { + call_user_func( MW_CMDLINE_CALLBACK ); + } + } + + /** + * Potentially debug globals. Originally a feature only + * for refreshLinks + */ + public function globals() { + if ( $this->hasOption( 'globals' ) ) { + print_r( $GLOBALS ); + } + } + + /** + * Generic setup for most installs. Returns the location of LocalSettings + * @return string + */ + public function loadSettings() { + global $wgCommandLineMode, $IP; + + if ( isset( $this->mOptions['conf'] ) ) { + $settingsFile = $this->mOptions['conf']; + } elseif ( defined( "MW_CONFIG_FILE" ) ) { + $settingsFile = MW_CONFIG_FILE; + } else { + $settingsFile = "$IP/LocalSettings.php"; + } + if ( isset( $this->mOptions['wiki'] ) ) { + $bits = explode( '-', $this->mOptions['wiki'] ); + if ( count( $bits ) == 1 ) { + $bits[] = ''; + } + define( 'MW_DB', $bits[0] ); + define( 'MW_PREFIX', $bits[1] ); + } + + if ( !is_readable( $settingsFile ) ) { + $this->error( "A copy of your installation's LocalSettings.php\n" . + "must exist and be readable in the source directory.\n" . + "Use --conf to specify it.", true ); + } + $wgCommandLineMode = true; + + return $settingsFile; + } + + /** + * Support function for cleaning up redundant text records + * @param bool $delete Whether or not to actually delete the records + * @author Rob Church <robchur@gmail.com> + */ + public function purgeRedundantText( $delete = true ) { + # Data should come off the master, wrapped in a transaction + $dbw = $this->getDB( DB_MASTER ); + $this->beginTransaction( $dbw, __METHOD__ ); + + # Get "active" text records from the revisions table + $cur = []; + $this->output( 'Searching for active text records in revisions table...' ); + $res = $dbw->select( 'revision', 'rev_text_id', [], __METHOD__, [ 'DISTINCT' ] ); + foreach ( $res as $row ) { + $cur[] = $row->rev_text_id; + } + $this->output( "done.\n" ); + + # Get "active" text records from the archive table + $this->output( 'Searching for active text records in archive table...' ); + $res = $dbw->select( 'archive', 'ar_text_id', [], __METHOD__, [ 'DISTINCT' ] ); + foreach ( $res as $row ) { + # old pre-MW 1.5 records can have null ar_text_id's. + if ( $row->ar_text_id !== null ) { + $cur[] = $row->ar_text_id; + } + } + $this->output( "done.\n" ); + + # Get the IDs of all text records not in these sets + $this->output( 'Searching for inactive text records...' ); + $cond = 'old_id NOT IN ( ' . $dbw->makeList( $cur ) . ' )'; + $res = $dbw->select( 'text', 'old_id', [ $cond ], __METHOD__, [ 'DISTINCT' ] ); + $old = []; + foreach ( $res as $row ) { + $old[] = $row->old_id; + } + $this->output( "done.\n" ); + + # Inform the user of what we're going to do + $count = count( $old ); + $this->output( "$count inactive items found.\n" ); + + # Delete as appropriate + if ( $delete && $count ) { + $this->output( 'Deleting...' ); + $dbw->delete( 'text', [ 'old_id' => $old ], __METHOD__ ); + $this->output( "done.\n" ); + } + + # Done + $this->commitTransaction( $dbw, __METHOD__ ); + } + + /** + * Get the maintenance directory. + * @return string + */ + protected function getDir() { + return __DIR__; + } + + /** + * Returns a database to be used by current maintenance script. It can be set by setDB(). + * If not set, wfGetDB() will be used. + * This function has the same parameters as wfGetDB() + * + * @param int $db DB index (DB_REPLICA/DB_MASTER) + * @param array $groups default: empty array + * @param string|bool $wiki default: current wiki + * @return IMaintainableDatabase + */ + protected function getDB( $db, $groups = [], $wiki = false ) { + if ( is_null( $this->mDb ) ) { + return wfGetDB( $db, $groups, $wiki ); + } else { + return $this->mDb; + } + } + + /** + * Sets database object to be returned by getDB(). + * + * @param IDatabase $db + */ + public function setDB( IDatabase $db ) { + $this->mDb = $db; + } + + /** + * Begin a transcation on a DB + * + * This method makes it clear that begin() is called from a maintenance script, + * which has outermost scope. This is safe, unlike $dbw->begin() called in other places. + * + * @param IDatabase $dbw + * @param string $fname Caller name + * @since 1.27 + */ + protected function beginTransaction( IDatabase $dbw, $fname ) { + $dbw->begin( $fname ); + } + + /** + * Commit the transcation on a DB handle and wait for replica DBs to catch up + * + * This method makes it clear that commit() is called from a maintenance script, + * which has outermost scope. This is safe, unlike $dbw->commit() called in other places. + * + * @param IDatabase $dbw + * @param string $fname Caller name + * @return bool Whether the replica DB wait succeeded + * @since 1.27 + */ + protected function commitTransaction( IDatabase $dbw, $fname ) { + $dbw->commit( $fname ); + try { + $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory(); + $lbFactory->waitForReplication( + [ 'timeout' => 30, 'ifWritesSince' => $this->lastReplicationWait ] + ); + $this->lastReplicationWait = microtime( true ); + + return true; + } catch ( DBReplicationWaitError $e ) { + return false; + } + } + + /** + * Rollback the transcation on a DB handle + * + * This method makes it clear that rollback() is called from a maintenance script, + * which has outermost scope. This is safe, unlike $dbw->rollback() called in other places. + * + * @param IDatabase $dbw + * @param string $fname Caller name + * @since 1.27 + */ + protected function rollbackTransaction( IDatabase $dbw, $fname ) { + $dbw->rollback( $fname ); + } + + /** + * Lock the search index + * @param IMaintainableDatabase &$db + */ + private function lockSearchindex( $db ) { + $write = [ 'searchindex' ]; + $read = [ + 'page', + 'revision', + 'text', + 'interwiki', + 'l10n_cache', + 'user', + 'page_restrictions' + ]; + $db->lockTables( $read, $write, __CLASS__ . '::' . __METHOD__ ); + } + + /** + * Unlock the tables + * @param IMaintainableDatabase &$db + */ + private function unlockSearchindex( $db ) { + $db->unlockTables( __CLASS__ . '::' . __METHOD__ ); + } + + /** + * Unlock and lock again + * Since the lock is low-priority, queued reads will be able to complete + * @param IMaintainableDatabase &$db + */ + private function relockSearchindex( $db ) { + $this->unlockSearchindex( $db ); + $this->lockSearchindex( $db ); + } + + /** + * Perform a search index update with locking + * @param int $maxLockTime The maximum time to keep the search index locked. + * @param string $callback The function that will update the function. + * @param IMaintainableDatabase $dbw + * @param array $results + */ + public function updateSearchIndex( $maxLockTime, $callback, $dbw, $results ) { + $lockTime = time(); + + # Lock searchindex + if ( $maxLockTime ) { + $this->output( " --- Waiting for lock ---" ); + $this->lockSearchindex( $dbw ); + $lockTime = time(); + $this->output( "\n" ); + } + + # Loop through the results and do a search update + foreach ( $results as $row ) { + # Allow reads to be processed + if ( $maxLockTime && time() > $lockTime + $maxLockTime ) { + $this->output( " --- Relocking ---" ); + $this->relockSearchindex( $dbw ); + $lockTime = time(); + $this->output( "\n" ); + } + call_user_func( $callback, $dbw, $row ); + } + + # Unlock searchindex + if ( $maxLockTime ) { + $this->output( " --- Unlocking --" ); + $this->unlockSearchindex( $dbw ); + $this->output( "\n" ); + } + } + + /** + * Update the searchindex table for a given pageid + * @param IDatabase $dbw A database write handle + * @param int $pageId The page ID to update. + * @return null|string + */ + public function updateSearchIndexForPage( $dbw, $pageId ) { + // Get current revision + $rev = Revision::loadFromPageId( $dbw, $pageId ); + $title = null; + if ( $rev ) { + $titleObj = $rev->getTitle(); + $title = $titleObj->getPrefixedDBkey(); + $this->output( "$title..." ); + # Update searchindex + $u = new SearchUpdate( $pageId, $titleObj->getText(), $rev->getContent() ); + $u->doUpdate(); + $this->output( "\n" ); + } + + return $title; + } + + /** + * Wrapper for posix_isatty() + * We default as considering stdin a tty (for nice readline methods) + * but treating stout as not a tty to avoid color codes + * + * @param mixed $fd File descriptor + * @return bool + */ + public static function posix_isatty( $fd ) { + if ( !function_exists( 'posix_isatty' ) ) { + return !$fd; + } else { + return posix_isatty( $fd ); + } + } + + /** + * Prompt the console for input + * @param string $prompt What to begin the line with, like '> ' + * @return string Response + */ + public static function readconsole( $prompt = '> ' ) { + static $isatty = null; + if ( is_null( $isatty ) ) { + $isatty = self::posix_isatty( 0 /*STDIN*/ ); + } + + if ( $isatty && function_exists( 'readline' ) ) { + $resp = readline( $prompt ); + if ( $resp === null ) { + // Workaround for https://github.com/facebook/hhvm/issues/4776 + return false; + } else { + return $resp; + } + } else { + if ( $isatty ) { + $st = self::readlineEmulation( $prompt ); + } else { + if ( feof( STDIN ) ) { + $st = false; + } else { + $st = fgets( STDIN, 1024 ); + } + } + if ( $st === false ) { + return false; + } + $resp = trim( $st ); + + return $resp; + } + } + + /** + * Emulate readline() + * @param string $prompt What to begin the line with, like '> ' + * @return string + */ + private static function readlineEmulation( $prompt ) { + $bash = Installer::locateExecutableInDefaultPaths( [ 'bash' ] ); + if ( !wfIsWindows() && $bash ) { + $retval = false; + $encPrompt = wfEscapeShellArg( $prompt ); + $command = "read -er -p $encPrompt && echo \"\$REPLY\""; + $encCommand = wfEscapeShellArg( $command ); + $line = wfShellExec( "$bash -c $encCommand", $retval, [], [ 'walltime' => 0 ] ); + + if ( $retval == 0 ) { + return $line; + } elseif ( $retval == 127 ) { + // Couldn't execute bash even though we thought we saw it. + // Shell probably spit out an error message, sorry :( + // Fall through to fgets()... + } else { + // EOF/ctrl+D + return false; + } + } + + // Fallback... we'll have no editing controls, EWWW + if ( feof( STDIN ) ) { + return false; + } + print $prompt; + + return fgets( STDIN, 1024 ); + } + + /** + * Get the terminal size as a two-element array where the first element + * is the width (number of columns) and the second element is the height + * (number of rows). + * + * @return array + */ + public static function getTermSize() { + $default = [ 80, 50 ]; + if ( wfIsWindows() ) { + return $default; + } + // It's possible to get the screen size with VT-100 terminal escapes, + // but reading the responses is not possible without setting raw mode + // (unless you want to require the user to press enter), and that + // requires an ioctl(), which we can't do. So we have to shell out to + // something that can do the relevant syscalls. There are a few + // options. Linux and Mac OS X both have "stty size" which does the + // job directly. + $retval = false; + $size = wfShellExec( 'stty size', $retval ); + if ( $retval !== 0 ) { + return $default; + } + if ( !preg_match( '/^(\d+) (\d+)$/', $size, $m ) ) { + return $default; + } + return [ intval( $m[2] ), intval( $m[1] ) ]; + } + + /** + * Call this to set up the autoloader to allow classes to be used from the + * tests directory. + */ + public static function requireTestsAutoloader() { + require_once __DIR__ . '/../tests/common/TestsAutoLoader.php'; + } +} + +/** + * Fake maintenance wrapper, mostly used for the web installer/updater + */ +class FakeMaintenance extends Maintenance { + protected $mSelf = "FakeMaintenanceScript"; + + public function execute() { + return; + } +} + +/** + * Class for scripts that perform database maintenance and want to log the + * update in `updatelog` so we can later skip it + */ +abstract class LoggedUpdateMaintenance extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addOption( 'force', 'Run the update even if it was completed already' ); + $this->setBatchSize( 200 ); + } + + public function execute() { + $db = $this->getDB( DB_MASTER ); + $key = $this->getUpdateKey(); + + if ( !$this->hasOption( 'force' ) + && $db->selectRow( 'updatelog', '1', [ 'ul_key' => $key ], __METHOD__ ) + ) { + $this->output( "..." . $this->updateSkippedMessage() . "\n" ); + + return true; + } + + if ( !$this->doDBUpdates() ) { + return false; + } + + if ( $db->insert( 'updatelog', [ 'ul_key' => $key ], __METHOD__, 'IGNORE' ) ) { + return true; + } else { + $this->output( $this->updatelogFailedMessage() . "\n" ); + + return false; + } + } + + /** + * Message to show that the update was done already and was just skipped + * @return string + */ + protected function updateSkippedMessage() { + $key = $this->getUpdateKey(); + + return "Update '{$key}' already logged as completed."; + } + + /** + * Message to show that the update log was unable to log the completion of this update + * @return string + */ + protected function updatelogFailedMessage() { + $key = $this->getUpdateKey(); + + return "Unable to log update '{$key}' as completed."; + } + + /** + * Do the actual work. All child classes will need to implement this. + * Return true to log the update as done or false (usually on failure). + * @return bool + */ + abstract protected function doDBUpdates(); + + /** + * Get the update key name to go in the update log table + * @return string + */ + abstract protected function getUpdateKey(); +} diff --git a/www/wiki/maintenance/Makefile b/www/wiki/maintenance/Makefile new file mode 100644 index 00000000..a348e856 --- /dev/null +++ b/www/wiki/maintenance/Makefile @@ -0,0 +1,19 @@ +help: + @echo "Run 'make test' to run the parser tests." + @echo "Run 'make doc' to run the doxygen generation." + @echo "Run 'make man' to run the doxygen generation with man pages." + +test: + php tests/parser/parserTests.php --quiet + +doc: + php mwdocgen.php --all + ./mwjsduck-gen + @echo 'PHP documentation (by Doxygen) in ./docs/html/' + @echo 'JS documentation (by JSDuck) in ./docs/js/' + +man: + php mwdocgen.php --all --generate-man + @echo 'Doc generation done. Look at ./docs/html/ and ./docs/man' + @echo 'You might want to update your MANPATH currently:' + @echo 'MANPATH: $(MANPATH)' diff --git a/www/wiki/maintenance/README b/www/wiki/maintenance/README new file mode 100644 index 00000000..8d0b1c45 --- /dev/null +++ b/www/wiki/maintenance/README @@ -0,0 +1,103 @@ +== MediaWiki Maintenance == + +The .sql scripts in this directory are not intended to be run standalone, +although this is appropriate in some cases, e.g. manual creation of blank tables +prior to an import. + +Most of the PHP scripts need to be run from the command line. Prior to doing so, +ensure that the LocalSettings.php file in the directory above points to the +proper installation. + +Certain scripts will require elevated access to the database. In order to +provide this, first create a MySQL user with "all" permissions on the wiki +database, and then set $wgDBadminuser and $wgDBadminpassword in your +LocalSettings.php + +=== Brief explanation of files === + +A lot of the files in this directory are PHP scripts used to perform various +maintenance tasks on the wiki database, e.g. rebuilding link tables, updating +the search indices, etc. The files in the "archives" directory are used to +upgrade the database schema when updating the software. Some schema definitions +for alternative (as yet unsupported) database management systems are stored +here too. + +The "storage" directory contains scripts and resources useful for working with +external storage clusters, and are not likely to be particularly useful to the +vast majority of installations. This directory does contain the compressOld +scripts, however, which can be useful for compacting old data. + +=== Maintenance scripts === + +As noted above, these should be run from the command line. Not all scripts are +listed, as some are Wikimedia-specific, and some are not applicable to most +installations. + + changePassword.php + Reset the password of a specified user + + cleanupSpam.php + Mass-revert insertion of linkspam + + createAndPromote.php + Create a user with administrator (and optionally, bureaucrat) permissions + + deleteOldRevisions.php + Erase old revisions of pages from the database + + dumpBackup.php + Backup dump script + + edit.php + Edit a page to change its content + + findHooks.php + Find hooks that aren't documented in docs/hooks.txt + + importDump.php + XML dump importer + + importImages.php + Import images into the wiki + + moveBatch.php + Move a batch of pages + + namespaceDupes.php + Check articles name to see if they conflict with new/existing namespaces + + nukePage.php + Wipe a page and all revisions from the database + + reassignEdits.php + Reassign edits from one user to another + + rebuildImages.php + Update image metadata records + + rebuildmessages.php + Update the MediaWiki namespace after changing site language + + rebuildtextindex.php + Rebuild the fulltext search indices + + refreshLinks.php + Rebuild the link tables + + removeUnusedAccounts.php + Remove user accounts which have made no edits + + runJobs.php + Immediately complete all jobs in the job queue + + undelete.php + Undelete all revisions of a page + + update.php + Check and upgrade the database schema to the current version + + updateRestrictions.php + Update pages restriction to the new schema + + userOptions.php + Change user options diff --git a/www/wiki/maintenance/addRFCandPMIDInterwiki.php b/www/wiki/maintenance/addRFCandPMIDInterwiki.php new file mode 100644 index 00000000..b21bfbb7 --- /dev/null +++ b/www/wiki/maintenance/addRFCandPMIDInterwiki.php @@ -0,0 +1,95 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Run automatically with update.php + * + * - Changes "rfc" URL to use tools.ietf.org domain + * - Adds "pmid" interwiki + * + * @since 1.28 + */ +class AddRFCAndPMIDInterwiki extends LoggedUpdateMaintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Add RFC and PMID to the interwiki database table' ); + } + + protected function getUpdateKey() { + return __CLASS__; + } + + protected function updateSkippedMessage() { + return 'RFC and PMID already added to interwiki database table.'; + } + + protected function doDBUpdates() { + $interwikiCache = $this->getConfig()->get( 'InterwikiCache' ); + // Using something other than the database, + if ( $interwikiCache !== false ) { + return true; + } + $dbw = $this->getDB( DB_MASTER ); + $rfc = $dbw->selectField( + 'interwiki', + 'iw_url', + [ 'iw_prefix' => 'rfc' ], + __METHOD__ + ); + + // Old pre-1.28 default value, or not set at all + if ( $rfc === false || $rfc === 'http://www.rfc-editor.org/rfc/rfc$1.txt' ) { + $dbw->replace( + 'interwiki', + [ 'iw_prefix' ], + [ + 'iw_prefix' => 'rfc', + 'iw_url' => 'https://tools.ietf.org/html/rfc$1', + 'iw_api' => '', + 'iw_wikiid' => '', + 'iw_local' => 0, + ], + __METHOD__ + ); + } + + $dbw->insert( + 'interwiki', + [ + 'iw_prefix' => 'pmid', + 'iw_url' => 'https://www.ncbi.nlm.nih.gov/pubmed/$1?dopt=Abstract', + 'iw_api' => '', + 'iw_wikiid' => '', + 'iw_local' => 0, + ], + __METHOD__, + // If there's already a pmid interwiki link, don't + // overwrite it + [ 'IGNORE' ] + ); + + return true; + } +} + +$maintClass = 'AddRFCAndPMIDInterwiki'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/addSite.php b/www/wiki/maintenance/addSite.php new file mode 100644 index 00000000..04158aee --- /dev/null +++ b/www/wiki/maintenance/addSite.php @@ -0,0 +1,92 @@ +<?php + +use MediaWiki\MediaWikiServices; + +$basePath = getenv( 'MW_INSTALL_PATH' ) !== false ? getenv( 'MW_INSTALL_PATH' ) : __DIR__ . '/..'; + +require_once $basePath . '/maintenance/Maintenance.php'; + +/** + * Maintenance script for adding a site definition into the sites table. + * + * @since 1.29 + * + * @license GNU GPL v2+ + * @author Florian Schmidt + */ +class AddSite extends Maintenance { + + public function __construct() { + $this->addDescription( 'Add a site definition into the sites table.' ); + + $this->addArg( 'globalid', 'The global id of the site to add, e.g. "wikipedia".', true ); + $this->addArg( 'group', 'In which group this site should be sorted in.', true ); + $this->addOption( 'language', 'The language code of the site, e.g. "de".' ); + $this->addOption( 'interwiki-id', 'The interwiki ID of the site.' ); + $this->addOption( 'navigation-id', 'The navigation ID of the site.' ); + $this->addOption( 'pagepath', 'The URL to pages of this site, e.g.' . + ' https://example.com/wiki/\$1.' ); + $this->addOption( 'filepath', 'The URL to files of this site, e.g. https://example + .com/w/\$1.' ); + + parent::__construct(); + } + + /** + * Imports the site described by the parameters (see self::__construct()) passed to this + * maintenance sccript into the sites table of MediaWiki. + * @return bool + */ + public function execute() { + $siteStore = MediaWikiServices::getInstance()->getSiteStore(); + $siteStore->reset(); + + $globalId = $this->getArg( 0 ); + $group = $this->getArg( 1 ); + $language = $this->getOption( 'language' ); + $interwikiId = $this->getOption( 'interwiki-id' ); + $navigationId = $this->getOption( 'navigation-id' ); + $pagepath = $this->getOption( 'pagepath' ); + $filepath = $this->getOption( 'filepath' ); + + if ( !is_string( $globalId ) || !is_string( $group ) ) { + echo "Arguments globalid and group need to be strings.\n"; + return false; + } + + if ( $siteStore->getSite( $globalId ) !== null ) { + echo "Site with global id $globalId already exists.\n"; + return false; + } + + $site = new MediaWikiSite(); + $site->setGlobalId( $globalId ); + $site->setGroup( $group ); + if ( $language !== null ) { + $site->setLanguageCode( $language ); + } + if ( $interwikiId !== null ) { + $site->addInterwikiId( $interwikiId ); + } + if ( $navigationId !== null ) { + $site->addNavigationId( $navigationId ); + } + if ( $pagepath !== null ) { + $site->setPagePath( $pagepath ); + } + if ( $filepath !== null ) { + $site->setFilePath( $filepath ); + } + + $siteStore->saveSites( [ $site ] ); + + if ( method_exists( $siteStore, 'reset' ) ) { + $siteStore->reset(); + } + + echo "Done.\n"; + } +} + +$maintClass = 'AddSite'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/archives/.htaccess b/www/wiki/maintenance/archives/.htaccess new file mode 100644 index 00000000..3a428827 --- /dev/null +++ b/www/wiki/maintenance/archives/.htaccess @@ -0,0 +1 @@ +Deny from all diff --git a/www/wiki/maintenance/archives/patch-add-3d.sql b/www/wiki/maintenance/archives/patch-add-3d.sql new file mode 100644 index 00000000..13ea4ed2 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-add-3d.sql @@ -0,0 +1,11 @@ +ALTER TABLE /*$wgDBprefix*/image + MODIFY img_media_type ENUM("UNKNOWN", "BITMAP", "DRAWING", "AUDIO", "VIDEO", "MULTIMEDIA", "OFFICE", "TEXT", "EXECUTABLE", "ARCHIVE", "3D") default NULL; + +ALTER TABLE /*$wgDBprefix*/oldimage + MODIFY oi_media_type ENUM("UNKNOWN", "BITMAP", "DRAWING", "AUDIO", "VIDEO", "MULTIMEDIA", "OFFICE", "TEXT", "EXECUTABLE", "ARCHIVE", "3D") default NULL; + +ALTER TABLE /*$wgDBprefix*/filearchive + MODIFY fa_media_type ENUM("UNKNOWN", "BITMAP", "DRAWING", "AUDIO", "VIDEO", "MULTIMEDIA", "OFFICE", "TEXT", "EXECUTABLE", "ARCHIVE", "3D") default NULL; + +ALTER TABLE /*$wgDBprefix*/uploadstash + MODIFY us_media_type ENUM("UNKNOWN", "BITMAP", "DRAWING", "AUDIO", "VIDEO", "MULTIMEDIA", "OFFICE", "TEXT", "EXECUTABLE", "ARCHIVE", "3D") default NULL; diff --git a/www/wiki/maintenance/archives/patch-add-cl_collation_ext_index.sql b/www/wiki/maintenance/archives/patch-add-cl_collation_ext_index.sql new file mode 100644 index 00000000..8137dc64 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-add-cl_collation_ext_index.sql @@ -0,0 +1,2 @@ +-- @since 1.27 +CREATE INDEX /*i*/cl_collation_ext ON /*_*/categorylinks (cl_collation, cl_to, cl_type, cl_from); diff --git a/www/wiki/maintenance/archives/patch-add-rc_name_type_patrolled_timestamp_index.sql b/www/wiki/maintenance/archives/patch-add-rc_name_type_patrolled_timestamp_index.sql new file mode 100644 index 00000000..aa54e753 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-add-rc_name_type_patrolled_timestamp_index.sql @@ -0,0 +1,2 @@ +-- @since 1.28 +CREATE INDEX /*i*/rc_name_type_patrolled_timestamp ON /*_*/recentchanges (rc_namespace, rc_type, rc_patrolled, rc_timestamp); diff --git a/www/wiki/maintenance/archives/patch-ar_deleted.sql b/www/wiki/maintenance/archives/patch-ar_deleted.sql new file mode 100644 index 00000000..2e5edc44 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-ar_deleted.sql @@ -0,0 +1,3 @@ +-- Adding ar_deleted field for revisiondelete +ALTER TABLE /*$wgDBprefix*/archive + ADD ar_deleted tinyint unsigned NOT NULL default '0'; diff --git a/www/wiki/maintenance/archives/patch-ar_len.sql b/www/wiki/maintenance/archives/patch-ar_len.sql new file mode 100644 index 00000000..1710e099 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-ar_len.sql @@ -0,0 +1,3 @@ +ALTER TABLE /*$wgDBprefix*/archive + ADD ar_len INT UNSIGNED; + diff --git a/www/wiki/maintenance/archives/patch-ar_parent_id.sql b/www/wiki/maintenance/archives/patch-ar_parent_id.sql new file mode 100644 index 00000000..b24cf46c --- /dev/null +++ b/www/wiki/maintenance/archives/patch-ar_parent_id.sql @@ -0,0 +1,3 @@ +-- Adding ar_deleted field for revisiondelete +ALTER TABLE /*$wgDBprefix*/archive + ADD ar_parent_id int unsigned default NULL; diff --git a/www/wiki/maintenance/archives/patch-ar_sha1.sql b/www/wiki/maintenance/archives/patch-ar_sha1.sql new file mode 100644 index 00000000..1c7d8e91 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-ar_sha1.sql @@ -0,0 +1,3 @@ +-- Adding ar_sha1 field +ALTER TABLE /*$wgDBprefix*/archive + ADD ar_sha1 varbinary(32) NOT NULL default ''; diff --git a/www/wiki/maintenance/archives/patch-archive-ar_content_format.sql b/www/wiki/maintenance/archives/patch-archive-ar_content_format.sql new file mode 100644 index 00000000..81f9fca8 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-archive-ar_content_format.sql @@ -0,0 +1,2 @@ +ALTER TABLE /*$wgDBprefix*/archive + ADD ar_content_format varbinary(64) DEFAULT NULL; diff --git a/www/wiki/maintenance/archives/patch-archive-ar_content_model.sql b/www/wiki/maintenance/archives/patch-archive-ar_content_model.sql new file mode 100644 index 00000000..1a8b630e --- /dev/null +++ b/www/wiki/maintenance/archives/patch-archive-ar_content_model.sql @@ -0,0 +1,2 @@ +ALTER TABLE /*$wgDBprefix*/archive + ADD ar_content_model varbinary(32) DEFAULT NULL; diff --git a/www/wiki/maintenance/archives/patch-archive-ar_id.sql b/www/wiki/maintenance/archives/patch-archive-ar_id.sql new file mode 100644 index 00000000..08287cd5 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-archive-ar_id.sql @@ -0,0 +1,8 @@ +-- +-- patch-archive-ar_id.sql +-- +-- T41675. Add archive.ar_id. + +ALTER TABLE /*$wgDBprefix*/archive + ADD COLUMN ar_id int unsigned NOT NULL AUTO_INCREMENT FIRST, + ADD PRIMARY KEY (ar_id); diff --git a/www/wiki/maintenance/archives/patch-archive-page_id.sql b/www/wiki/maintenance/archives/patch-archive-page_id.sql new file mode 100644 index 00000000..47a1c47e --- /dev/null +++ b/www/wiki/maintenance/archives/patch-archive-page_id.sql @@ -0,0 +1,6 @@ +-- Reference to page_id. Useful for sysadmin fixing of large +-- pages merged together in the archives +-- Added 2007-07-21 + +ALTER TABLE /*$wgDBprefix*/archive + ADD ar_page_id int unsigned; diff --git a/www/wiki/maintenance/archives/patch-archive-rev_id.sql b/www/wiki/maintenance/archives/patch-archive-rev_id.sql new file mode 100644 index 00000000..b9d789ee --- /dev/null +++ b/www/wiki/maintenance/archives/patch-archive-rev_id.sql @@ -0,0 +1,6 @@ +-- New field in archive table to preserve revision IDs across undeletion. +-- Added 2005-03-10 + +ALTER TABLE /*$wgDBprefix*/archive + ADD + ar_rev_id int unsigned; diff --git a/www/wiki/maintenance/archives/patch-archive-text_id.sql b/www/wiki/maintenance/archives/patch-archive-text_id.sql new file mode 100644 index 00000000..8557f2ad --- /dev/null +++ b/www/wiki/maintenance/archives/patch-archive-text_id.sql @@ -0,0 +1,14 @@ +-- New field in archive table to preserve text source IDs across undeletion. +-- +-- Older entries containing NULL in this field will contain text in the +-- ar_text and ar_flags fields, and will cause the (re)creation of a new +-- text record upon undeletion. +-- +-- Newer ones will reference a text.old_id with this field, and the existing +-- entries will be used as-is; only a revision record need be created. +-- +-- Added 2005-05-01 + +ALTER TABLE /*$wgDBprefix*/archive + ADD + ar_text_id int unsigned; diff --git a/www/wiki/maintenance/archives/patch-archive-user-index.sql b/www/wiki/maintenance/archives/patch-archive-user-index.sql new file mode 100644 index 00000000..997b4a97 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-archive-user-index.sql @@ -0,0 +1,4 @@ +-- Adds a user,timestamp index to the archive table +-- Used for browsing deleted contributions and renames +ALTER TABLE /*$wgDBprefix*/archive + ADD INDEX usertext_timestamp ( ar_user_text , ar_timestamp ); diff --git a/www/wiki/maintenance/archives/patch-archive_ar_revid.sql b/www/wiki/maintenance/archives/patch-archive_ar_revid.sql new file mode 100644 index 00000000..f3b9c3c9 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-archive_ar_revid.sql @@ -0,0 +1,3 @@ +-- Hopefully temporary index. +-- For https://phabricator.wikimedia.org/T23279 +CREATE INDEX /*i*/ar_revid ON /*$wgDBprefix*/archive ( ar_rev_id );
\ No newline at end of file diff --git a/www/wiki/maintenance/archives/patch-archive_kill_ar_page_revid.sql b/www/wiki/maintenance/archives/patch-archive_kill_ar_page_revid.sql new file mode 100644 index 00000000..2e6fe453 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-archive_kill_ar_page_revid.sql @@ -0,0 +1,4 @@ +-- Used for killing the wrong index added during SVN for 1.17 +-- Won't affect most people, but it doesn't need to exist +ALTER TABLE /*$wgDBprefix*/archive + DROP INDEX ar_page_revid;
\ No newline at end of file diff --git a/www/wiki/maintenance/archives/patch-backlinkindexes.sql b/www/wiki/maintenance/archives/patch-backlinkindexes.sql new file mode 100644 index 00000000..9a991b81 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-backlinkindexes.sql @@ -0,0 +1,19 @@ +-- +-- patch-backlinkindexes.sql +-- +-- Per task T8440 / https://phabricator.wikimedia.org/T8440 +-- +-- Improve performance of the "what links here"-type queries +-- + +ALTER TABLE /*$wgDBprefix*/pagelinks + DROP INDEX pl_namespace, + ADD INDEX pl_namespace(pl_namespace, pl_title, pl_from); + +ALTER TABLE /*$wgDBprefix*/templatelinks + DROP INDEX tl_namespace, + ADD INDEX tl_namespace(tl_namespace, tl_title, tl_from); + +ALTER TABLE /*$wgDBprefix*/imagelinks + DROP INDEX il_to, + ADD INDEX il_to(il_to, il_from); diff --git a/www/wiki/maintenance/archives/patch-bot.sql b/www/wiki/maintenance/archives/patch-bot.sql new file mode 100644 index 00000000..7625889c --- /dev/null +++ b/www/wiki/maintenance/archives/patch-bot.sql @@ -0,0 +1,11 @@ +-- Add field to recentchanges for easy filtering of bot entries +-- edits by a user with 'bot' in user.user_rights should be +-- marked 1 in rc_bot. + +-- Change made 2002-12-15 by Brion VIBBER <brion@pobox.com> +-- this affects code in Article.php, User.php SpecialRecentchanges.php +-- column also added to buildTables.inc + +ALTER TABLE /*$wgDBprefix*/recentchanges + ADD COLUMN rc_bot tinyint unsigned NOT NULL default '0' + AFTER rc_minor; diff --git a/www/wiki/maintenance/archives/patch-bot_passwords-bp_user-unsigned.sql b/www/wiki/maintenance/archives/patch-bot_passwords-bp_user-unsigned.sql new file mode 100644 index 00000000..163609ab --- /dev/null +++ b/www/wiki/maintenance/archives/patch-bot_passwords-bp_user-unsigned.sql @@ -0,0 +1 @@ +ALTER TABLE /*_*/bot_passwords MODIFY bp_user int unsigned NOT NULL;
\ No newline at end of file diff --git a/www/wiki/maintenance/archives/patch-bot_passwords.sql b/www/wiki/maintenance/archives/patch-bot_passwords.sql new file mode 100644 index 00000000..bd60ff72 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-bot_passwords.sql @@ -0,0 +1,25 @@ +-- +-- This table contains a user's bot passwords: passwords that allow access to +-- the account via the API with limited rights. +-- +CREATE TABLE /*_*/bot_passwords ( + -- Foreign key to user.user_id + bp_user int NOT NULL, + + -- Application identifier + bp_app_id varbinary(32) NOT NULL, + + -- Password hashes, like user.user_password + bp_password tinyblob NOT NULL, + + -- Like user.user_token + bp_token binary(32) NOT NULL default '', + + -- JSON blob for MWRestrictions + bp_restrictions blob NOT NULL, + + -- Grants allowed to the account when authenticated with this bot-password + bp_grants blob NOT NULL, + + PRIMARY KEY ( bp_user, bp_app_id ) +) /*$wgDBTableOptions*/; diff --git a/www/wiki/maintenance/archives/patch-cache.sql b/www/wiki/maintenance/archives/patch-cache.sql new file mode 100644 index 00000000..0545da8b --- /dev/null +++ b/www/wiki/maintenance/archives/patch-cache.sql @@ -0,0 +1,41 @@ +-- patch-cache.sql +-- 2003-03-22 <brion@pobox.com> +-- +-- Add 'last touched' fields to cur and user tables. +-- These are useful for maintaining cache consistency. +-- (Updates to OutputPage.php and elsewhere.) +-- +-- cur_touched should be set to the current time whenever: +-- * the page is updated +-- * a linked page is created +-- * a linked page is destroyed +-- +-- The cur_touched time will then be compared against the +-- timestamps of cached pages to ensure consistency; if +-- cur_touched is later, the page must be regenerated. + +ALTER TABLE /*$wgDBprefix*/cur + ADD COLUMN cur_touched binary(14) NOT NULL default ''; + +-- Existing pages should be initialized to the current +-- time so they don't needlessly rerender until they are +-- changed for the first time: + +UPDATE /*$wgDBprefix*/cur + SET cur_touched=NOW()+0; + +-- user_touched should be set to the current time whenever: +-- * the user logs in +-- * the user saves preferences (if no longer default...?) +-- * the user's newtalk status is altered +-- +-- The user_touched time should also be checked against the +-- timestamp reported by a browser requesting revalidation. +-- If user_touched is later than the reported last modified +-- time, the page should be rerendered with new options and +-- sent again. + +ALTER TABLE /*$wgDBprefix*/user + ADD COLUMN user_touched binary(14) NOT NULL default ''; +UPDATE /*$wgDBprefix*/user + SET user_touched=NOW()+0; diff --git a/www/wiki/maintenance/archives/patch-cat_hidden.sql b/www/wiki/maintenance/archives/patch-cat_hidden.sql new file mode 100644 index 00000000..933188ce --- /dev/null +++ b/www/wiki/maintenance/archives/patch-cat_hidden.sql @@ -0,0 +1,3 @@ +-- cat_hidden is no longer used, delete it + +ALTER TABLE /*$wgDBprefix*/category DROP COLUMN cat_hidden; diff --git a/www/wiki/maintenance/archives/patch-category.sql b/www/wiki/maintenance/archives/patch-category.sql new file mode 100644 index 00000000..97a5690d --- /dev/null +++ b/www/wiki/maintenance/archives/patch-category.sql @@ -0,0 +1,17 @@ +CREATE TABLE /*$wgDBprefix*/category ( + cat_id int unsigned NOT NULL auto_increment, + + cat_title varchar(255) binary NOT NULL, + + cat_pages int signed NOT NULL default 0, + cat_subcats int signed NOT NULL default 0, + cat_files int signed NOT NULL default 0, + + cat_hidden tinyint(1) unsigned NOT NULL default 0, + + PRIMARY KEY (cat_id), + UNIQUE KEY (cat_title), + + KEY (cat_pages) +) /*$wgDBTableOptions*/; + diff --git a/www/wiki/maintenance/archives/patch-categorylinks-better-collation.sql b/www/wiki/maintenance/archives/patch-categorylinks-better-collation.sql new file mode 100644 index 00000000..f8b63405 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-categorylinks-better-collation.sql @@ -0,0 +1,19 @@ +-- +-- patch-categorylinks-better-collation.sql +-- +-- T2164, T3211, T25682. This is the second version of this patch; the +-- changes are also incorporated into patch-categorylinks-better-collation2.sql, +-- for the benefit of trunk users who applied the original. +-- +-- Due to T27254, the length limit of 255 bytes for cl_sortkey_prefix +-- is also enforced in php. If you change the length of that field, make +-- sure to also change the check in LinksUpdate.php. +ALTER TABLE /*$wgDBprefix*/categorylinks + CHANGE COLUMN cl_sortkey cl_sortkey varbinary(230) NOT NULL default '', + ADD COLUMN cl_sortkey_prefix varchar(255) binary NOT NULL default '', + ADD COLUMN cl_collation varbinary(32) NOT NULL default '', + ADD COLUMN cl_type ENUM('page', 'subcat', 'file') NOT NULL default 'page', +-- rm'd in 1.27 ADD INDEX (cl_collation), + DROP INDEX cl_sortkey, + ADD INDEX cl_sortkey (cl_to, cl_type, cl_sortkey, cl_from); +INSERT IGNORE INTO /*$wgDBprefix*/updatelog (ul_key) VALUES ('cl_fields_update'); diff --git a/www/wiki/maintenance/archives/patch-categorylinks-better-collation2.sql b/www/wiki/maintenance/archives/patch-categorylinks-better-collation2.sql new file mode 100644 index 00000000..e9574693 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-categorylinks-better-collation2.sql @@ -0,0 +1,12 @@ +-- +-- patch-categorylinks-better-collation2.sql +-- +-- Bugs 164, 1211, 23682. This patch exists for trunk users who already +-- applied the first patch in its original version. The first patch was +-- updated to incorporate the changes as well, so as not to do two alters on a +-- large table unnecessarily for people upgrading from 1.16, so this will be +-- skipped if unneeded. +ALTER TABLE /*$wgDBprefix*/categorylinks + CHANGE COLUMN cl_sortkey cl_sortkey varbinary(230) NOT NULL default '', + CHANGE COLUMN cl_collation cl_collation varbinary(32) NOT NULL default ''; +INSERT IGNORE INTO /*$wgDBprefix*/updatelog (ul_key) VALUES ('cl_fields_update'); diff --git a/www/wiki/maintenance/archives/patch-categorylinks-fix-pk.sql b/www/wiki/maintenance/archives/patch-categorylinks-fix-pk.sql new file mode 100644 index 00000000..20bc7160 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-categorylinks-fix-pk.sql @@ -0,0 +1 @@ +ALTER TABLE /*_*/categorylinks DROP KEY /*i*/cl_from, ADD PRIMARY KEY (cl_from,cl_to); diff --git a/www/wiki/maintenance/archives/patch-categorylinks.sql b/www/wiki/maintenance/archives/patch-categorylinks.sql new file mode 100644 index 00000000..0af0cf91 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-categorylinks.sql @@ -0,0 +1,37 @@ +-- +-- Track category inclusions *used inline* +-- This tracks a single level of category membership +-- (folksonomic tagging, really). +-- +CREATE TABLE /*$wgDBprefix*/categorylinks ( + -- Key to page_id of the page defined as a category member. + cl_from int unsigned NOT NULL default '0', + + -- Name of the category. + -- This is also the page_title of the category's description page; + -- all such pages are in namespace 14 (NS_CATEGORY). + cl_to varchar(255) binary NOT NULL default '', + + -- The title of the linking page, or an optional override + -- to determine sort order. Sorting is by binary order, which + -- isn't always ideal, but collations seem to be an exciting + -- and dangerous new world in MySQL... + -- + -- Truncate so that the cl_sortkey key fits in 1000 bytes + -- (MyISAM 5 with server_character_set=utf8) + cl_sortkey varchar(70) binary NOT NULL default '', + + -- This isn't really used at present. Provided for an optional + -- sorting method by approximate addition time. + cl_timestamp timestamp NOT NULL, + + UNIQUE KEY cl_from(cl_from,cl_to), + + -- This key is trouble. It's incomplete, AND it's too big + -- when collation is set to UTF-8. Bleeeacch! + KEY cl_sortkey(cl_to,cl_sortkey), + + -- Not really used? + KEY cl_timestamp(cl_to,cl_timestamp) + +) /*$wgDBTableOptions*/; diff --git a/www/wiki/maintenance/archives/patch-categorylinksindex.sql b/www/wiki/maintenance/archives/patch-categorylinksindex.sql new file mode 100644 index 00000000..e2b2c3ac --- /dev/null +++ b/www/wiki/maintenance/archives/patch-categorylinksindex.sql @@ -0,0 +1,11 @@ +-- +-- patch-categorylinksindex.sql +-- +-- Per task T12280 / https://phabricator.wikimedia.org/T12280 +-- +-- Improve enum continuation performance of the what pages belong to a category query +-- + +ALTER TABLE /*$wgDBprefix*/categorylinks + DROP INDEX cl_sortkey, + ADD INDEX cl_sortkey(cl_to, cl_sortkey, cl_from); diff --git a/www/wiki/maintenance/archives/patch-change_tag-ct_id.sql b/www/wiki/maintenance/archives/patch-change_tag-ct_id.sql new file mode 100644 index 00000000..7b986d67 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-change_tag-ct_id.sql @@ -0,0 +1,5 @@ +-- Primary key in change_tag table + +ALTER TABLE /*$wgDBprefix*/change_tag + ADD COLUMN ct_id INT UNSIGNED NOT NULL AUTO_INCREMENT FIRST, + ADD PRIMARY KEY (ct_id); diff --git a/www/wiki/maintenance/archives/patch-change_tag-ct_log_id-unsigned.sql b/www/wiki/maintenance/archives/patch-change_tag-ct_log_id-unsigned.sql new file mode 100644 index 00000000..1371c474 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-change_tag-ct_log_id-unsigned.sql @@ -0,0 +1 @@ +ALTER TABLE /*_*/change_tag MODIFY ct_log_id int unsigned NULL;
\ No newline at end of file diff --git a/www/wiki/maintenance/archives/patch-change_tag-ct_rev_id-unsigned.sql b/www/wiki/maintenance/archives/patch-change_tag-ct_rev_id-unsigned.sql new file mode 100644 index 00000000..b7e1f02e --- /dev/null +++ b/www/wiki/maintenance/archives/patch-change_tag-ct_rev_id-unsigned.sql @@ -0,0 +1 @@ +ALTER TABLE /*_*/change_tag MODIFY ct_rev_id int unsigned NULL;
\ No newline at end of file diff --git a/www/wiki/maintenance/archives/patch-change_tag-indexes.sql b/www/wiki/maintenance/archives/patch-change_tag-indexes.sql new file mode 100644 index 00000000..9f8d8f80 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-change_tag-indexes.sql @@ -0,0 +1,21 @@ +-- +-- Rename indexes on change_tag from implicit to explicit names +-- + +DROP INDEX ct_rc_id ON /*_*/change_tag; +DROP INDEX ct_log_id ON /*_*/change_tag; +DROP INDEX ct_rev_id ON /*_*/change_tag; +DROP INDEX ct_tag ON /*_*/change_tag; + +DROP INDEX ts_rc_id ON /*_*/tag_summary; +DROP INDEX ts_log_id ON /*_*/tag_summary; +DROP INDEX ts_rev_id ON /*_*/tag_summary; + +CREATE UNIQUE INDEX /*i*/change_tag_rc_tag ON /*_*/change_tag (ct_rc_id,ct_tag); +CREATE UNIQUE INDEX /*i*/change_tag_log_tag ON /*_*/change_tag (ct_log_id,ct_tag); +CREATE UNIQUE INDEX /*i*/change_tag_rev_tag ON /*_*/change_tag (ct_rev_id,ct_tag); +CREATE INDEX /*i*/change_tag_tag_id ON /*_*/change_tag (ct_tag,ct_rc_id,ct_rev_id,ct_log_id); + +CREATE UNIQUE INDEX /*i*/tag_summary_rc_id ON /*_*/tag_summary (ts_rc_id); +CREATE UNIQUE INDEX /*i*/tag_summary_log_id ON /*_*/tag_summary (ts_log_id); +CREATE UNIQUE INDEX /*i*/tag_summary_rev_id ON /*_*/tag_summary (ts_rev_id); diff --git a/www/wiki/maintenance/archives/patch-change_tag.sql b/www/wiki/maintenance/archives/patch-change_tag.sql new file mode 100644 index 00000000..3079a5bb --- /dev/null +++ b/www/wiki/maintenance/archives/patch-change_tag.sql @@ -0,0 +1,15 @@ +-- A table to track tags for revisions, logs and recent changes. +-- Andrew Garrett, 2009-01 +CREATE TABLE /*_*/change_tag ( + ct_rc_id int NULL, + ct_log_id int NULL, + ct_rev_id int NULL, + ct_tag varchar(255) NOT NULL, + ct_params BLOB NULL +) /*$wgDBTableOptions*/; + +CREATE UNIQUE INDEX /*i*/change_tag_rc_tag ON /*_*/change_tag (ct_rc_id,ct_tag); +CREATE UNIQUE INDEX /*i*/change_tag_log_tag ON /*_*/change_tag (ct_log_id,ct_tag); +CREATE UNIQUE INDEX /*i*/change_tag_rev_tag ON /*_*/change_tag (ct_rev_id,ct_tag); +-- Covering index, so we can pull all the info only out of the index. +CREATE INDEX /*i*/change_tag_tag_id ON /*_*/change_tag (ct_tag,ct_rc_id,ct_rev_id,ct_log_id); diff --git a/www/wiki/maintenance/archives/patch-comment-table.sql b/www/wiki/maintenance/archives/patch-comment-table.sql new file mode 100644 index 00000000..c8bf9580 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-comment-table.sql @@ -0,0 +1,59 @@ +-- +-- patch-comment-table.sql +-- +-- T166732. Add a `comment` table and various columns (and temporary tables) to reference it. + +CREATE TABLE /*_*/comment ( + comment_id bigint unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + comment_hash INT NOT NULL, + comment_text BLOB NOT NULL, + comment_data BLOB +) /*$wgDBTableOptions*/; +CREATE INDEX /*i*/comment_hash ON /*_*/comment (comment_hash); + +CREATE TABLE /*_*/revision_comment_temp ( + revcomment_rev int unsigned NOT NULL, + revcomment_comment_id bigint unsigned NOT NULL, + PRIMARY KEY (revcomment_rev, revcomment_comment_id) +) /*$wgDBTableOptions*/; +CREATE UNIQUE INDEX /*i*/revcomment_rev ON /*_*/revision_comment_temp (revcomment_rev); + +CREATE TABLE /*_*/image_comment_temp ( + imgcomment_name varchar(255) binary NOT NULL, + imgcomment_description_id bigint unsigned NOT NULL, + PRIMARY KEY (imgcomment_name, imgcomment_description_id) +) /*$wgDBTableOptions*/; +CREATE UNIQUE INDEX /*i*/imgcomment_name ON /*_*/image_comment_temp (imgcomment_name); + +ALTER TABLE /*_*/revision + ALTER COLUMN rev_comment SET DEFAULT ''; + +ALTER TABLE /*_*/archive + ALTER COLUMN ar_comment SET DEFAULT '', + ADD COLUMN ar_comment_id bigint unsigned NOT NULL DEFAULT 0 AFTER ar_comment; + +ALTER TABLE /*_*/ipblocks + ALTER COLUMN ipb_reason SET DEFAULT '', + ADD COLUMN ipb_reason_id bigint unsigned NOT NULL DEFAULT 0 AFTER ipb_reason; + +ALTER TABLE /*_*/image + ALTER COLUMN img_description SET DEFAULT ''; + +ALTER TABLE /*_*/oldimage + ALTER COLUMN oi_description SET DEFAULT '', + ADD COLUMN oi_description_id bigint unsigned NOT NULL DEFAULT 0 AFTER oi_description; + +ALTER TABLE /*_*/filearchive + ADD COLUMN fa_deleted_reason_id bigint unsigned NOT NULL DEFAULT 0 AFTER fa_deleted_reason, + ALTER COLUMN fa_description SET DEFAULT '', + ADD COLUMN fa_description_id bigint unsigned NOT NULL DEFAULT 0 AFTER fa_description; + +ALTER TABLE /*_*/recentchanges + ADD COLUMN rc_comment_id bigint unsigned NOT NULL DEFAULT 0 AFTER rc_comment; + +ALTER TABLE /*_*/logging + ADD COLUMN log_comment_id bigint unsigned NOT NULL DEFAULT 0 AFTER log_comment; + +ALTER TABLE /*_*/protected_titles + ALTER COLUMN pt_reason SET DEFAULT '', + ADD COLUMN pt_reason_id bigint unsigned NOT NULL DEFAULT 0 AFTER pt_reason; diff --git a/www/wiki/maintenance/archives/patch-drop-page_counter.sql b/www/wiki/maintenance/archives/patch-drop-page_counter.sql new file mode 100644 index 00000000..1d8e701b --- /dev/null +++ b/www/wiki/maintenance/archives/patch-drop-page_counter.sql @@ -0,0 +1,2 @@ +-- field is deprecated and no longer updated as of 1.25 +ALTER TABLE /*_*/page DROP COLUMN page_counter; diff --git a/www/wiki/maintenance/archives/patch-drop-rc_cur_time.sql b/www/wiki/maintenance/archives/patch-drop-rc_cur_time.sql new file mode 100644 index 00000000..f1bc9e8b --- /dev/null +++ b/www/wiki/maintenance/archives/patch-drop-rc_cur_time.sql @@ -0,0 +1,2 @@ +-- rc_cur_time is no longer used, delete the field +ALTER TABLE /*$wgDBprefix*/recentchanges DROP COLUMN rc_cur_time;
\ No newline at end of file diff --git a/www/wiki/maintenance/archives/patch-drop-ss_admins.sql b/www/wiki/maintenance/archives/patch-drop-ss_admins.sql new file mode 100644 index 00000000..13c3d3b0 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-drop-ss_admins.sql @@ -0,0 +1,2 @@ +-- field is deprecated and no longer updated as of 1.5 +ALTER TABLE /*_*/site_stats DROP COLUMN ss_admins;
\ No newline at end of file diff --git a/www/wiki/maintenance/archives/patch-drop-ss_total_views.sql b/www/wiki/maintenance/archives/patch-drop-ss_total_views.sql new file mode 100644 index 00000000..00591939 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-drop-ss_total_views.sql @@ -0,0 +1,2 @@ +-- field is deprecated and no longer updated as of 1.24 +ALTER TABLE /*_*/site_stats DROP COLUMN ss_total_views;
\ No newline at end of file diff --git a/www/wiki/maintenance/archives/patch-drop-user_newtalk.sql b/www/wiki/maintenance/archives/patch-drop-user_newtalk.sql new file mode 100644 index 00000000..6ec84fb3 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-drop-user_newtalk.sql @@ -0,0 +1,3 @@ +-- Patch for email authentication T.Gries/M.Arndt 27.11.2004 +-- Table user_newtalk is dropped, as the table watchlist is now also used for storing user_talk-page notifications +DROP TABLE /*$wgDBprefix*/user_newtalk; diff --git a/www/wiki/maintenance/archives/patch-drop-user_options.sql b/www/wiki/maintenance/archives/patch-drop-user_options.sql new file mode 100644 index 00000000..15b7d278 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-drop-user_options.sql @@ -0,0 +1 @@ +ALTER TABLE /*_*/user DROP COLUMN user_options;
\ No newline at end of file diff --git a/www/wiki/maintenance/archives/patch-drop_img_type.sql b/www/wiki/maintenance/archives/patch-drop_img_type.sql new file mode 100644 index 00000000..e3737617 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-drop_img_type.sql @@ -0,0 +1,3 @@ +-- img_type is no longer used, delete it + +ALTER TABLE /*$wgDBprefix*/image DROP COLUMN img_type; diff --git a/www/wiki/maintenance/archives/patch-editsummary-length.sql b/www/wiki/maintenance/archives/patch-editsummary-length.sql new file mode 100644 index 00000000..996d562d --- /dev/null +++ b/www/wiki/maintenance/archives/patch-editsummary-length.sql @@ -0,0 +1,11 @@ +ALTER TABLE /*_*/revision MODIFY rev_comment varbinary(767) NOT NULL default ''; +ALTER TABLE /*_*/archive MODIFY ar_comment varbinary(767) NOT NULL default ''; +ALTER TABLE /*_*/image MODIFY img_description varbinary(767) NOT NULL default ''; +ALTER TABLE /*_*/oldimage MODIFY oi_description varbinary(767) NOT NULL default ''; +ALTER TABLE /*_*/filearchive MODIFY fa_description varbinary(767) default ''; +ALTER TABLE /*_*/filearchive MODIFY fa_deleted_reason varbinary(767) default ''; +ALTER TABLE /*_*/recentchanges MODIFY rc_comment varbinary(767) NOT NULL default ''; +ALTER TABLE /*_*/logging MODIFY log_comment varbinary(767) NOT NULL default ''; +ALTER TABLE /*_*/ipblocks MODIFY ipb_reason varbinary(767) NOT NULL default ''; +ALTER TABLE /*_*/protected_titles MODIFY pt_reason varbinary(767) default ''; + diff --git a/www/wiki/maintenance/archives/patch-email-authentication.sql b/www/wiki/maintenance/archives/patch-email-authentication.sql new file mode 100644 index 00000000..b35b10f1 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-email-authentication.sql @@ -0,0 +1,3 @@ +-- Added early in 1.5 alpha development, removed 2005-04-25 + +ALTER TABLE /*$wgDBprefix*/user DROP COLUMN user_emailauthenticationtimestamp; diff --git a/www/wiki/maintenance/archives/patch-email-notification.sql b/www/wiki/maintenance/archives/patch-email-notification.sql new file mode 100644 index 00000000..337e1ac2 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-email-notification.sql @@ -0,0 +1,11 @@ +-- Patch for email notification on page changes T.Gries/M.Arndt 11.09.2004 + +-- A new column 'wl_notificationtimestamp' is added to the table 'watchlist'. +-- When a page watched by a user X is changed by someone else, an email is sent to the watching user X +-- if and only if the field 'wl_notificationtimestamp' is '0'. The time/date of sending the mail is then stored in that field. +-- Further pages changes do not trigger new notification mails as long as user X has not re-visited that page. +-- The field is reset to '0' when user X re-visits the page or when he or she resets all notification timestamps +-- ("notification flags") at once by clicking the new button on his/her watchlist page. +-- T. Gries/M. Arndt 11.09.2004 - December 2004 + +ALTER TABLE /*$wgDBprefix*/watchlist ADD (wl_notificationtimestamp varbinary(14)); diff --git a/www/wiki/maintenance/archives/patch-externallinks-el_id.sql b/www/wiki/maintenance/archives/patch-externallinks-el_id.sql new file mode 100644 index 00000000..ded84543 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-externallinks-el_id.sql @@ -0,0 +1,8 @@ +-- +-- patch-extenallinks-el_id.sql +-- +-- T17441. Add externallinks.el_id. + +ALTER TABLE /*$wgDBprefix*/externallinks + ADD COLUMN el_id int unsigned NOT NULL AUTO_INCREMENT FIRST, + ADD PRIMARY KEY (el_id); diff --git a/www/wiki/maintenance/archives/patch-externallinks-el_index_60.sql b/www/wiki/maintenance/archives/patch-externallinks-el_index_60.sql new file mode 100644 index 00000000..eacb1074 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-externallinks-el_index_60.sql @@ -0,0 +1,4 @@ +-- @since 1.29 +ALTER TABLE /*$wgDBprefix*/externallinks ADD COLUMN el_index_60 varbinary(60) NOT NULL DEFAULT ''; +CREATE INDEX /*i*/el_index_60 ON /*_*/externallinks (el_index_60, el_id); +CREATE INDEX /*i*/el_from_index_60 ON /*_*/externallinks (el_from, el_index_60, el_id); diff --git a/www/wiki/maintenance/archives/patch-externallinks.sql b/www/wiki/maintenance/archives/patch-externallinks.sql new file mode 100644 index 00000000..fc5017db --- /dev/null +++ b/www/wiki/maintenance/archives/patch-externallinks.sql @@ -0,0 +1,13 @@ +-- +-- Track links to external URLs +-- +CREATE TABLE /*$wgDBprefix*/externallinks ( + el_from int(8) unsigned NOT NULL default '0', + el_to blob NOT NULL, + el_index blob NOT NULL, + + KEY (el_from, el_to(40)), + KEY (el_to(60), el_from), + KEY (el_index(60)) +) /*$wgDBTableOptions*/; + diff --git a/www/wiki/maintenance/archives/patch-fa_deleted.sql b/www/wiki/maintenance/archives/patch-fa_deleted.sql new file mode 100644 index 00000000..7ab65239 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-fa_deleted.sql @@ -0,0 +1,3 @@ +-- Adding fa_deleted field for additional content suppression +ALTER TABLE /*$wgDBprefix*/filearchive + ADD fa_deleted tinyint unsigned NOT NULL default '0'; diff --git a/www/wiki/maintenance/archives/patch-fa_major_mime-chemical.sql b/www/wiki/maintenance/archives/patch-fa_major_mime-chemical.sql new file mode 100644 index 00000000..be9b0ff5 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-fa_major_mime-chemical.sql @@ -0,0 +1,3 @@ +ALTER TABLE /*$wgDBprefix*/filearchive + CHANGE fa_major_mime fa_major_mime ENUM('unknown','application','audio','image','text','video','message','model','multipart','chemical'); + diff --git a/www/wiki/maintenance/archives/patch-fa_sha1.sql b/www/wiki/maintenance/archives/patch-fa_sha1.sql new file mode 100644 index 00000000..931bc44d --- /dev/null +++ b/www/wiki/maintenance/archives/patch-fa_sha1.sql @@ -0,0 +1,4 @@ +-- Add fa_sha1 and related index +ALTER TABLE /*$wgDBprefix*/filearchive + ADD COLUMN fa_sha1 varbinary(32) NOT NULL default ''; +CREATE INDEX /*i*/fa_sha1 ON /*$wgDBprefix*/filearchive (fa_sha1(10)); diff --git a/www/wiki/maintenance/archives/patch-filearchive-user-index.sql b/www/wiki/maintenance/archives/patch-filearchive-user-index.sql new file mode 100644 index 00000000..0d8c3ab1 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-filearchive-user-index.sql @@ -0,0 +1,5 @@ +-- Adding index to sort by uploader +ALTER TABLE /*$wgDBprefix*/filearchive + ADD INDEX fa_user_timestamp (fa_user_text,fa_timestamp), + -- Remove useless, incomplete index + DROP INDEX fa_deleted_user; diff --git a/www/wiki/maintenance/archives/patch-filearchive.sql b/www/wiki/maintenance/archives/patch-filearchive.sql new file mode 100644 index 00000000..f75da8be --- /dev/null +++ b/www/wiki/maintenance/archives/patch-filearchive.sql @@ -0,0 +1,51 @@ +-- +-- Record of deleted file data +-- +CREATE TABLE /*$wgDBprefix*/filearchive ( + -- Unique row id + fa_id int not null auto_increment, + + -- Original base filename; key to image.img_name, page.page_title, etc + fa_name varchar(255) binary NOT NULL default '', + + -- Filename of archived file, if an old revision + fa_archive_name varchar(255) binary default '', + + -- Which storage bin (directory tree or object store) the file data + -- is stored in. Should be 'deleted' for files that have been deleted; + -- any other bin is not yet in use. + fa_storage_group varbinary(16), + + -- SHA-1 of the file contents plus extension, used as a key for storage. + -- eg 8f8a562add37052a1848ff7771a2c515db94baa9.jpg + -- + -- If NULL, the file was missing at deletion time or has been purged + -- from the archival storage. + fa_storage_key varbinary(64) default '', + + -- Deletion information, if this file is deleted. + fa_deleted_user int, + fa_deleted_timestamp binary(14) default '', + fa_deleted_reason text, + + -- Duped fields from image + fa_size int unsigned default '0', + fa_width int default '0', + fa_height int default '0', + fa_metadata mediumblob, + fa_bits int default '0', + fa_media_type ENUM("UNKNOWN", "BITMAP", "DRAWING", "AUDIO", "VIDEO", "MULTIMEDIA", "OFFICE", "TEXT", "EXECUTABLE", "ARCHIVE") default NULL, + fa_major_mime ENUM("unknown", "application", "audio", "image", "text", "video", "message", "model", "multipart") default "unknown", + fa_minor_mime varbinary(32) default "unknown", + fa_description tinyblob, + fa_user int unsigned default '0', + fa_user_text varchar(255) binary default '', + fa_timestamp binary(14) default '', + + PRIMARY KEY (fa_id), + INDEX (fa_name, fa_timestamp), -- pick out by image name + INDEX (fa_storage_group, fa_storage_key), -- pick out dupe files + INDEX (fa_deleted_timestamp), -- sort by deletion time + INDEX (fa_deleted_user) -- sort by deleter + +) /*$wgDBTableOptions*/; diff --git a/www/wiki/maintenance/archives/patch-filejournal.sql b/www/wiki/maintenance/archives/patch-filejournal.sql new file mode 100644 index 00000000..4356d70d --- /dev/null +++ b/www/wiki/maintenance/archives/patch-filejournal.sql @@ -0,0 +1,20 @@ +-- File backend operation journal +CREATE TABLE /*_*/filejournal ( + -- Unique ID for each file operation + fj_id bigint unsigned NOT NULL PRIMARY KEY auto_increment, + -- UUID of the batch this operation belongs to + fj_batch_uuid varbinary(32) NOT NULL, + -- The registered file backend name + fj_backend varchar(255) NOT NULL, + -- The storage path that was affected (may be internal paths) + fj_path blob NOT NULL, + -- Primitive operation description (create/update/delete) + fj_op varchar(16) NOT NULL default '', + -- SHA-1 file content hash in base-36 + fj_new_sha1 varbinary(32) NOT NULL default '', + -- Timestamp of the batch operation + fj_timestamp varbinary(14) NOT NULL default '' +) /*$wgDBTableOptions*/; + +CREATE INDEX /*i*/fj_batch_id ON /*_*/filejournal (fj_batch_uuid); +CREATE INDEX /*i*/fj_timestamp ON /*_*/filejournal (fj_timestamp); diff --git a/www/wiki/maintenance/archives/patch-fix-il_from.sql b/www/wiki/maintenance/archives/patch-fix-il_from.sql new file mode 100644 index 00000000..0a199e4d --- /dev/null +++ b/www/wiki/maintenance/archives/patch-fix-il_from.sql @@ -0,0 +1,11 @@ +-- Fix a bug from the 1.2 -> 1.3 upgrader by moving away the imagelinks table +-- and recreating it. +RENAME TABLE /*_*/imagelinks TO /*_*/imagelinks_old; +CREATE TABLE /*_*/imagelinks ( + il_from int unsigned NOT NULL default 0, + il_to varchar(255) binary NOT NULL default '' +) /*$wgDBTableOptions*/; + +CREATE UNIQUE INDEX /*i*/il_from ON /*_*/imagelinks (il_from,il_to); +CREATE UNIQUE INDEX /*i*/il_to ON /*_*/imagelinks (il_to,il_from); + diff --git a/www/wiki/maintenance/archives/patch-il_from_namespace.sql b/www/wiki/maintenance/archives/patch-il_from_namespace.sql new file mode 100644 index 00000000..2a2d361b --- /dev/null +++ b/www/wiki/maintenance/archives/patch-il_from_namespace.sql @@ -0,0 +1,4 @@ +ALTER TABLE /*_*/imagelinks + ADD COLUMN il_from_namespace int NOT NULL default 0; + +CREATE INDEX /*i*/il_backlinks_namespace ON /*_*/imagelinks (il_from_namespace,il_to,il_from);
\ No newline at end of file diff --git a/www/wiki/maintenance/archives/patch-image-user-index-2.sql b/www/wiki/maintenance/archives/patch-image-user-index-2.sql new file mode 100644 index 00000000..8b19d820 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-image-user-index-2.sql @@ -0,0 +1 @@ +CREATE INDEX /*i*/img_user_timestamp ON /*_*/image (img_user,img_timestamp); diff --git a/www/wiki/maintenance/archives/patch-image-user-index.sql b/www/wiki/maintenance/archives/patch-image-user-index.sql new file mode 100644 index 00000000..b44930fc --- /dev/null +++ b/www/wiki/maintenance/archives/patch-image-user-index.sql @@ -0,0 +1,8 @@ +-- +-- image-user-index.sql +-- +-- Add user_text/timestamp index to current image versions +-- + +ALTER TABLE /*$wgDBprefix*/image + ADD INDEX img_usertext_timestamp (img_user_text,img_timestamp); diff --git a/www/wiki/maintenance/archives/patch-image_name_primary.sql b/www/wiki/maintenance/archives/patch-image_name_primary.sql new file mode 100644 index 00000000..5bd88264 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-image_name_primary.sql @@ -0,0 +1,6 @@ +-- Make the image name index unique + +ALTER TABLE /*$wgDBprefix*/image DROP INDEX img_name; + +ALTER TABLE /*$wgDBprefix*/image + ADD PRIMARY KEY img_name (img_name); diff --git a/www/wiki/maintenance/archives/patch-image_name_unique.sql b/www/wiki/maintenance/archives/patch-image_name_unique.sql new file mode 100644 index 00000000..5cf02d41 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-image_name_unique.sql @@ -0,0 +1,6 @@ +-- Make the image name index unique + +ALTER TABLE /*$wgDBprefix*/image DROP INDEX img_name; + +ALTER TABLE /*$wgDBprefix*/image + ADD UNIQUE INDEX img_name (img_name); diff --git a/www/wiki/maintenance/archives/patch-imagelinks-fix-pk.sql b/www/wiki/maintenance/archives/patch-imagelinks-fix-pk.sql new file mode 100644 index 00000000..e66500f7 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-imagelinks-fix-pk.sql @@ -0,0 +1 @@ +ALTER TABLE /*_*/imagelinks DROP KEY /*i*/il_from, ADD PRIMARY KEY (il_from,il_to); diff --git a/www/wiki/maintenance/archives/patch-img_exif.sql b/www/wiki/maintenance/archives/patch-img_exif.sql new file mode 100644 index 00000000..2fd78f76 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-img_exif.sql @@ -0,0 +1,3 @@ +-- Extra image exif metadata, added for 1.5 but quickly removed. + +ALTER TABLE /*$wgDBprefix*/image DROP img_exif; diff --git a/www/wiki/maintenance/archives/patch-img_major_mime-chemical.sql b/www/wiki/maintenance/archives/patch-img_major_mime-chemical.sql new file mode 100644 index 00000000..4bde446e --- /dev/null +++ b/www/wiki/maintenance/archives/patch-img_major_mime-chemical.sql @@ -0,0 +1,3 @@ +ALTER TABLE /*$wgDBprefix*/image + CHANGE img_major_mime img_major_mime ENUM('unknown','application','audio','image','text','video','message','model','multipart','chemical'); + diff --git a/www/wiki/maintenance/archives/patch-img_media_mime-index.sql b/www/wiki/maintenance/archives/patch-img_media_mime-index.sql new file mode 100644 index 00000000..bfaf84f9 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-img_media_mime-index.sql @@ -0,0 +1,4 @@ +-- New index on image table to allow searches for types i.e. video webm +-- Added 2013-01-08 + +CREATE INDEX /*i*/img_media_mime ON /*_*/image (img_media_type,img_major_mime,img_minor_mime); diff --git a/www/wiki/maintenance/archives/patch-img_media_type.sql b/www/wiki/maintenance/archives/patch-img_media_type.sql new file mode 100644 index 00000000..b0f9ece5 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-img_media_type.sql @@ -0,0 +1,17 @@ +-- media type columns, added for 1.5 +-- this alters the scheme for 1.5, img_type is no longer used. + +ALTER TABLE /*$wgDBprefix*/image ADD ( + -- Media type as defined by the MEDIATYPE_xxx constants + img_media_type ENUM("UNKNOWN", "BITMAP", "DRAWING", "AUDIO", "VIDEO", "MULTIMEDIA", "OFFICE", "TEXT", "EXECUTABLE", "ARCHIVE") default NULL, + + -- major part of a MIME media type as defined by IANA + -- see https://www.iana.org/assignments/media-types/ + img_major_mime ENUM("unknown", "application", "audio", "image", "text", "video", "message", "model", "multipart") NOT NULL default "unknown", + + -- minor part of a MIME media type as defined by IANA + -- the minor parts are not required to adher to any standard + -- but should be consistent throughout the database + -- see https://www.iana.org/assignments/media-types/ + img_minor_mime varbinary(32) NOT NULL default "unknown" +); diff --git a/www/wiki/maintenance/archives/patch-img_metadata.sql b/www/wiki/maintenance/archives/patch-img_metadata.sql new file mode 100644 index 00000000..407e4325 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-img_metadata.sql @@ -0,0 +1,6 @@ +-- Moving img_exif to img_metadata, so the name won't be so confusing when we +-- Use it for Ogg metadata or something like that. + +ALTER TABLE /*$wgDBprefix*/image ADD ( + img_metadata mediumblob NOT NULL +); diff --git a/www/wiki/maintenance/archives/patch-img_sha1.sql b/www/wiki/maintenance/archives/patch-img_sha1.sql new file mode 100644 index 00000000..0a375c4f --- /dev/null +++ b/www/wiki/maintenance/archives/patch-img_sha1.sql @@ -0,0 +1,8 @@ +-- Add img_sha1, oi_sha1 and related indexes +ALTER TABLE /*$wgDBprefix*/image + ADD COLUMN img_sha1 varbinary(32) NOT NULL default '', + ADD INDEX img_sha1 (img_sha1(10)); + +ALTER TABLE /*$wgDBprefix*/oldimage + ADD COLUMN oi_sha1 varbinary(32) NOT NULL default '', + ADD INDEX oi_sha1 (oi_sha1(10)); diff --git a/www/wiki/maintenance/archives/patch-img_width.sql b/www/wiki/maintenance/archives/patch-img_width.sql new file mode 100644 index 00000000..06889ea6 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-img_width.sql @@ -0,0 +1,18 @@ +-- Extra image metadata, added for 1.5 + +-- NOTE: as by patch-img_media_type.sql, the img_type +-- column is no longer used and has therefore be removed from this patch + +ALTER TABLE /*$wgDBprefix*/image ADD ( + img_width int NOT NULL default 0, + img_height int NOT NULL default 0, + img_bits int NOT NULL default 0 +); + +ALTER TABLE /*$wgDBprefix*/oldimage ADD ( + oi_width int NOT NULL default 0, + oi_height int NOT NULL default 0, + oi_bits int NOT NULL default 0 +); + + diff --git a/www/wiki/maintenance/archives/patch-indexes.sql b/www/wiki/maintenance/archives/patch-indexes.sql new file mode 100644 index 00000000..c24d9953 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-indexes.sql @@ -0,0 +1,24 @@ +-- +-- patch-indexes.sql +-- +-- Fix up table indexes; new to stable release in November 2003 +-- + +ALTER TABLE IF EXISTS /*$wgDBprefix*/links + DROP INDEX l_from, + ADD INDEX l_from (l_from); + +ALTER TABLE /*$wgDBprefix*/brokenlinks + DROP INDEX bl_to, + ADD INDEX bl_to (bL_to); + +ALTER TABLE /*$wgDBprefix*/recentchanges + ADD INDEX rc_timestamp (rc_timestamp), + ADD INDEX rc_namespace_title (rc_namespace, rc_title), + ADD INDEX rc_cur_id (rc_cur_id); + +ALTER TABLE /*$wgDBprefix*/archive + ADD KEY name_title_timestamp (ar_namespace,ar_title,ar_timestamp); + +ALTER TABLE /*$wgDBprefix*/watchlist + ADD KEY namespace_title (wl_namespace,wl_title); diff --git a/www/wiki/maintenance/archives/patch-interwiki-trans.sql b/www/wiki/maintenance/archives/patch-interwiki-trans.sql new file mode 100644 index 00000000..5cc4d0b5 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-interwiki-trans.sql @@ -0,0 +1,2 @@ +ALTER TABLE /*$wgDBprefix*/interwiki + ADD COLUMN iw_trans TINYINT NOT NULL DEFAULT 0; diff --git a/www/wiki/maintenance/archives/patch-interwiki.sql b/www/wiki/maintenance/archives/patch-interwiki.sql new file mode 100644 index 00000000..57b79456 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-interwiki.sql @@ -0,0 +1,20 @@ +-- Creates interwiki prefix<->url mapping table +-- used from 2003-08-21 dev version. +-- Import the default mappings from maintenance/interwiki.sql + +CREATE TABLE /*$wgDBprefix*/interwiki ( + -- The interwiki prefix, (e.g. "Meatball", or the language prefix "de") + iw_prefix varchar(32) NOT NULL, + + -- The URL of the wiki, with "$1" as a placeholder for an article name. + -- Any spaces in the name will be transformed to underscores before + -- insertion. + iw_url blob NOT NULL, + + -- A boolean value indicating whether the wiki is in this project + -- (used, for example, to detect redirect loops) + iw_local BOOL NOT NULL, + + UNIQUE KEY iw_prefix (iw_prefix) + +) /*$wgDBTableOptions*/; diff --git a/www/wiki/maintenance/archives/patch-inverse_timestamp.sql b/www/wiki/maintenance/archives/patch-inverse_timestamp.sql new file mode 100644 index 00000000..0f7d66f1 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-inverse_timestamp.sql @@ -0,0 +1,15 @@ +-- Removes the inverse_timestamp field from early 1.5 alphas. +-- This field was used in the olden days as a crutch for sorting +-- limitations in MySQL 3.x, but is being dropped now as an +-- unnecessary burden. Serious wikis should be running on 4.x. +-- +-- Updater added 2005-03-13 + +ALTER TABLE /*$wgDBprefix*/revision + DROP COLUMN inverse_timestamp, + DROP INDEX page_timestamp, + DROP INDEX user_timestamp, + DROP INDEX usertext_timestamp, + ADD INDEX page_timestamp (rev_page,rev_timestamp), + ADD INDEX user_timestamp (rev_user,rev_timestamp), + ADD INDEX usertext_timestamp (rev_user_text,rev_timestamp); diff --git a/www/wiki/maintenance/archives/patch-ip_changes.sql b/www/wiki/maintenance/archives/patch-ip_changes.sql new file mode 100644 index 00000000..5f05672e --- /dev/null +++ b/www/wiki/maintenance/archives/patch-ip_changes.sql @@ -0,0 +1,23 @@ +-- +-- Every time an edit by a logged out user is saved, +-- a row is created in ip_changes. This stores +-- the IP as a hex representation so that we can more +-- easily find edits within an IP range. +-- +CREATE TABLE /*_*/ip_changes ( + -- Foreign key to the revision table, also serves as the unique primary key + ipc_rev_id int unsigned NOT NULL PRIMARY KEY DEFAULT '0', + + -- The timestamp of the revision + ipc_rev_timestamp binary(14) NOT NULL DEFAULT '', + + -- Hex representation of the IP address, as returned by IP::toHex() + -- For IPv4 it will resemble: ABCD1234 + -- For IPv6: v6-ABCD1234000000000000000000000000 + -- BETWEEN is then used to identify revisions within a given range + ipc_hex varbinary(35) NOT NULL DEFAULT '' + +) /*$wgDBTableOptions*/; + +CREATE INDEX /*i*/ipc_rev_timestamp ON /*_*/ip_changes (ipc_rev_timestamp); +CREATE INDEX /*i*/ipc_hex_time ON /*_*/ip_changes (ipc_hex,ipc_rev_timestamp); diff --git a/www/wiki/maintenance/archives/patch-ipb-parent-block-id-index.sql b/www/wiki/maintenance/archives/patch-ipb-parent-block-id-index.sql new file mode 100644 index 00000000..1f413f37 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-ipb-parent-block-id-index.sql @@ -0,0 +1,2 @@ +-- index for ipblocks.ipb_parent_block_id +CREATE INDEX /*i*/ipb_parent_block_id ON /*_*/ipblocks (ipb_parent_block_id); diff --git a/www/wiki/maintenance/archives/patch-ipb-parent-block-id.sql b/www/wiki/maintenance/archives/patch-ipb-parent-block-id.sql new file mode 100644 index 00000000..8ebcf786 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-ipb-parent-block-id.sql @@ -0,0 +1,3 @@ +-- Adding ipb_parent_block_id to track the block that caused an autoblock +ALTER TABLE /*$wgDBprefix*/ipblocks + ADD ipb_parent_block_id int DEFAULT NULL; diff --git a/www/wiki/maintenance/archives/patch-ipb_allow_usertalk.sql b/www/wiki/maintenance/archives/patch-ipb_allow_usertalk.sql new file mode 100644 index 00000000..92e7d9a4 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-ipb_allow_usertalk.sql @@ -0,0 +1,3 @@ +-- Adding ipb_allow_usertalk for blocks +ALTER TABLE /*$wgDBprefix*/ipblocks + ADD ipb_allow_usertalk bool NOT NULL default 1; diff --git a/www/wiki/maintenance/archives/patch-ipb_anon_only.sql b/www/wiki/maintenance/archives/patch-ipb_anon_only.sql new file mode 100644 index 00000000..bb39c1d9 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-ipb_anon_only.sql @@ -0,0 +1,44 @@ +-- Add extra option fields to the ipblocks table, add some extra indexes, +-- convert infinity values in ipb_expiry to something that sorts better, +-- extend ipb_address and range fields, add a unique index for block conflict +-- detection. + +-- Conflicts in the new unique index can be handled by creating a new +-- table and inserting into it instead of doing an ALTER TABLE. + + +DROP TABLE IF EXISTS /*$wgDBprefix*/ipblocks_newunique; + +CREATE TABLE /*$wgDBprefix*/ipblocks_newunique ( + ipb_id int NOT NULL auto_increment, + ipb_address tinyblob NOT NULL, + ipb_user int unsigned NOT NULL default '0', + ipb_by int unsigned NOT NULL default '0', + ipb_reason tinyblob NOT NULL, + ipb_timestamp binary(14) NOT NULL default '', + ipb_auto bool NOT NULL default 0, + ipb_anon_only bool NOT NULL default 0, + ipb_create_account bool NOT NULL default 1, + ipb_expiry varbinary(14) NOT NULL default '', + ipb_range_start tinyblob NOT NULL, + ipb_range_end tinyblob NOT NULL, + + PRIMARY KEY ipb_id (ipb_id), + UNIQUE INDEX ipb_address_unique (ipb_address(255), ipb_user, ipb_auto), + INDEX ipb_user (ipb_user), + INDEX ipb_range (ipb_range_start(8), ipb_range_end(8)), + INDEX ipb_timestamp (ipb_timestamp), + INDEX ipb_expiry (ipb_expiry) + +) /*$wgDBTableOptions*/; + +INSERT IGNORE INTO /*$wgDBprefix*/ipblocks_newunique + (ipb_id, ipb_address, ipb_user, ipb_by, ipb_reason, ipb_timestamp, ipb_auto, ipb_expiry, ipb_range_start, ipb_range_end, ipb_anon_only, ipb_create_account) + SELECT ipb_id, ipb_address, ipb_user, ipb_by, ipb_reason, ipb_timestamp, ipb_auto, ipb_expiry, ipb_range_start, ipb_range_end, 0 , ipb_user=0 + FROM /*$wgDBprefix*/ipblocks; + +DROP TABLE IF EXISTS /*$wgDBprefix*/ipblocks_old; +RENAME TABLE /*$wgDBprefix*/ipblocks TO /*$wgDBprefix*/ipblocks_old; +RENAME TABLE /*$wgDBprefix*/ipblocks_newunique TO /*$wgDBprefix*/ipblocks; + +UPDATE /*$wgDBprefix*/ipblocks SET ipb_expiry='infinity' WHERE ipb_expiry=''; diff --git a/www/wiki/maintenance/archives/patch-ipb_by_text.sql b/www/wiki/maintenance/archives/patch-ipb_by_text.sql new file mode 100644 index 00000000..e809d102 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-ipb_by_text.sql @@ -0,0 +1,10 @@ +-- Adding colomn with username of blocker and sets it. +-- Required for crosswiki blocks. + +ALTER TABLE /*$wgDBprefix*/ipblocks + ADD ipb_by_text varchar(255) binary NOT NULL default ''; + +UPDATE /*$wgDBprefix*/ipblocks + JOIN /*$wgDBprefix*/user ON ipb_by = user_id + SET ipb_by_text = user_name + WHERE ipb_by != 0;
\ No newline at end of file diff --git a/www/wiki/maintenance/archives/patch-ipb_deleted.sql b/www/wiki/maintenance/archives/patch-ipb_deleted.sql new file mode 100644 index 00000000..b12ddaaa --- /dev/null +++ b/www/wiki/maintenance/archives/patch-ipb_deleted.sql @@ -0,0 +1,3 @@ +-- Adding ipb_deleted field for hiding usernames +ALTER TABLE /*$wgDBprefix*/ipblocks + ADD ipb_deleted bool NOT NULL default 0; diff --git a/www/wiki/maintenance/archives/patch-ipb_emailban.sql b/www/wiki/maintenance/archives/patch-ipb_emailban.sql new file mode 100644 index 00000000..e05c20b3 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-ipb_emailban.sql @@ -0,0 +1,4 @@ +-- Add row for email blocks -- + +ALTER TABLE /*$wgDBprefix*/ipblocks + ADD ipb_block_email tinyint NOT NULL default '0'; diff --git a/www/wiki/maintenance/archives/patch-ipb_expiry.sql b/www/wiki/maintenance/archives/patch-ipb_expiry.sql new file mode 100644 index 00000000..f3b6a82b --- /dev/null +++ b/www/wiki/maintenance/archives/patch-ipb_expiry.sql @@ -0,0 +1,8 @@ +-- Adds the ipb_expiry field to ipblocks + +ALTER TABLE /*$wgDBprefix*/ipblocks ADD ipb_expiry varbinary(14) NOT NULL default ''; + +-- All IP blocks have one day expiry +UPDATE /*$wgDBprefix*/ipblocks SET ipb_expiry = date_format(date_add(ipb_timestamp,INTERVAL 1 DAY),"%Y%m%d%H%i%s") WHERE ipb_user = 0; + +-- Null string is fine for user blocks, since this indicates infinity diff --git a/www/wiki/maintenance/archives/patch-ipb_optional_autoblock.sql b/www/wiki/maintenance/archives/patch-ipb_optional_autoblock.sql new file mode 100644 index 00000000..f31b8359 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-ipb_optional_autoblock.sql @@ -0,0 +1,3 @@ +-- Add an extra option field "ipb_enable_autoblock" into the ipblocks table. This allows a block to be placed that does not trigger any autoblocks. + +ALTER TABLE /*$wgDBprefix*/ipblocks ADD COLUMN ipb_enable_autoblock bool NOT NULL default '1'; diff --git a/www/wiki/maintenance/archives/patch-ipb_range_start.sql b/www/wiki/maintenance/archives/patch-ipb_range_start.sql new file mode 100644 index 00000000..84cba8f6 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-ipb_range_start.sql @@ -0,0 +1,25 @@ +-- Add the range handling fields +ALTER TABLE /*$wgDBprefix*/ipblocks + ADD ipb_range_start tinyblob NOT NULL default '', + ADD ipb_range_end tinyblob NOT NULL default '', + ADD INDEX ipb_range (ipb_range_start(8), ipb_range_end(8)); + + +-- Initialise fields +-- Only range blocks match ipb_address LIKE '%/%', this fact is used in the code already +UPDATE /*$wgDBprefix*/ipblocks + SET + ipb_range_start = LPAD(HEX( + (SUBSTRING_INDEX(ipb_address, '.', 1) << 24) + + (SUBSTRING_INDEX(SUBSTRING_INDEX(ipb_address, '.', 2), '.', -1) << 16) + + (SUBSTRING_INDEX(SUBSTRING_INDEX(ipb_address, '.', 3), '.', -1) << 24) + + (SUBSTRING_INDEX(SUBSTRING_INDEX(ipb_address, '/', 1), '.', -1)) ), 8, '0' ), + + ipb_range_end = LPAD(HEX( + (SUBSTRING_INDEX(ipb_address, '.', 1) << 24) + + (SUBSTRING_INDEX(SUBSTRING_INDEX(ipb_address, '.', 2), '.', -1) << 16) + + (SUBSTRING_INDEX(SUBSTRING_INDEX(ipb_address, '.', 3), '.', -1) << 24) + + (SUBSTRING_INDEX(SUBSTRING_INDEX(ipb_address, '/', 1), '.', -1)) + + ((1 << (32 - SUBSTRING_INDEX(ipb_address, '/', -1))) - 1) ), 8, '0' ) + + WHERE ipb_address LIKE '%/%'; diff --git a/www/wiki/maintenance/archives/patch-ipblocks.sql b/www/wiki/maintenance/archives/patch-ipblocks.sql new file mode 100644 index 00000000..634fa78c --- /dev/null +++ b/www/wiki/maintenance/archives/patch-ipblocks.sql @@ -0,0 +1,6 @@ +-- For auto-expiring blocks -- + +ALTER TABLE /*$wgDBprefix*/ipblocks + ADD ipb_auto tinyint NOT NULL default '0', + ADD ipb_id int NOT NULL auto_increment, + ADD PRIMARY KEY (ipb_id); diff --git a/www/wiki/maintenance/archives/patch-iw_api_and_wikiid.sql b/www/wiki/maintenance/archives/patch-iw_api_and_wikiid.sql new file mode 100644 index 00000000..4384a715 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-iw_api_and_wikiid.sql @@ -0,0 +1,9 @@ +-- +-- Add iw_api and iw_wikiid to interwiki table +-- + +ALTER TABLE /*_*/interwiki + ADD iw_api BLOB NOT NULL; +ALTER TABLE /*_*/interwiki + ADD iw_wikiid varchar(64) NOT NULL; + diff --git a/www/wiki/maintenance/archives/patch-iwl_prefix_title_from-non-unique.sql b/www/wiki/maintenance/archives/patch-iwl_prefix_title_from-non-unique.sql new file mode 100644 index 00000000..bff63c74 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-iwl_prefix_title_from-non-unique.sql @@ -0,0 +1,5 @@ +-- +-- Makes the iwl_prefix_title_from index for the iwlinks table non-unique +-- +DROP INDEX /*i*/iwl_prefix_title_from ON /*_*/iwlinks; +CREATE INDEX /*i*/iwl_prefix_title_from ON /*_*/iwlinks (iwl_prefix, iwl_title, iwl_from); diff --git a/www/wiki/maintenance/archives/patch-iwlinks-fix-pk.sql b/www/wiki/maintenance/archives/patch-iwlinks-fix-pk.sql new file mode 100644 index 00000000..1dd5220d --- /dev/null +++ b/www/wiki/maintenance/archives/patch-iwlinks-fix-pk.sql @@ -0,0 +1 @@ +ALTER TABLE /*_*/iwlinks DROP KEY /*i*/iwl_from, ADD PRIMARY KEY (iwl_from,iwl_prefix,iwl_title); diff --git a/www/wiki/maintenance/archives/patch-iwlinks-from-title-index.sql b/www/wiki/maintenance/archives/patch-iwlinks-from-title-index.sql new file mode 100644 index 00000000..8b73f9e3 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-iwlinks-from-title-index.sql @@ -0,0 +1,4 @@ +-- +-- Recreates the iwl_prefix_from_title index for the iwlinks table +-- +CREATE INDEX /*i*/iwl_prefix_from_title ON /*_*/iwlinks (iwl_prefix, iwl_from, iwl_title); diff --git a/www/wiki/maintenance/archives/patch-iwlinks.sql b/www/wiki/maintenance/archives/patch-iwlinks.sql new file mode 100644 index 00000000..b7bd3f13 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-iwlinks.sql @@ -0,0 +1,16 @@ +-- +-- Track inline interwiki links +-- +CREATE TABLE /*_*/iwlinks ( + -- page_id of the referring page + iwl_from int unsigned NOT NULL default 0, + + -- Interwiki prefix code of the target + iwl_prefix varbinary(20) NOT NULL default '', + + -- Title of the target, including namespace + iwl_title varchar(255) binary NOT NULL default '' +) /*$wgDBTableOptions*/; + +CREATE UNIQUE INDEX /*i*/iwl_from ON /*_*/iwlinks (iwl_from, iwl_prefix, iwl_title); +CREATE UNIQUE INDEX /*i*/iwl_prefix_title_from ON /*_*/iwlinks (iwl_prefix, iwl_title, iwl_from); diff --git a/www/wiki/maintenance/archives/patch-job.sql b/www/wiki/maintenance/archives/patch-job.sql new file mode 100644 index 00000000..662f5d27 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-job.sql @@ -0,0 +1,20 @@ +-- Jobs performed by parallel apache threads or a command-line daemon +CREATE TABLE /*_*/job ( + job_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + + -- Command name + -- Limited to 60 to prevent key length overflow + job_cmd varbinary(60) NOT NULL default '', + + -- Namespace and title to act on + -- Should be 0 and '' if the command does not operate on a title + job_namespace int NOT NULL, + job_title varchar(255) binary NOT NULL, + + -- Any other parameters to the command + -- Stored as a PHP serialized array, or an empty string if there are no parameters + job_params blob NOT NULL +) /*$wgDBTableOptions*/; + +CREATE INDEX /*i*/job_cmd ON /*_*/job (job_cmd, job_namespace, job_title, job_params(128)); + diff --git a/www/wiki/maintenance/archives/patch-job_attempts.sql b/www/wiki/maintenance/archives/patch-job_attempts.sql new file mode 100644 index 00000000..47b73e81 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-job_attempts.sql @@ -0,0 +1,4 @@ +ALTER TABLE /*_*/job + ADD COLUMN job_attempts integer unsigned NOT NULL default 0; + +CREATE INDEX /*i*/job_cmd_token_id ON /*_*/job (job_cmd,job_token,job_id); diff --git a/www/wiki/maintenance/archives/patch-job_token.sql b/www/wiki/maintenance/archives/patch-job_token.sql new file mode 100644 index 00000000..080fa97c --- /dev/null +++ b/www/wiki/maintenance/archives/patch-job_token.sql @@ -0,0 +1,9 @@ +ALTER TABLE /*_*/job + ADD COLUMN job_random integer unsigned NOT NULL default 0, + ADD COLUMN job_token varbinary(32) NOT NULL default '', + ADD COLUMN job_token_timestamp varbinary(14) NULL default NULL, + ADD COLUMN job_sha1 varbinary(32) NOT NULL default ''; + +CREATE INDEX /*i*/job_sha1 ON /*_*/job (job_sha1); +CREATE INDEX /*i*/job_cmd_token ON /*_*/job (job_cmd,job_token,job_random); + diff --git a/www/wiki/maintenance/archives/patch-jobs-add-timestamp.sql b/www/wiki/maintenance/archives/patch-jobs-add-timestamp.sql new file mode 100644 index 00000000..c5e6e711 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-jobs-add-timestamp.sql @@ -0,0 +1,2 @@ +ALTER TABLE /*_*/job ADD COLUMN job_timestamp varbinary(14) NULL default NULL; +CREATE INDEX /*i*/job_timestamp ON /*_*/job(job_timestamp); diff --git a/www/wiki/maintenance/archives/patch-kill-cl_collation_index.sql b/www/wiki/maintenance/archives/patch-kill-cl_collation_index.sql new file mode 100644 index 00000000..7f75a623 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-kill-cl_collation_index.sql @@ -0,0 +1,7 @@ +-- +-- Kill cl_collation index. +-- @since 1.27 +-- + +DROP INDEX /*i*/cl_collation ON /*_*/categorylinks; + diff --git a/www/wiki/maintenance/archives/patch-kill-iwl_prefix.sql b/www/wiki/maintenance/archives/patch-kill-iwl_prefix.sql new file mode 100644 index 00000000..1cd9b454 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-kill-iwl_prefix.sql @@ -0,0 +1,7 @@ +-- +-- Kill the old iwl_prefix index, which may be present on some +-- installs if they ran update.php between it being added and being renamed +-- + +DROP INDEX /*i*/iwl_prefix ON /*_*/iwlinks; + diff --git a/www/wiki/maintenance/archives/patch-l10n_cache-primary-key.sql b/www/wiki/maintenance/archives/patch-l10n_cache-primary-key.sql new file mode 100644 index 00000000..d5830396 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-l10n_cache-primary-key.sql @@ -0,0 +1,8 @@ +-- +-- patch-l10n_cache-primary-key.sql +-- +-- Bug T146591. Add l10n_cache primary key + +DELETE FROM /*$wgDBprefix*/l10n_cache; + +ALTER TABLE /*$wgDBprefix*/l10n_cache DROP KEY /*i*/lc_lang_key, ADD PRIMARY KEY(lc_lang, lc_key); diff --git a/www/wiki/maintenance/archives/patch-l10n_cache.sql b/www/wiki/maintenance/archives/patch-l10n_cache.sql new file mode 100644 index 00000000..4c865d71 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-l10n_cache.sql @@ -0,0 +1,8 @@ +-- Table for storing localisation data +CREATE TABLE /*_*/l10n_cache ( + lc_lang varbinary(32) NOT NULL, + lc_key varchar(255) NOT NULL, + lc_value mediumblob NOT NULL +) /*$wgDBTableOptions*/; +CREATE INDEX /*i*/lc_lang_key ON /*_*/l10n_cache (lc_lang, lc_key); + diff --git a/www/wiki/maintenance/archives/patch-langlinks-fix-pk.sql b/www/wiki/maintenance/archives/patch-langlinks-fix-pk.sql new file mode 100644 index 00000000..e3ac3125 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-langlinks-fix-pk.sql @@ -0,0 +1 @@ +ALTER TABLE /*_*/langlinks DROP KEY /*i*/ll_from, ADD PRIMARY KEY (ll_from,ll_lang); diff --git a/www/wiki/maintenance/archives/patch-langlinks-ll_lang-20.sql b/www/wiki/maintenance/archives/patch-langlinks-ll_lang-20.sql new file mode 100644 index 00000000..ce026382 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-langlinks-ll_lang-20.sql @@ -0,0 +1,3 @@ +ALTER TABLE /*$wgDBprefix*/langlinks + MODIFY `ll_lang` + VARBINARY(20) NOT NULL DEFAULT '';
\ No newline at end of file diff --git a/www/wiki/maintenance/archives/patch-langlinks.sql b/www/wiki/maintenance/archives/patch-langlinks.sql new file mode 100644 index 00000000..5594acd5 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-langlinks.sql @@ -0,0 +1,14 @@ +CREATE TABLE /*$wgDBprefix*/langlinks ( + -- page_id of the referring page + ll_from int unsigned NOT NULL default '0', + + -- Language code of the target + ll_lang varbinary(20) NOT NULL default '', + + -- Title of the target, including namespace + ll_title varchar(255) binary NOT NULL default '', + + UNIQUE KEY (ll_from, ll_lang), + KEY (ll_lang, ll_title) +) /*$wgDBTableOptions*/; + diff --git a/www/wiki/maintenance/archives/patch-linkscc-1.3.sql b/www/wiki/maintenance/archives/patch-linkscc-1.3.sql new file mode 100644 index 00000000..e397fcb9 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-linkscc-1.3.sql @@ -0,0 +1,6 @@ +-- +-- linkscc table used to cache link lists in easier to digest form. +-- New schema for 1.3 - removes old lcc_title column. +-- May 2004 +-- +ALTER TABLE /*$wgDBprefix*/linkscc DROP COLUMN lcc_title;
\ No newline at end of file diff --git a/www/wiki/maintenance/archives/patch-linkscc.sql b/www/wiki/maintenance/archives/patch-linkscc.sql new file mode 100644 index 00000000..684384f5 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-linkscc.sql @@ -0,0 +1,12 @@ +-- +-- linkscc table used to cache link lists in easier to digest form +-- November 2003 +-- +-- Format later updated. +-- + +CREATE TABLE /*$wgDBprefix*/linkscc ( + lcc_pageid INT UNSIGNED NOT NULL UNIQUE KEY, + lcc_cacheobj MEDIUMBLOB NOT NULL + +) /*$wgDBTableOptions*/; diff --git a/www/wiki/maintenance/archives/patch-linktables.sql b/www/wiki/maintenance/archives/patch-linktables.sql new file mode 100644 index 00000000..d53d2ea3 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-linktables.sql @@ -0,0 +1,70 @@ +-- +-- Track links that do exist +-- l_from and l_to key to cur_id +-- +DROP TABLE IF EXISTS /*$wgDBprefix*/links; +CREATE TABLE /*$wgDBprefix*/links ( + -- Key to the page_id of the page containing the link. + l_from int unsigned NOT NULL default '0', + + -- Key to the page_id of the link target. + -- An unfortunate consequence of this is that rename + -- operations require changing the links entries for + -- all links to the moved page. + l_to int unsigned NOT NULL default '0', + + UNIQUE KEY l_from(l_from,l_to), + KEY (l_to) + +) /*$wgDBTableOptions*/; + +-- +-- Track links to pages that don't yet exist. +-- bl_from keys to cur_id +-- bl_to is a text link (namespace:title) +-- +DROP TABLE IF EXISTS /*$wgDBprefix*/brokenlinks; +CREATE TABLE /*$wgDBprefix*/brokenlinks ( + -- Key to the page_id of the page containing the link. + bl_from int unsigned NOT NULL default '0', + + -- Text of the target page title ("namesapce:title"). + -- Unfortunately this doesn't split the namespace index + -- key and therefore can't easily be joined to anything. + bl_to varchar(255) binary NOT NULL default '', + UNIQUE KEY bl_from(bl_from,bl_to), + KEY (bl_to) + +) /*$wgDBTableOptions*/; + +-- +-- Track links to images *used inline* +-- il_from keys to cur_id, il_to keys to image_name. +-- We don't distinguish live from broken links. +-- +DROP TABLE IF EXISTS /*$wgDBprefix*/imagelinks; +CREATE TABLE /*$wgDBprefix*/imagelinks ( + -- Key to page_id of the page containing the image / media link. + il_from int unsigned NOT NULL default '0', + + -- Filename of target image. + -- This is also the page_title of the file's description page; + -- all such pages are in namespace 6 (NS_FILE). + il_to varchar(255) binary NOT NULL default '', + + UNIQUE KEY il_from(il_from,il_to), + KEY (il_to) + +) /*$wgDBTableOptions*/; + +-- +-- Stores (possibly gzipped) serialized objects with +-- cache arrays to reduce database load slurping up +-- from links and brokenlinks. +-- +DROP TABLE IF EXISTS /*$wgDBprefix*/linkscc; +CREATE TABLE /*$wgDBprefix*/linkscc ( + lcc_pageid INT UNSIGNED NOT NULL UNIQUE KEY, + lcc_cacheobj MEDIUMBLOB NOT NULL + +) /*$wgDBTableOptions*/; diff --git a/www/wiki/maintenance/archives/patch-log_deleted.sql b/www/wiki/maintenance/archives/patch-log_deleted.sql new file mode 100644 index 00000000..0fce0f51 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-log_deleted.sql @@ -0,0 +1,3 @@ +-- Adding ar_deleted field for revisiondelete +ALTER TABLE /*$wgDBprefix*/logging + ADD log_deleted tinyint unsigned NOT NULL default '0'; diff --git a/www/wiki/maintenance/archives/patch-log_id.sql b/www/wiki/maintenance/archives/patch-log_id.sql new file mode 100644 index 00000000..bd69ddb6 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-log_id.sql @@ -0,0 +1,8 @@ +-- Log_id field that means one log entry can be referred to with a single number, +-- rather than a dirty great big mess of features. +-- This might be useful for single-log-entry deletion, et cetera. +-- Andrew Garrett, February 2007. + +ALTER TABLE /*$wgDBprefix*/logging + ADD COLUMN log_id int unsigned not null auto_increment, + ADD PRIMARY KEY log_id (log_id); diff --git a/www/wiki/maintenance/archives/patch-log_params.sql b/www/wiki/maintenance/archives/patch-log_params.sql new file mode 100644 index 00000000..ff6527ec --- /dev/null +++ b/www/wiki/maintenance/archives/patch-log_params.sql @@ -0,0 +1 @@ +ALTER TABLE /*$wgDBprefix*/logging ADD log_params blob NOT NULL; diff --git a/www/wiki/maintenance/archives/patch-log_search-fix-pk.sql b/www/wiki/maintenance/archives/patch-log_search-fix-pk.sql new file mode 100644 index 00000000..51bfdf59 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-log_search-fix-pk.sql @@ -0,0 +1 @@ +ALTER TABLE /*_*/log_search DROP KEY /*i*/ls_field_val, ADD PRIMARY KEY (ls_field,ls_value,ls_log_id); diff --git a/www/wiki/maintenance/archives/patch-log_search-rename-index.sql b/www/wiki/maintenance/archives/patch-log_search-rename-index.sql new file mode 100644 index 00000000..7e1113e6 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-log_search-rename-index.sql @@ -0,0 +1,7 @@ +-- Rename the primary unique index from PRIMARY to ls_field_val +-- This is for MySQL only and is necessary only for databases which were updated +-- between MW 1.16 development revisions r50567 and r51465. +ALTER TABLE /*_*/log_search + DROP PRIMARY KEY, + ADD UNIQUE INDEX ls_field_val (ls_field,ls_value,ls_log_id); + diff --git a/www/wiki/maintenance/archives/patch-log_search.sql b/www/wiki/maintenance/archives/patch-log_search.sql new file mode 100644 index 00000000..8d92030b --- /dev/null +++ b/www/wiki/maintenance/archives/patch-log_search.sql @@ -0,0 +1,10 @@ +CREATE TABLE /*_*/log_search ( + -- The type of ID (rev ID, log ID, rev timestamp, username) + ls_field varbinary(32) NOT NULL, + -- The value of the ID + ls_value varchar(255) NOT NULL, + -- Key to log_id + ls_log_id int unsigned NOT NULL default 0 +) /*$wgDBTableOptions*/; +CREATE UNIQUE INDEX /*i*/ls_field_val ON /*_*/log_search (ls_field,ls_value,ls_log_id); +CREATE INDEX /*i*/ls_log_id ON /*_*/log_search (ls_log_id); diff --git a/www/wiki/maintenance/archives/patch-log_user_text.sql b/www/wiki/maintenance/archives/patch-log_user_text.sql new file mode 100644 index 00000000..12ca75e5 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-log_user_text.sql @@ -0,0 +1,8 @@ +ALTER TABLE /*$wgDBprefix*/logging + ADD log_user_text varchar(255) binary NOT NULL default '', + ADD log_page int unsigned NULL, + CHANGE log_type log_type varbinary(32) NOT NULL, + CHANGE log_action log_action varbinary(32) NOT NULL; + +CREATE INDEX /*i*/log_user_type_time ON /*_*/logging (log_user, log_type, log_timestamp); +CREATE INDEX /*i*/log_page_id_time ON /*_*/logging (log_page,log_timestamp); diff --git a/www/wiki/maintenance/archives/patch-logging-times-index.sql b/www/wiki/maintenance/archives/patch-logging-times-index.sql new file mode 100644 index 00000000..5f24f5c3 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-logging-times-index.sql @@ -0,0 +1,9 @@ +-- +-- patch-logging-times-index.sql +-- +-- Add a very humble index on logging times +-- + +ALTER TABLE /*$wgDBprefix*/logging + ADD INDEX times (log_timestamp); + diff --git a/www/wiki/maintenance/archives/patch-logging-title.sql b/www/wiki/maintenance/archives/patch-logging-title.sql new file mode 100644 index 00000000..c5da0dc0 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-logging-title.sql @@ -0,0 +1,6 @@ +-- 1.4 betas were missing the 'binary' marker from logging.log_title, +-- which causes a collation mismatch error on joins in MySQL 4.1. + +ALTER TABLE /*$wgDBprefix*/logging + CHANGE COLUMN log_title + log_title varchar(255) binary NOT NULL default ''; diff --git a/www/wiki/maintenance/archives/patch-logging-type-action-index.sql b/www/wiki/maintenance/archives/patch-logging-type-action-index.sql new file mode 100644 index 00000000..5edc61a5 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-logging-type-action-index.sql @@ -0,0 +1 @@ +CREATE INDEX /*i*/type_action ON /*_*/logging(log_type, log_action, log_timestamp); diff --git a/www/wiki/maintenance/archives/patch-logging.sql b/www/wiki/maintenance/archives/patch-logging.sql new file mode 100644 index 00000000..79df0dd4 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-logging.sql @@ -0,0 +1,37 @@ +-- Add the logging table and adjust recentchanges to accomodate special pages +-- 2004-08-24 + +CREATE TABLE /*$wgDBprefix*/logging ( + -- Symbolic keys for the general log type and the action type + -- within the log. The output format will be controlled by the + -- action field, but only the type controls categorization. + log_type varbinary(10) NOT NULL default '', + log_action varbinary(10) NOT NULL default '', + + -- Timestamp. Duh. + log_timestamp binary(14) NOT NULL default '19700101000000', + + -- The user who performed this action; key to user_id + log_user int unsigned NOT NULL default 0, + + -- Key to the page affected. Where a user is the target, + -- this will point to the user page. + log_namespace int NOT NULL default 0, + log_title varchar(255) binary NOT NULL default '', + + -- Freeform text. Interpreted as edit history comments. + log_comment varchar(255) NOT NULL default '', + + -- LF separated list of miscellaneous parameters + log_params blob NOT NULL, + + KEY type_time (log_type, log_timestamp), + KEY user_time (log_user, log_timestamp), + KEY page_time (log_namespace, log_title, log_timestamp) + +) /*$wgDBTableOptions*/; + + +-- Change from unsigned to signed so we can store special pages +ALTER TABLE recentchanges + MODIFY rc_namespace tinyint(3) NOT NULL default '0'; diff --git a/www/wiki/maintenance/archives/patch-logging_user_text_time_index.sql b/www/wiki/maintenance/archives/patch-logging_user_text_time_index.sql new file mode 100644 index 00000000..06f29861 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-logging_user_text_time_index.sql @@ -0,0 +1 @@ +CREATE INDEX /*i*/log_user_text_time ON /*_*/logging (log_user_text, log_timestamp); diff --git a/www/wiki/maintenance/archives/patch-logging_user_text_type_time_index.sql b/www/wiki/maintenance/archives/patch-logging_user_text_type_time_index.sql new file mode 100644 index 00000000..2801bc86 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-logging_user_text_type_time_index.sql @@ -0,0 +1 @@ +CREATE INDEX /*i*/log_user_text_type_time ON /*_*/logging (log_user_text, log_type, log_timestamp); diff --git a/www/wiki/maintenance/archives/patch-mime_minor_length.sql b/www/wiki/maintenance/archives/patch-mime_minor_length.sql new file mode 100644 index 00000000..88dd64cf --- /dev/null +++ b/www/wiki/maintenance/archives/patch-mime_minor_length.sql @@ -0,0 +1,10 @@ +ALTER TABLE /*_*/filearchive + MODIFY COLUMN fa_minor_mime varbinary(100) default "unknown"; + +ALTER TABLE /*_*/image + MODIFY COLUMN img_minor_mime varbinary(100) NOT NULL default "unknown"; + +ALTER TABLE /*_*/oldimage + MODIFY COLUMN oi_minor_mime varbinary(100) NOT NULL default "unknown"; + +INSERT INTO /*_*/updatelog(ul_key) VALUES ('mime_minor_length'); diff --git a/www/wiki/maintenance/archives/patch-mimesearch-indexes.sql b/www/wiki/maintenance/archives/patch-mimesearch-indexes.sql new file mode 100644 index 00000000..8d9426ea --- /dev/null +++ b/www/wiki/maintenance/archives/patch-mimesearch-indexes.sql @@ -0,0 +1,22 @@ +-- Add indexes to the MIME types in image for use on Special:MIMEsearch, +-- changes a query like +-- +-- SELECT img_name FROM image WHERE img_major_mime = "image" AND img_minor_mime = "svg"; +-- from: +-- +-------+------+---------------+------+---------+------+------+-------------+ +-- | table | type | possible_keys | key | key_len | ref | rows | Extra | +-- +-------+------+---------------+------+---------+------+------+-------------+ +-- | image | ALL | NULL | NULL | NULL | NULL | 194 | Using where | +-- +-------+------+---------------+------+---------+------+------+-------------+ +-- to: +-- +-------+------+-------------------------------+----------------+---------+-------+------+-------------+ +-- | table | type | possible_keys | key | key_len | ref | rows | Extra | +-- +-------+------+-------------------------------+----------------+---------+-------+------+-------------+ +-- | image | ref | img_major_mime,img_minor_mime | img_minor_mime | 32 | const | 4 | Using where | +-- +-------+------+-------------------------------+----------------+---------+-------+------+-------------+ + +ALTER TABLE /*$wgDBprefix*/image + ADD INDEX img_major_mime (img_major_mime); +ALTER TABLE /*$wgDBprefix*/image + ADD INDEX img_minor_mime (img_minor_mime); + diff --git a/www/wiki/maintenance/archives/patch-module_deps-fix-pk.sql b/www/wiki/maintenance/archives/patch-module_deps-fix-pk.sql new file mode 100644 index 00000000..2338df0a --- /dev/null +++ b/www/wiki/maintenance/archives/patch-module_deps-fix-pk.sql @@ -0,0 +1 @@ +ALTER TABLE /*_*/module_deps DROP KEY /*i*/md_module_skin, ADD PRIMARY KEY (md_module,md_skin); diff --git a/www/wiki/maintenance/archives/patch-module_deps.sql b/www/wiki/maintenance/archives/patch-module_deps.sql new file mode 100644 index 00000000..ffc94829 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-module_deps.sql @@ -0,0 +1,12 @@ +-- Table for tracking which local files a module depends on that aren't +-- registered directly. +-- Currently only used for tracking images that CSS depends on +CREATE TABLE /*_*/module_deps ( + -- Module name + md_module varbinary(255) NOT NULL, + -- Skin name + md_skin varbinary(32) NOT NULL, + -- JSON blob with file dependencies + md_deps mediumblob NOT NULL +) /*$wgDBTableOptions*/; +CREATE UNIQUE INDEX /*i*/md_module_skin ON /*_*/module_deps (md_module, md_skin); diff --git a/www/wiki/maintenance/archives/patch-objectcache-fix-pk.sql b/www/wiki/maintenance/archives/patch-objectcache-fix-pk.sql new file mode 100644 index 00000000..cd557160 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-objectcache-fix-pk.sql @@ -0,0 +1 @@ +ALTER TABLE /*_*/objectcache DROP KEY /*i*/keyname, ADD PRIMARY KEY (keyname); diff --git a/www/wiki/maintenance/archives/patch-objectcache.sql b/www/wiki/maintenance/archives/patch-objectcache.sql new file mode 100644 index 00000000..5edf305b --- /dev/null +++ b/www/wiki/maintenance/archives/patch-objectcache.sql @@ -0,0 +1,9 @@ +-- For a few generic cache operations if not using Memcached +CREATE TABLE /*$wgDBprefix*/objectcache ( + keyname varbinary(255) NOT NULL default '', + value mediumblob, + exptime datetime, + UNIQUE KEY (keyname), + KEY (exptime) + +) /*$wgDBTableOptions*/; diff --git a/www/wiki/maintenance/archives/patch-oi_major_mime-chemical.sql b/www/wiki/maintenance/archives/patch-oi_major_mime-chemical.sql new file mode 100644 index 00000000..e3b4552d --- /dev/null +++ b/www/wiki/maintenance/archives/patch-oi_major_mime-chemical.sql @@ -0,0 +1,3 @@ +ALTER TABLE /*$wgDBprefix*/oldimage + CHANGE oi_major_mime oi_major_mime ENUM('unknown','application','audio','image','text','video','message','model','multipart','chemical'); + diff --git a/www/wiki/maintenance/archives/patch-oi_metadata.sql b/www/wiki/maintenance/archives/patch-oi_metadata.sql new file mode 100644 index 00000000..df043c55 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-oi_metadata.sql @@ -0,0 +1,17 @@ +-- +-- patch-oi_metadata.sql +-- +-- Add data to allow for direct reference to old images +-- Some re-indexing here. +-- Old images can be included into pages effeciently now. +-- + +ALTER TABLE /*$wgDBprefix*/oldimage + DROP INDEX oi_name, + ADD INDEX oi_name_timestamp (oi_name,oi_timestamp), + ADD INDEX oi_name_archive_name (oi_name,oi_archive_name(14)), + ADD oi_metadata mediumblob NOT NULL, + ADD oi_media_type ENUM("UNKNOWN", "BITMAP", "DRAWING", "AUDIO", "VIDEO", "MULTIMEDIA", "OFFICE", "TEXT", "EXECUTABLE", "ARCHIVE") default NULL, + ADD oi_major_mime ENUM("unknown", "application", "audio", "image", "text", "video", "message", "model", "multipart") NOT NULL default "unknown", + ADD oi_minor_mime varbinary(32) NOT NULL default "unknown", + ADD oi_deleted tinyint unsigned NOT NULL default '0'; diff --git a/www/wiki/maintenance/archives/patch-oldestindex.sql b/www/wiki/maintenance/archives/patch-oldestindex.sql new file mode 100644 index 00000000..930214fd --- /dev/null +++ b/www/wiki/maintenance/archives/patch-oldestindex.sql @@ -0,0 +1,5 @@ +-- Add index for "Oldest articles" (Special:Ancientpages) +-- 2003-05-23 Erik Moeller <moeller@scireview.de> + +ALTER TABLE /*$wgDBprefix*/cur + ADD INDEX namespace_redirect_timestamp(cur_namespace,cur_is_redirect,cur_timestamp); diff --git a/www/wiki/maintenance/archives/patch-oldimage-user-index.sql b/www/wiki/maintenance/archives/patch-oldimage-user-index.sql new file mode 100644 index 00000000..2c7f8071 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-oldimage-user-index.sql @@ -0,0 +1,8 @@ +-- +-- oldimage-user-index.sql +-- +-- Add user/timestamp index to old image versions +-- + +ALTER TABLE /*$wgDBprefix*/oldimage + ADD INDEX oi_usertext_timestamp (oi_user_text,oi_timestamp); diff --git a/www/wiki/maintenance/archives/patch-page-page_content_model.sql b/www/wiki/maintenance/archives/patch-page-page_content_model.sql new file mode 100644 index 00000000..30434d93 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-page-page_content_model.sql @@ -0,0 +1,2 @@ +ALTER TABLE /*$wgDBprefix*/page + ADD page_content_model varbinary(32) DEFAULT NULL; diff --git a/www/wiki/maintenance/archives/patch-page_lang.sql b/www/wiki/maintenance/archives/patch-page_lang.sql new file mode 100644 index 00000000..c792b4ad --- /dev/null +++ b/www/wiki/maintenance/archives/patch-page_lang.sql @@ -0,0 +1,2 @@ +ALTER TABLE /*$wgDBprefix*/page + ADD page_lang varbinary(35) DEFAULT NULL; diff --git a/www/wiki/maintenance/archives/patch-page_len.sql b/www/wiki/maintenance/archives/patch-page_len.sql new file mode 100644 index 00000000..7d01d90a --- /dev/null +++ b/www/wiki/maintenance/archives/patch-page_len.sql @@ -0,0 +1,16 @@ +-- Page length field (in bytes) for current revision of page. +-- Since page text is now stored separately, it may be compressed +-- or otherwise difficult to calculate. Additionally, the field +-- can be indexed for handy 'long' and 'short' page lists. +-- +-- Added 2005-03-12 + +ALTER TABLE /*$wgDBprefix*/page + ADD page_len int unsigned NOT NULL, + ADD INDEX (page_len); + +-- Not accurate if upgrading from intermediate +-- 1.5 alpha and have revision compression on. +UPDATE /*$wgDBprefix*/page, /*$wgDBprefix*/text + SET page_len=LENGTH(old_text) + WHERE page_latest=old_id; diff --git a/www/wiki/maintenance/archives/patch-page_links_updated.sql b/www/wiki/maintenance/archives/patch-page_links_updated.sql new file mode 100644 index 00000000..18d9e2d9 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-page_links_updated.sql @@ -0,0 +1,2 @@ +ALTER TABLE /*$wgDBprefix*/page + ADD page_links_updated varbinary(14) NULL default NULL; diff --git a/www/wiki/maintenance/archives/patch-page_props-propname-page-index.sql b/www/wiki/maintenance/archives/patch-page_props-propname-page-index.sql new file mode 100644 index 00000000..822fa04d --- /dev/null +++ b/www/wiki/maintenance/archives/patch-page_props-propname-page-index.sql @@ -0,0 +1,4 @@ +-- +-- Creates the pp_propname_page index on page_props +-- +CREATE UNIQUE INDEX /*i*/pp_propname_page ON /*_*/page_props (pp_propname, pp_page); diff --git a/www/wiki/maintenance/archives/patch-page_props.sql b/www/wiki/maintenance/archives/patch-page_props.sql new file mode 100644 index 00000000..15a35581 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-page_props.sql @@ -0,0 +1,9 @@ +-- Name/value pairs indexed by page_id +CREATE TABLE /*$wgDBprefix*/page_props ( + pp_page int NOT NULL, + pp_propname varbinary(60) NOT NULL, + pp_value blob NOT NULL, + + PRIMARY KEY (pp_page,pp_propname) +) /*$wgDBTableOptions*/; + diff --git a/www/wiki/maintenance/archives/patch-page_redirect_namespace_len.sql b/www/wiki/maintenance/archives/patch-page_redirect_namespace_len.sql new file mode 100644 index 00000000..392945fb --- /dev/null +++ b/www/wiki/maintenance/archives/patch-page_redirect_namespace_len.sql @@ -0,0 +1,6 @@ +-- +-- Add the page_redirect_namespace_len index +-- + +CREATE INDEX /*i*/page_redirect_namespace_len ON /*_*/page (page_is_redirect, page_namespace, page_len); + diff --git a/www/wiki/maintenance/archives/patch-page_restrictions-pr_user-unsigned.sql b/www/wiki/maintenance/archives/patch-page_restrictions-pr_user-unsigned.sql new file mode 100644 index 00000000..2337ff0c --- /dev/null +++ b/www/wiki/maintenance/archives/patch-page_restrictions-pr_user-unsigned.sql @@ -0,0 +1 @@ +ALTER TABLE /*_*/page_restrictions MODIFY pr_user int unsigned NULL; diff --git a/www/wiki/maintenance/archives/patch-page_restrictions.sql b/www/wiki/maintenance/archives/patch-page_restrictions.sql new file mode 100644 index 00000000..6813c1e7 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-page_restrictions.sql @@ -0,0 +1,20 @@ +--- Used for storing page restrictions (i.e. protection levels) +CREATE TABLE /*$wgDBprefix*/page_restrictions ( + -- Page to apply restrictions to (Foreign Key to page). + pr_page int NOT NULL, + -- The protection type (edit, move, etc) + pr_type varbinary(60) NOT NULL, + -- The protection level (Sysop, autoconfirmed, etc) + pr_level varbinary(60) NOT NULL, + -- Whether or not to cascade the protection down to pages transcluded. + pr_cascade tinyint NOT NULL, + -- Field for future support of per-user restriction. + pr_user int NULL, + -- Field for time-limited protection. + pr_expiry varbinary(14) NULL, + + PRIMARY KEY pr_pagetype (pr_page,pr_type), + KEY pr_typelevel (pr_type,pr_level), + KEY pr_level (pr_level), + KEY pr_cascade (pr_cascade) +) /*$wgDBTableOptions*/; diff --git a/www/wiki/maintenance/archives/patch-page_restrictions_sortkey.sql b/www/wiki/maintenance/archives/patch-page_restrictions_sortkey.sql new file mode 100644 index 00000000..6b24e3a5 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-page_restrictions_sortkey.sql @@ -0,0 +1,8 @@ +-- Add a sort-key to page_restrictions table. +-- First immediate use of this is as a sort-key for coming modifications +-- of Special:Protectedpages. +-- Andrew Garrett, February 2007 + +ALTER TABLE /*$wgDBprefix*/page_restrictions + ADD COLUMN pr_id int unsigned not null auto_increment, + ADD UNIQUE KEY pr_id (pr_id); diff --git a/www/wiki/maintenance/archives/patch-pagelinks-fix-pk.sql b/www/wiki/maintenance/archives/patch-pagelinks-fix-pk.sql new file mode 100644 index 00000000..e2691439 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-pagelinks-fix-pk.sql @@ -0,0 +1 @@ +ALTER TABLE /*_*/pagelinks DROP INDEX /*i*/pl_from, ADD PRIMARY KEY (pl_from,pl_namespace,pl_title); diff --git a/www/wiki/maintenance/archives/patch-pagelinks.sql b/www/wiki/maintenance/archives/patch-pagelinks.sql new file mode 100644 index 00000000..cea89b52 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-pagelinks.sql @@ -0,0 +1,56 @@ +-- +-- Create the new pagelinks table to merge links and brokenlinks data, +-- and populate it. +-- +-- Unlike the old links and brokenlinks, these records will not need to be +-- altered when target pages are created, deleted, or renamed. This should +-- reduce the amount of severe database frustration that happens when widely- +-- linked pages are altered. +-- +-- Fixups for brokenlinks to pages in namespaces need to be run after this; +-- this is done by updaters.inc if run through the regular update scripts. +-- +-- 2005-05-26 +-- + +-- +-- Track page-to-page hyperlinks within the wiki. +-- +CREATE TABLE /*$wgDBprefix*/pagelinks ( + -- Key to the page_id of the page containing the link. + pl_from int unsigned NOT NULL default '0', + + -- Key to page_namespace/page_title of the target page. + -- The target page may or may not exist, and due to renames + -- and deletions may refer to different page records as time + -- goes by. + pl_namespace int NOT NULL default '0', + pl_title varchar(255) binary NOT NULL default '', + + UNIQUE KEY pl_from(pl_from,pl_namespace,pl_title), + KEY (pl_namespace,pl_title) + +) /*$wgDBTableOptions*/; + + +-- Import existing-page links +INSERT + INTO /*$wgDBprefix*/pagelinks (pl_from,pl_namespace,pl_title) + SELECT l_from,page_namespace,page_title + FROM /*$wgDBprefix*/links, /*$wgDBprefix*/page + WHERE l_to=page_id; + +-- import brokenlinks +-- NOTE: We'll have to fix up individual entries that aren't in main NS +INSERT INTO /*$wgDBprefix*/pagelinks (pl_from,pl_namespace,pl_title) + SELECT bl_from, 0, bl_to + FROM /*$wgDBprefix*/brokenlinks; + +-- For each namespace do something like: +-- +-- UPDATE /*$wgDBprefix*/pagelinks +-- SET pl_namespace=$ns, +-- pl_title=TRIM(LEADING '$prefix:' FROM pl_title) +-- WHERE pl_namespace=0 +-- AND pl_title LIKE '$likeprefix:%'"; +-- diff --git a/www/wiki/maintenance/archives/patch-parsercache.sql b/www/wiki/maintenance/archives/patch-parsercache.sql new file mode 100644 index 00000000..5fe241c3 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-parsercache.sql @@ -0,0 +1,15 @@ +-- +-- parsercache table, for cacheing complete parsed articles +-- before they are imbedded in the skin. +-- + +CREATE TABLE /*$wgDBprefix*/parsercache ( + pc_pageid INT(11) NOT NULL, + pc_title VARCHAR(255) NOT NULL, + pc_prefhash CHAR(32) NOT NULL, + pc_expire DATETIME NOT NULL, + pc_data MEDIUMBLOB NOT NULL, + PRIMARY KEY (pc_pageid, pc_prefhash), + KEY(pc_title), + KEY(pc_expire) +) /*$wgDBTableOptions*/; diff --git a/www/wiki/maintenance/archives/patch-pl-tl-il-nonunique.sql b/www/wiki/maintenance/archives/patch-pl-tl-il-nonunique.sql new file mode 100644 index 00000000..8e1715b3 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-pl-tl-il-nonunique.sql @@ -0,0 +1,11 @@ +-- Make reorderings of UNIQUE indices non-UNIQUE +-- Since 1.24, these indices have been non-UNIQUE in tables.sql. +-- However, an earlier update from 1.15 that made the indices +-- UNIQUE was not removed until 1.28 (T78513). + +DROP INDEX /*i*/pl_namespace ON /*_*/pagelinks; +CREATE INDEX /*i*/pl_namespace ON /*_*/pagelinks (pl_namespace, pl_title, pl_from); +DROP INDEX /*i*/tl_namespace ON /*_*/templatelinks; +CREATE INDEX /*i*/tl_namespace ON /*_*/templatelinks (tl_namespace, tl_title, tl_from); +DROP INDEX /*i*/il_to ON /*_*/imagelinks; +CREATE INDEX /*i*/il_to ON /*_*/imagelinks (il_to, il_from); diff --git a/www/wiki/maintenance/archives/patch-pl-tl-il-unique.sql b/www/wiki/maintenance/archives/patch-pl-tl-il-unique.sql new file mode 100644 index 00000000..a3566705 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-pl-tl-il-unique.sql @@ -0,0 +1,11 @@ +-- +-- patch-pl-tl-il-unique-index.sql +-- +-- Make reorderings of UNIQUE indices UNIQUE as well + +DROP INDEX /*i*/pl_namespace ON /*_*/pagelinks; +CREATE UNIQUE INDEX /*i*/pl_namespace ON /*_*/pagelinks (pl_namespace, pl_title, pl_from); +DROP INDEX /*i*/tl_namespace ON /*_*/templatelinks; +CREATE UNIQUE INDEX /*i*/tl_namespace ON /*_*/templatelinks (tl_namespace, tl_title, tl_from); +DROP INDEX /*i*/il_to ON /*_*/imagelinks; +CREATE UNIQUE INDEX /*i*/il_to ON /*_*/imagelinks (il_to, il_from); diff --git a/www/wiki/maintenance/archives/patch-pl_from_namespace.sql b/www/wiki/maintenance/archives/patch-pl_from_namespace.sql new file mode 100644 index 00000000..dcf2b60a --- /dev/null +++ b/www/wiki/maintenance/archives/patch-pl_from_namespace.sql @@ -0,0 +1,4 @@ +ALTER TABLE /*_*/pagelinks + ADD COLUMN pl_from_namespace int NOT NULL default 0; + +CREATE INDEX /*i*/pl_backlinks_namespace ON /*_*/pagelinks (pl_from_namespace,pl_namespace,pl_title,pl_from); diff --git a/www/wiki/maintenance/archives/patch-pp_sortkey.sql b/www/wiki/maintenance/archives/patch-pp_sortkey.sql new file mode 100644 index 00000000..b13b6055 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-pp_sortkey.sql @@ -0,0 +1,8 @@ +-- Add a 'sortkey' field to page_props so pages can be efficiently +-- queried by the numeric value of a property. + +ALTER TABLE /*_*/page_props + ADD pp_sortkey float DEFAULT NULL; + +CREATE UNIQUE INDEX /*i*/pp_propname_sortkey_page + ON /*_*/page_props ( pp_propname, pp_sortkey, pp_page ); diff --git a/www/wiki/maintenance/archives/patch-profiling-memory.sql b/www/wiki/maintenance/archives/patch-profiling-memory.sql new file mode 100644 index 00000000..ddd851e1 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-profiling-memory.sql @@ -0,0 +1,2 @@ +ALTER TABLE /*$wgDBprefix*/profiling + ADD pf_memory float NOT NULL default 0; diff --git a/www/wiki/maintenance/archives/patch-profiling.sql b/www/wiki/maintenance/archives/patch-profiling.sql new file mode 100644 index 00000000..6ad16224 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-profiling.sql @@ -0,0 +1,12 @@ +-- profiling table +-- This is optional + +CREATE TABLE /*_*/profiling ( + pf_count int NOT NULL default 0, + pf_time float NOT NULL default 0, + pf_memory float NOT NULL default 0, + pf_name varchar(255) NOT NULL default '', + pf_server varchar(30) NOT NULL default '' +) ENGINE=MEMORY; + +CREATE UNIQUE INDEX /*i*/pf_name_server ON /*_*/profiling (pf_name, pf_server);
\ No newline at end of file diff --git a/www/wiki/maintenance/archives/patch-protected_titles.sql b/www/wiki/maintenance/archives/patch-protected_titles.sql new file mode 100644 index 00000000..20b6035d --- /dev/null +++ b/www/wiki/maintenance/archives/patch-protected_titles.sql @@ -0,0 +1,12 @@ +-- Protected titles - nonexistent pages that have been protected +CREATE TABLE /*$wgDBprefix*/protected_titles ( + pt_namespace int NOT NULL, + pt_title varchar(255) binary NOT NULL, + pt_user int unsigned NOT NULL, + pt_reason tinyblob, + pt_timestamp binary(14) NOT NULL, + pt_expiry varbinary(14) NOT NULL default '', + pt_create_perm varbinary(60) NOT NULL, + PRIMARY KEY (pt_namespace,pt_title), + KEY pt_timestamp (pt_timestamp) +) /*$wgDBTableOptions*/; diff --git a/www/wiki/maintenance/archives/patch-pt_title-encoding.sql b/www/wiki/maintenance/archives/patch-pt_title-encoding.sql new file mode 100644 index 00000000..b0a23932 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-pt_title-encoding.sql @@ -0,0 +1,5 @@ +-- pt_title was accidentally left with the wrong collation. +-- This might cause failures with JOINs, and could protect the wrong pages +-- with different case variants or unrelated UTF-8 chars. +ALTER TABLE /*$wgDBprefix*/protected_titles + CHANGE COLUMN pt_title pt_title varchar(255) binary NOT NULL; diff --git a/www/wiki/maintenance/archives/patch-querycache.sql b/www/wiki/maintenance/archives/patch-querycache.sql new file mode 100644 index 00000000..8e1a5188 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-querycache.sql @@ -0,0 +1,16 @@ +-- Used for caching expensive grouped queries + +CREATE TABLE /*$wgDBprefix*/querycache ( + -- A key name, generally the base name of of the special page. + qc_type varbinary(32) NOT NULL, + + -- Some sort of stored value. Sizes, counts... + qc_value int unsigned NOT NULL default '0', + + -- Target namespace+title + qc_namespace int NOT NULL default '0', + qc_title varchar(255) binary NOT NULL default '', + + KEY (qc_type,qc_value) + +) /*$wgDBTableOptions*/; diff --git a/www/wiki/maintenance/archives/patch-querycache_info-fix-pk.sql b/www/wiki/maintenance/archives/patch-querycache_info-fix-pk.sql new file mode 100644 index 00000000..94f3c1d6 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-querycache_info-fix-pk.sql @@ -0,0 +1 @@ +ALTER TABLE /*_*/querycache_info DROP KEY /*i*/qci_type, ADD PRIMARY KEY (qci_type); diff --git a/www/wiki/maintenance/archives/patch-querycacheinfo.sql b/www/wiki/maintenance/archives/patch-querycacheinfo.sql new file mode 100644 index 00000000..7ad2bca6 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-querycacheinfo.sql @@ -0,0 +1,12 @@ +CREATE TABLE /*$wgDBprefix*/querycache_info ( + + -- Special page name + -- Corresponds to a qc_type value + qci_type varbinary(32) NOT NULL default '', + + -- Timestamp of last update + qci_timestamp binary(14) NOT NULL default '19700101000000', + + UNIQUE KEY ( qci_type ) + +) /*$wgDBTableOptions*/; diff --git a/www/wiki/maintenance/archives/patch-querycachetwo.sql b/www/wiki/maintenance/archives/patch-querycachetwo.sql new file mode 100644 index 00000000..79131310 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-querycachetwo.sql @@ -0,0 +1,22 @@ +-- Used for caching expensive grouped queries that need two links (for example double-redirects) + +CREATE TABLE /*$wgDBprefix*/querycachetwo ( + -- A key name, generally the base name of of the special page. + qcc_type varbinary(32) NOT NULL, + + -- Some sort of stored value. Sizes, counts... + qcc_value int unsigned NOT NULL default '0', + + -- Target namespace+title + qcc_namespace int NOT NULL default '0', + qcc_title varchar(255) binary NOT NULL default '', + + -- Target namespace+title2 + qcc_namespacetwo int NOT NULL default '0', + qcc_titletwo varchar(255) binary NOT NULL default '', + + KEY qcc_type (qcc_type,qcc_value), + KEY qcc_title (qcc_type,qcc_namespace,qcc_title), + KEY qcc_titletwo (qcc_type,qcc_namespacetwo,qcc_titletwo) + +) /*$wgDBTableOptions*/; diff --git a/www/wiki/maintenance/archives/patch-random-dateindex.sql b/www/wiki/maintenance/archives/patch-random-dateindex.sql new file mode 100644 index 00000000..5d514cc3 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-random-dateindex.sql @@ -0,0 +1,54 @@ +-- patch-random-dateindex.sql +-- 2003-02-09 +-- +-- This patch does two things: +-- * Adds cur_random column to replace random table +-- (Requires change to SpecialRandom.php) +-- random table no longer needs refilling +-- Note: short-term duplicate results *are* possible, but very unlikely on large wiki +-- +-- * Adds inverse_timestamp columns to cur and old and indexes +-- to allow descending timestamp sort in history, contribs, etc +-- (Requires changes to Article.php, DatabaseFunctions.php, +-- ... ) +-- cur_timestamp inverse_timestamp +-- 99999999999999 - 20030209222556 = 79969790777443 +-- 99999999999999 - 20030211083412 = 79969788916587 +-- +-- We won't need this on MySQL 4; there will be a removal patch later. + +-- Indexes: +-- cur needs (cur_random) for random sort +-- cur and old need (namespace,title,timestamp) index for history,watchlist,rclinked +-- cur and old need (user,timestamp) index for contribs +-- cur and old need (user_text,timestamp) index for contribs + +ALTER TABLE /*$wgDBprefix*/cur + DROP INDEX cur_user, + DROP INDEX cur_user_text, + ADD COLUMN cur_random real unsigned NOT NULL, + ADD COLUMN inverse_timestamp char(14) binary NOT NULL default '', + ADD INDEX (cur_random), + ADD INDEX name_title_timestamp (cur_namespace,cur_title,inverse_timestamp), + ADD INDEX user_timestamp (cur_user,inverse_timestamp), + ADD INDEX usertext_timestamp (cur_user_text,inverse_timestamp); + +UPDATE /*$wgDBprefix*/cur SET + inverse_timestamp=99999999999999-cur_timestamp, + cur_random=RAND(); + +ALTER TABLE /*$wgDBprefix*/old + DROP INDEX old_user, + DROP INDEX old_user_text, + ADD COLUMN inverse_timestamp char(14) binary NOT NULL default '', + ADD INDEX name_title_timestamp (old_namespace,old_title,inverse_timestamp), + ADD INDEX user_timestamp (old_user,inverse_timestamp), + ADD INDEX usertext_timestamp (old_user_text,inverse_timestamp); + +UPDATE /*$wgDBprefix*/old SET + inverse_timestamp=99999999999999-old_timestamp; + +-- If leaving wiki publicly accessible in read-only mode during +-- the upgrade, comment out the below line; leave 'random' table +-- in place until the new software is installed. +DROP TABLE /*$wgDBprefix*/random; diff --git a/www/wiki/maintenance/archives/patch-rc-newindex.sql b/www/wiki/maintenance/archives/patch-rc-newindex.sql new file mode 100644 index 00000000..2315ff37 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-rc-newindex.sql @@ -0,0 +1,9 @@ +-- +-- patch-rc-newindex.sql +-- Adds an index to recentchanges to optimize Special:Newpages +-- 2004-01-25 +-- + +ALTER TABLE /*$wgDBprefix*/recentchanges + ADD INDEX new_name_timestamp(rc_new,rc_namespace,rc_timestamp); + diff --git a/www/wiki/maintenance/archives/patch-rc-patrol.sql b/www/wiki/maintenance/archives/patch-rc-patrol.sql new file mode 100644 index 00000000..1839c1ee --- /dev/null +++ b/www/wiki/maintenance/archives/patch-rc-patrol.sql @@ -0,0 +1,9 @@ +-- +-- patch-rc-patrol.sql +-- Adds a row to recentchanges for the patrolling feature +-- 2004-08-09 +-- + +ALTER TABLE /*$wgDBprefix*/recentchanges + ADD COLUMN rc_patrolled tinyint(3) unsigned NOT NULL default '0'; + diff --git a/www/wiki/maintenance/archives/patch-rc_deleted.sql b/www/wiki/maintenance/archives/patch-rc_deleted.sql new file mode 100644 index 00000000..f4bbd0f9 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-rc_deleted.sql @@ -0,0 +1,8 @@ +-- Adding rc_deleted field for revisiondelete +-- Add rc_logid to match log_id +ALTER TABLE /*$wgDBprefix*/recentchanges + ADD rc_deleted tinyint unsigned NOT NULL default '0', + ADD rc_logid int unsigned NOT NULL default '0', + ADD rc_log_type varbinary(255) NULL default NULL, + ADD rc_log_action varbinary(255) NULL default NULL, + ADD rc_params BLOB NULL; diff --git a/www/wiki/maintenance/archives/patch-rc_id.sql b/www/wiki/maintenance/archives/patch-rc_id.sql new file mode 100644 index 00000000..28caee0e --- /dev/null +++ b/www/wiki/maintenance/archives/patch-rc_id.sql @@ -0,0 +1,7 @@ +-- Primary key in recentchanges + +ALTER TABLE /*$wgDBprefix*/recentchanges + ADD rc_id int NOT NULL auto_increment, + ADD PRIMARY KEY rc_id (rc_id); + + diff --git a/www/wiki/maintenance/archives/patch-rc_ip.sql b/www/wiki/maintenance/archives/patch-rc_ip.sql new file mode 100644 index 00000000..4d93300f --- /dev/null +++ b/www/wiki/maintenance/archives/patch-rc_ip.sql @@ -0,0 +1,7 @@ +-- Adding the rc_ip field for logging of IP addresses in recentchanges + +ALTER TABLE /*$wgDBprefix*/recentchanges + ADD rc_ip varbinary(40) NOT NULL default '', + ADD INDEX rc_ip (rc_ip); + + diff --git a/www/wiki/maintenance/archives/patch-rc_ip_modify.sql b/www/wiki/maintenance/archives/patch-rc_ip_modify.sql new file mode 100644 index 00000000..e889b5c5 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-rc_ip_modify.sql @@ -0,0 +1 @@ +ALTER TABLE /*_*/recentchanges MODIFY COLUMN rc_ip varbinary(40) NOT NULL default ''; diff --git a/www/wiki/maintenance/archives/patch-rc_len.sql b/www/wiki/maintenance/archives/patch-rc_len.sql new file mode 100644 index 00000000..6c781a00 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-rc_len.sql @@ -0,0 +1,9 @@ +-- +-- patch-rc_len.sql +-- Adds two rows to recentchanges to hold the text size befor and after the edit +-- 2006-12-03 +-- + +ALTER TABLE /*$wgDBprefix*/recentchanges + ADD COLUMN rc_old_len int, ADD COLUMN rc_new_len int; + diff --git a/www/wiki/maintenance/archives/patch-rc_moved.sql b/www/wiki/maintenance/archives/patch-rc_moved.sql new file mode 100644 index 00000000..2fa1de6b --- /dev/null +++ b/www/wiki/maintenance/archives/patch-rc_moved.sql @@ -0,0 +1,4 @@ +-- rc_moved_to_ns and rc_moved_to_title is no longer used, delete the fields + +ALTER TABLE /*$wgDBprefix*/recentchanges DROP COLUMN rc_moved_to_ns, + DROP COLUMN rc_moved_to_title; diff --git a/www/wiki/maintenance/archives/patch-rc_source.sql b/www/wiki/maintenance/archives/patch-rc_source.sql new file mode 100644 index 00000000..7dedd745 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-rc_source.sql @@ -0,0 +1,16 @@ +-- first step of migrating recentchanges rc_type to rc_source +ALTER TABLE /*$wgDBprefix*/recentchanges + ADD rc_source varbinary(16) NOT NULL default ''; + +-- Populate rc_source field with the data from rc_type +-- Large wiki's might prefer the PopulateRecentChangeSource maintenance +-- script to batch updates into groups rather than all at once. +UPDATE /*$wgDBprefix*/recentchanges + SET rc_source = CASE + WHEN rc_type = 0 THEN 'mw.edit' + WHEN rc_type = 1 THEN 'mw.new' + WHEN rc_type = 3 THEN 'mw.log' + WHEN rc_type = 5 THEN 'mw.external' + ELSE '' + END +WHERE rc_source = ''; diff --git a/www/wiki/maintenance/archives/patch-rc_type.sql b/www/wiki/maintenance/archives/patch-rc_type.sql new file mode 100644 index 00000000..f1fb18e5 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-rc_type.sql @@ -0,0 +1,9 @@ +-- recentchanges improvements -- + +ALTER TABLE /*$wgDBprefix*/recentchanges + ADD rc_type tinyint unsigned NOT NULL default '0', + ADD rc_moved_to_ns tinyint unsigned NOT NULL default '0', + ADD rc_moved_to_title varchar(255) binary NOT NULL default ''; + +UPDATE /*$wgDBprefix*/recentchanges SET rc_type=1 WHERE rc_new; +UPDATE /*$wgDBprefix*/recentchanges SET rc_type=3 WHERE rc_namespace=4 AND (rc_title='Deletion_log' OR rc_title='Upload_log'); diff --git a/www/wiki/maintenance/archives/patch-rc_user_text-index.sql b/www/wiki/maintenance/archives/patch-rc_user_text-index.sql new file mode 100644 index 00000000..f6acc992 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-rc_user_text-index.sql @@ -0,0 +1,7 @@ +-- Add an index to recentchanges on rc_user_text +-- +-- Added 2006-11-08 +-- + + ALTER TABLE /*$wgDBprefix*/recentchanges +ADD INDEX rc_user_text(rc_user_text, rc_timestamp);
\ No newline at end of file diff --git a/www/wiki/maintenance/archives/patch-rd_interwiki.sql b/www/wiki/maintenance/archives/patch-rd_interwiki.sql new file mode 100644 index 00000000..a12f1a7d --- /dev/null +++ b/www/wiki/maintenance/archives/patch-rd_interwiki.sql @@ -0,0 +1,6 @@ +-- Add interwiki and fragment columns to redirect table + +ALTER TABLE /*$wgDBprefix*/redirect + ADD rd_interwiki varchar(32) default NULL, + ADD rd_fragment varchar(255) binary default NULL; + diff --git a/www/wiki/maintenance/archives/patch-recentchanges-utindex.sql b/www/wiki/maintenance/archives/patch-recentchanges-utindex.sql new file mode 100644 index 00000000..4ebe3165 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-recentchanges-utindex.sql @@ -0,0 +1,4 @@ +--- July 2006 +--- Index on recentchanges.( rc_namespace, rc_user_text ) +--- Helps the username filtering in Special:Newpages +ALTER TABLE /*$wgDBprefix*/recentchanges ADD INDEX `rc_ns_usertext` ( `rc_namespace` , `rc_user_text` );
\ No newline at end of file diff --git a/www/wiki/maintenance/archives/patch-redirect.sql b/www/wiki/maintenance/archives/patch-redirect.sql new file mode 100644 index 00000000..d2957df4 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-redirect.sql @@ -0,0 +1,28 @@ +-- +-- Create the new redirect table. +-- For each redirect, this table contains exactly one row defining its target +-- +CREATE TABLE /*$wgDBprefix*/redirect ( + -- Key to the page_id of the redirect page + rd_from int unsigned NOT NULL default '0', + + -- Key to page_namespace/page_title of the target page. + -- The target page may or may not exist, and due to renames + -- and deletions may refer to different page records as time + -- goes by. + rd_namespace int NOT NULL default '0', + rd_title varchar(255) binary NOT NULL default '', + + PRIMARY KEY rd_from (rd_from), + KEY rd_ns_title (rd_namespace,rd_title,rd_from) +) /*$wgDBTableOptions*/; + +-- Import existing redirects +-- Using ignore because some of the redirect pages contain more than one link +INSERT IGNORE + INTO /*$wgDBprefix*/redirect (rd_from,rd_namespace,rd_title) + SELECT pl_from,pl_namespace,pl_title + FROM /*$wgDBprefix*/pagelinks, /*$wgDBprefix*/page + WHERE pl_from=page_id AND page_is_redirect=1; + + diff --git a/www/wiki/maintenance/archives/patch-rename-ar_usertext_timestamp.sql b/www/wiki/maintenance/archives/patch-rename-ar_usertext_timestamp.sql new file mode 100644 index 00000000..658c179a --- /dev/null +++ b/www/wiki/maintenance/archives/patch-rename-ar_usertext_timestamp.sql @@ -0,0 +1,7 @@ +-- Rename the archive.ar_usertext_timestamp index to usertext_timestamp. +-- This is for MySQL only and is only necessary on wikis freshly installed on +-- 1.28.0 when bug T154872 was present. The patch will probably be removed in +-- 1.29 since we plan on renaming the index properly to ar_usertext_timestamp. +ALTER TABLE /*$wgDBprefix*/archive + DROP INDEX ar_usertext_timestamp, + ADD INDEX usertext_timestamp (ar_user_text,ar_timestamp); diff --git a/www/wiki/maintenance/archives/patch-rename-iwl_prefix.sql b/www/wiki/maintenance/archives/patch-rename-iwl_prefix.sql new file mode 100644 index 00000000..4a410037 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-rename-iwl_prefix.sql @@ -0,0 +1,4 @@ +-- +-- Recreates the iwl_prefix index for the iwlinks table +-- +CREATE UNIQUE INDEX /*i*/iwl_prefix_title_from ON /*_*/iwlinks (iwl_prefix, iwl_title, iwl_from); diff --git a/www/wiki/maintenance/archives/patch-rename-user_groups-and_rights.sql b/www/wiki/maintenance/archives/patch-rename-user_groups-and_rights.sql new file mode 100644 index 00000000..978b31f7 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-rename-user_groups-and_rights.sql @@ -0,0 +1,9 @@ + +ALTER TABLE /*$wgDBprefix*/user_groups + CHANGE user_id ug_user INT UNSIGNED NOT NULL DEFAULT '0', + CHANGE group_id ug_group INT UNSIGNED NOT NULL DEFAULT '0'; + +ALTER TABLE /*$wgDBprefix*/user_rights + CHANGE user_id ur_user INT UNSIGNED NOT NULL, + CHANGE user_rights ur_rights TINYBLOB NOT NULL; + diff --git a/www/wiki/maintenance/archives/patch-rev_deleted.sql b/www/wiki/maintenance/archives/patch-rev_deleted.sql new file mode 100644 index 00000000..ba47f789 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-rev_deleted.sql @@ -0,0 +1,11 @@ +-- +-- Add rev_deleted flag to revision table. +-- Deleted revisions can thus continue to be listed in history +-- and user contributions, and their text storage doesn't have +-- to be disturbed. +-- +-- 2005-03-31 +-- + +ALTER TABLE /*$wgDBprefix*/revision + ADD rev_deleted tinyint unsigned NOT NULL default '0'; diff --git a/www/wiki/maintenance/archives/patch-rev_len.sql b/www/wiki/maintenance/archives/patch-rev_len.sql new file mode 100644 index 00000000..ccdae8b8 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-rev_len.sql @@ -0,0 +1,3 @@ +ALTER TABLE /*$wgDBprefix*/revision + ADD rev_len INT UNSIGNED; + diff --git a/www/wiki/maintenance/archives/patch-rev_parent_id.sql b/www/wiki/maintenance/archives/patch-rev_parent_id.sql new file mode 100644 index 00000000..4baf7927 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-rev_parent_id.sql @@ -0,0 +1,9 @@ +-- +-- Key to revision.rev_id +-- This field is used to add support for a tree structure (The Adjacency List Model) +-- +-- 2007-03-04 +-- + +ALTER TABLE /*$wgDBprefix*/revision + ADD rev_parent_id int unsigned default NULL; diff --git a/www/wiki/maintenance/archives/patch-rev_sha1.sql b/www/wiki/maintenance/archives/patch-rev_sha1.sql new file mode 100644 index 00000000..0100c365 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-rev_sha1.sql @@ -0,0 +1,3 @@ +-- Adding rev_sha1 field +ALTER TABLE /*$wgDBprefix*/revision + ADD rev_sha1 varbinary(32) NOT NULL default ''; diff --git a/www/wiki/maintenance/archives/patch-rev_text_id.sql b/www/wiki/maintenance/archives/patch-rev_text_id.sql new file mode 100644 index 00000000..3dd9127d --- /dev/null +++ b/www/wiki/maintenance/archives/patch-rev_text_id.sql @@ -0,0 +1,17 @@ +-- +-- Adds rev_text_id field to revision table. +-- This is a key to text.old_id, so that revisions can be stored +-- for non-save operations without duplicating text, and so that +-- a back-end storage system can provide its own numbering system +-- if necessary. +-- +-- rev.rev_id and text.old_id are no longer assumed to be the same. +-- +-- 2005-03-28 +-- + +ALTER TABLE /*$wgDBprefix*/revision + ADD rev_text_id int unsigned NOT NULL; + +UPDATE /*$wgDBprefix*/revision + SET rev_text_id=rev_id; diff --git a/www/wiki/maintenance/archives/patch-revision-page-rev-index-nonunique.sql b/www/wiki/maintenance/archives/patch-revision-page-rev-index-nonunique.sql new file mode 100644 index 00000000..dbb03257 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-revision-page-rev-index-nonunique.sql @@ -0,0 +1,5 @@ +-- Makes rev_page_id index non-unique +ALTER TABLE /*_*/revision +DROP INDEX /*i*/rev_page_id; + +CREATE INDEX /*i*/rev_page_id ON /*_*/revision (rev_page, rev_id); diff --git a/www/wiki/maintenance/archives/patch-revision-rev_content_format.sql b/www/wiki/maintenance/archives/patch-revision-rev_content_format.sql new file mode 100644 index 00000000..22aeb8a7 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-revision-rev_content_format.sql @@ -0,0 +1,2 @@ +ALTER TABLE /*$wgDBprefix*/revision + ADD rev_content_format varbinary(64) DEFAULT NULL; diff --git a/www/wiki/maintenance/archives/patch-revision-rev_content_model.sql b/www/wiki/maintenance/archives/patch-revision-rev_content_model.sql new file mode 100644 index 00000000..1ba05721 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-revision-rev_content_model.sql @@ -0,0 +1,2 @@ +ALTER TABLE /*$wgDBprefix*/revision + ADD rev_content_model varbinary(32) DEFAULT NULL; diff --git a/www/wiki/maintenance/archives/patch-revision-user-page-index.sql b/www/wiki/maintenance/archives/patch-revision-user-page-index.sql new file mode 100644 index 00000000..a4554c8f --- /dev/null +++ b/www/wiki/maintenance/archives/patch-revision-user-page-index.sql @@ -0,0 +1,4 @@ +-- New index on revision table to allow searches for all edits by a given user +-- to a given page. Added 2007-08-28 + +CREATE INDEX /*i*/page_user_timestamp ON /*_*/revision (rev_page,rev_user,rev_timestamp); diff --git a/www/wiki/maintenance/archives/patch-searchindex.sql b/www/wiki/maintenance/archives/patch-searchindex.sql new file mode 100644 index 00000000..36507a2b --- /dev/null +++ b/www/wiki/maintenance/archives/patch-searchindex.sql @@ -0,0 +1,40 @@ +-- Break fulltext search index out to separate table from cur +-- This is being done mainly to allow us to use InnoDB tables +-- for the main db while keeping the MyISAM fulltext index for +-- search. + +-- 2002-12-16, 2003-01-25 Brion VIBBER <brion@pobox.com> + +-- Creating searchindex table... +DROP TABLE IF EXISTS /*$wgDBprefix*/searchindex; +CREATE TABLE /*$wgDBprefix*/searchindex ( + -- Key to page_id + si_page int unsigned NOT NULL, + + -- Munged version of title + si_title varchar(255) NOT NULL default '', + + -- Munged version of body text + si_text mediumtext NOT NULL, + + UNIQUE KEY (si_page) + +) ENGINE=MyISAM; + +-- Copying data into new table... +INSERT INTO /*$wgDBprefix*/searchindex + (si_page,si_title,si_text) + SELECT + cur_id,cur_ind_title,cur_ind_text + FROM /*$wgDBprefix*/cur; + + +-- Creating fulltext index... +ALTER TABLE /*$wgDBprefix*/searchindex + ADD FULLTEXT si_title (si_title), + ADD FULLTEXT si_text (si_text); + +-- Dropping index columns from cur table. +ALTER TABLE /*$wgDBprefix*/cur + DROP COLUMN cur_ind_title, + DROP COLUMN cur_ind_text; diff --git a/www/wiki/maintenance/archives/patch-site_stats-fix-pk.sql b/www/wiki/maintenance/archives/patch-site_stats-fix-pk.sql new file mode 100644 index 00000000..d32adf34 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-site_stats-fix-pk.sql @@ -0,0 +1 @@ +ALTER TABLE /*_*/site_stats DROP KEY /*i*/ss_row_id, ADD PRIMARY KEY (ss_row_id); diff --git a/www/wiki/maintenance/archives/patch-sites.sql b/www/wiki/maintenance/archives/patch-sites.sql new file mode 100644 index 00000000..88392748 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-sites.sql @@ -0,0 +1,71 @@ +-- Patch to add the sites and site_identifiers tables. +-- Licence: GNU GPL v2+ +-- Author: Jeroen De Dauw < jeroendedauw@gmail.com > + + +-- Holds all the sites known to the wiki. +CREATE TABLE IF NOT EXISTS /*_*/sites ( +-- Numeric id of the site + site_id INT UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT, + + -- Global identifier for the site, ie 'enwiktionary' + site_global_key varbinary(32) NOT NULL, + + -- Type of the site, ie 'mediawiki' + site_type varbinary(32) NOT NULL, + + -- Group of the site, ie 'wikipedia' + site_group varbinary(32) NOT NULL, + + -- Source of the site data, ie 'local', 'wikidata', 'my-magical-repo' + site_source varbinary(32) NOT NULL, + + -- Language code of the sites primary language. + site_language varbinary(32) NOT NULL, + + -- Protocol of the site, ie 'http://', 'irc://', '//' + -- This field is an index for lookups and is build from type specific data in site_data. + site_protocol varbinary(32) NOT NULL, + + -- Domain of the site in reverse order, ie 'org.mediawiki.www.' + -- This field is an index for lookups and is build from type specific data in site_data. + site_domain VARCHAR(255) NOT NULL, + + -- Type dependent site data. + site_data BLOB NOT NULL, + + -- If site.tld/path/key:pageTitle should forward users to the page on + -- the actual site, where "key" is the local identifier. + site_forward bool NOT NULL, + + -- Type dependent site config. + -- For instance if template transclusion should be allowed if it's a MediaWiki. + site_config BLOB NOT NULL +) /*$wgDBTableOptions*/; + +CREATE UNIQUE INDEX /*i*/sites_global_key ON /*_*/sites (site_global_key); +CREATE INDEX /*i*/sites_type ON /*_*/sites (site_type); +CREATE INDEX /*i*/sites_group ON /*_*/sites (site_group); +CREATE INDEX /*i*/sites_source ON /*_*/sites (site_source); +CREATE INDEX /*i*/sites_language ON /*_*/sites (site_language); +CREATE INDEX /*i*/sites_protocol ON /*_*/sites (site_protocol); +CREATE INDEX /*i*/sites_domain ON /*_*/sites (site_domain); +CREATE INDEX /*i*/sites_forward ON /*_*/sites (site_forward); + + + +-- Links local site identifiers to their corresponding site. +CREATE TABLE IF NOT EXISTS /*_*/site_identifiers ( + -- Key on site.site_id + si_site INT UNSIGNED NOT NULL, + + -- local key type, ie 'interwiki' or 'langlink' + si_type varbinary(32) NOT NULL, + + -- local key value, ie 'en' or 'wiktionary' + si_key varbinary(32) NOT NULL +) /*$wgDBTableOptions*/; + +CREATE UNIQUE INDEX /*i*/site_ids_type ON /*_*/site_identifiers (si_type, si_key); +CREATE INDEX /*i*/site_ids_site ON /*_*/site_identifiers (si_site); +CREATE INDEX /*i*/site_ids_key ON /*_*/site_identifiers (si_key);
\ No newline at end of file diff --git a/www/wiki/maintenance/archives/patch-ss_active_users.sql b/www/wiki/maintenance/archives/patch-ss_active_users.sql new file mode 100644 index 00000000..a583cdc8 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-ss_active_users.sql @@ -0,0 +1,3 @@ +-- More statistics, for version 1.14 + +ALTER TABLE /*$wgDBprefix*/site_stats ADD ss_active_users bigint default '-1'; diff --git a/www/wiki/maintenance/archives/patch-ss_images.sql b/www/wiki/maintenance/archives/patch-ss_images.sql new file mode 100644 index 00000000..80f1295f --- /dev/null +++ b/www/wiki/maintenance/archives/patch-ss_images.sql @@ -0,0 +1,5 @@ +-- More statistics, for version 1.6 + +ALTER TABLE /*$wgDBprefix*/site_stats ADD ss_images int default '0'; +SELECT @images := COUNT(*) FROM /*$wgDBprefix*/image; +UPDATE /*$wgDBprefix*/site_stats SET ss_images=@images; diff --git a/www/wiki/maintenance/archives/patch-ss_total_articles.sql b/www/wiki/maintenance/archives/patch-ss_total_articles.sql new file mode 100644 index 00000000..ce804ce5 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-ss_total_articles.sql @@ -0,0 +1,6 @@ +-- Faster statistics, as of 1.4.3 + +ALTER TABLE /*$wgDBprefix*/site_stats + ADD ss_total_pages bigint default -1, + ADD ss_users bigint default -1, + ADD ss_admins int default -1; diff --git a/www/wiki/maintenance/archives/patch-tag_summary-ts_id.sql b/www/wiki/maintenance/archives/patch-tag_summary-ts_id.sql new file mode 100644 index 00000000..66fa72e1 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-tag_summary-ts_id.sql @@ -0,0 +1,5 @@ +-- Primary key in tag_summary table + +ALTER TABLE /*$wgDBprefix*/tag_summary + ADD COLUMN ts_id INT UNSIGNED NOT NULL AUTO_INCREMENT FIRST, + ADD PRIMARY KEY (ts_id); diff --git a/www/wiki/maintenance/archives/patch-tag_summary-ts_log_id-unsigned.sql b/www/wiki/maintenance/archives/patch-tag_summary-ts_log_id-unsigned.sql new file mode 100644 index 00000000..617073db --- /dev/null +++ b/www/wiki/maintenance/archives/patch-tag_summary-ts_log_id-unsigned.sql @@ -0,0 +1 @@ +ALTER TABLE /*_*/tag_summary MODIFY ts_log_id int unsigned NULL;
\ No newline at end of file diff --git a/www/wiki/maintenance/archives/patch-tag_summary-ts_rev_id-unsigned.sql b/www/wiki/maintenance/archives/patch-tag_summary-ts_rev_id-unsigned.sql new file mode 100644 index 00000000..e6a5bcde --- /dev/null +++ b/www/wiki/maintenance/archives/patch-tag_summary-ts_rev_id-unsigned.sql @@ -0,0 +1 @@ +ALTER TABLE /*_*/tag_summary MODIFY ts_rev_id int unsigned NULL;
\ No newline at end of file diff --git a/www/wiki/maintenance/archives/patch-tag_summary.sql b/www/wiki/maintenance/archives/patch-tag_summary.sql new file mode 100644 index 00000000..a81b3680 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-tag_summary.sql @@ -0,0 +1,12 @@ +-- Rollup table to pull a LIST of tags simply without ugly GROUP_CONCAT that only works on MySQL 4.1+ +-- Andrew Garrett, 2009-01 +CREATE TABLE /*_*/tag_summary ( + ts_rc_id int NULL, + ts_log_id int NULL, + ts_rev_id int NULL, + ts_tags BLOB NOT NULL +) /*$wgDBTableOptions*/; + +CREATE UNIQUE INDEX /*i*/tag_summary_rc_id ON /*_*/tag_summary (ts_rc_id); +CREATE UNIQUE INDEX /*i*/tag_summary_log_id ON /*_*/tag_summary (ts_log_id); +CREATE UNIQUE INDEX /*i*/tag_summary_rev_id ON /*_*/tag_summary (ts_rev_id); diff --git a/www/wiki/maintenance/archives/patch-tc-timestamp.sql b/www/wiki/maintenance/archives/patch-tc-timestamp.sql new file mode 100644 index 00000000..3f7dde41 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-tc-timestamp.sql @@ -0,0 +1,4 @@ +ALTER TABLE /*_*/transcache MODIFY tc_time binary(14); +UPDATE /*_*/transcache SET tc_time = DATE_FORMAT(FROM_UNIXTIME(tc_time), "%Y%c%d%H%i%s"); + +INSERT INTO /*_*/updatelog(ul_key) VALUES ('convert transcache field'); diff --git a/www/wiki/maintenance/archives/patch-templatelinks-fix-pk.sql b/www/wiki/maintenance/archives/patch-templatelinks-fix-pk.sql new file mode 100644 index 00000000..8aca5105 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-templatelinks-fix-pk.sql @@ -0,0 +1 @@ +ALTER TABLE /*_*/templatelinks DROP INDEX /*i*/tl_from, ADD PRIMARY KEY (tl_from,tl_namespace,tl_title); diff --git a/www/wiki/maintenance/archives/patch-templatelinks.sql b/www/wiki/maintenance/archives/patch-templatelinks.sql new file mode 100644 index 00000000..086b6a1b --- /dev/null +++ b/www/wiki/maintenance/archives/patch-templatelinks.sql @@ -0,0 +1,18 @@ +-- +-- Track template inclusions. +-- +CREATE TABLE /*$wgDBprefix*/templatelinks ( + -- Key to the page_id of the page containing the link. + tl_from int unsigned NOT NULL default '0', + + -- Key to page_namespace/page_title of the target page. + -- The target page may or may not exist, and due to renames + -- and deletions may refer to different page records as time + -- goes by. + tl_namespace int NOT NULL default '0', + tl_title varchar(255) binary NOT NULL default '', + + UNIQUE KEY tl_from(tl_from,tl_namespace,tl_title), + KEY (tl_namespace,tl_title) +) /*$wgDBTableOptions*/; + diff --git a/www/wiki/maintenance/archives/patch-testrun.sql b/www/wiki/maintenance/archives/patch-testrun.sql new file mode 100644 index 00000000..6699b554 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-testrun.sql @@ -0,0 +1,35 @@ +-- +-- Optional tables for parserTests recording mode +-- With --record option, success data will be saved to these tables, +-- and comparisons of what's changed from the previous run will be +-- displayed at the end of each run. +-- +-- These tables currently require MySQL 5 (or maybe 4.1?) for subselects. +-- + +drop table if exists /*$wgDBprefix*/testitem; +drop table if exists /*$wgDBprefix*/testrun; + +create table /*$wgDBprefix*/testrun ( + tr_id int not null auto_increment, + + tr_date char(14) binary, + tr_mw_version blob, + tr_php_version blob, + tr_db_version blob, + tr_uname blob, + + primary key (tr_id) +) engine=InnoDB; + +create table /*$wgDBprefix*/testitem ( + ti_run int not null, + ti_name varchar(255), + ti_success bool, + + unique key (ti_run, ti_name), + key (ti_run, ti_success), + + foreign key (ti_run) references /*$wgDBprefix*/testrun(tr_id) + on delete cascade +) engine=InnoDB; diff --git a/www/wiki/maintenance/archives/patch-text-fix-pk.sql b/www/wiki/maintenance/archives/patch-text-fix-pk.sql new file mode 100644 index 00000000..b546333b --- /dev/null +++ b/www/wiki/maintenance/archives/patch-text-fix-pk.sql @@ -0,0 +1 @@ +ALTER TABLE /*_*/text DROP KEY /*i*/old_id, ADD PRIMARY KEY (old_id); diff --git a/www/wiki/maintenance/archives/patch-tl_from_namespace.sql b/www/wiki/maintenance/archives/patch-tl_from_namespace.sql new file mode 100644 index 00000000..edfb7a52 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-tl_from_namespace.sql @@ -0,0 +1,4 @@ +ALTER TABLE /*_*/templatelinks + ADD COLUMN tl_from_namespace int NOT NULL default 0; + +CREATE INDEX /*i*/tl_backlinks_namespace ON /*_*/templatelinks (tl_from_namespace,tl_namespace,tl_title,tl_from); diff --git a/www/wiki/maintenance/archives/patch-transcache-fix-pk.sql b/www/wiki/maintenance/archives/patch-transcache-fix-pk.sql new file mode 100644 index 00000000..2e8fea1b --- /dev/null +++ b/www/wiki/maintenance/archives/patch-transcache-fix-pk.sql @@ -0,0 +1 @@ +ALTER TABLE /*_*/transcache DROP KEY /*i*/tc_url_idx, ADD PRIMARY KEY (tc_url); diff --git a/www/wiki/maintenance/archives/patch-transcache.sql b/www/wiki/maintenance/archives/patch-transcache.sql new file mode 100644 index 00000000..70870efa --- /dev/null +++ b/www/wiki/maintenance/archives/patch-transcache.sql @@ -0,0 +1,7 @@ +CREATE TABLE /*$wgDBprefix*/transcache ( + tc_url varbinary(255) NOT NULL, + tc_contents TEXT, + tc_time binary(14) NOT NULL, + UNIQUE INDEX tc_url_idx(tc_url) +) /*$wgDBTableOptions*/; + diff --git a/www/wiki/maintenance/archives/patch-ufg_group-length-increase-255.sql b/www/wiki/maintenance/archives/patch-ufg_group-length-increase-255.sql new file mode 100644 index 00000000..4b7f0d38 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-ufg_group-length-increase-255.sql @@ -0,0 +1,2 @@ +ALTER TABLE /*_*/user_former_groups + MODIFY COLUMN ufg_group varbinary(255) NOT NULL default ''; diff --git a/www/wiki/maintenance/archives/patch-ug_group-length-increase-255.sql b/www/wiki/maintenance/archives/patch-ug_group-length-increase-255.sql new file mode 100644 index 00000000..79e17ac0 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-ug_group-length-increase-255.sql @@ -0,0 +1,2 @@ +ALTER TABLE /*_*/user_groups + MODIFY COLUMN ug_group varbinary(255) NOT NULL default ''; diff --git a/www/wiki/maintenance/archives/patch-ul_value.sql b/www/wiki/maintenance/archives/patch-ul_value.sql new file mode 100644 index 00000000..50f4e9a8 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-ul_value.sql @@ -0,0 +1,4 @@ +-- Add the ul_value column to updatelog + +ALTER TABLE /*_*/updatelog + add ul_value blob; diff --git a/www/wiki/maintenance/archives/patch-up_property.sql b/www/wiki/maintenance/archives/patch-up_property.sql new file mode 100644 index 00000000..c516aafd --- /dev/null +++ b/www/wiki/maintenance/archives/patch-up_property.sql @@ -0,0 +1,4 @@ +-- Increase the length of up_property from 32 -> 255 bytes. T21408 + +ALTER TABLE /*_*/user_properties + MODIFY up_property varbinary(255); diff --git a/www/wiki/maintenance/archives/patch-updatelog.sql b/www/wiki/maintenance/archives/patch-updatelog.sql new file mode 100644 index 00000000..168ad082 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-updatelog.sql @@ -0,0 +1,4 @@ +CREATE TABLE /*$wgDBprefix*/updatelog ( + ul_key varchar(255) NOT NULL, + PRIMARY KEY (ul_key) +) /*$wgDBTableOptions*/; diff --git a/www/wiki/maintenance/archives/patch-uploadstash-us_props.sql b/www/wiki/maintenance/archives/patch-uploadstash-us_props.sql new file mode 100644 index 00000000..d64515a8 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-uploadstash-us_props.sql @@ -0,0 +1,2 @@ +ALTER TABLE /*$wgDBprefix*/uploadstash + ADD COLUMN us_props blob; diff --git a/www/wiki/maintenance/archives/patch-uploadstash.sql b/www/wiki/maintenance/archives/patch-uploadstash.sql new file mode 100644 index 00000000..c1d93ef3 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-uploadstash.sql @@ -0,0 +1,48 @@ +-- +-- Store information about newly uploaded files before they're +-- moved into the actual filestore +-- +CREATE TABLE /*_*/uploadstash ( + us_id int unsigned NOT NULL PRIMARY KEY auto_increment, + + -- the user who uploaded the file. + us_user int unsigned NOT NULL, + + -- file key. this is how applications actually search for the file. + -- this might go away, or become the primary key. + us_key varchar(255) NOT NULL, + + -- the original path + us_orig_path varchar(255) NOT NULL, + + -- the temporary path at which the file is actually stored + us_path varchar(255) NOT NULL, + + -- which type of upload the file came from (sometimes) + us_source_type varchar(50), + + -- the date/time on which the file was added + us_timestamp varbinary(14) not null, + + us_status varchar(50) not null, + + -- file properties from FSFile::getProps(). these may prove unnecessary. + -- + us_size int unsigned NOT NULL, + -- this hash comes from FSFile::getSha1Base36(), and is 31 characters + us_sha1 varchar(31) NOT NULL, + us_mime varchar(255), + -- Media type as defined by the MEDIATYPE_xxx constants, should duplicate definition in the image table + us_media_type ENUM("UNKNOWN", "BITMAP", "DRAWING", "AUDIO", "VIDEO", "MULTIMEDIA", "OFFICE", "TEXT", "EXECUTABLE", "ARCHIVE") default NULL, + -- image-specific properties + us_image_width int unsigned, + us_image_height int unsigned, + us_image_bits smallint unsigned +) /*$wgDBTableOptions*/; + +-- sometimes there's a delete for all of a user's stuff. +CREATE INDEX /*i*/us_user ON /*_*/uploadstash (us_user); +-- pick out files by key, enforce key uniqueness +CREATE UNIQUE INDEX /*i*/us_key ON /*_*/uploadstash (us_key); +-- the abandoned upload cleanup script needs this +CREATE INDEX /*i*/us_timestamp ON /*_*/uploadstash (us_timestamp); diff --git a/www/wiki/maintenance/archives/patch-uploadstash_chunk.sql b/www/wiki/maintenance/archives/patch-uploadstash_chunk.sql new file mode 100644 index 00000000..29e41870 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-uploadstash_chunk.sql @@ -0,0 +1,3 @@ +-- Adding us_chunk_inx field +ALTER TABLE /*$wgDBprefix*/uploadstash + ADD us_chunk_inx int unsigned NULL; diff --git a/www/wiki/maintenance/archives/patch-user-newtalk-timestamp-null.sql b/www/wiki/maintenance/archives/patch-user-newtalk-timestamp-null.sql new file mode 100644 index 00000000..7234362d --- /dev/null +++ b/www/wiki/maintenance/archives/patch-user-newtalk-timestamp-null.sql @@ -0,0 +1 @@ +ALTER TABLE /*_*/user_newtalk MODIFY user_last_timestamp varbinary(14) NULL default NULL; diff --git a/www/wiki/maintenance/archives/patch-user-newtalk-userid-unsigned.sql b/www/wiki/maintenance/archives/patch-user-newtalk-userid-unsigned.sql new file mode 100644 index 00000000..a83e03b9 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-user-newtalk-userid-unsigned.sql @@ -0,0 +1 @@ +ALTER TABLE /*_*/user_newtalk MODIFY user_id int unsigned NOT NULL default 0; diff --git a/www/wiki/maintenance/archives/patch-user-realname.sql b/www/wiki/maintenance/archives/patch-user-realname.sql new file mode 100644 index 00000000..de7cee75 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-user-realname.sql @@ -0,0 +1,5 @@ +-- Add a 'real name' field where users can specify the name they want +-- used for author attribution or other places that real names matter. + +ALTER TABLE user + ADD (user_real_name varchar(255) binary NOT NULL default ''); diff --git a/www/wiki/maintenance/archives/patch-user_editcount.sql b/www/wiki/maintenance/archives/patch-user_editcount.sql new file mode 100644 index 00000000..cdde36dc --- /dev/null +++ b/www/wiki/maintenance/archives/patch-user_editcount.sql @@ -0,0 +1,5 @@ +ALTER TABLE /*$wgDBprefix*/user + ADD COLUMN user_editcount int; + +-- Don't initialize values immediately... or should we? +-- They will be lazy-evaluated, or batch-filled via maintenance/initEditCount.php diff --git a/www/wiki/maintenance/archives/patch-user_email_index.sql b/www/wiki/maintenance/archives/patch-user_email_index.sql new file mode 100644 index 00000000..6a3d6208 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-user_email_index.sql @@ -0,0 +1 @@ +CREATE INDEX /*i*/user_email ON /*_*/user (user_email(50)); diff --git a/www/wiki/maintenance/archives/patch-user_email_token.sql b/www/wiki/maintenance/archives/patch-user_email_token.sql new file mode 100644 index 00000000..f8e66ca4 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-user_email_token.sql @@ -0,0 +1,12 @@ +-- +-- E-mail confirmation token and expiration timestamp, +-- for verification of e-mail addresses. +-- +-- 2005-04-25 +-- + +ALTER TABLE /*$wgDBprefix*/user + ADD COLUMN user_email_authenticated binary(14), + ADD COLUMN user_email_token binary(32), + ADD COLUMN user_email_token_expires binary(14), + ADD INDEX (user_email_token); diff --git a/www/wiki/maintenance/archives/patch-user_former_groups-fix-pk.sql b/www/wiki/maintenance/archives/patch-user_former_groups-fix-pk.sql new file mode 100644 index 00000000..9a776caf --- /dev/null +++ b/www/wiki/maintenance/archives/patch-user_former_groups-fix-pk.sql @@ -0,0 +1 @@ +ALTER TABLE /*_*/user_former_groups DROP KEY /*i*/ufg_user_group, ADD PRIMARY KEY (ufg_user,ufg_group); diff --git a/www/wiki/maintenance/archives/patch-user_former_groups.sql b/www/wiki/maintenance/archives/patch-user_former_groups.sql new file mode 100644 index 00000000..b043196d --- /dev/null +++ b/www/wiki/maintenance/archives/patch-user_former_groups.sql @@ -0,0 +1,9 @@ +-- Stores the groups the user has once belonged to. +-- The user may still belong these groups. Check user_groups. +CREATE TABLE /*_*/user_former_groups ( + -- Key to user_id + ufg_user int unsigned NOT NULL default 0, + ufg_group varbinary(255) NOT NULL default '' +) /*$wgDBTableOptions*/; + +CREATE UNIQUE INDEX /*i*/ufg_user_group ON /*_*/user_former_groups (ufg_user,ufg_group); diff --git a/www/wiki/maintenance/archives/patch-user_groups-primary-key.sql b/www/wiki/maintenance/archives/patch-user_groups-primary-key.sql new file mode 100644 index 00000000..e3c87356 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-user_groups-primary-key.sql @@ -0,0 +1,5 @@ +-- Convert unique index into a primary key on user_groups + +ALTER TABLE /*$wgDBprefix*/user_groups + DROP INDEX ug_user_group, + ADD PRIMARY KEY (ug_user, ug_group); diff --git a/www/wiki/maintenance/archives/patch-user_groups-ug_expiry.sql b/www/wiki/maintenance/archives/patch-user_groups-ug_expiry.sql new file mode 100644 index 00000000..b329f948 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-user_groups-ug_expiry.sql @@ -0,0 +1,5 @@ +-- Add expiry column in user_groups table + +ALTER TABLE /*$wgDBprefix*/user_groups + ADD COLUMN ug_expiry varbinary(14) NULL default NULL, + ADD INDEX ug_expiry (ug_expiry); diff --git a/www/wiki/maintenance/archives/patch-user_groups.sql b/www/wiki/maintenance/archives/patch-user_groups.sql new file mode 100644 index 00000000..1683cf2a --- /dev/null +++ b/www/wiki/maintenance/archives/patch-user_groups.sql @@ -0,0 +1,25 @@ +-- +-- User permissions have been broken out to a separate table; +-- this allows sites with a shared user table to have different +-- permissions assigned to a user in each project. +-- +-- This table replaces the old user_rights field which used a +-- comma-separated blob. +-- +CREATE TABLE /*$wgDBprefix*/user_groups ( + -- Key to user_id + ug_user int unsigned NOT NULL default '0', + + -- Group names are short symbolic string keys. + -- The set of group names is open-ended, though in practice + -- only some predefined ones are likely to be used. + -- + -- At runtime $wgGroupPermissions will associate group keys + -- with particular permissions. A user will have the combined + -- permissions of any group they're explicitly in, plus + -- the implicit '*' and 'user' groups. + ug_group varbinary(16) NOT NULL default '', + + PRIMARY KEY (ug_user,ug_group), + KEY (ug_group) +) /*$wgDBTableOptions*/; diff --git a/www/wiki/maintenance/archives/patch-user_last_timestamp.sql b/www/wiki/maintenance/archives/patch-user_last_timestamp.sql new file mode 100644 index 00000000..e5f85bf1 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-user_last_timestamp.sql @@ -0,0 +1,3 @@ +-- For getting diff since last view +ALTER TABLE /*$wgDBprefix*/user_newtalk + ADD user_last_timestamp varbinary(14) NULL default NULL; diff --git a/www/wiki/maintenance/archives/patch-user_nameindex.sql b/www/wiki/maintenance/archives/patch-user_nameindex.sql new file mode 100644 index 00000000..9bf0aab1 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-user_nameindex.sql @@ -0,0 +1,13 @@ +-- +-- Change the index on user_name to a unique index to prevent +-- duplicate registrations from creeping in. +-- +-- Run maintenance/userDupes.php or through the updater first +-- to clean up any prior duplicate accounts. +-- +-- Added 2005-06-05 +-- + + ALTER TABLE /*$wgDBprefix*/user + DROP INDEX user_name, +ADD UNIQUE INDEX user_name(user_name); diff --git a/www/wiki/maintenance/archives/patch-user_newpass_time.sql b/www/wiki/maintenance/archives/patch-user_newpass_time.sql new file mode 100644 index 00000000..c323f238 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-user_newpass_time.sql @@ -0,0 +1,4 @@ +-- Timestamp of the last time when a new password was +-- sent, for throttling purposes +ALTER TABLE /*$wgDBprefix*/user ADD user_newpass_time binary(14); + diff --git a/www/wiki/maintenance/archives/patch-user_newtalk-user_id-unsigned.sql b/www/wiki/maintenance/archives/patch-user_newtalk-user_id-unsigned.sql new file mode 100644 index 00000000..a83e03b9 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-user_newtalk-user_id-unsigned.sql @@ -0,0 +1 @@ +ALTER TABLE /*_*/user_newtalk MODIFY user_id int unsigned NOT NULL default 0; diff --git a/www/wiki/maintenance/archives/patch-user_password_expire.sql b/www/wiki/maintenance/archives/patch-user_password_expire.sql new file mode 100644 index 00000000..3e716d33 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-user_password_expire.sql @@ -0,0 +1,3 @@ +-- For setting a password expiration date for users +ALTER TABLE /*$wgDBprefix*/user + ADD COLUMN user_password_expires varbinary(14) DEFAULT NULL; diff --git a/www/wiki/maintenance/archives/patch-user_properties-fix-pk.sql b/www/wiki/maintenance/archives/patch-user_properties-fix-pk.sql new file mode 100644 index 00000000..5d51b785 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-user_properties-fix-pk.sql @@ -0,0 +1 @@ +ALTER TABLE /*_*/user_properties DROP KEY /*i*/user_properties_user_property, ADD PRIMARY KEY (up_user,up_property); diff --git a/www/wiki/maintenance/archives/patch-user_properties-up_user-unsigned.sql b/www/wiki/maintenance/archives/patch-user_properties-up_user-unsigned.sql new file mode 100644 index 00000000..f4f563f8 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-user_properties-up_user-unsigned.sql @@ -0,0 +1 @@ +ALTER TABLE /*_*/user_properties MODIFY up_user int unsigned NOT NULL;
\ No newline at end of file diff --git a/www/wiki/maintenance/archives/patch-user_properties.sql b/www/wiki/maintenance/archives/patch-user_properties.sql new file mode 100644 index 00000000..85b00616 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-user_properties.sql @@ -0,0 +1,22 @@ +-- +-- User preferences and perhaps other fun stuff. :) +-- Replaces the old user.user_options blob, with a couple nice properties: +-- +-- 1) We only store non-default settings, so changes to the defauls +-- are now reflected for everybody, not just new accounts. +-- 2) We can more easily do bulk lookups, statistics, or modifications of +-- saved options since it's a sane table structure. +-- +CREATE TABLE /*_*/user_properties( + -- Foreign key to user.user_id + up_user int not null, + + -- Name of the option being saved. This is indexed for bulk lookup. + up_property varbinary(32) not null, + + -- Property value as a string. + up_value blob +) /*$wgDBTableOptions*/; + +CREATE UNIQUE INDEX /*i*/user_properties_user_property on /*_*/user_properties (up_user,up_property); +CREATE INDEX /*i*/user_properties_property on /*_*/user_properties (up_property); diff --git a/www/wiki/maintenance/archives/patch-user_registration.sql b/www/wiki/maintenance/archives/patch-user_registration.sql new file mode 100644 index 00000000..906a6954 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-user_registration.sql @@ -0,0 +1,9 @@ +-- +-- New user field for tracking registration time +-- 2005-12-21 +-- + +ALTER TABLE /*$wgDBprefix*/user + -- Timestamp of account registration. + -- Accounts predating this schema addition may contain NULL. + ADD user_registration binary(14); diff --git a/www/wiki/maintenance/archives/patch-user_rights.sql b/www/wiki/maintenance/archives/patch-user_rights.sql new file mode 100644 index 00000000..a39ac0af --- /dev/null +++ b/www/wiki/maintenance/archives/patch-user_rights.sql @@ -0,0 +1,21 @@ +-- Split user table into two parts: +-- user +-- user_rights +-- The latter contains only the permissions of the user. This way, +-- you can store the accounts for several wikis in one central +-- database but keep user rights local to the wiki. + +CREATE TABLE /*$wgDBprefix*/user_rights ( + -- Key to user_id + ur_user int unsigned NOT NULL, + + -- Comma-separated list of permission keys + ur_rights tinyblob NOT NULL, + + UNIQUE KEY ur_user (ur_user) + +) /*$wgDBTableOptions*/; + +INSERT INTO /*$wgDBprefix*/user_rights SELECT user_id,user_rights FROM /*$wgDBprefix*/user; + +ALTER TABLE /*$wgDBprefix*/user DROP COLUMN user_rights; diff --git a/www/wiki/maintenance/archives/patch-user_token.sql b/www/wiki/maintenance/archives/patch-user_token.sql new file mode 100644 index 00000000..a3eb0bfd --- /dev/null +++ b/www/wiki/maintenance/archives/patch-user_token.sql @@ -0,0 +1,15 @@ +-- user_token patch +-- 2004-09-23 + +ALTER TABLE /*$wgDBprefix*/user ADD user_token binary(32) NOT NULL default ''; + +UPDATE /*$wgDBprefix*/user SET user_token = concat( + substring(rand(),3,4), + substring(rand(),3,4), + substring(rand(),3,4), + substring(rand(),3,4), + substring(rand(),3,4), + substring(rand(),3,4), + substring(rand(),3,4), + substring(rand(),3,4) +); diff --git a/www/wiki/maintenance/archives/patch-userindex.sql b/www/wiki/maintenance/archives/patch-userindex.sql new file mode 100644 index 00000000..c039b2f3 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-userindex.sql @@ -0,0 +1 @@ + ALTER TABLE /*$wgDBprefix*/user ADD INDEX ( `user_name` );
\ No newline at end of file diff --git a/www/wiki/maintenance/archives/patch-userlevels.sql b/www/wiki/maintenance/archives/patch-userlevels.sql new file mode 100644 index 00000000..399d6cb2 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-userlevels.sql @@ -0,0 +1,8 @@ + +-- Relation table between user and groups +CREATE TABLE /*$wgDBprefix*/user_groups ( + ug_user int unsigned NOT NULL default '0', + ug_group varbinary(16) NOT NULL default '0', + PRIMARY KEY (ug_user,ug_group) + KEY (ug_group) +) /*$wgDBTableOptions*/; diff --git a/www/wiki/maintenance/archives/patch-usernewtalk.sql b/www/wiki/maintenance/archives/patch-usernewtalk.sql new file mode 100644 index 00000000..34fae946 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-usernewtalk.sql @@ -0,0 +1,20 @@ +--- This table stores all the IDs of users whose talk +--- page has been changed (the respective row is deleted +--- when the user looks at the page). +--- The respective column in the user table is no longer +--- required and therefore dropped. + +CREATE TABLE /*$wgDBprefix*/user_newtalk ( + user_id int NOT NULL default '0', + user_ip varbinary(40) NOT NULL default '', + KEY user_id (user_id), + KEY user_ip (user_ip) +) /*$wgDBTableOptions*/; + +INSERT INTO + /*$wgDBprefix*/user_newtalk (user_id, user_ip) + SELECT user_id, '' + FROM user + WHERE user_newtalk != 0; + +ALTER TABLE /*$wgDBprefix*/user DROP COLUMN user_newtalk; diff --git a/www/wiki/maintenance/archives/patch-valid_tag.sql b/www/wiki/maintenance/archives/patch-valid_tag.sql new file mode 100644 index 00000000..994a5d53 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-valid_tag.sql @@ -0,0 +1,4 @@ +-- Andrew Garrett, 2009-01 +CREATE TABLE /*_*/valid_tag ( + vt_tag varchar(255) NOT NULL PRIMARY KEY +) /*$wgDBTableOptions*/; diff --git a/www/wiki/maintenance/archives/patch-watchlist-null.sql b/www/wiki/maintenance/archives/patch-watchlist-null.sql new file mode 100644 index 00000000..d4869a02 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-watchlist-null.sql @@ -0,0 +1,9 @@ +-- Set up wl_notificationtimestamp with NULL support. +-- 2005-08-17 + +ALTER TABLE /*$wgDBprefix*/watchlist + CHANGE wl_notificationtimestamp wl_notificationtimestamp varbinary(14); + +UPDATE /*$wgDBprefix*/watchlist + SET wl_notificationtimestamp=NULL + WHERE wl_notificationtimestamp='0'; diff --git a/www/wiki/maintenance/archives/patch-watchlist-user-notificationtimestamp-index.sql b/www/wiki/maintenance/archives/patch-watchlist-user-notificationtimestamp-index.sql new file mode 100644 index 00000000..22ae44f1 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-watchlist-user-notificationtimestamp-index.sql @@ -0,0 +1,4 @@ +-- +-- Creates the wl_user_notificationtimestamp index for the watchlist table +-- +CREATE INDEX /*i*/wl_user_notificationtimestamp ON /*_*/watchlist (wl_user, wl_notificationtimestamp); diff --git a/www/wiki/maintenance/archives/patch-watchlist-wl_id.sql b/www/wiki/maintenance/archives/patch-watchlist-wl_id.sql new file mode 100644 index 00000000..a73e514c --- /dev/null +++ b/www/wiki/maintenance/archives/patch-watchlist-wl_id.sql @@ -0,0 +1,5 @@ +-- Primary key in watchlist + +ALTER TABLE /*$wgDBprefix*/watchlist + ADD COLUMN wl_id int unsigned NOT NULL AUTO_INCREMENT FIRST, + ADD PRIMARY KEY (wl_id); diff --git a/www/wiki/maintenance/archives/patch-watchlist.sql b/www/wiki/maintenance/archives/patch-watchlist.sql new file mode 100644 index 00000000..83826b72 --- /dev/null +++ b/www/wiki/maintenance/archives/patch-watchlist.sql @@ -0,0 +1,30 @@ +-- Convert watchlists to new new format ;) + +-- Ids just aren't convenient when what we want is to +-- treat article and talk pages as equivalent. +-- Better to use namespace (drop the 1 bit!) and title + +-- 2002-12-17 by Brion Vibber <brion@pobox.com> +-- affects, affected by changes to SpecialWatchlist.php, User.php, +-- Article.php, Title.php, SpecialRecentchanges.php + +DROP TABLE IF EXISTS watchlist2; +CREATE TABLE watchlist2 ( + wl_user int unsigned NOT NULL, + wl_namespace int unsigned NOT NULL default '0', + wl_title varchar(255) binary NOT NULL default '', + UNIQUE KEY (wl_user, wl_namespace, wl_title) +) /*$wgDBTableOptions*/; + +INSERT INTO watchlist2 (wl_user,wl_namespace,wl_title) + SELECT DISTINCT wl_user,(cur_namespace | 1) - 1,cur_title + FROM watchlist,cur WHERE wl_page=cur_id; + +ALTER TABLE watchlist RENAME TO oldwatchlist; +ALTER TABLE watchlist2 RENAME TO watchlist; + +-- Check that the new one is correct, then: +-- DROP TABLE oldwatchlist; + +-- Also should probably drop the ancient and now unused: +ALTER TABLE user DROP COLUMN user_watch; diff --git a/www/wiki/maintenance/archives/upgradeLogging.php b/www/wiki/maintenance/archives/upgradeLogging.php new file mode 100644 index 00000000..13362e09 --- /dev/null +++ b/www/wiki/maintenance/archives/upgradeLogging.php @@ -0,0 +1,219 @@ +<?php +/** + * Replication-safe online upgrade for log_id/log_deleted fields. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup MaintenanceArchive + */ + +require __DIR__ . '/../commandLine.inc'; + +use Wikimedia\Rdbms\IMaintainableDatabase; + +/** + * Maintenance script that upgrade for log_id/log_deleted fields in a + * replication-safe way. + * + * @ingroup Maintenance + */ +class UpdateLogging { + + /** + * @var IMaintainableDatabase + */ + public $dbw; + public $batchSize = 1000; + public $minTs = false; + + function execute() { + $this->dbw = $this->getDB( DB_MASTER ); + $logging = $this->dbw->tableName( 'logging' ); + $logging_1_10 = $this->dbw->tableName( 'logging_1_10' ); + $logging_pre_1_10 = $this->dbw->tableName( 'logging_pre_1_10' ); + + if ( $this->dbw->tableExists( 'logging_pre_1_10' ) && !$this->dbw->tableExists( 'logging' ) ) { + # Fix previous aborted run + echo "Cleaning up from previous aborted run\n"; + $this->dbw->query( "RENAME TABLE $logging_pre_1_10 TO $logging", __METHOD__ ); + } + + if ( $this->dbw->tableExists( 'logging_pre_1_10' ) ) { + echo "This script has already been run to completion\n"; + + return; + } + + # Create the target table + if ( !$this->dbw->tableExists( 'logging_1_10' ) ) { + global $wgDBTableOptions; + + $sql = <<<EOT +CREATE TABLE $logging_1_10 ( + -- Log ID, for referring to this specific log entry, probably for deletion and such. + log_id int unsigned NOT NULL auto_increment, + + -- Symbolic keys for the general log type and the action type + -- within the log. The output format will be controlled by the + -- action field, but only the type controls categorization. + log_type varbinary(10) NOT NULL default '', + log_action varbinary(10) NOT NULL default '', + + -- Timestamp. Duh. + log_timestamp binary(14) NOT NULL default '19700101000000', + + -- The user who performed this action; key to user_id + log_user int unsigned NOT NULL default 0, + + -- Key to the page affected. Where a user is the target, + -- this will point to the user page. + log_namespace int NOT NULL default 0, + log_title varchar(255) binary NOT NULL default '', + + -- Freeform text. Interpreted as edit history comments. + log_comment varchar(255) NOT NULL default '', + + -- LF separated list of miscellaneous parameters + log_params blob NOT NULL, + + -- rev_deleted for logs + log_deleted tinyint unsigned NOT NULL default '0', + + PRIMARY KEY log_id (log_id), + KEY type_time (log_type, log_timestamp), + KEY user_time (log_user, log_timestamp), + KEY page_time (log_namespace, log_title, log_timestamp), + KEY times (log_timestamp) + +) $wgDBTableOptions +EOT; + echo "Creating table logging_1_10\n"; + $this->dbw->query( $sql, __METHOD__ ); + } + + # Synchronise the tables + echo "Doing initial sync...\n"; + $this->sync( 'logging', 'logging_1_10' ); + echo "Sync done\n\n"; + + # Rename the old table away + echo "Renaming the old table to $logging_pre_1_10\n"; + $this->dbw->query( "RENAME TABLE $logging TO $logging_pre_1_10", __METHOD__ ); + + # Copy remaining old rows + # Done before the new table is active so that $copyPos is accurate + echo "Doing final sync...\n"; + $this->sync( 'logging_pre_1_10', 'logging_1_10' ); + + # Move the new table in + echo "Moving the new table in...\n"; + $this->dbw->query( "RENAME TABLE $logging_1_10 TO $logging", __METHOD__ ); + echo "Finished.\n"; + } + + /** + * Copy all rows from $srcTable to $dstTable + * @param string $srcTable + * @param string $dstTable + */ + function sync( $srcTable, $dstTable ) { + $batchSize = 1000; + $minTs = $this->dbw->selectField( $srcTable, 'MIN(log_timestamp)', false, __METHOD__ ); + $minTsUnix = wfTimestamp( TS_UNIX, $minTs ); + $numRowsCopied = 0; + + while ( true ) { + $maxTs = $this->dbw->selectField( $srcTable, 'MAX(log_timestamp)', false, __METHOD__ ); + $copyPos = $this->dbw->selectField( $dstTable, 'MAX(log_timestamp)', false, __METHOD__ ); + $maxTsUnix = wfTimestamp( TS_UNIX, $maxTs ); + $copyPosUnix = wfTimestamp( TS_UNIX, $copyPos ); + + if ( $copyPos === null ) { + $percent = 0; + } else { + $percent = ( $copyPosUnix - $minTsUnix ) / ( $maxTsUnix - $minTsUnix ) * 100; + } + printf( "%s %.2f%%\n", $copyPos, $percent ); + + # Handle all entries with timestamp equal to $copyPos + if ( $copyPos !== null ) { + $numRowsCopied += $this->copyExactMatch( $srcTable, $dstTable, $copyPos ); + } + + # Now copy a batch of rows + if ( $copyPos === null ) { + $conds = false; + } else { + $conds = [ 'log_timestamp > ' . $this->dbw->addQuotes( $copyPos ) ]; + } + $srcRes = $this->dbw->select( $srcTable, '*', $conds, __METHOD__, + [ 'LIMIT' => $batchSize, 'ORDER BY' => 'log_timestamp' ] ); + + if ( !$srcRes->numRows() ) { + # All done + break; + } + + $batch = []; + foreach ( $srcRes as $srcRow ) { + $batch[] = (array)$srcRow; + } + $this->dbw->insert( $dstTable, $batch, __METHOD__ ); + $numRowsCopied += count( $batch ); + + wfWaitForSlaves(); + } + echo "Copied $numRowsCopied rows\n"; + } + + function copyExactMatch( $srcTable, $dstTable, $copyPos ) { + $numRowsCopied = 0; + $srcRes = $this->dbw->select( $srcTable, '*', [ 'log_timestamp' => $copyPos ], __METHOD__ ); + $dstRes = $this->dbw->select( $dstTable, '*', [ 'log_timestamp' => $copyPos ], __METHOD__ ); + + if ( $srcRes->numRows() ) { + $srcRow = $srcRes->fetchObject(); + $srcFields = array_keys( (array)$srcRow ); + $srcRes->seek( 0 ); + $dstRowsSeen = []; + + # Make a hashtable of rows that already exist in the destination + foreach ( $dstRes as $dstRow ) { + $reducedDstRow = []; + foreach ( $srcFields as $field ) { + $reducedDstRow[$field] = $dstRow->$field; + } + $hash = md5( serialize( $reducedDstRow ) ); + $dstRowsSeen[$hash] = true; + } + + # Copy all the source rows that aren't already in the destination + foreach ( $srcRes as $srcRow ) { + $hash = md5( serialize( (array)$srcRow ) ); + if ( !isset( $dstRowsSeen[$hash] ) ) { + $this->dbw->insert( $dstTable, (array)$srcRow, __METHOD__ ); + $numRowsCopied++; + } + } + } + + return $numRowsCopied; + } +} + +$ul = new UpdateLogging; +$ul->execute(); diff --git a/www/wiki/maintenance/attachLatest.php b/www/wiki/maintenance/attachLatest.php new file mode 100644 index 00000000..36060d83 --- /dev/null +++ b/www/wiki/maintenance/attachLatest.php @@ -0,0 +1,92 @@ +<?php +/** + * Corrects wrong values in the `page_latest` field in the database. + * + * Copyright © 2005 Brion Vibber <brion@pobox.com> + * https://www.mediawiki.org/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to correct wrong values in the `page_latest` field + * in the database. + * + * @ingroup Maintenance + */ +class AttachLatest extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addOption( "fix", "Actually fix the entries, will dry run otherwise" ); + $this->addOption( "regenerate-all", + "Regenerate the page_latest field for all records in table page" ); + $this->addDescription( 'Fix page_latest entries in the page table' ); + } + + public function execute() { + $this->output( "Looking for pages with page_latest set to 0...\n" ); + $dbw = $this->getDB( DB_MASTER ); + $conds = [ 'page_latest' => 0 ]; + if ( $this->hasOption( 'regenerate-all' ) ) { + $conds = ''; + } + $result = $dbw->select( 'page', + [ 'page_id', 'page_namespace', 'page_title' ], + $conds, + __METHOD__ ); + + $n = 0; + foreach ( $result as $row ) { + $pageId = intval( $row->page_id ); + $title = Title::makeTitle( $row->page_namespace, $row->page_title ); + $name = $title->getPrefixedText(); + $latestTime = $dbw->selectField( 'revision', + 'MAX(rev_timestamp)', + [ 'rev_page' => $pageId ], + __METHOD__ ); + if ( !$latestTime ) { + $this->output( wfWikiID() . " $pageId [[$name]] can't find latest rev time?!\n" ); + continue; + } + + $revision = Revision::loadFromTimestamp( $dbw, $title, $latestTime ); + if ( is_null( $revision ) ) { + $this->output( wfWikiID() + . " $pageId [[$name]] latest time $latestTime, can't find revision id\n" ); + continue; + } + $id = $revision->getId(); + $this->output( wfWikiID() . " $pageId [[$name]] latest time $latestTime, rev id $id\n" ); + if ( $this->hasOption( 'fix' ) ) { + $page = WikiPage::factory( $title ); + $page->updateRevisionOn( $dbw, $revision ); + } + $n++; + } + $this->output( "Done! Processed $n pages.\n" ); + if ( !$this->hasOption( 'fix' ) ) { + $this->output( "This was a dry run; rerun with --fix to update page_latest.\n" ); + } + } +} + +$maintClass = "AttachLatest"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/backup.inc b/www/wiki/maintenance/backup.inc new file mode 100644 index 00000000..60b8a7a9 --- /dev/null +++ b/www/wiki/maintenance/backup.inc @@ -0,0 +1,443 @@ +<?php +/** + * Base classes for database dumpers + * + * Copyright © 2005 Brion Vibber <brion@pobox.com> + * https://www.mediawiki.org/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Dump Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; +require_once __DIR__ . '/../includes/export/DumpFilter.php'; + +use Wikimedia\Rdbms\LoadBalancer; +use Wikimedia\Rdbms\IDatabase; + +/** + * @ingroup Dump Maintenance + */ +class BackupDumper extends Maintenance { + public $reporting = true; + public $pages = null; // all pages + public $skipHeader = false; // don't output <mediawiki> and <siteinfo> + public $skipFooter = false; // don't output </mediawiki> + public $startId = 0; + public $endId = 0; + public $revStartId = 0; + public $revEndId = 0; + public $dumpUploads = false; + public $dumpUploadFileContents = false; + public $orderRevs = false; + + protected $reportingInterval = 100; + protected $pageCount = 0; + protected $revCount = 0; + protected $server = null; // use default + protected $sink = null; // Output filters + protected $lastTime = 0; + protected $pageCountLast = 0; + protected $revCountLast = 0; + + protected $outputTypes = []; + protected $filterTypes = []; + + protected $ID = 0; + + /** + * The dependency-injected database to use. + * + * @var IDatabase|null + * + * @see self::setDB + */ + protected $forcedDb = null; + + /** @var LoadBalancer */ + protected $lb; + + // @todo Unused? + private $stubText = false; // include rev_text_id instead of text; for 2-pass dump + + /** + * @param array $args For backward compatibility + */ + function __construct( $args = null ) { + parent::__construct(); + $this->stderr = fopen( "php://stderr", "wt" ); + + // Built-in output and filter plugins + $this->registerOutput( 'file', 'DumpFileOutput' ); + $this->registerOutput( 'gzip', 'DumpGZipOutput' ); + $this->registerOutput( 'bzip2', 'DumpBZip2Output' ); + $this->registerOutput( 'dbzip2', 'DumpDBZip2Output' ); + $this->registerOutput( '7zip', 'Dump7ZipOutput' ); + + $this->registerFilter( 'latest', 'DumpLatestFilter' ); + $this->registerFilter( 'notalk', 'DumpNotalkFilter' ); + $this->registerFilter( 'namespace', 'DumpNamespaceFilter' ); + + // These three can be specified multiple times + $this->addOption( 'plugin', 'Load a dump plugin class. Specify as <class>[:<file>].', + false, true, false, true ); + $this->addOption( 'output', 'Begin a filtered output stream; Specify as <type>:<file>. ' . + '<type>s: file, gzip, bzip2, 7zip, dbzip2', false, true, false, true ); + $this->addOption( 'filter', 'Add a filter on an output branch. Specify as ' . + '<type>[:<options>]. <types>s: latest, notalk, namespace', false, true, false, true ); + $this->addOption( 'report', 'Report position and speed after every n pages processed. ' . + 'Default: 100.', false, true ); + $this->addOption( 'server', 'Force reading from MySQL server', false, true ); + $this->addOption( '7ziplevel', '7zip compression level for all 7zip outputs. Used for ' . + '-mx option to 7za command.', false, true ); + + if ( $args ) { + // Args should be loaded and processed so that dump() can be called directly + // instead of execute() + $this->loadWithArgv( $args ); + $this->processOptions(); + } + } + + /** + * @param string $name + * @param string $class Name of output filter plugin class + */ + function registerOutput( $name, $class ) { + $this->outputTypes[$name] = $class; + } + + /** + * @param string $name + * @param string $class Name of filter plugin class + */ + function registerFilter( $name, $class ) { + $this->filterTypes[$name] = $class; + } + + /** + * Load a plugin and register it + * + * @param string $class Name of plugin class; must have a static 'register' + * method that takes a BackupDumper as a parameter. + * @param string $file Full or relative path to the PHP file to load, or empty + */ + function loadPlugin( $class, $file ) { + if ( $file != '' ) { + require_once $file; + } + $register = [ $class, 'register' ]; + call_user_func_array( $register, [ $this ] ); + } + + function execute() { + throw new MWException( 'execute() must be overridden in subclasses' ); + } + + /** + * Processes arguments and sets $this->$sink accordingly + */ + function processOptions() { + $sink = null; + $sinks = []; + + $options = $this->orderedOptions; + foreach ( $options as $arg ) { + $opt = $arg[0]; + $param = $arg[1]; + + switch ( $opt ) { + case 'plugin': + $val = explode( ':', $param ); + + if ( count( $val ) === 1 ) { + $this->loadPlugin( $val[0], '' ); + } elseif ( count( $val ) === 2 ) { + $this->loadPlugin( $val[0], $val[1] ); + } else { + $this->fatalError( 'Invalid plugin parameter' ); + return; + } + + break; + case 'output': + $split = explode( ':', $param, 2 ); + if ( count( $split ) !== 2 ) { + $this->fatalError( 'Invalid output parameter' ); + } + list( $type, $file ) = $split; + if ( !is_null( $sink ) ) { + $sinks[] = $sink; + } + if ( !isset( $this->outputTypes[$type] ) ) { + $this->fatalError( "Unrecognized output sink type '$type'" ); + } + $class = $this->outputTypes[$type]; + if ( $type === "7zip" ) { + $sink = new $class( $file, intval( $this->getOption( '7ziplevel' ) ) ); + } else { + $sink = new $class( $file ); + } + + break; + case 'filter': + if ( is_null( $sink ) ) { + $sink = new DumpOutput(); + } + + $split = explode( ':', $param ); + $key = $split[0]; + + if ( !isset( $this->filterTypes[$key] ) ) { + $this->fatalError( "Unrecognized filter type '$key'" ); + } + + $type = $this->filterTypes[$key]; + + if ( count( $split ) === 1 ) { + $filter = new $type( $sink ); + } elseif ( count( $split ) === 2 ) { + $filter = new $type( $sink, $split[1] ); + } else { + $this->fatalError( 'Invalid filter parameter' ); + } + + // references are lame in php... + unset( $sink ); + $sink = $filter; + + break; + } + } + + if ( $this->hasOption( 'report' ) ) { + $this->reportingInterval = intval( $this->getOption( 'report' ) ); + } + + if ( $this->hasOption( 'server' ) ) { + $this->server = $this->getOption( 'server' ); + } + + if ( is_null( $sink ) ) { + $sink = new DumpOutput(); + } + $sinks[] = $sink; + + if ( count( $sinks ) > 1 ) { + $this->sink = new DumpMultiWriter( $sinks ); + } else { + $this->sink = $sink; + } + } + + function dump( $history, $text = WikiExporter::TEXT ) { + # Notice messages will foul up your XML output even if they're + # relatively harmless. + if ( ini_get( 'display_errors' ) ) { + ini_set( 'display_errors', 'stderr' ); + } + + $this->initProgress( $history ); + + $db = $this->backupDb(); + $exporter = new WikiExporter( $db, $history, WikiExporter::STREAM, $text ); + $exporter->dumpUploads = $this->dumpUploads; + $exporter->dumpUploadFileContents = $this->dumpUploadFileContents; + + $wrapper = new ExportProgressFilter( $this->sink, $this ); + $exporter->setOutputSink( $wrapper ); + + if ( !$this->skipHeader ) { + $exporter->openStream(); + } + # Log item dumps: all or by range + if ( $history & WikiExporter::LOGS ) { + if ( $this->startId || $this->endId ) { + $exporter->logsByRange( $this->startId, $this->endId ); + } else { + $exporter->allLogs(); + } + } elseif ( is_null( $this->pages ) ) { + # Page dumps: all or by page ID range + if ( $this->startId || $this->endId ) { + $exporter->pagesByRange( $this->startId, $this->endId, $this->orderRevs ); + } elseif ( $this->revStartId || $this->revEndId ) { + $exporter->revsByRange( $this->revStartId, $this->revEndId ); + } else { + $exporter->allPages(); + } + } else { + # Dump of specific pages + $exporter->pagesByName( $this->pages ); + } + + if ( !$this->skipFooter ) { + $exporter->closeStream(); + } + + $this->report( true ); + } + + /** + * Initialise starting time and maximum revision count. + * We'll make ETA calculations based an progress, assuming relatively + * constant per-revision rate. + * @param int $history WikiExporter::CURRENT or WikiExporter::FULL + */ + function initProgress( $history = WikiExporter::FULL ) { + $table = ( $history == WikiExporter::CURRENT ) ? 'page' : 'revision'; + $field = ( $history == WikiExporter::CURRENT ) ? 'page_id' : 'rev_id'; + + $dbr = $this->forcedDb; + if ( $this->forcedDb === null ) { + $dbr = wfGetDB( DB_REPLICA ); + } + $this->maxCount = $dbr->selectField( $table, "MAX($field)", '', __METHOD__ ); + $this->startTime = microtime( true ); + $this->lastTime = $this->startTime; + $this->ID = getmypid(); + } + + /** + * @todo Fixme: the --server parameter is currently not respected, as it + * doesn't seem terribly easy to ask the load balancer for a particular + * connection by name. + * @return IDatabase + */ + function backupDb() { + if ( $this->forcedDb !== null ) { + return $this->forcedDb; + } + + $this->lb = wfGetLBFactory()->newMainLB(); + $db = $this->lb->getConnection( DB_REPLICA, 'dump' ); + + // Discourage the server from disconnecting us if it takes a long time + // to read out the big ol' batch query. + $db->setSessionOptions( [ 'connTimeout' => 3600 * 24 ] ); + + return $db; + } + + /** + * Force the dump to use the provided database connection for database + * operations, wherever possible. + * + * @param IDatabase|null $db (Optional) the database connection to use. If null, resort to + * use the globally provided ways to get database connections. + */ + function setDB( IDatabase $db = null ) { + parent::setDB( $db ); + $this->forcedDb = $db; + } + + function __destruct() { + if ( isset( $this->lb ) ) { + $this->lb->closeAll(); + } + } + + function backupServer() { + global $wgDBserver; + + return $this->server + ? $this->server + : $wgDBserver; + } + + function reportPage() { + $this->pageCount++; + } + + function revCount() { + $this->revCount++; + $this->report(); + } + + function report( $final = false ) { + if ( $final xor ( $this->revCount % $this->reportingInterval == 0 ) ) { + $this->showReport(); + } + } + + function showReport() { + if ( $this->reporting ) { + $now = wfTimestamp( TS_DB ); + $nowts = microtime( true ); + $deltaAll = $nowts - $this->startTime; + $deltaPart = $nowts - $this->lastTime; + $this->pageCountPart = $this->pageCount - $this->pageCountLast; + $this->revCountPart = $this->revCount - $this->revCountLast; + + if ( $deltaAll ) { + $portion = $this->revCount / $this->maxCount; + $eta = $this->startTime + $deltaAll / $portion; + $etats = wfTimestamp( TS_DB, intval( $eta ) ); + $pageRate = $this->pageCount / $deltaAll; + $revRate = $this->revCount / $deltaAll; + } else { + $pageRate = '-'; + $revRate = '-'; + $etats = '-'; + } + if ( $deltaPart ) { + $pageRatePart = $this->pageCountPart / $deltaPart; + $revRatePart = $this->revCountPart / $deltaPart; + } else { + $pageRatePart = '-'; + $revRatePart = '-'; + } + $this->progress( sprintf( + "%s: %s (ID %d) %d pages (%0.1f|%0.1f/sec all|curr), " + . "%d revs (%0.1f|%0.1f/sec all|curr), ETA %s [max %d]", + $now, wfWikiID(), $this->ID, $this->pageCount, $pageRate, + $pageRatePart, $this->revCount, $revRate, $revRatePart, $etats, + $this->maxCount + ) ); + $this->lastTime = $nowts; + $this->revCountLast = $this->revCount; + } + } + + function progress( $string ) { + if ( $this->reporting ) { + fwrite( $this->stderr, $string . "\n" ); + } + } + + function fatalError( $msg ) { + $this->error( "$msg\n", 1 ); + } +} + +class ExportProgressFilter extends DumpFilter { + function __construct( &$sink, &$progress ) { + parent::__construct( $sink ); + $this->progress = $progress; + } + + function writeClosePage( $string ) { + parent::writeClosePage( $string ); + $this->progress->reportPage(); + } + + function writeRevision( $rev, $string ) { + parent::writeRevision( $rev, $string ); + $this->progress->revCount(); + } +} diff --git a/www/wiki/maintenance/backupPrefetch.inc b/www/wiki/maintenance/backupPrefetch.inc new file mode 100644 index 00000000..6a2d3bf6 --- /dev/null +++ b/www/wiki/maintenance/backupPrefetch.inc @@ -0,0 +1,219 @@ +<?php +/** + * Helper class for the --prefetch option of dumpTextPass.php + * + * Copyright © 2005 Brion Vibber <brion@pobox.com> + * https://www.mediawiki.org/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +/** + * Readahead helper for making large MediaWiki data dumps; + * reads in a previous XML dump to sequentially prefetch text + * records already normalized and decompressed. + * + * This can save load on the external database servers, hopefully. + * + * Assumes that dumps will be recorded in the canonical order: + * - ascending by page_id + * - ascending by rev_id within each page + * - text contents are immutable and should not change once + * recorded, so the previous dump is a reliable source + * + * @ingroup Maintenance + */ +class BaseDump { + /** @var XMLReader */ + protected $reader = null; + protected $atEnd = false; + protected $atPageEnd = false; + protected $lastPage = 0; + protected $lastRev = 0; + protected $infiles = null; + + public function __construct( $infile ) { + $this->infiles = explode( ';', $infile ); + $this->reader = new XMLReader(); + $infile = array_shift( $this->infiles ); + if ( defined( 'LIBXML_PARSEHUGE' ) ) { + $this->reader->open( $infile, null, LIBXML_PARSEHUGE ); + } else { + $this->reader->open( $infile ); + } + } + + /** + * Attempts to fetch the text of a particular page revision + * from the dump stream. May return null if the page is + * unavailable. + * + * @param int $page ID number of page to read + * @param int $rev ID number of revision to read + * @return string|null + */ + function prefetch( $page, $rev ) { + $page = intval( $page ); + $rev = intval( $rev ); + while ( $this->lastPage < $page && !$this->atEnd ) { + $this->debug( "BaseDump::prefetch at page $this->lastPage, looking for $page" ); + $this->nextPage(); + } + if ( $this->lastPage > $page || $this->atEnd ) { + $this->debug( "BaseDump::prefetch already past page $page " + . "looking for rev $rev [$this->lastPage, $this->lastRev]" ); + + return null; + } + while ( $this->lastRev < $rev && !$this->atEnd && !$this->atPageEnd ) { + $this->debug( "BaseDump::prefetch at page $this->lastPage, rev $this->lastRev, " + . "looking for $page, $rev" ); + $this->nextRev(); + } + if ( $this->lastRev == $rev && !$this->atEnd ) { + $this->debug( "BaseDump::prefetch hit on $page, $rev [$this->lastPage, $this->lastRev]" ); + + return $this->nextText(); + } else { + $this->debug( "BaseDump::prefetch already past rev $rev on page $page " + . "[$this->lastPage, $this->lastRev]" ); + + return null; + } + } + + function debug( $str ) { + wfDebug( $str . "\n" ); + // global $dumper; + // $dumper->progress( $str ); + } + + /** + * @access private + */ + function nextPage() { + if ( $this->skipTo( 'page', 'mediawiki' ) ) { + if ( $this->skipTo( 'id' ) ) { + $this->lastPage = intval( $this->nodeContents() ); + $this->lastRev = 0; + $this->atPageEnd = false; + } + } else { + $this->close(); + if ( count( $this->infiles ) ) { + $infile = array_shift( $this->infiles ); + $this->reader->open( $infile ); + $this->atEnd = false; + } + } + } + + /** + * @access private + */ + function nextRev() { + if ( $this->skipTo( 'revision' ) ) { + if ( $this->skipTo( 'id' ) ) { + $this->lastRev = intval( $this->nodeContents() ); + } + } else { + $this->atPageEnd = true; + } + } + + /** + * @access private + * @return string + */ + function nextText() { + $this->skipTo( 'text' ); + + return strval( $this->nodeContents() ); + } + + /** + * @access private + * @param string $name + * @param string $parent + * @return bool|null + */ + function skipTo( $name, $parent = 'page' ) { + if ( $this->atEnd ) { + return false; + } + while ( $this->reader->read() ) { + if ( $this->reader->nodeType == XMLReader::ELEMENT + && $this->reader->name == $name + ) { + return true; + } + if ( $this->reader->nodeType == XMLReader::END_ELEMENT + && $this->reader->name == $parent + ) { + $this->debug( "BaseDump::skipTo found </$parent> searching for <$name>" ); + + return false; + } + } + + return $this->close(); + } + + /** + * Shouldn't something like this be built-in to XMLReader? + * Fetches text contents of the current element, assuming + * no sub-elements or such scary things. + * + * @return string + * @access private + */ + function nodeContents() { + if ( $this->atEnd ) { + return null; + } + if ( $this->reader->isEmptyElement ) { + return ""; + } + $buffer = ""; + while ( $this->reader->read() ) { + switch ( $this->reader->nodeType ) { + case XMLReader::TEXT: + // case XMLReader::WHITESPACE: + case XMLReader::SIGNIFICANT_WHITESPACE: + $buffer .= $this->reader->value; + break; + case XMLReader::END_ELEMENT: + return $buffer; + } + } + + return $this->close(); + } + + /** + * @access private + * @return null + */ + function close() { + $this->reader->close(); + $this->atEnd = true; + + return null; + } +} diff --git a/www/wiki/maintenance/benchmarks/Benchmarker.php b/www/wiki/maintenance/benchmarks/Benchmarker.php new file mode 100644 index 00000000..832da4db --- /dev/null +++ b/www/wiki/maintenance/benchmarks/Benchmarker.php @@ -0,0 +1,169 @@ +<?php +/** + * @defgroup Benchmark Benchmark + * @ingroup Maintenance + */ + +/** + * Base code for benchmark scripts. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Benchmark + */ + +require_once __DIR__ . '/../Maintenance.php'; + +/** + * Base class for benchmark scripts. + * + * @ingroup Benchmark + */ +abstract class Benchmarker extends Maintenance { + protected $defaultCount = 100; + private $lang; + + public function __construct() { + parent::__construct(); + $this->addOption( 'count', 'How many times to run a benchmark', false, true ); + $this->addOption( 'verbose', 'Verbose logging of resource usage', false, false, 'v' ); + } + + public function bench( array $benchs ) { + $this->lang = Language::factory( 'en' ); + + $this->startBench(); + $count = $this->getOption( 'count', $this->defaultCount ); + $verbose = $this->hasOption( 'verbose' ); + foreach ( $benchs as $key => $bench ) { + // Shortcut for simple functions + if ( is_callable( $bench ) ) { + $bench = [ 'function' => $bench ]; + } + + // Default to no arguments + if ( !isset( $bench['args'] ) ) { + $bench['args'] = []; + } + + // Optional setup called outside time measure + if ( isset( $bench['setup'] ) ) { + call_user_func( $bench['setup'] ); + } + + // Run benchmarks + $times = []; + for ( $i = 0; $i < $count; $i++ ) { + $t = microtime( true ); + call_user_func_array( $bench['function'], $bench['args'] ); + $t = ( microtime( true ) - $t ) * 1000; + if ( $verbose ) { + $this->verboseRun( $i ); + } + $times[] = $t; + } + + // Collect metrics + sort( $times, SORT_NUMERIC ); + $min = $times[0]; + $max = end( $times ); + if ( $count % 2 ) { + $median = $times[ ( $count - 1 ) / 2 ]; + } else { + $median = ( $times[$count / 2] + $times[$count / 2 - 1] ) / 2; + } + $total = array_sum( $times ); + $mean = $total / $count; + + // Name defaults to name of called function + if ( is_string( $key ) ) { + $name = $key; + } else { + if ( is_array( $bench['function'] ) ) { + $name = get_class( $bench['function'][0] ) . '::' . $bench['function'][1]; + } else { + $name = strval( $bench['function'] ); + } + $name = sprintf( "%s(%s)", + $name, + implode( ', ', $bench['args'] ) + ); + } + + $this->addResult( [ + 'name' => $name, + 'count' => $count, + 'total' => $total, + 'min' => $min, + 'median' => $median, + 'mean' => $mean, + 'max' => $max, + 'usage' => [ + 'mem' => memory_get_usage( true ), + 'mempeak' => memory_get_peak_usage( true ), + ], + ] ); + } + } + + public function startBench() { + $this->output( + sprintf( "Running PHP version %s (%s) on %s %s %s\n\n", + phpversion(), + php_uname( 'm' ), + php_uname( 's' ), + php_uname( 'r' ), + php_uname( 'v' ) + ) + ); + } + + public function addResult( $res ) { + $ret = sprintf( "%s\n %' 6s: %d\n", + $res['name'], + 'times', + $res['count'] + ); + + foreach ( [ 'total', 'min', 'median', 'mean', 'max' ] as $metric ) { + $ret .= sprintf( " %' 6s: %6.2fms\n", + $metric, + $res[$metric] + ); + } + + foreach ( [ + 'mem' => 'Current memory usage', + 'mempeak' => 'Peak memory usage' + ] as $key => $label ) { + $ret .= sprintf( "%' 20s: %s\n", + $label, + $this->lang->formatSize( $res['usage'][$key] ) + ); + } + + $this->output( "$ret\n" ); + } + + protected function verboseRun( $iteration ) { + $this->output( sprintf( "#%3d - memory: %-10s - peak: %-10s\n", + $iteration, + $this->lang->formatSize( memory_get_usage( true ) ), + $this->lang->formatSize( memory_get_peak_usage( true ) ) + ) ); + } +} diff --git a/www/wiki/maintenance/benchmarks/README b/www/wiki/maintenance/benchmarks/README new file mode 100644 index 00000000..27da9def --- /dev/null +++ b/www/wiki/maintenance/benchmarks/README @@ -0,0 +1,10 @@ +This directory hold several benchmarking scripts used as a proof of speed +or to track PHP performances over time. + +To get somehow accurate result, you might want to bound the PHP process +to a specific CPU with `taskset` and raise its priority with `nice`. Example: + + $ taskset 1 nice -n-10 php bench_wfIsWindows.php + +australia-untidy.html.gz contains representative input text for +benchmarkTidy.php. It needs to be decompressed before use. diff --git a/www/wiki/maintenance/benchmarks/australia-untidy.html.gz b/www/wiki/maintenance/benchmarks/australia-untidy.html.gz Binary files differnew file mode 100644 index 00000000..148481da --- /dev/null +++ b/www/wiki/maintenance/benchmarks/australia-untidy.html.gz diff --git a/www/wiki/maintenance/benchmarks/bench_HTTP_HTTPS.php b/www/wiki/maintenance/benchmarks/bench_HTTP_HTTPS.php new file mode 100644 index 00000000..0e3cd73d --- /dev/null +++ b/www/wiki/maintenance/benchmarks/bench_HTTP_HTTPS.php @@ -0,0 +1,63 @@ +<?php +/** + * Benchmark HTTP request vs HTTPS request. + * + * This come from r75429 message. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Benchmark + * @author Platonides + */ + +require_once __DIR__ . '/Benchmarker.php'; + +/** + * Maintenance script that benchmarks HTTP request vs HTTPS request. + * + * @ingroup Benchmark + */ +class BenchHttpHttps extends Benchmarker { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Benchmark HTTP request vs HTTPS request.' ); + } + + public function execute() { + $this->bench( [ + [ 'function' => [ $this, 'getHTTP' ] ], + [ 'function' => [ $this, 'getHTTPS' ] ], + ] ); + } + + private function doRequest( $proto ) { + Http::get( "$proto://localhost/", [], __METHOD__ ); + } + + // bench function 1 + protected function getHTTP() { + $this->doRequest( 'http' ); + } + + // bench function 2 + protected function getHTTPS() { + $this->doRequest( 'https' ); + } +} + +$maintClass = 'BenchHttpHttps'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/benchmarks/bench_Wikimedia_base_convert.php b/www/wiki/maintenance/benchmarks/bench_Wikimedia_base_convert.php new file mode 100644 index 00000000..86bcc8a3 --- /dev/null +++ b/www/wiki/maintenance/benchmarks/bench_Wikimedia_base_convert.php @@ -0,0 +1,77 @@ +<?php +/** + * Benchmark for Wikimedia\base_convert() + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Benchmark + * @author Tyler Romeo + */ + +require_once __DIR__ . '/Benchmarker.php'; + +/** + * Maintenance script that benchmarks Wikimedia\base_convert(). + * + * Code exists in vendor repository brought in via composer. + * + * @ingroup Benchmark + */ +class BenchWikimediaBaseConvert extends Benchmarker { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Benchmark for Wikimedia\base_convert.' ); + $this->addOption( "inbase", "Input base", false, true ); + $this->addOption( "outbase", "Output base", false, true ); + $this->addOption( "length", "Size in digits to generate for input", false, true ); + } + + public function execute() { + $inbase = $this->getOption( "inbase", 36 ); + $outbase = $this->getOption( "outbase", 16 ); + $length = $this->getOption( "length", 128 ); + $number = self::makeRandomNumber( $inbase, $length ); + + $this->bench( [ + [ + 'function' => 'Wikimedia\base_convert', + 'args' => [ $number, $inbase, $outbase, 0, true, 'php' ] + ], + [ + 'function' => 'Wikimedia\base_convert', + 'args' => [ $number, $inbase, $outbase, 0, true, 'bcmath' ] + ], + [ + 'function' => 'Wikimedia\base_convert', + 'args' => [ $number, $inbase, $outbase, 0, true, 'gmp' ] + ], + ] ); + } + + protected static function makeRandomNumber( $base, $length ) { + $baseChars = '0123456789abcdefghijklmnopqrstuvwxyz'; + $res = ''; + for ( $i = 0; $i < $length; $i++ ) { + $res .= $baseChars[mt_rand( 0, $base - 1 )]; + } + + return $res; + } +} + +$maintClass = 'BenchWikimediaBaseConvert'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/benchmarks/bench_delete_truncate.php b/www/wiki/maintenance/benchmarks/bench_delete_truncate.php new file mode 100644 index 00000000..0a999ecc --- /dev/null +++ b/www/wiki/maintenance/benchmarks/bench_delete_truncate.php @@ -0,0 +1,105 @@ +<?php +/** + * Benchmark SQL DELETE vs SQL TRUNCATE. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Benchmark + */ + +require_once __DIR__ . '/Benchmarker.php'; + +use Wikimedia\Rdbms\IDatabase; +use Wikimedia\Rdbms\IMaintainableDatabase; + +/** + * Maintenance script that benchmarks SQL DELETE vs SQL TRUNCATE. + * + * @ingroup Benchmark + */ +class BenchmarkDeleteTruncate extends Benchmarker { + protected $defaultCount = 10; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Benchmarks SQL DELETE vs SQL TRUNCATE.' ); + } + + public function execute() { + $dbw = $this->getDB( DB_MASTER ); + + $test = $dbw->tableName( 'test' ); + $dbw->query( "CREATE TABLE IF NOT EXISTS /*_*/$test ( + test_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + text varbinary(255) NOT NULL +);" ); + + $this->bench( [ + 'Delete' => [ + 'setup' => function () use ( $dbw ) { + $this->insertData( $dbw ); + }, + 'function' => function () use ( $dbw ) { + $this->delete( $dbw ); + } + ], + 'Truncate' => [ + 'setup' => function () use ( $dbw ) { + $this->insertData( $dbw ); + }, + 'function' => function () use ( $dbw ) { + $this->truncate( $dbw ); + } + ] + ] ); + + $dbw->dropTable( 'test' ); + } + + /** + * @param IDatabase $dbw + * @return void + */ + private function insertData( $dbw ) { + $range = range( 0, 1024 ); + $data = []; + foreach ( $range as $r ) { + $data[] = [ 'text' => $r ]; + } + $dbw->insert( 'test', $data, __METHOD__ ); + } + + /** + * @param IDatabase $dbw + * @return void + */ + private function delete( $dbw ) { + $dbw->delete( 'text', '*', __METHOD__ ); + } + + /** + * @param IMaintainableDatabase $dbw + * @return void + */ + private function truncate( $dbw ) { + $test = $dbw->tableName( 'test' ); + $dbw->query( "TRUNCATE TABLE $test" ); + } +} + +$maintClass = 'BenchmarkDeleteTruncate'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/benchmarks/bench_if_switch.php b/www/wiki/maintenance/benchmarks/bench_if_switch.php new file mode 100644 index 00000000..843ef7cd --- /dev/null +++ b/www/wiki/maintenance/benchmarks/bench_if_switch.php @@ -0,0 +1,110 @@ +<?php +/** + * Benchmark if elseif... versus switch case. + * + * This come from r75429 message + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Benchmark + * @author Platonides + */ + +require_once __DIR__ . '/Benchmarker.php'; + +/** + * Maintenance script that benchmark if elseif... versus switch case. + * + * @ingroup Maintenance + */ +class BenchIfSwitch extends Benchmarker { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Benchmark if elseif... versus switch case.' ); + } + + public function execute() { + $this->bench( [ + [ 'function' => [ $this, 'doElseIf' ] ], + [ 'function' => [ $this, 'doSwitch' ] ], + ] ); + } + + // bench function 1 + protected function doElseIf() { + $a = 'z'; + if ( $a == 'a' ) { + } elseif ( $a == 'b' ) { + } elseif ( $a == 'c' ) { + } elseif ( $a == 'd' ) { + } elseif ( $a == 'e' ) { + } elseif ( $a == 'f' ) { + } elseif ( $a == 'g' ) { + } elseif ( $a == 'h' ) { + } elseif ( $a == 'i' ) { + } elseif ( $a == 'j' ) { + } elseif ( $a == 'k' ) { + } elseif ( $a == 'l' ) { + } elseif ( $a == 'm' ) { + } elseif ( $a == 'n' ) { + } elseif ( $a == 'o' ) { + } elseif ( $a == 'p' ) { + } else { + } + } + + // bench function 2 + protected function doSwitch() { + $a = 'z'; + switch ( $a ) { + case 'b': + break; + case 'c': + break; + case 'd': + break; + case 'e': + break; + case 'f': + break; + case 'g': + break; + case 'h': + break; + case 'i': + break; + case 'j': + break; + case 'k': + break; + case 'l': + break; + case 'm': + break; + case 'n': + break; + case 'o': + break; + case 'p': + break; + default: + } + } +} + +$maintClass = 'BenchIfSwitch'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/benchmarks/bench_strtr_str_replace.php b/www/wiki/maintenance/benchmarks/bench_strtr_str_replace.php new file mode 100644 index 00000000..55c7159b --- /dev/null +++ b/www/wiki/maintenance/benchmarks/bench_strtr_str_replace.php @@ -0,0 +1,74 @@ +<?php +/** + * Benchmark for strtr() vs str_replace(). + * + * This come from r75429 message. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Benchmark + */ + +require_once __DIR__ . '/Benchmarker.php'; + +function bfNormalizeTitleStrTr( $str ) { + return strtr( $str, '_', ' ' ); +} + +function bfNormalizeTitleStrReplace( $str ) { + return str_replace( '_', ' ', $str ); +} + +/** + * Maintenance script that benchmarks for strtr() vs str_replace(). + * + * @ingroup Benchmark + */ +class BenchStrtrStrReplace extends Benchmarker { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Benchmark for strtr() vs str_replace().' ); + } + + public function execute() { + $this->bench( [ + [ 'function' => [ $this, 'benchstrtr' ] ], + [ 'function' => [ $this, 'benchstr_replace' ] ], + [ 'function' => [ $this, 'benchstrtr_indirect' ] ], + [ 'function' => [ $this, 'benchstr_replace_indirect' ] ], + ] ); + } + + protected function benchstrtr() { + strtr( "[[MediaWiki:Some_random_test_page]]", "_", " " ); + } + + protected function benchstr_replace() { + str_replace( "_", " ", "[[MediaWiki:Some_random_test_page]]" ); + } + + protected function benchstrtr_indirect() { + bfNormalizeTitleStrTr( "[[MediaWiki:Some_random_test_page]]" ); + } + + protected function benchstr_replace_indirect() { + bfNormalizeTitleStrReplace( "[[MediaWiki:Some_random_test_page]]" ); + } +} + +$maintClass = 'BenchStrtrStrReplace'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/benchmarks/bench_utf8_title_check.php b/www/wiki/maintenance/benchmarks/bench_utf8_title_check.php new file mode 100644 index 00000000..3091de62 --- /dev/null +++ b/www/wiki/maintenance/benchmarks/bench_utf8_title_check.php @@ -0,0 +1,114 @@ +<?php +/** + * Benchmark for using a regexp vs. mb_check_encoding to check for UTF-8 encoding. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Benchmark + */ + +require_once __DIR__ . '/Benchmarker.php'; + +/** + * This little benchmark executes the regexp formerly used in Language->checkTitleEncoding() + * and compares its execution time against that of mb_check_encoding. + * + * @ingroup Benchmark + */ +class BenchUtf8TitleCheck extends Benchmarker { + private $data; + + private $isutf8; + + public function __construct() { + parent::__construct(); + + // @codingStandardsIgnoreStart Ignore long line warnings. + $this->data = [ + "", + "United States of America", // 7bit ASCII + "S%C3%A9rie%20t%C3%A9l%C3%A9vis%C3%A9e", + "Acteur%7CAlbert%20Robbins%7CAnglais%7CAnn%20Donahue%7CAnthony%20E.%20Zuiker%7CCarol%20Mendelsohn", + // This comes from T38839 + "Acteur%7CAlbert%20Robbins%7CAnglais%7CAnn%20Donahue%7CAnthony%20E.%20Zuiker%7CCarol%20Mendelsohn%7C" + . "Catherine%20Willows%7CDavid%20Hodges%7CDavid%20Phillips%7CGil%20Grissom%7CGreg%20Sanders%7CHodges%7C" + . "Internet%20Movie%20Database%7CJim%20Brass%7CLady%20Heather%7C" + . "Les%20Experts%20(s%C3%A9rie%20t%C3%A9l%C3%A9vis%C3%A9e)%7CLes%20Experts%20:%20Manhattan%7C" + . "Les%20Experts%20:%20Miami%7CListe%20des%20personnages%20des%20Experts%7C" + . "Liste%20des%20%C3%A9pisodes%20des%20Experts%7CMod%C3%A8le%20discussion:Palette%20Les%20Experts%7C" + . "Nick%20Stokes%7CPersonnage%20de%20fiction%7CPersonnage%20fictif%7CPersonnage%20de%20fiction%7C" + . "Personnages%20r%C3%A9currents%20dans%20Les%20Experts%7CRaymond%20Langston%7CRiley%20Adams%7C" + . "Saison%201%20des%20Experts%7CSaison%2010%20des%20Experts%7CSaison%2011%20des%20Experts%7C" + . "Saison%2012%20des%20Experts%7CSaison%202%20des%20Experts%7CSaison%203%20des%20Experts%7C" + . "Saison%204%20des%20Experts%7CSaison%205%20des%20Experts%7CSaison%206%20des%20Experts%7C" + . "Saison%207%20des%20Experts%7CSaison%208%20des%20Experts%7CSaison%209%20des%20Experts%7C" + . "Sara%20Sidle%7CSofia%20Curtis%7CS%C3%A9rie%20t%C3%A9l%C3%A9vis%C3%A9e%7CWallace%20Langham%7C" + . "Warrick%20Brown%7CWendy%20Simms%7C%C3%89tats-Unis" + ]; + // @codingStandardsIgnoreEnd + + $this->addDescription( "Benchmark for using a regexp vs. mb_check_encoding " . + "to check for UTF-8 encoding." ); + } + + public function execute() { + $benchmarks = []; + foreach ( $this->data as $val ) { + $benchmarks[] = [ + 'function' => [ $this, 'use_regexp' ], + 'args' => [ rawurldecode( $val ) ] + ]; + $benchmarks[] = [ + 'function' => [ $this, 'use_regexp_non_capturing' ], + 'args' => [ rawurldecode( $val ) ] + ]; + $benchmarks[] = [ + 'function' => [ $this, 'use_regexp_once_only' ], + 'args' => [ rawurldecode( $val ) ] + ]; + $benchmarks[] = [ + 'function' => [ $this, 'use_mb_check_encoding' ], + 'args' => [ rawurldecode( $val ) ] + ]; + } + $this->bench( $benchmarks ); + } + + protected function use_regexp( $s ) { + $this->isutf8 = preg_match( '/^([\x00-\x7f]|[\xc0-\xdf][\x80-\xbf]|' . + '[\xe0-\xef][\x80-\xbf]{2}|[\xf0-\xf7][\x80-\xbf]{3})+$/', $s ); + } + + protected function use_regexp_non_capturing( $s ) { + // Same as above with a non-capturing subgroup. + $this->isutf8 = preg_match( '/^(?:[\x00-\x7f]|[\xc0-\xdf][\x80-\xbf]|' . + '[\xe0-\xef][\x80-\xbf]{2}|[\xf0-\xf7][\x80-\xbf]{3})+$/', $s ); + } + + protected function use_regexp_once_only( $s ) { + // Same as above with a once-only subgroup. + $this->isutf8 = preg_match( '/^(?>[\x00-\x7f]|[\xc0-\xdf][\x80-\xbf]|' . + '[\xe0-\xef][\x80-\xbf]{2}|[\xf0-\xf7][\x80-\xbf]{3})+$/', $s ); + } + + protected function use_mb_check_encoding( $s ) { + $this->isutf8 = mb_check_encoding( $s, 'UTF-8' ); + } +} + +$maintClass = 'BenchUtf8TitleCheck'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/benchmarks/bench_wfIsWindows.php b/www/wiki/maintenance/benchmarks/bench_wfIsWindows.php new file mode 100644 index 00000000..960ef0e8 --- /dev/null +++ b/www/wiki/maintenance/benchmarks/bench_wfIsWindows.php @@ -0,0 +1,68 @@ +<?php +/** + * Benchmark for wfIsWindows(). + * + * This come from r75429 message. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Benchmark + * @author Platonides + */ + +require_once __DIR__ . '/Benchmarker.php'; + +/** + * Maintenance script that benchmarks wfIsWindows(). + * + * @ingroup Benchmark + */ +class BenchWfIsWindows extends Benchmarker { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Benchmark for wfIsWindows.' ); + } + + public function execute() { + $this->bench( [ + [ 'function' => [ $this, 'wfIsWindows' ] ], + [ 'function' => [ $this, 'wfIsWindowsCached' ] ], + ] ); + } + + protected static function is_win() { + return substr( php_uname(), 0, 7 ) == 'Windows'; + } + + // bench function 1 + protected function wfIsWindows() { + return self::is_win(); + } + + // bench function 2 + protected function wfIsWindowsCached() { + static $isWindows = null; + if ( $isWindows == null ) { + $isWindows = self::is_win(); + } + + return $isWindows; + } +} + +$maintClass = 'BenchWfIsWindows'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/benchmarks/benchmarkCSSMin.php b/www/wiki/maintenance/benchmarks/benchmarkCSSMin.php new file mode 100644 index 00000000..3eaa88dc --- /dev/null +++ b/www/wiki/maintenance/benchmarks/benchmarkCSSMin.php @@ -0,0 +1,76 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Benchmark + * @author Timo Tijhof + */ + +require_once __DIR__ . '/Benchmarker.php'; + +/** + * Maintenance script that benchmarks CSSMin. + * + * @ingroup Benchmark + */ +class BenchmarkCSSMin extends Benchmarker { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Benchmarks CSSMin.' ); + $this->addOption( 'file', 'Path to CSS file (may be gzipped)', false, true ); + $this->addOption( 'out', 'Echo output of one run to stdout for inspection', false, false ); + } + + public function execute() { + $file = $this->getOption( 'file', __DIR__ . '/cssmin/styles.css' ); + $filename = basename( $file ); + $css = $this->loadFile( $file ); + + if ( $this->hasOption( 'out' ) ) { + echo "## minify\n\n", + CSSMin::minify( $css ), + "\n\n"; + echo "## remap\n\n", + CSSMin::remap( $css, dirname( $file ), 'https://example.org/test/', true ), + "\n"; + return; + } + + $this->bench( [ + "minify ($filename)" => [ + 'function' => [ 'CSSMin', 'minify' ], + 'args' => [ $css ] + ], + "remap ($filename)" => [ + 'function' => [ 'CSSMin', 'remap' ], + 'args' => [ $css, dirname( $file ), 'https://example.org/test/', true ] + ], + ] ); + } + + private function loadFile( $file ) { + $css = file_get_contents( $file ); + // Detect GZIP compression header + if ( substr( $css, 0, 2 ) === "\037\213" ) { + $css = gzdecode( $css ); + } + return $css; + } +} + +$maintClass = 'BenchmarkCSSMin'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/benchmarks/benchmarkHooks.php b/www/wiki/maintenance/benchmarks/benchmarkHooks.php new file mode 100644 index 00000000..d49fa1d4 --- /dev/null +++ b/www/wiki/maintenance/benchmarks/benchmarkHooks.php @@ -0,0 +1,73 @@ +<?php +/** + * Benchmark %MediaWiki hooks. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Benchmark + */ + +require_once __DIR__ . '/Benchmarker.php'; + +/** + * Maintenance script that benchmarks %MediaWiki hooks. + * + * @ingroup Benchmark + */ +class BenchmarkHooks extends Benchmarker { + protected $defaultCount = 10; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Benchmark MediaWiki Hooks.' ); + } + + public function execute() { + $cases = [ + 'Loaded 0 hooks' => 0, + 'Loaded 1 hook' => 1, + 'Loaded 10 hooks' => 10, + 'Loaded 100 hooks' => 100, + ]; + $benches = []; + foreach ( $cases as $label => $load ) { + $benches[$label] = [ + 'setup' => function () use ( $load ) { + global $wgHooks; + $wgHooks['Test'] = []; + for ( $i = 1; $i <= $load; $i++ ) { + $wgHooks['Test'][] = [ $this, 'test' ]; + } + }, + 'function' => function () { + Hooks::run( 'Test' ); + } + ]; + } + $this->bench( $benches ); + } + + /** + * @return bool + */ + public function test() { + return true; + } +} + +$maintClass = 'BenchmarkHooks'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/benchmarks/benchmarkJSMinPlus.php b/www/wiki/maintenance/benchmarks/benchmarkJSMinPlus.php new file mode 100644 index 00000000..dc925160 --- /dev/null +++ b/www/wiki/maintenance/benchmarks/benchmarkJSMinPlus.php @@ -0,0 +1,62 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Benchmark + * @author Timo Tijhof + */ + +require_once __DIR__ . '/Benchmarker.php'; + +/** + * Maintenance script that benchmarks JSMinPlus. + * + * @ingroup Benchmark + */ +class BenchmarkJSMinPlus extends Benchmarker { + protected $defaultCount = 10; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Benchmarks JSMinPlus.' ); + $this->addOption( 'file', 'Path to JS file', true, true ); + } + + public function execute() { + MediaWiki\suppressWarnings(); + $content = file_get_contents( $this->getOption( 'file' ) ); + MediaWiki\restoreWarnings(); + if ( $content === false ) { + $this->error( 'Unable to open input file', 1 ); + } + + $filename = basename( $this->getOption( 'file' ) ); + $parser = new JSParser(); + + $this->bench( [ + "JSParser::parse ($filename)" => [ + 'function' => function ( $parser, $content, $filename ) { + $parser->parse( $content, $filename, 1 ); + }, + 'args' => [ $parser, $content, $filename ] + ] + ] ); + } +} + +$maintClass = 'BenchmarkJSMinPlus'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/benchmarks/benchmarkLruHash.php b/www/wiki/maintenance/benchmarks/benchmarkLruHash.php new file mode 100644 index 00000000..1541f827 --- /dev/null +++ b/www/wiki/maintenance/benchmarks/benchmarkLruHash.php @@ -0,0 +1,97 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Benchmark + */ + +require_once __DIR__ . '/Benchmarker.php'; + +/** + * Maintenance script that benchmarks HashBagOStuff and MapCacheLRU. + * + * @ingroup Benchmark + */ +class BenchmarkLruHash extends Benchmarker { + protected $defaultCount = 1000; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Benchmarks HashBagOStuff and MapCacheLRU.' ); + $this->addOption( 'construct', 'Run construct only', false, false ); + $this->addOption( 'fill', 'Run fill only', false, false ); + } + + public function execute() { + $exampleKeys = []; + $max = 100; + $count = 500; + while ( $count-- ) { + $exampleKeys[] = wfRandomString(); + } + // 1000 keys (1...500, 500...1) + $keys = array_merge( $exampleKeys, array_reverse( $exampleKeys ) ); + + $fill = $this->hasOption( 'fill' ) || !$this->hasOption( 'construct' ); + $construct = $this->hasOption( 'construct' ) || !$this->hasOption( 'fill' ); + $benches = []; + + if ( $construct ) { + $benches['HashBagOStuff-construct'] = [ + 'function' => function () use ( $max ) { + $obj = new HashBagOStuff( [ 'maxKeys' => $max ] ); + }, + ]; + $benches['MapCacheLRU-construct'] = [ + 'function' => function () use ( $max ) { + $obj = new MapCacheLRU( $max ); + }, + ]; + } + + if ( $fill ) { + // For the fill bechmark, ensure object creation is not measured. + $hObj = null; + $benches['HashBagOStuff-fill'] = [ + 'setup' => function () use ( &$hObj, $max ) { + $hObj = new HashBagOStuff( [ 'maxKeys' => $max ] ); + }, + 'function' => function () use ( &$hObj, &$keys ) { + foreach ( $keys as $i => $key ) { + $hObj->set( $key, $i ); + } + } + ]; + $mObj = null; + $benches['MapCacheLRU-fill'] = [ + 'setup' => function () use ( &$mObj, $max ) { + $mObj = new MapCacheLRU( $max ); + }, + 'function' => function () use ( &$mObj, &$keys ) { + foreach ( $keys as $i => $key ) { + $mObj->set( $key, $i ); + } + } + ]; + } + + $this->bench( $benches ); + } +} + +$maintClass = BenchmarkLruHash::class; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/benchmarks/benchmarkParse.php b/www/wiki/maintenance/benchmarks/benchmarkParse.php new file mode 100644 index 00000000..1753250b --- /dev/null +++ b/www/wiki/maintenance/benchmarks/benchmarkParse.php @@ -0,0 +1,192 @@ +<?php +/** + * Benchmark script for parse operations + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @author Tim Starling <tstarling@wikimedia.org> + * @ingroup Benchmark + */ + +require __DIR__ . '/../Maintenance.php'; + +use MediaWiki\MediaWikiServices; + +/** + * Maintenance script to benchmark how long it takes to parse a given title at an optionally + * specified timestamp + * + * @since 1.23 + */ +class BenchmarkParse extends Maintenance { + /** @var string MediaWiki concatenated string timestamp (YYYYMMDDHHMMSS) */ + private $templateTimestamp = null; + + private $clearLinkCache = false; + + /** + * @var LinkCache + */ + private $linkCache; + + /** @var array Cache that maps a Title DB key to revision ID for the requested timestamp */ + private $idCache = []; + + function __construct() { + parent::__construct(); + $this->addDescription( 'Benchmark parse operation' ); + $this->addArg( 'title', 'The name of the page to parse' ); + $this->addOption( 'warmup', 'Repeat the parse operation this number of times to warm the cache', + false, true ); + $this->addOption( 'loops', 'Number of times to repeat parse operation post-warmup', + false, true ); + $this->addOption( 'page-time', + 'Use the version of the page which was current at the given time', + false, true ); + $this->addOption( 'tpl-time', + 'Use templates which were current at the given time (except that moves and ' . + 'deletes are not handled properly)', + false, true ); + $this->addOption( 'reset-linkcache', 'Reset the LinkCache after every parse.', + false, false ); + } + + function execute() { + if ( $this->hasOption( 'tpl-time' ) ) { + $this->templateTimestamp = wfTimestamp( TS_MW, strtotime( $this->getOption( 'tpl-time' ) ) ); + Hooks::register( 'BeforeParserFetchTemplateAndtitle', [ $this, 'onFetchTemplate' ] ); + } + + $this->clearLinkCache = $this->hasOption( 'reset-linkcache' ); + // Set as a member variable to avoid function calls when we're timing the parse + $this->linkCache = MediaWikiServices::getInstance()->getLinkCache(); + + $title = Title::newFromText( $this->getArg() ); + if ( !$title ) { + $this->error( "Invalid title" ); + exit( 1 ); + } + + if ( $this->hasOption( 'page-time' ) ) { + $pageTimestamp = wfTimestamp( TS_MW, strtotime( $this->getOption( 'page-time' ) ) ); + $id = $this->getRevIdForTime( $title, $pageTimestamp ); + if ( !$id ) { + $this->error( "The page did not exist at that time" ); + exit( 1 ); + } + + $revision = Revision::newFromId( $id ); + } else { + $revision = Revision::newFromTitle( $title ); + } + + if ( !$revision ) { + $this->error( "Unable to load revision, incorrect title?" ); + exit( 1 ); + } + + $warmup = $this->getOption( 'warmup', 1 ); + for ( $i = 0; $i < $warmup; $i++ ) { + $this->runParser( $revision ); + } + + $loops = $this->getOption( 'loops', 1 ); + if ( $loops < 1 ) { + $this->error( 'Invalid number of loops specified', true ); + } + $startUsage = getrusage(); + $startTime = microtime( true ); + for ( $i = 0; $i < $loops; $i++ ) { + $this->runParser( $revision ); + } + $endUsage = getrusage(); + $endTime = microtime( true ); + + printf( "CPU time = %.3f s, wall clock time = %.3f s\n", + // CPU time + ( $endUsage['ru_utime.tv_sec'] + $endUsage['ru_utime.tv_usec'] * 1e-6 + - $startUsage['ru_utime.tv_sec'] - $startUsage['ru_utime.tv_usec'] * 1e-6 ) / $loops, + // Wall clock time + ( $endTime - $startTime ) / $loops + ); + } + + /** + * Fetch the ID of the revision of a Title that occurred + * + * @param Title $title + * @param string $timestamp + * @return bool|string Revision ID, or false if not found or error + */ + function getRevIdForTime( Title $title, $timestamp ) { + $dbr = $this->getDB( DB_REPLICA ); + + $id = $dbr->selectField( + [ 'revision', 'page' ], + 'rev_id', + [ + 'page_namespace' => $title->getNamespace(), + 'page_title' => $title->getDBkey(), + 'rev_timestamp <= ' . $dbr->addQuotes( $timestamp ) + ], + __METHOD__, + [ 'ORDER BY' => 'rev_timestamp DESC', 'LIMIT' => 1 ], + [ 'revision' => [ 'INNER JOIN', 'rev_page=page_id' ] ] + ); + + return $id; + } + + /** + * Parse the text from a given Revision + * + * @param Revision $revision + */ + function runParser( Revision $revision ) { + $content = $revision->getContent(); + $content->getParserOutput( $revision->getTitle(), $revision->getId() ); + if ( $this->clearLinkCache ) { + $this->linkCache->clear(); + } + } + + /** + * Hook into the parser's revision ID fetcher. Make sure that the parser only + * uses revisions around the specified timestamp. + * + * @param Parser $parser + * @param Title $title + * @param bool &$skip + * @param string|bool &$id + * @return bool + */ + function onFetchTemplate( Parser $parser, Title $title, &$skip, &$id ) { + $pdbk = $title->getPrefixedDBkey(); + if ( !isset( $this->idCache[$pdbk] ) ) { + $proposedId = $this->getRevIdForTime( $title, $this->templateTimestamp ); + $this->idCache[$pdbk] = $proposedId; + } + if ( $this->idCache[$pdbk] !== false ) { + $id = $this->idCache[$pdbk]; + } + + return true; + } +} + +$maintClass = 'BenchmarkParse'; +require RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/benchmarks/benchmarkPurge.php b/www/wiki/maintenance/benchmarks/benchmarkPurge.php new file mode 100644 index 00000000..e006cf53 --- /dev/null +++ b/www/wiki/maintenance/benchmarks/benchmarkPurge.php @@ -0,0 +1,118 @@ +<?php +/** + * Benchmark for Squid purge. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Benchmark + */ + +require_once __DIR__ . '/Benchmarker.php'; + +/** + * Maintenance script that benchmarks Squid purge. + * + * @ingroup Benchmark + */ +class BenchmarkPurge extends Benchmarker { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Benchmark the Squid purge functions.' ); + } + + public function execute() { + global $wgUseSquid, $wgSquidServers; + if ( !$wgUseSquid ) { + $this->error( "Squid purge benchmark doesn't do much without squid support on.", true ); + } else { + $this->output( "There are " . count( $wgSquidServers ) . " defined squid servers:\n" ); + if ( $this->hasOption( 'count' ) ) { + $lengths = [ intval( $this->getOption( 'count' ) ) ]; + } else { + $lengths = [ 1, 10, 100 ]; + } + foreach ( $lengths as $length ) { + $urls = $this->randomUrlList( $length ); + $trial = $this->benchSquid( $urls ); + $this->output( $trial . "\n" ); + } + } + } + + /** + * Run a bunch of URLs through SquidUpdate::purge() + * to benchmark Squid response times. + * @param array $urls A bunch of URLs to purge + * @param int $trials How many times to run the test? + * @return string + */ + private function benchSquid( $urls, $trials = 1 ) { + $start = microtime( true ); + for ( $i = 0; $i < $trials; $i++ ) { + CdnCacheUpdate::purge( $urls ); + } + $delta = microtime( true ) - $start; + $pertrial = $delta / $trials; + $pertitle = $pertrial / count( $urls ); + + return sprintf( "%4d titles in %6.2fms (%6.2fms each)", + count( $urls ), $pertrial * 1000.0, $pertitle * 1000.0 ); + } + + /** + * Get an array of randomUrl()'s. + * @param int $length How many urls to add to the array + * @return array + */ + private function randomUrlList( $length ) { + $list = []; + for ( $i = 0; $i < $length; $i++ ) { + $list[] = $this->randomUrl(); + } + + return $list; + } + + /** + * Return a random URL of the wiki. Not necessarily an actual title in the + * database, but at least a URL that looks like one. + * @return string + */ + private function randomUrl() { + global $wgServer, $wgArticlePath; + + return $wgServer . str_replace( '$1', $this->randomTitle(), $wgArticlePath ); + } + + /** + * Create a random title string (not necessarily a Title object). + * For use with randomUrl(). + * @return string + */ + private function randomTitle() { + $str = ''; + $length = mt_rand( 1, 20 ); + for ( $i = 0; $i < $length; $i++ ) { + $str .= chr( mt_rand( ord( 'a' ), ord( 'z' ) ) ); + } + + return ucfirst( $str ); + } +} + +$maintClass = "BenchmarkPurge"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/benchmarks/benchmarkTidy.php b/www/wiki/maintenance/benchmarks/benchmarkTidy.php new file mode 100644 index 00000000..14791745 --- /dev/null +++ b/www/wiki/maintenance/benchmarks/benchmarkTidy.php @@ -0,0 +1,78 @@ +<?php + +require __DIR__ . '/../Maintenance.php'; + +class BenchmarkTidy extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addOption( 'file', 'A filename which contains the input text', true, true ); + $this->addOption( 'driver', 'The Tidy driver name, or false to use the configured instance', + false, true ); + $this->addOption( 'tidy-config', 'JSON encoded value for the tidy configuration array', + false, true ); + } + + public function execute() { + $html = file_get_contents( $this->getOption( 'file' ) ); + if ( $html === false ) { + $this->error( "Unable to open input file", 1 ); + } + if ( $this->hasOption( 'driver' ) || $this->hasOption( 'tidy-config' ) ) { + $config = json_decode( $this->getOption( 'tidy-config', '{}' ), true ); + if ( !is_array( $config ) ) { + $this->error( "Invalid JSON tidy config", 1 ); + } + $config += [ 'driver' => $this->getOption( 'driver', 'RemexHtml' ) ]; + $driver = MWTidy::factory( $config ); + } else { + $driver = MWTidy::singleton(); + if ( !$driver ) { + $this->error( "Tidy disabled or not installed", 1 ); + } + } + + $this->benchmark( $driver, $html ); + } + + private function benchmark( $driver, $html ) { + global $wgContLang; + + $times = []; + $innerCount = 10; + $outerCount = 10; + for ( $j = 1; $j <= $outerCount; $j++ ) { + $t = microtime( true ); + for ( $i = 0; $i < $innerCount; $i++ ) { + $driver->tidy( $html ); + print $wgContLang->formatSize( memory_get_usage( true ) ) . "\n"; + } + $t = ( ( microtime( true ) - $t ) / $innerCount ) * 1000; + $times[] = $t; + print "Run $j: $t\n"; + } + print "\n"; + + sort( $times, SORT_NUMERIC ); + $n = $outerCount; + $min = $times[0]; + $max = end( $times ); + if ( $n % 2 ) { + $median = $times[ ( $n - 1 ) / 2 ]; + } else { + $median = ( $times[$n / 2] + $times[$n / 2 - 1] ) / 2; + } + $mean = array_sum( $times ) / $n; + + print "Minimum: $min ms\n"; + print "Median: $median ms\n"; + print "Mean: $mean ms\n"; + print "Maximum: $max ms\n"; + print "Memory usage: " . + $wgContLang->formatSize( memory_get_usage( true ) ) . "\n"; + print "Peak memory usage: " . + $wgContLang->formatSize( memory_get_peak_usage( true ) ) . "\n"; + } +} + +$maintClass = 'BenchmarkTidy'; +require RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/benchmarks/cssmin/circle.svg b/www/wiki/maintenance/benchmarks/cssmin/circle.svg new file mode 100644 index 00000000..4f7af217 --- /dev/null +++ b/www/wiki/maintenance/benchmarks/cssmin/circle.svg @@ -0,0 +1,4 @@ +<?xml version="1.0" encoding="UTF-8"?> +<svg xmlns="http://www.w3.org/2000/svg" width="8" height="8"> + <circle cx="4" cy="4" r="2"/> +</svg> diff --git a/www/wiki/maintenance/benchmarks/cssmin/styles.css b/www/wiki/maintenance/benchmarks/cssmin/styles.css new file mode 100644 index 00000000..3cc15206 --- /dev/null +++ b/www/wiki/maintenance/benchmarks/cssmin/styles.css @@ -0,0 +1,32 @@ +/** + * Header + */ + +.foo { + background: url(wiki.png); +} + +.foo { + background: url(unknown.png); +} + +.foo { + background: url(https://example.org/foo.png); + background: url('https://example.org/foo.png'); + background: url("https://example.org/foo.png"); +} + +.foo { + /* @embed */ + background: url(wiki.png); +} + +.foo { + /* @embed */ + background: url(circle.svg); +} + +.foo { + /* @embed */ + background: url(wiki.png), url(wiki.png); +} diff --git a/www/wiki/maintenance/benchmarks/cssmin/wiki.png b/www/wiki/maintenance/benchmarks/cssmin/wiki.png Binary files differnew file mode 100644 index 00000000..8c421183 --- /dev/null +++ b/www/wiki/maintenance/benchmarks/cssmin/wiki.png diff --git a/www/wiki/maintenance/cdb.php b/www/wiki/maintenance/cdb.php new file mode 100644 index 00000000..bff2c13f --- /dev/null +++ b/www/wiki/maintenance/cdb.php @@ -0,0 +1,131 @@ +<?php +/** + * cdb inspector tool + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @todo document + * @ingroup Maintenance + */ +use \Cdb\Exception as CdbException; +use \Cdb\Reader as CdbReader; + +require_once __DIR__ . '/commandLine.inc'; + +function cdbShowHelp( $command ) { + $commandList = [ + 'load' => 'load a cdb file for reading', + 'get' => 'get a value for a key', + 'exit' => 'exit cdb', + 'quit' => 'exit cdb', + 'help' => 'help about a command', + ]; + if ( !$command ) { + $command = 'fullhelp'; + } + if ( $command === 'fullhelp' ) { + $max_cmd_len = max( array_map( 'strlen', array_keys( $commandList ) ) ); + foreach ( $commandList as $cmd => $desc ) { + printf( "%-{$max_cmd_len}s: %s\n", $cmd, $desc ); + } + } elseif ( isset( $commandList[$command] ) ) { + print "$command: $commandList[$command]\n"; + } else { + print "$command: command does not exist or no help for it\n"; + } +} + +do { + $bad = false; + $showhelp = false; + $quit = false; + static $fileHandle = false; + + $line = Maintenance::readconsole(); + if ( $line === false ) { + exit; + } + + $args = explode( ' ', $line, 2 ); + $command = array_shift( $args ); + + // process command + switch ( $command ) { + case 'help': + // show an help message + cdbShowHelp( array_shift( $args ) ); + break; + case 'load': + if ( !isset( $args[0] ) ) { + print "Need a filename there buddy\n"; + break; + } + $file = $args[0]; + print "Loading cdb file $file..."; + try { + $fileHandle = CdbReader::open( $file ); + } catch ( CdbException $e ) { + } + + if ( !$fileHandle ) { + print "not a cdb file or unable to read it\n"; + } else { + print "ok\n"; + } + break; + case 'get': + if ( !$fileHandle ) { + print "Need to load a cdb file first\n"; + break; + } + if ( !isset( $args[0] ) ) { + print "Need to specify a key, Luke\n"; + break; + } + try { + $res = $fileHandle->get( $args[0] ); + } catch ( CdbException $e ) { + print "Unable to read key from file\n"; + break; + } + if ( $res === false ) { + print "No such key/value pair\n"; + } elseif ( is_string( $res ) ) { + print "$res\n"; + } else { + var_dump( $res ); + } + break; + case 'quit': + case 'exit': + $quit = true; + break; + + default: + $bad = true; + } // switch() end + + if ( $bad ) { + if ( $command ) { + print "Bad command\n"; + } + } else { + if ( function_exists( 'readline_add_history' ) ) { + readline_add_history( $line ); + } + } +} while ( !$quit ); diff --git a/www/wiki/maintenance/changePassword.php b/www/wiki/maintenance/changePassword.php new file mode 100644 index 00000000..9fa66324 --- /dev/null +++ b/www/wiki/maintenance/changePassword.php @@ -0,0 +1,73 @@ +<?php +/** + * Change the password of a given user + * + * Copyright © 2005, Ævar Arnfjörð Bjarmason + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @author Ævar Arnfjörð Bjarmason <avarab@gmail.com> + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to change the password of a given user. + * + * @ingroup Maintenance + */ +class ChangePassword extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addOption( "user", "The username to operate on", false, true ); + $this->addOption( "userid", "The user id to operate on", false, true ); + $this->addOption( "password", "The password to use", true, true ); + $this->addDescription( "Change a user's password" ); + } + + public function execute() { + if ( $this->hasOption( "user" ) ) { + $user = User::newFromName( $this->getOption( 'user' ) ); + } elseif ( $this->hasOption( "userid" ) ) { + $user = User::newFromId( $this->getOption( 'userid' ) ); + } else { + $this->error( "A \"user\" or \"userid\" must be set to change the password for", true ); + } + if ( !$user || !$user->getId() ) { + $this->error( "No such user: " . $this->getOption( 'user' ), true ); + } + $password = $this->getOption( 'password' ); + try { + $status = $user->changeAuthenticationData( [ + 'username' => $user->getName(), + 'password' => $password, + 'retype' => $password, + ] ); + if ( !$status->isGood() ) { + throw new PasswordError( $status->getWikiText( null, null, 'en' ) ); + } + $user->saveSettings(); + $this->output( "Password set for " . $user->getName() . "\n" ); + } catch ( PasswordError $pwe ) { + $this->error( $pwe->getText(), true ); + } + } +} + +$maintClass = "ChangePassword"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/checkBadRedirects.php b/www/wiki/maintenance/checkBadRedirects.php new file mode 100644 index 00000000..6eafc96e --- /dev/null +++ b/www/wiki/maintenance/checkBadRedirects.php @@ -0,0 +1,64 @@ +<?php +/** + * Check that pages marked as being redirects really are. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to check that pages marked as being redirects really are. + * + * @ingroup Maintenance + */ +class CheckBadRedirects extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Check for bad redirects' ); + } + + public function execute() { + $this->output( "Fetching redirects...\n" ); + $dbr = $this->getDB( DB_REPLICA ); + $result = $dbr->select( + [ 'page' ], + [ 'page_namespace', 'page_title', 'page_latest' ], + [ 'page_is_redirect' => 1 ] ); + + $count = $result->numRows(); + $this->output( "Found $count redirects.\n" . + "Checking for bad redirects:\n\n" ); + + foreach ( $result as $row ) { + $title = Title::makeTitle( $row->page_namespace, $row->page_title ); + $rev = Revision::newFromId( $row->page_latest ); + if ( $rev ) { + $target = $rev->getContent()->getRedirectTarget(); + if ( !$target ) { + $this->output( $title->getPrefixedText() . "\n" ); + } + } + } + $this->output( "\nDone.\n" ); + } +} + +$maintClass = "CheckBadRedirects"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/checkComposerLockUpToDate.php b/www/wiki/maintenance/checkComposerLockUpToDate.php new file mode 100644 index 00000000..e5b4c13e --- /dev/null +++ b/www/wiki/maintenance/checkComposerLockUpToDate.php @@ -0,0 +1,67 @@ +<?php + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Checks whether your composer-installed dependencies are up to date + * + * Composer creates a "composer.lock" file which specifies which versions are installed + * (via `composer install`). It has a hash, which can be compared to the value of + * the composer.json file to see if dependencies are up to date. + */ +class CheckComposerLockUpToDate extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( + 'Checks whether your composer.lock file is up to date with the current composer.json' ); + } + + public function execute() { + global $IP; + $lockLocation = "$IP/composer.lock"; + $jsonLocation = "$IP/composer.json"; + if ( !file_exists( $lockLocation ) ) { + // Maybe they're using mediawiki/vendor? + $lockLocation = "$IP/vendor/composer.lock"; + if ( !file_exists( $lockLocation ) ) { + $this->error( + 'Could not find composer.lock file. Have you run "composer install --no-dev"?', + 1 + ); + } + } + + $lock = new ComposerLock( $lockLocation ); + $json = new ComposerJson( $jsonLocation ); + + // Check all the dependencies to see if any are old + $found = false; + $installed = $lock->getInstalledDependencies(); + foreach ( $json->getRequiredDependencies() as $name => $version ) { + if ( isset( $installed[$name] ) ) { + if ( $installed[$name]['version'] !== $version ) { + $this->output( + "$name: {$installed[$name]['version']} installed, $version required.\n" + ); + $found = true; + } + } else { + $this->output( "$name: not installed, $version required.\n" ); + $found = true; + } + } + if ( $found ) { + $this->error( + 'Error: your composer.lock file is not up to date. ' . + 'Run "composer update --no-dev" to install newer dependencies', + 1 + ); + } else { + // We couldn't find any out-of-date dependencies, so assume everything is ok! + $this->output( "Your composer.lock file is up to date with current dependencies!\n" ); + } + } +} + +$maintClass = 'CheckComposerLockUpToDate'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/checkImages.php b/www/wiki/maintenance/checkImages.php new file mode 100644 index 00000000..3e573930 --- /dev/null +++ b/www/wiki/maintenance/checkImages.php @@ -0,0 +1,84 @@ +<?php +/** + * Check images to see if they exist, are readable, etc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to check images to see if they exist, are readable, etc. + * + * @ingroup Maintenance + */ +class CheckImages extends Maintenance { + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Check images to see if they exist, are readable, etc' ); + $this->setBatchSize( 1000 ); + } + + public function execute() { + $start = ''; + $dbr = $this->getDB( DB_REPLICA ); + + $numImages = 0; + $numGood = 0; + + $repo = RepoGroup::singleton()->getLocalRepo(); + do { + $res = $dbr->select( 'image', '*', [ 'img_name > ' . $dbr->addQuotes( $start ) ], + __METHOD__, [ 'LIMIT' => $this->mBatchSize ] ); + foreach ( $res as $row ) { + $numImages++; + $start = $row->img_name; + $file = $repo->newFileFromRow( $row ); + $path = $file->getPath(); + if ( !$path ) { + $this->output( "{$row->img_name}: not locally accessible\n" ); + continue; + } + $size = $repo->getFileSize( $file->getPath() ); + if ( $size === false ) { + $this->output( "{$row->img_name}: missing\n" ); + continue; + } + + if ( $size == 0 && $row->img_size != 0 ) { + $this->output( "{$row->img_name}: truncated, was {$row->img_size}\n" ); + continue; + } + + if ( $size != $row->img_size ) { + $this->output( "{$row->img_name}: size mismatch DB={$row->img_size}, " + . "actual={$size}\n" ); + continue; + } + + $numGood++; + } + } while ( $res->numRows() ); + + $this->output( "Good images: $numGood/$numImages\n" ); + } +} + +$maintClass = "CheckImages"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/checkLess.php b/www/wiki/maintenance/checkLess.php new file mode 100644 index 00000000..8416c8ab --- /dev/null +++ b/www/wiki/maintenance/checkLess.php @@ -0,0 +1,66 @@ +<?php +/** + * Checks LESS files in known resources for errors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * @ingroup Maintenance + */ +class CheckLess extends Maintenance { + + public function __construct() { + parent::__construct(); + $this->addDescription( + 'Checks LESS files for errors by running the LessTestSuite PHPUnit test suite' ); + } + + public function execute() { + global $IP; + + // NOTE (phuedx, 2014-03-26) wgAutoloadClasses isn't set up + // by either of the dependencies at the top of the file, so + // require it here. + self::requireTestsAutoloader(); + + // If phpunit isn't available by autoloader try pulling it in + if ( !class_exists( 'PHPUnit_Framework_TestCase' ) ) { + require_once 'PHPUnit/Autoload.php'; + } + + // RequestContext::resetMain() will print warnings unless this + // is defined. + if ( !defined( 'MW_PHPUNIT_TEST' ) ) { + define( 'MW_PHPUNIT_TEST', true ); + } + + $textUICommand = new PHPUnit_TextUI_Command(); + $argv = [ + "$IP/tests/phpunit/phpunit.php", + "$IP/tests/phpunit/suites/LessTestSuite.php" + ]; + $textUICommand->run( $argv ); + } +} + +$maintClass = 'CheckLess'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/checkSyntax.php b/www/wiki/maintenance/checkSyntax.php new file mode 100644 index 00000000..3910f29d --- /dev/null +++ b/www/wiki/maintenance/checkSyntax.php @@ -0,0 +1,349 @@ +<?php +/** + * Check syntax of all PHP files in MediaWiki + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to check syntax of all PHP files in MediaWiki. + * + * @ingroup Maintenance + */ +class CheckSyntax extends Maintenance { + + // List of files we're going to check + private $mFiles = [], $mFailures = [], $mWarnings = []; + private $mIgnorePaths = [], $mNoStyleCheckPaths = []; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Check syntax for all PHP files in MediaWiki' ); + $this->addOption( 'with-extensions', 'Also recurse the extensions folder' ); + $this->addOption( + 'path', + 'Specific path (file or directory) to check, either with absolute path or ' + . 'relative to the root of this MediaWiki installation', + false, + true + ); + $this->addOption( + 'list-file', + 'Text file containing list of files or directories to check', + false, + true + ); + $this->addOption( + 'modified', + 'Check only files that were modified (requires Git command-line client)' + ); + $this->addOption( 'syntax-only', 'Check for syntax validity only, skip code style warnings' ); + } + + public function getDbType() { + return Maintenance::DB_NONE; + } + + public function execute() { + $this->buildFileList(); + + $this->output( "Checking syntax (using php -l, this can take a long time)\n" ); + foreach ( $this->mFiles as $f ) { + $this->checkFileWithCli( $f ); + if ( !$this->hasOption( 'syntax-only' ) ) { + $this->checkForMistakes( $f ); + } + } + $this->output( "\nDone! " . count( $this->mFiles ) . " files checked, " . + count( $this->mFailures ) . " failures and " . count( $this->mWarnings ) . + " warnings found\n" ); + } + + /** + * Build the list of files we'll check for syntax errors + */ + private function buildFileList() { + global $IP; + + $this->mIgnorePaths = [ + ]; + + $this->mNoStyleCheckPaths = [ + // Third-party code we don't care about + "/activemq_stomp/", + "EmailPage/PHPMailer", + "FCKeditor/fckeditor/", + '\bphplot-', + "/svggraph/", + "\bjsmin.php$", + "PEAR/File_Ogg/", + "QPoll/Excel/", + "/geshi/", + "/smarty/", + ]; + + if ( $this->hasOption( 'path' ) ) { + $path = $this->getOption( 'path' ); + if ( !$this->addPath( $path ) ) { + $this->error( "Error: can't find file or directory $path\n", true ); + } + + return; // process only this path + } elseif ( $this->hasOption( 'list-file' ) ) { + $file = $this->getOption( 'list-file' ); + MediaWiki\suppressWarnings(); + $f = fopen( $file, 'r' ); + MediaWiki\restoreWarnings(); + if ( !$f ) { + $this->error( "Can't open file $file\n", true ); + } + $path = trim( fgets( $f ) ); + while ( $path ) { + $this->addPath( $path ); + } + fclose( $f ); + + return; + } elseif ( $this->hasOption( 'modified' ) ) { + $this->output( "Retrieving list from Git... " ); + $files = $this->getGitModifiedFiles( $IP ); + $this->output( "done\n" ); + foreach ( $files as $file ) { + if ( $this->isSuitableFile( $file ) && !is_dir( $file ) ) { + $this->mFiles[] = $file; + } + } + + return; + } + + $this->output( 'Building file list...', 'listfiles' ); + + // Only check files in these directories. + // Don't just put $IP, because the recursive dir thingie goes into all subdirs + $dirs = [ + $IP . '/includes', + $IP . '/mw-config', + $IP . '/languages', + $IP . '/maintenance', + $IP . '/skins', + ]; + if ( $this->hasOption( 'with-extensions' ) ) { + $dirs[] = $IP . '/extensions'; + } + + foreach ( $dirs as $d ) { + $this->addDirectoryContent( $d ); + } + + // Manually add two user-editable files that are usually sources of problems + if ( file_exists( "$IP/LocalSettings.php" ) ) { + $this->mFiles[] = "$IP/LocalSettings.php"; + } + + $this->output( 'done.', 'listfiles' ); + } + + /** + * Returns a list of tracked files in a Git work tree differing from the master branch. + * @param string $path Path to the repository + * @return array Resulting list of changed files + */ + private function getGitModifiedFiles( $path ) { + global $wgMaxShellMemory; + + if ( !is_dir( "$path/.git" ) ) { + $this->error( "Error: Not a Git repository!\n", true ); + } + + // git diff eats memory. + $oldMaxShellMemory = $wgMaxShellMemory; + if ( $wgMaxShellMemory < 1024000 ) { + $wgMaxShellMemory = 1024000; + } + + $ePath = wfEscapeShellArg( $path ); + + // Find an ancestor in common with master (rather than just using its HEAD) + // to prevent files only modified there from showing up in the list. + $cmd = "cd $ePath && git merge-base master HEAD"; + $retval = 0; + $output = wfShellExec( $cmd, $retval ); + if ( $retval !== 0 ) { + $this->error( "Error retrieving base SHA1 from Git!\n", true ); + } + + // Find files in the working tree that changed since then. + $eBase = wfEscapeShellArg( rtrim( $output, "\n" ) ); + $cmd = "cd $ePath && git diff --name-only --diff-filter AM $eBase"; + $retval = 0; + $output = wfShellExec( $cmd, $retval ); + if ( $retval !== 0 ) { + $this->error( "Error retrieving list from Git!\n", true ); + } + + $wgMaxShellMemory = $oldMaxShellMemory; + + $arr = []; + $filename = strtok( $output, "\n" ); + while ( $filename !== false ) { + if ( $filename !== '' ) { + $arr[] = "$path/$filename"; + } + $filename = strtok( "\n" ); + } + + return $arr; + } + + /** + * Returns true if $file is of a type we can check + * @param string $file + * @return bool + */ + private function isSuitableFile( $file ) { + $file = str_replace( '\\', '/', $file ); + $ext = pathinfo( $file, PATHINFO_EXTENSION ); + if ( $ext != 'php' && $ext != 'inc' && $ext != 'php5' ) { + return false; + } + foreach ( $this->mIgnorePaths as $regex ) { + $m = []; + if ( preg_match( "~{$regex}~", $file, $m ) ) { + return false; + } + } + + return true; + } + + /** + * Add given path to file list, searching it in include path if needed + * @param string $path + * @return bool + */ + private function addPath( $path ) { + global $IP; + + return $this->addFileOrDir( $path ) || $this->addFileOrDir( "$IP/$path" ); + } + + /** + * Add given file to file list, or, if it's a directory, add its content + * @param string $path + * @return bool + */ + private function addFileOrDir( $path ) { + if ( is_dir( $path ) ) { + $this->addDirectoryContent( $path ); + } elseif ( file_exists( $path ) ) { + $this->mFiles[] = $path; + } else { + return false; + } + + return true; + } + + /** + * Add all suitable files in given directory or its subdirectories to the file list + * + * @param string $dir Directory to process + */ + private function addDirectoryContent( $dir ) { + $iterator = new RecursiveIteratorIterator( + new RecursiveDirectoryIterator( $dir ), + RecursiveIteratorIterator::SELF_FIRST + ); + foreach ( $iterator as $file ) { + if ( $this->isSuitableFile( $file->getRealPath() ) ) { + $this->mFiles[] = $file->getRealPath(); + } + } + } + + /** + * Check a file for syntax errors using php -l + * @param string $file Path to a file to check for syntax errors + * @return bool + */ + private function checkFileWithCli( $file ) { + $res = exec( 'php -l ' . wfEscapeShellArg( $file ) ); + if ( strpos( $res, 'No syntax errors detected' ) === false ) { + $this->mFailures[$file] = $res; + $this->output( $res . "\n" ); + + return false; + } + + return true; + } + + /** + * Check a file for non-fatal coding errors, such as byte-order marks in the beginning + * or pointless ?> closing tags at the end. + * + * @param string $file String Path to a file to check for errors + */ + private function checkForMistakes( $file ) { + foreach ( $this->mNoStyleCheckPaths as $regex ) { + $m = []; + if ( preg_match( "~{$regex}~", $file, $m ) ) { + return; + } + } + + $text = file_get_contents( $file ); + $tokens = token_get_all( $text ); + + $this->checkEvilToken( $file, $tokens, '@', 'Error supression operator (@)' ); + $this->checkRegex( $file, $text, '/^[\s\r\n]+<\?/', 'leading whitespace' ); + $this->checkRegex( $file, $text, '/\?>[\s\r\n]*$/', 'trailing ?>' ); + $this->checkRegex( $file, $text, '/^[\xFF\xFE\xEF]/', 'byte-order mark' ); + } + + private function checkRegex( $file, $text, $regex, $desc ) { + if ( !preg_match( $regex, $text ) ) { + return; + } + + if ( !isset( $this->mWarnings[$file] ) ) { + $this->mWarnings[$file] = []; + } + $this->mWarnings[$file][] = $desc; + $this->output( "Warning in file $file: $desc found.\n" ); + } + + private function checkEvilToken( $file, $tokens, $evilToken, $desc ) { + if ( !in_array( $evilToken, $tokens ) ) { + return; + } + + if ( !isset( $this->mWarnings[$file] ) ) { + $this->mWarnings[$file] = []; + } + $this->mWarnings[$file][] = $desc; + $this->output( "Warning in file $file: $desc found.\n" ); + } +} + +$maintClass = "CheckSyntax"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/checkUsernames.php b/www/wiki/maintenance/checkUsernames.php new file mode 100644 index 00000000..e6d95477 --- /dev/null +++ b/www/wiki/maintenance/checkUsernames.php @@ -0,0 +1,69 @@ +<?php +/** + * Check that database usernames are actually valid. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to check that database usernames are actually valid. + * + * An existing usernames can become invalid if User::isValidUserName() + * is altered or if we change the $wgMaxNameChars + * + * @ingroup Maintenance + */ +class CheckUsernames extends Maintenance { + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Verify that database usernames are actually valid' ); + $this->setBatchSize( 1000 ); + } + + function execute() { + $dbr = $this->getDB( DB_REPLICA ); + + $maxUserId = 0; + do { + $res = $dbr->select( 'user', + [ 'user_id', 'user_name' ], + [ 'user_id > ' . $maxUserId ], + __METHOD__, + [ + 'ORDER BY' => 'user_id', + 'LIMIT' => $this->mBatchSize, + ] + ); + + foreach ( $res as $row ) { + if ( !User::isValidUserName( $row->user_name ) ) { + $this->output( sprintf( "Found: %6d: '%s'\n", $row->user_id, $row->user_name ) ); + wfDebugLog( 'checkUsernames', $row->user_name ); + } + } + $maxUserId = $row->user_id; + } while ( $res->numRows() ); + } +} + +$maintClass = "CheckUsernames"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/cleanupAncientTables.php b/www/wiki/maintenance/cleanupAncientTables.php new file mode 100644 index 00000000..add967ad --- /dev/null +++ b/www/wiki/maintenance/cleanupAncientTables.php @@ -0,0 +1,114 @@ +<?php +/** + * Cleans up old database tables, dropping old indexes and fields. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to cleans up old database tables, dropping old indexes + * and fields. + * + * @ingroup Maintenance + */ +class CleanupAncientTables extends Maintenance { + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Cleanup ancient tables and indexes' ); + $this->addOption( 'force', 'Actually run this script' ); + } + + public function execute() { + if ( !$this->hasOption( 'force' ) ) { + $this->error( "This maintenance script will remove old columns and indexes.\n" + . "It is recommended to backup your database first, and ensure all your data has\n" + . "been migrated to newer tables. If you want to continue, run this script again\n" + . "with --force.\n" + ); + } + + $db = $this->getDB( DB_MASTER ); + $ancientTables = [ + 'blobs', // 1.4 + 'brokenlinks', // 1.4 + 'cur', // 1.4 + 'ip_blocks_old', // Temporary in 1.6 + 'links', // 1.4 + 'linkscc', // 1.4 + // 'math', // 1.18, but don't want to drop if math extension is enabled... + 'old', // 1.4 + 'oldwatchlist', // pre 1.1? + 'trackback', // 1.19 + 'user_rights', // 1.5 + 'validate', // 1.6 + ]; + + foreach ( $ancientTables as $table ) { + if ( $db->tableExists( $table, __METHOD__ ) ) { + $this->output( "Dropping table $table..." ); + $db->dropTable( $table, __METHOD__ ); + $this->output( "done.\n" ); + } + } + + $this->output( "Cleaning up text table\n" ); + + $oldIndexes = [ + 'old_namespace', + 'old_timestamp', + 'name_title_timestamp', + 'user_timestamp', + 'usertext_timestamp', + ]; + foreach ( $oldIndexes as $index ) { + if ( $db->indexExists( 'text', $index, __METHOD__ ) ) { + $this->output( "Dropping index $index from the text table..." ); + $db->query( "DROP INDEX " . $db->addIdentifierQuotes( $index ) + . " ON " . $db->tableName( 'text' ) ); + $this->output( "done.\n" ); + } + } + + $oldFields = [ + 'old_namespace', + 'old_title', + 'old_comment', + 'old_user', + 'old_user_text', + 'old_timestamp', + 'old_minor_edit', + 'inverse_timestamp', + ]; + foreach ( $oldFields as $field ) { + if ( $db->fieldExists( 'text', $field, __METHOD__ ) ) { + $this->output( "Dropping the $field field from the text table..." ); + $db->query( "ALTER TABLE " . $db->tableName( 'text' ) + . " DROP COLUMN " . $db->addIdentifierQuotes( $field ) ); + $this->output( "done.\n" ); + } + } + $this->output( "Done!\n" ); + } +} + +$maintClass = "CleanupAncientTables"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/cleanupBlocks.php b/www/wiki/maintenance/cleanupBlocks.php new file mode 100644 index 00000000..7a3034fb --- /dev/null +++ b/www/wiki/maintenance/cleanupBlocks.php @@ -0,0 +1,147 @@ +<?php +/** + * Cleans up user blocks with user names not matching the 'user' table + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to clean up user blocks with user names not matching the + * 'user' table. + * + * @ingroup Maintenance + */ +class CleanupBlocks extends Maintenance { + + public function __construct() { + parent::__construct(); + $this->addDescription( "Cleanup user blocks with user names not matching the 'user' table" ); + $this->setBatchSize( 1000 ); + } + + public function execute() { + $db = $this->getDB( DB_MASTER ); + + $max = $db->selectField( 'ipblocks', 'MAX(ipb_user)' ); + + // Step 1: Clean up any duplicate user blocks + for ( $from = 1; $from <= $max; $from += $this->mBatchSize ) { + $to = min( $max, $from + $this->mBatchSize - 1 ); + $this->output( "Cleaning up duplicate ipb_user ($from-$to of $max)\n" ); + + $delete = []; + + $res = $db->select( + 'ipblocks', + [ 'ipb_user' ], + [ + "ipb_user >= $from", + "ipb_user <= $to", + ], + __METHOD__, + [ + 'GROUP BY' => 'ipb_user', + 'HAVING' => 'COUNT(*) > 1', + ] + ); + foreach ( $res as $row ) { + $bestBlock = null; + $res2 = $db->select( + 'ipblocks', + '*', + [ + 'ipb_user' => $row->ipb_user, + ] + ); + foreach ( $res2 as $row2 ) { + $block = Block::newFromRow( $row2 ); + if ( !$bestBlock ) { + $bestBlock = $block; + continue; + } + + // Find the most-restrictive block. Can't use + // Block::chooseBlock because that's for IP blocks, not + // user blocks. + $keep = null; + if ( $keep === null && $block->getExpiry() !== $bestBlock->getExpiry() ) { + // This works for infinite blocks because 'infinity' > '20141024234513' + $keep = $block->getExpiry() > $bestBlock->getExpiry(); + } + if ( $keep === null ) { + foreach ( [ 'createaccount', 'sendemail', 'editownusertalk' ] as $action ) { + if ( $block->prevents( $action ) xor $bestBlock->prevents( $action ) ) { + $keep = $block->prevents( $action ); + break; + } + } + } + + if ( $keep ) { + $delete[] = $bestBlock->getId(); + $bestBlock = $block; + } else { + $delete[] = $block->getId(); + } + } + } + + if ( $delete ) { + $db->delete( + 'ipblocks', + [ 'ipb_id' => $delete ], + __METHOD__ + ); + } + } + + // Step 2: Update the user name in any blocks where it doesn't match + for ( $from = 1; $from <= $max; $from += $this->mBatchSize ) { + $to = min( $max, $from + $this->mBatchSize - 1 ); + $this->output( "Cleaning up mismatched user name ($from-$to of $max)\n" ); + + $res = $db->select( + [ 'ipblocks', 'user' ], + [ 'ipb_id', 'user_name' ], + [ + 'ipb_user = user_id', + "ipb_user >= $from", + "ipb_user <= $to", + 'ipb_address != user_name', + ], + __METHOD__ + ); + foreach ( $res as $row ) { + $db->update( + 'ipblocks', + [ 'ipb_address' => $row->user_name ], + [ 'ipb_id' => $row->ipb_id ], + __METHOD__ + ); + } + } + + $this->output( "Done!\n" ); + } +} + +$maintClass = "CleanupBlocks"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/cleanupCaps.php b/www/wiki/maintenance/cleanupCaps.php new file mode 100644 index 00000000..2da45ca1 --- /dev/null +++ b/www/wiki/maintenance/cleanupCaps.php @@ -0,0 +1,173 @@ +<?php +/** + * Clean up broken page links when somebody turns on $wgCapitalLinks. + * + * Usage: php cleanupCaps.php [--dry-run] + * Options: + * --dry-run don't actually try moving them + * + * Copyright © 2005 Brion Vibber <brion@pobox.com> + * https://www.mediawiki.org/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @author Brion Vibber <brion at pobox.com> + * @ingroup Maintenance + */ + +require_once __DIR__ . '/cleanupTable.inc'; + +/** + * Maintenance script to clean up broken page links when somebody turns + * on or off $wgCapitalLinks. + * + * @ingroup Maintenance + */ +class CapsCleanup extends TableCleanup { + + private $user; + private $namespace; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Script to cleanup capitalization' ); + $this->addOption( 'namespace', 'Namespace number to run caps cleanup on', false, true ); + } + + public function execute() { + $this->user = User::newSystemUser( 'Conversion script', [ 'steal' => true ] ); + + $this->namespace = intval( $this->getOption( 'namespace', 0 ) ); + + if ( MWNamespace::isCapitalized( $this->namespace ) ) { + $this->output( "Will be moving pages to first letter capitalized titles" ); + $callback = 'processRowToUppercase'; + } else { + $this->output( "Will be moving pages to first letter lowercase titles" ); + $callback = 'processRowToLowercase'; + } + + $this->dryrun = $this->hasOption( 'dry-run' ); + + $this->runTable( [ + 'table' => 'page', + 'conds' => [ 'page_namespace' => $this->namespace ], + 'index' => 'page_id', + 'callback' => $callback ] ); + } + + protected function processRowToUppercase( $row ) { + global $wgContLang; + + $current = Title::makeTitle( $row->page_namespace, $row->page_title ); + $display = $current->getPrefixedText(); + $lower = $row->page_title; + $upper = $wgContLang->ucfirst( $row->page_title ); + if ( $upper == $lower ) { + $this->output( "\"$display\" already uppercase.\n" ); + + return $this->progress( 0 ); + } + + $target = Title::makeTitle( $row->page_namespace, $upper ); + if ( $target->exists() ) { + // Prefix "CapsCleanup" to bypass the conflict + $target = Title::newFromText( __CLASS__ . '/' . $display ); + } + $ok = $this->movePage( + $current, + $target, + 'Converting page title to first-letter uppercase', + false + ); + if ( $ok ) { + $this->progress( 1 ); + if ( $row->page_namespace == $this->namespace ) { + $talk = $target->getTalkPage(); + $row->page_namespace = $talk->getNamespace(); + if ( $talk->exists() ) { + return $this->processRowToUppercase( $row ); + } + } + } + + return $this->progress( 0 ); + } + + protected function processRowToLowercase( $row ) { + global $wgContLang; + + $current = Title::makeTitle( $row->page_namespace, $row->page_title ); + $display = $current->getPrefixedText(); + $upper = $row->page_title; + $lower = $wgContLang->lcfirst( $row->page_title ); + if ( $upper == $lower ) { + $this->output( "\"$display\" already lowercase.\n" ); + + return $this->progress( 0 ); + } + + $target = Title::makeTitle( $row->page_namespace, $lower ); + if ( $target->exists() ) { + $targetDisplay = $target->getPrefixedText(); + $this->output( "\"$display\" skipped; \"$targetDisplay\" already exists\n" ); + + return $this->progress( 0 ); + } + + $ok = $this->movePage( $current, $target, 'Converting page titles to lowercase', true ); + if ( $ok === true ) { + $this->progress( 1 ); + if ( $row->page_namespace == $this->namespace ) { + $talk = $target->getTalkPage(); + $row->page_namespace = $talk->getNamespace(); + if ( $talk->exists() ) { + return $this->processRowToLowercase( $row ); + } + } + } + + return $this->progress( 0 ); + } + + /** + * @param Title $current + * @param Title $target + * @param string $reason + * @param bool $createRedirect + * @return bool Success + */ + private function movePage( Title $current, Title $target, $reason, $createRedirect ) { + $display = $current->getPrefixedText(); + $targetDisplay = $target->getPrefixedText(); + + if ( $this->dryrun ) { + $this->output( "\"$display\" -> \"$targetDisplay\": DRY RUN, NOT MOVED\n" ); + $ok = 'OK'; + } else { + $mp = new MovePage( $current, $target ); + $status = $mp->move( $this->user, $reason, $createRedirect ); + $ok = $status->isOK() ? 'OK' : $status->getWikiText( false, false, 'en' ); + $this->output( "\"$display\" -> \"$targetDisplay\": $ok\n" ); + } + + return $ok === 'OK'; + } +} + +$maintClass = "CapsCleanup"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/cleanupEmptyCategories.php b/www/wiki/maintenance/cleanupEmptyCategories.php new file mode 100644 index 00000000..86722238 --- /dev/null +++ b/www/wiki/maintenance/cleanupEmptyCategories.php @@ -0,0 +1,203 @@ +<?php +/** + * Clean up empty categories in the category table. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to clean up empty categories in the category table. + * + * @ingroup Maintenance + * @since 1.28 + */ +class CleanupEmptyCategories extends LoggedUpdateMaintenance { + + public function __construct() { + parent::__construct(); + $this->addDescription( + <<<TEXT +This script will clean up the category table by removing entries for empty +categories without a description page and adding entries for empty categories +with a description page. It will print out progress indicators every batch. The +script is perfectly safe to run on large, live wikis, and running it multiple +times is harmless. You may want to use the throttling options if it's causing +too much load; they will not affect correctness. + +If the script is stopped and later resumed, you can use the --mode and --begin +options with the last printed progress indicator to pick up where you left off. + +When the script has finished, it will make a note of this in the database, and +will not run again without the --force option. +TEXT + ); + + $this->addOption( + 'mode', + '"add" empty categories with description pages, "remove" empty categories ' + . 'without description pages, or "both"', + false, + true + ); + $this->addOption( + 'begin', + 'Only do categories whose names are alphabetically after the provided name', + false, + true + ); + $this->addOption( + 'throttle', + 'Wait this many milliseconds after each batch. Default: 0', + false, + true + ); + } + + protected function getUpdateKey() { + return 'cleanup empty categories'; + } + + protected function doDBUpdates() { + $mode = $this->getOption( 'mode', 'both' ); + $begin = $this->getOption( 'begin', '' ); + $throttle = $this->getOption( 'throttle', 0 ); + + if ( !in_array( $mode, [ 'add', 'remove', 'both' ] ) ) { + $this->output( "--mode must be 'add', 'remove', or 'both'.\n" ); + return false; + } + + $dbw = $this->getDB( DB_MASTER ); + + $throttle = intval( $throttle ); + + if ( $mode === 'add' || $mode === 'both' ) { + if ( $begin !== '' ) { + $where = [ 'page_title > ' . $dbw->addQuotes( $begin ) ]; + } else { + $where = []; + } + + $this->output( "Adding empty categories with description pages...\n" ); + while ( true ) { + # Find which category to update + $rows = $dbw->select( + [ 'page', 'category' ], + 'page_title', + array_merge( $where, [ + 'page_namespace' => NS_CATEGORY, + 'cat_title' => null, + ] ), + __METHOD__, + [ + 'ORDER BY' => 'page_title', + 'LIMIT' => $this->mBatchSize, + ], + [ + 'category' => [ 'LEFT JOIN', 'page_title = cat_title' ], + ] + ); + if ( !$rows || $rows->numRows() <= 0 ) { + # Done, hopefully. + break; + } + + foreach ( $rows as $row ) { + $name = $row->page_title; + $where = [ 'page_title > ' . $dbw->addQuotes( $name ) ]; + + # Use the row to update the category count + $cat = Category::newFromName( $name ); + if ( !is_object( $cat ) ) { + $this->output( "The category named $name is not valid?!\n" ); + } else { + $cat->refreshCounts(); + } + } + $this->output( "--mode=$mode --begin=$name\n" ); + + wfWaitForSlaves(); + usleep( $throttle * 1000 ); + } + + $begin = ''; + } + + if ( $mode === 'remove' || $mode === 'both' ) { + if ( $begin !== '' ) { + $where = [ 'cat_title > ' . $dbw->addQuotes( $begin ) ]; + } else { + $where = []; + } + + $this->output( "Removing empty categories without description pages...\n" ); + while ( true ) { + # Find which category to update + $rows = $dbw->select( + [ 'category', 'page' ], + 'cat_title', + array_merge( $where, [ + 'page_title' => null, + 'cat_pages' => 0, + ] ), + __METHOD__, + [ + 'ORDER BY' => 'cat_title', + 'LIMIT' => $this->mBatchSize, + ], + [ + 'page' => [ 'LEFT JOIN', [ + 'page_namespace' => NS_CATEGORY, 'page_title = cat_title' + ] ], + ] + ); + if ( !$rows || $rows->numRows() <= 0 ) { + # Done, hopefully. + break; + } + foreach ( $rows as $row ) { + $name = $row->cat_title; + $where = [ 'cat_title > ' . $dbw->addQuotes( $name ) ]; + + # Use the row to update the category count + $cat = Category::newFromName( $name ); + if ( !is_object( $cat ) ) { + $this->output( "The category named $name is not valid?!\n" ); + } else { + $cat->refreshCounts(); + } + } + + $this->output( "--mode=remove --begin=$name\n" ); + + wfWaitForSlaves(); + usleep( $throttle * 1000 ); + } + } + + $this->output( "Category cleanup complete.\n" ); + + return true; + } +} + +$maintClass = 'CleanupEmptyCategories'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/cleanupImages.php b/www/wiki/maintenance/cleanupImages.php new file mode 100644 index 00000000..e0da027f --- /dev/null +++ b/www/wiki/maintenance/cleanupImages.php @@ -0,0 +1,224 @@ +<?php +/** + * Clean up broken, unparseable upload filenames. + * + * Copyright © 2005-2006 Brion Vibber <brion@pobox.com> + * https://www.mediawiki.org/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @author Brion Vibber <brion at pobox.com> + * @ingroup Maintenance + */ + +require_once __DIR__ . '/cleanupTable.inc'; + +/** + * Maintenance script to clean up broken, unparseable upload filenames. + * + * @ingroup Maintenance + */ +class ImageCleanup extends TableCleanup { + protected $defaultParams = [ + 'table' => 'image', + 'conds' => [], + 'index' => 'img_name', + 'callback' => 'processRow', + ]; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Script to clean up broken, unparseable upload filenames' ); + } + + protected function processRow( $row ) { + global $wgContLang; + + $source = $row->img_name; + if ( $source == '' ) { + // Ye olde empty rows. Just kill them. + $this->killRow( $source ); + + return $this->progress( 1 ); + } + + $cleaned = $source; + + // About half of old bad image names have percent-codes + $cleaned = rawurldecode( $cleaned ); + + // We also have some HTML entities there + $cleaned = Sanitizer::decodeCharReferences( $cleaned ); + + // Some are old latin-1 + $cleaned = $wgContLang->checkTitleEncoding( $cleaned ); + + // Many of remainder look like non-normalized unicode + $cleaned = $wgContLang->normalize( $cleaned ); + + $title = Title::makeTitleSafe( NS_FILE, $cleaned ); + + if ( is_null( $title ) ) { + $this->output( "page $source ($cleaned) is illegal.\n" ); + $safe = $this->buildSafeTitle( $cleaned ); + if ( $safe === false ) { + return $this->progress( 0 ); + } + $this->pokeFile( $source, $safe ); + + return $this->progress( 1 ); + } + + if ( $title->getDBkey() !== $source ) { + $munged = $title->getDBkey(); + $this->output( "page $source ($munged) doesn't match self.\n" ); + $this->pokeFile( $source, $munged ); + + return $this->progress( 1 ); + } + + return $this->progress( 0 ); + } + + /** + * @param string $name + */ + private function killRow( $name ) { + if ( $this->dryrun ) { + $this->output( "DRY RUN: would delete bogus row '$name'\n" ); + } else { + $this->output( "deleting bogus row '$name'\n" ); + $db = $this->getDB( DB_MASTER ); + $db->delete( 'image', + [ 'img_name' => $name ], + __METHOD__ ); + } + } + + private function filePath( $name ) { + if ( !isset( $this->repo ) ) { + $this->repo = RepoGroup::singleton()->getLocalRepo(); + } + + return $this->repo->getRootDirectory() . '/' . $this->repo->getHashPath( $name ) . $name; + } + + private function imageExists( $name, $db ) { + return $db->selectField( 'image', '1', [ 'img_name' => $name ], __METHOD__ ); + } + + private function pageExists( $name, $db ) { + return $db->selectField( + 'page', + '1', + [ 'page_namespace' => NS_FILE, 'page_title' => $name ], + __METHOD__ + ); + } + + private function pokeFile( $orig, $new ) { + $path = $this->filePath( $orig ); + if ( !file_exists( $path ) ) { + $this->output( "missing file: $path\n" ); + $this->killRow( $orig ); + + return; + } + + $db = $this->getDB( DB_MASTER ); + + /* + * To prevent key collisions in the update() statements below, + * if the target title exists in the image table, or if both the + * original and target titles exist in the page table, append + * increasing version numbers until the target title exists in + * neither. (See also T18916.) + */ + $version = 0; + $final = $new; + $conflict = ( $this->imageExists( $final, $db ) || + ( $this->pageExists( $orig, $db ) && $this->pageExists( $final, $db ) ) ); + + while ( $conflict ) { + $this->output( "Rename conflicts with '$final'...\n" ); + $version++; + $final = $this->appendTitle( $new, "_$version" ); + $conflict = ( $this->imageExists( $final, $db ) || $this->pageExists( $final, $db ) ); + } + + $finalPath = $this->filePath( $final ); + + if ( $this->dryrun ) { + $this->output( "DRY RUN: would rename $path to $finalPath\n" ); + } else { + $this->output( "renaming $path to $finalPath\n" ); + // @todo FIXME: Should this use File::move()? + $this->beginTransaction( $db, __METHOD__ ); + $db->update( 'image', + [ 'img_name' => $final ], + [ 'img_name' => $orig ], + __METHOD__ ); + $db->update( 'oldimage', + [ 'oi_name' => $final ], + [ 'oi_name' => $orig ], + __METHOD__ ); + $db->update( 'page', + [ 'page_title' => $final ], + [ 'page_title' => $orig, 'page_namespace' => NS_FILE ], + __METHOD__ ); + $dir = dirname( $finalPath ); + if ( !file_exists( $dir ) ) { + if ( !wfMkdirParents( $dir, null, __METHOD__ ) ) { + $this->output( "RENAME FAILED, COULD NOT CREATE $dir" ); + $this->rollbackTransaction( $db, __METHOD__ ); + + return; + } + } + if ( rename( $path, $finalPath ) ) { + $this->commitTransaction( $db, __METHOD__ ); + } else { + $this->error( "RENAME FAILED" ); + $this->rollbackTransaction( $db, __METHOD__ ); + } + } + } + + private function appendTitle( $name, $suffix ) { + return preg_replace( '/^(.*)(\..*?)$/', + "\\1$suffix\\2", $name ); + } + + private function buildSafeTitle( $name ) { + $x = preg_replace_callback( + '/([^' . Title::legalChars() . ']|~)/', + [ $this, 'hexChar' ], + $name ); + + $test = Title::makeTitleSafe( NS_FILE, $x ); + if ( is_null( $test ) || $test->getDBkey() !== $x ) { + $this->error( "Unable to generate safe title from '$name', got '$x'" ); + + return false; + } + + return $x; + } +} + +$maintClass = "ImageCleanup"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/cleanupInvalidDbKeys.php b/www/wiki/maintenance/cleanupInvalidDbKeys.php new file mode 100644 index 00000000..b487f896 --- /dev/null +++ b/www/wiki/maintenance/cleanupInvalidDbKeys.php @@ -0,0 +1,311 @@ +<?php +/** + * Cleans up invalid titles in various tables. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that cleans up invalid titles in various tables. + * + * @since 1.29 + * @ingroup Maintenance + */ +class CleanupInvalidDbKeys extends Maintenance { + /** @var array List of tables to clean up, and the field prefix for that table */ + protected static $tables = [ + // Data tables + [ 'page', 'page' ], + [ 'redirect', 'rd', 'idField' => 'rd_from' ], + [ 'archive', 'ar' ], + [ 'logging', 'log' ], + [ 'protected_titles', 'pt', 'idField' => 0 ], + [ 'category', 'cat', 'nsField' => 14 ], + [ 'recentchanges', 'rc' ], + [ 'watchlist', 'wl' ], + // The querycache tables' qc(c)_title and qcc_titletwo may contain titles, + // but also usernames or other things like that, so we leave them alone + + // Links tables + [ 'pagelinks', 'pl', 'idField' => 'pl_from' ], + [ 'templatelinks', 'tl', 'idField' => 'tl_from' ], + [ 'categorylinks', 'cl', 'idField' => 'cl_from', 'nsField' => 14, 'titleField' => 'cl_to' ], + ]; + + public function __construct() { + parent::__construct(); + $this->addDescription( <<<'TEXT' +This script cleans up the title fields in various tables to remove entries that +will be rejected by the constructor of TitleValue. This constructor throws an +exception when invalid data is encountered, which will not normally occur on +regular page views, but can happen on query special pages. + +The script targets titles matching the regular expression /^_|[ \r\n\t]|_$/. +Because any foreign key relationships involving these titles will already be +broken, the titles are corrected to a valid version or the rows are deleted +entirely, depending on the table. + +The script runs with the expectation that STDOUT is redirected to a file. +TEXT + ); + $this->addOption( 'fix', 'Actually clean up invalid titles. If this parameter is ' . + 'not specified, the script will report invalid titles but not clean them up.', + false, false ); + $this->addOption( 'table', 'The table(s) to process. This option can be specified ' . + 'more than once (e.g. -t category -t watchlist). If not specified, all available ' . + 'tables will be processed. Available tables are: ' . + implode( ', ', array_column( static::$tables, 0 ) ), false, true, 't', true ); + + $this->setBatchSize( 500 ); + } + + public function execute() { + $tablesToProcess = $this->getOption( 'table' ); + foreach ( static::$tables as $tableParams ) { + if ( !$tablesToProcess || in_array( $tableParams[0], $tablesToProcess ) ) { + $this->cleanupTable( $tableParams ); + } + } + + $this->outputStatus( 'Done!' ); + if ( $this->hasOption( 'fix' ) ) { + $this->outputStatus( ' Cleaned up invalid DB keys on ' . wfWikiID() . "!\n" ); + } + } + + /** + * Prints text to STDOUT, and STDERR if STDOUT was redirected to a file. + * Used for progress reporting. + * + * @param string $str Text to write to both places + * @param string|null $channel Ignored + */ + protected function outputStatus( $str, $channel = null ) { + // Make it easier to find progress lines in the STDOUT log + if ( trim( $str ) ) { + fwrite( STDOUT, '*** ' . trim( $str ) . "\n" ); + } + fwrite( STDERR, $str ); + } + + /** + * Prints text to STDOUT. Used for logging output. + * + * @param string $str Text to write + */ + protected function writeToReport( $str ) { + fwrite( STDOUT, $str ); + } + + /** + * Identifies, and optionally cleans up, invalid titles. + * + * @param array $tableParams A child array of self::$tables + */ + protected function cleanupTable( $tableParams ) { + $table = $tableParams[0]; + $prefix = $tableParams[1]; + $idField = isset( $tableParams['idField'] ) ? + $tableParams['idField'] : + "{$prefix}_id"; + $nsField = isset( $tableParams['nsField'] ) ? + $tableParams['nsField'] : + "{$prefix}_namespace"; + $titleField = isset( $tableParams['titleField'] ) ? + $tableParams['titleField'] : + "{$prefix}_title"; + + $this->outputStatus( "Looking for invalid $titleField entries in $table...\n" ); + + // Do all the select queries on the replicas, as they are slow (they use + // unanchored LIKEs). Naturally this could cause problems if rows are + // modified after selecting and before deleting/updating, but working on + // the hypothesis that invalid rows will be old and in all likelihood + // unreferenced, we should be fine to do it like this. + $dbr = $this->getDB( DB_REPLICA, 'vslow' ); + + // Find all TitleValue-invalid titles. + $percent = $dbr->anyString(); // DBMS-agnostic equivalent of '%' LIKE wildcard + $res = $dbr->select( + $table, + [ + 'id' => $idField, + 'ns' => $nsField, + 'title' => $titleField, + ], + // The REGEXP operator is not cross-DBMS, so we have to use lots of LIKEs + [ $dbr->makeList( [ + $titleField . $dbr->buildLike( $percent, ' ', $percent ), + $titleField . $dbr->buildLike( $percent, "\r", $percent ), + $titleField . $dbr->buildLike( $percent, "\n", $percent ), + $titleField . $dbr->buildLike( $percent, "\t", $percent ), + $titleField . $dbr->buildLike( '_', $percent ), + $titleField . $dbr->buildLike( $percent, '_' ), + ], LIST_OR ) ], + __METHOD__, + [ 'LIMIT' => $this->mBatchSize ] + ); + + $this->outputStatus( "Number of invalid rows: " . $res->numRows() . "\n" ); + if ( !$res->numRows() ) { + $this->outputStatus( "\n" ); + return; + } + + // Write a table of titles to the report file. Also keep a list of the found + // IDs, as we might need it later for DB updates + $this->writeToReport( sprintf( "%10s | ns | dbkey\n", $idField ) ); + $ids = []; + foreach ( $res as $row ) { + $this->writeToReport( sprintf( "%10d | %3d | %s\n", $row->id, $row->ns, $row->title ) ); + $ids[] = $row->id; + } + + // If we're doing a dry run, output the new titles we would use for the UPDATE + // queries (if relevant), and finish + if ( !$this->hasOption( 'fix' ) ) { + if ( $table === 'logging' || $table === 'archive' ) { + $this->writeToReport( "The following updates would be run with the --fix flag:\n" ); + foreach ( $res as $row ) { + $newTitle = self::makeValidTitle( $row->title ); + $this->writeToReport( + "$idField={$row->id}: update '{$row->title}' to '$newTitle'\n" ); + } + } + + if ( $table !== 'page' && $table !== 'redirect' ) { + $this->outputStatus( "Run with --fix to clean up these rows\n" ); + } + $this->outputStatus( "\n" ); + return; + } + + // Fix the bad data, using different logic for the various tables + $dbw = $this->getDB( DB_MASTER ); + switch ( $table ) { + case 'page': + case 'redirect': + // This shouldn't happen on production wikis, and we already have a script + // to handle 'page' rows anyway, so just notify the user and let them decide + // what to do next. + $this->outputStatus( <<<TEXT +IMPORTANT: This script does not fix invalid entries in the $table table. +Consider repairing these rows, and rows in related tables, by hand. +You may like to run, or borrow logic from, the cleanupTitles.php script. + +TEXT + ); + break; + + case 'archive': + case 'logging': + // Rename the title to a corrected equivalent. Any foreign key relationships + // to the page_title field are already broken, so this will just make sure + // users can still access the log entries/deleted revisions from the interface + // using a valid page title. + $this->outputStatus( + "Updating these rows, setting $titleField to the closest valid DB key...\n" ); + $affectedRowCount = 0; + foreach ( $res as $row ) { + $newTitle = self::makeValidTitle( $row->title ); + $this->writeToReport( + "$idField={$row->id}: updating '{$row->title}' to '$newTitle'\n" ); + + $dbw->update( $table, + [ $titleField => $newTitle ], + [ $idField => $row->id ], + __METHOD__ ); + $affectedRowCount += $dbw->affectedRows(); + } + wfWaitForSlaves(); + $this->outputStatus( "Updated $affectedRowCount rows on $table.\n" ); + + break; + + case 'recentchanges': + case 'watchlist': + case 'category': + // Since these broken titles can't exist, there's really nothing to watch, + // nothing can be categorised in them, and they can't have been changed + // recently, so we can just remove these rows. + $this->outputStatus( "Deleting invalid $table rows...\n" ); + $dbw->delete( $table, [ $idField => $ids ], __METHOD__ ); + wfWaitForSlaves(); + $this->outputStatus( 'Deleted ' . $dbw->affectedRows() . " rows from $table.\n" ); + break; + + case 'protected_titles': + // Since these broken titles can't exist, there's really nothing to protect, + // so we can just remove these rows. Made more complicated by this table + // not having an ID field + $this->outputStatus( "Deleting invalid $table rows...\n" ); + $affectedRowCount = 0; + foreach ( $res as $row ) { + $dbw->delete( $table, + [ $nsField => $row->ns, $titleField => $row->title ], + __METHOD__ ); + $affectedRowCount += $dbw->affectedRows(); + } + wfWaitForSlaves(); + $this->outputStatus( "Deleted $affectedRowCount rows from $table.\n" ); + break; + + case 'pagelinks': + case 'templatelinks': + case 'categorylinks': + // Update links tables for each page where these bogus links are supposedly + // located. If the invalid rows don't go away after these jobs go through, + // they're probably being added by a buggy hook. + $this->outputStatus( "Queueing link update jobs for the pages in $idField...\n" ); + foreach ( $res as $row ) { + $wp = WikiPage::newFromID( $row->id ); + if ( $wp ) { + RefreshLinks::fixLinksFromArticle( $row->id ); + } else { + // This link entry points to a nonexistent page, so just get rid of it + $dbw->delete( $table, + [ $idField => $row->id, $nsField => $row->ns, $titleField => $row->title ], + __METHOD__ ); + } + } + wfWaitForSlaves(); + $this->outputStatus( "Link update jobs have been added to the job queue.\n" ); + break; + } + + $this->outputStatus( "\n" ); + return; + } + + /** + * Fix possible validation issues in the given title (DB key). + * + * @param string $invalidTitle + * @return string + */ + protected static function makeValidTitle( $invalidTitle ) { + return strtr( trim( $invalidTitle, '_' ), + [ ' ' => '_', "\r" => '', "\n" => '', "\t" => '_' ] ); + } +} + +$maintClass = 'CleanupInvalidDbKeys'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/cleanupPreferences.php b/www/wiki/maintenance/cleanupPreferences.php new file mode 100644 index 00000000..6e58ae97 --- /dev/null +++ b/www/wiki/maintenance/cleanupPreferences.php @@ -0,0 +1,52 @@ +<?php +/** + * Remove hidden preferences from the database. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @author TyA <tya.wiki@gmail.com> + * @see https://phabricator.wikimedia.org/T32976 + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that removes hidden preferences from the database. + * + * @ingroup Maintenance + */ +class CleanupPreferences extends Maintenance { + public function execute() { + global $wgHiddenPrefs; + + $dbw = $this->getDB( DB_MASTER ); + $this->beginTransaction( $dbw, __METHOD__ ); + foreach ( $wgHiddenPrefs as $item ) { + $dbw->delete( + 'user_properties', + [ 'up_property' => $item ], + __METHOD__ + ); + }; + $this->commitTransaction( $dbw, __METHOD__ ); + $this->output( "Finished!\n" ); + } +} + +$maintClass = 'CleanupPreferences'; // Tells it to run the class +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/cleanupRemovedModules.php b/www/wiki/maintenance/cleanupRemovedModules.php new file mode 100644 index 00000000..dbaf6438 --- /dev/null +++ b/www/wiki/maintenance/cleanupRemovedModules.php @@ -0,0 +1,81 @@ +<?php +/** + * Remove cache entries for removed ResourceLoader modules from the database. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @author Roan Kattouw + */ + +use MediaWiki\MediaWikiServices; +use Wikimedia\Rdbms\IDatabase; + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to remove cache entries for removed ResourceLoader modules + * from the database. + * + * @ingroup Maintenance + */ +class CleanupRemovedModules extends Maintenance { + + public function __construct() { + parent::__construct(); + $this->addDescription( + 'Remove cache entries for removed ResourceLoader modules from the database' ); + $this->setBatchSize( 500 ); + } + + public function execute() { + $this->output( "Cleaning up module_deps table...\n" ); + + $dbw = $this->getDB( DB_MASTER ); + $rl = new ResourceLoader( MediaWikiServices::getInstance()->getMainConfig() ); + $moduleNames = $rl->getModuleNames(); + $res = $dbw->select( 'module_deps', + [ 'md_module', 'md_skin' ], + $moduleNames ? 'md_module NOT IN (' . $dbw->makeList( $moduleNames ) . ')' : '1=1', + __METHOD__ + ); + $rows = iterator_to_array( $res, false ); + + $modDeps = $dbw->tableName( 'module_deps' ); + $i = 1; + foreach ( array_chunk( $rows, $this->mBatchSize ) as $chunk ) { + // WHERE ( mod=A AND skin=A ) OR ( mod=A AND skin=B) .. + $conds = array_map( function ( stdClass $row ) use ( $dbw ) { + return $dbw->makeList( (array)$row, IDatabase::LIST_AND ); + }, $chunk ); + $conds = $dbw->makeList( $conds, IDatabase::LIST_OR ); + + $this->beginTransaction( $dbw, __METHOD__ ); + $dbw->query( "DELETE FROM $modDeps WHERE $conds", __METHOD__ ); + $numRows = $dbw->affectedRows(); + $this->output( "Batch $i: $numRows rows\n" ); + $this->commitTransaction( $dbw, __METHOD__ ); + + $i++; + } + + $this->output( "Done\n" ); + } +} + +$maintClass = 'CleanupRemovedModules'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/cleanupSpam.php b/www/wiki/maintenance/cleanupSpam.php new file mode 100644 index 00000000..4e47cfba --- /dev/null +++ b/www/wiki/maintenance/cleanupSpam.php @@ -0,0 +1,160 @@ +<?php +/** + * Cleanup all spam from a given hostname. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to cleanup all spam from a given hostname. + * + * @ingroup Maintenance + */ +class CleanupSpam extends Maintenance { + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Cleanup all spam from a given hostname' ); + $this->addOption( 'all', 'Check all wikis in $wgLocalDatabases' ); + $this->addOption( 'delete', 'Delete pages containing only spam instead of blanking them' ); + $this->addArg( + 'hostname', + 'Hostname that was spamming, single * wildcard in the beginning allowed' + ); + } + + public function execute() { + global $IP, $wgLocalDatabases, $wgUser; + + $username = wfMessage( 'spambot_username' )->text(); + $wgUser = User::newSystemUser( $username ); + if ( !$wgUser ) { + $this->error( "Invalid username specified in 'spambot_username' message: $username", true ); + } + // Create the user if necessary + if ( !$wgUser->getId() ) { + $wgUser->addToDatabase(); + } + $spec = $this->getArg(); + $like = LinkFilter::makeLikeArray( $spec ); + if ( !$like ) { + $this->error( "Not a valid hostname specification: $spec", true ); + } + + if ( $this->hasOption( 'all' ) ) { + // Clean up spam on all wikis + $this->output( "Finding spam on " . count( $wgLocalDatabases ) . " wikis\n" ); + $found = false; + foreach ( $wgLocalDatabases as $wikiID ) { + $dbr = $this->getDB( DB_REPLICA, [], $wikiID ); + + $count = $dbr->selectField( 'externallinks', 'COUNT(*)', + [ 'el_index' . $dbr->buildLike( $like ) ], __METHOD__ ); + if ( $count ) { + $found = true; + $cmd = wfShellWikiCmd( "$IP/maintenance/cleanupSpam.php", + [ '--wiki', $wikiID, $spec ] ); + passthru( "$cmd | sed 's/^/$wikiID: /'" ); + } + } + if ( $found ) { + $this->output( "All done\n" ); + } else { + $this->output( "None found\n" ); + } + } else { + // Clean up spam on this wiki + + $dbr = $this->getDB( DB_REPLICA ); + $res = $dbr->select( 'externallinks', [ 'DISTINCT el_from' ], + [ 'el_index' . $dbr->buildLike( $like ) ], __METHOD__ ); + $count = $dbr->numRows( $res ); + $this->output( "Found $count articles containing $spec\n" ); + foreach ( $res as $row ) { + $this->cleanupArticle( $row->el_from, $spec ); + } + if ( $count ) { + $this->output( "Done\n" ); + } + } + } + + private function cleanupArticle( $id, $domain ) { + $title = Title::newFromID( $id ); + if ( !$title ) { + $this->error( "Internal error: no page for ID $id" ); + + return; + } + + $this->output( $title->getPrefixedDBkey() . " ..." ); + $rev = Revision::newFromTitle( $title ); + $currentRevId = $rev->getId(); + + while ( $rev && ( $rev->isDeleted( Revision::DELETED_TEXT ) + || LinkFilter::matchEntry( $rev->getContent( Revision::RAW ), $domain ) ) + ) { + $rev = $rev->getPrevious(); + } + + if ( $rev && $rev->getId() == $currentRevId ) { + // The regex didn't match the current article text + // This happens e.g. when a link comes from a template rather than the page itself + $this->output( "False match\n" ); + } else { + $dbw = $this->getDB( DB_MASTER ); + $this->beginTransaction( $dbw, __METHOD__ ); + $page = WikiPage::factory( $title ); + if ( $rev ) { + // Revert to this revision + $content = $rev->getContent( Revision::RAW ); + + $this->output( "reverting\n" ); + $page->doEditContent( + $content, + wfMessage( 'spam_reverting', $domain )->inContentLanguage()->text(), + EDIT_UPDATE, + $rev->getId() + ); + } elseif ( $this->hasOption( 'delete' ) ) { + // Didn't find a non-spammy revision, blank the page + $this->output( "deleting\n" ); + $page->doDeleteArticle( + wfMessage( 'spam_deleting', $domain )->inContentLanguage()->text() + ); + } else { + // Didn't find a non-spammy revision, blank the page + $handler = ContentHandler::getForTitle( $title ); + $content = $handler->makeEmptyContent(); + + $this->output( "blanking\n" ); + $page->doEditContent( + $content, + wfMessage( 'spam_blanking', $domain )->inContentLanguage()->text() + ); + } + $this->commitTransaction( $dbw, __METHOD__ ); + } + } +} + +$maintClass = "CleanupSpam"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/cleanupTable.inc b/www/wiki/maintenance/cleanupTable.inc new file mode 100644 index 00000000..3ace09cb --- /dev/null +++ b/www/wiki/maintenance/cleanupTable.inc @@ -0,0 +1,174 @@ +<?php +/** + * Generic class to cleanup a database table. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Generic class to cleanup a database table. Already subclasses Maintenance. + * + * @ingroup Maintenance + */ +class TableCleanup extends Maintenance { + protected $defaultParams = [ + 'table' => 'page', + 'conds' => [], + 'index' => 'page_id', + 'callback' => 'processRow', + ]; + + protected $dryrun = false; + public $batchSize = 100; + public $reportInterval = 100; + + protected $processed, $updated, $count, $startTime, $table; + + public function __construct() { + parent::__construct(); + $this->addOption( 'dry-run', 'Perform a dry run' ); + } + + public function execute() { + global $wgUser; + $this->dryrun = $this->hasOption( 'dry-run' ); + if ( $this->dryrun ) { + $wgUser = User::newFromName( 'Conversion script' ); + $this->output( "Checking for bad titles...\n" ); + } else { + $wgUser = User::newSystemUser( 'Conversion script', [ 'steal' => true ] ); + $this->output( "Checking and fixing bad titles...\n" ); + } + $this->runTable( $this->defaultParams ); + } + + protected function init( $count, $table ) { + $this->processed = 0; + $this->updated = 0; + $this->count = $count; + $this->startTime = microtime( true ); + $this->table = $table; + } + + /** + * @param int $updated + */ + protected function progress( $updated ) { + $this->updated += $updated; + $this->processed++; + if ( $this->processed % $this->reportInterval != 0 ) { + return; + } + $portion = $this->processed / $this->count; + $updateRate = $this->updated / $this->processed; + + $now = microtime( true ); + $delta = $now - $this->startTime; + $estimatedTotalTime = $delta / $portion; + $eta = $this->startTime + $estimatedTotalTime; + + $this->output( + sprintf( "%s %s: %6.2f%% done on %s; ETA %s [%d/%d] %.2f/sec <%.2f%% updated>\n", + wfWikiID(), + wfTimestamp( TS_DB, intval( $now ) ), + $portion * 100.0, + $this->table, + wfTimestamp( TS_DB, intval( $eta ) ), + $this->processed, + $this->count, + $this->processed / $delta, + $updateRate * 100.0 + ) + ); + flush(); + } + + /** + * @param array $params + * @throws MWException + */ + public function runTable( $params ) { + $dbr = $this->getDB( DB_REPLICA ); + + if ( array_diff( array_keys( $params ), + [ 'table', 'conds', 'index', 'callback' ] ) + ) { + throw new MWException( __METHOD__ . ': Missing parameter ' . implode( ', ', $params ) ); + } + + $table = $params['table']; + // count(*) would melt the DB for huge tables, we can estimate here + $count = $dbr->estimateRowCount( $table, '*', '', __METHOD__ ); + $this->init( $count, $table ); + $this->output( "Processing $table...\n" ); + + $index = (array)$params['index']; + $indexConds = []; + $options = [ + 'ORDER BY' => implode( ',', $index ), + 'LIMIT' => $this->batchSize + ]; + $callback = [ $this, $params['callback'] ]; + + while ( true ) { + $conds = array_merge( $params['conds'], $indexConds ); + $res = $dbr->select( $table, '*', $conds, __METHOD__, $options ); + if ( !$res->numRows() ) { + // Done + break; + } + + foreach ( $res as $row ) { + call_user_func( $callback, $row ); + } + + if ( $res->numRows() < $this->batchSize ) { + // Done + break; + } + + // Update the conditions to select the next batch. + // Construct a condition string by starting with the least significant part + // of the index, and adding more significant parts progressively to the left + // of the string. + $nextCond = ''; + foreach ( array_reverse( $index ) as $field ) { + $encValue = $dbr->addQuotes( $row->$field ); + if ( $nextCond === '' ) { + $nextCond = "$field > $encValue"; + } else { + $nextCond = "$field > $encValue OR ($field = $encValue AND ($nextCond))"; + } + } + $indexConds = [ $nextCond ]; + } + + $this->output( "Finished $table... $this->updated of $this->processed rows updated\n" ); + } + + /** + * @param array $matches + * @return string + */ + protected function hexChar( $matches ) { + return sprintf( "\\x%02x", ord( $matches[1] ) ); + } +} diff --git a/www/wiki/maintenance/cleanupTitles.php b/www/wiki/maintenance/cleanupTitles.php new file mode 100644 index 00000000..50e17d8d --- /dev/null +++ b/www/wiki/maintenance/cleanupTitles.php @@ -0,0 +1,193 @@ +<?php +/** + * Clean up broken, unparseable titles. + * + * Copyright © 2005 Brion Vibber <brion@pobox.com> + * https://www.mediawiki.org/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @author Brion Vibber <brion at pobox.com> + * @ingroup Maintenance + */ + +use MediaWiki\MediaWikiServices; + +require_once __DIR__ . '/cleanupTable.inc'; + +/** + * Maintenance script to clean up broken, unparseable titles. + * + * @ingroup Maintenance + */ +class TitleCleanup extends TableCleanup { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Script to clean up broken, unparseable titles' ); + } + + /** + * @param object $row + */ + protected function processRow( $row ) { + global $wgContLang; + $display = Title::makeName( $row->page_namespace, $row->page_title ); + $verified = $wgContLang->normalize( $display ); + $title = Title::newFromText( $verified ); + + if ( !is_null( $title ) + && $title->canExist() + && $title->getNamespace() == $row->page_namespace + && $title->getDBkey() === $row->page_title + ) { + $this->progress( 0 ); // all is fine + + return; + } + + if ( $row->page_namespace == NS_FILE && $this->fileExists( $row->page_title ) ) { + $this->output( "file $row->page_title needs cleanup, please run cleanupImages.php.\n" ); + $this->progress( 0 ); + } elseif ( is_null( $title ) ) { + $this->output( "page $row->page_id ($display) is illegal.\n" ); + $this->moveIllegalPage( $row ); + $this->progress( 1 ); + } else { + $this->output( "page $row->page_id ($display) doesn't match self.\n" ); + $this->moveInconsistentPage( $row, $title ); + $this->progress( 1 ); + } + } + + /** + * @param string $name + * @return bool + */ + protected function fileExists( $name ) { + // XXX: Doesn't actually check for file existence, just presence of image record. + // This is reasonable, since cleanupImages.php only iterates over the image table. + $dbr = $this->getDB( DB_REPLICA ); + $row = $dbr->selectRow( 'image', [ 'img_name' ], [ 'img_name' => $name ], __METHOD__ ); + + return $row !== false; + } + + /** + * @param object $row + */ + protected function moveIllegalPage( $row ) { + $legal = 'A-Za-z0-9_/\\\\-'; + $legalized = preg_replace_callback( "!([^$legal])!", + [ $this, 'hexChar' ], + $row->page_title ); + if ( $legalized == '.' ) { + $legalized = '(dot)'; + } + if ( $legalized == '_' ) { + $legalized = '(space)'; + } + $legalized = 'Broken/' . $legalized; + + $title = Title::newFromText( $legalized ); + if ( is_null( $title ) ) { + $clean = 'Broken/id:' . $row->page_id; + $this->output( "Couldn't legalize; form '$legalized' still invalid; using '$clean'\n" ); + $title = Title::newFromText( $clean ); + } elseif ( $title->exists() ) { + $clean = 'Broken/id:' . $row->page_id; + $this->output( "Legalized for '$legalized' exists; using '$clean'\n" ); + $title = Title::newFromText( $clean ); + } + + $dest = $title->getDBkey(); + if ( $this->dryrun ) { + $this->output( "DRY RUN: would rename $row->page_id ($row->page_namespace," . + "'$row->page_title') to ($row->page_namespace,'$dest')\n" ); + } else { + $this->output( "renaming $row->page_id ($row->page_namespace," . + "'$row->page_title') to ($row->page_namespace,'$dest')\n" ); + $dbw = $this->getDB( DB_MASTER ); + $dbw->update( 'page', + [ 'page_title' => $dest ], + [ 'page_id' => $row->page_id ], + __METHOD__ ); + } + } + + /** + * @param object $row + * @param Title $title + */ + protected function moveInconsistentPage( $row, Title $title ) { + if ( $title->exists( Title::GAID_FOR_UPDATE ) + || $title->getInterwiki() + || !$title->canExist() + ) { + if ( $title->getInterwiki() || !$title->canExist() ) { + $prior = $title->getPrefixedDBkey(); + } else { + $prior = $title->getDBkey(); + } + + # Old cleanupTitles could move articles there. See T25147. + $ns = $row->page_namespace; + if ( $ns < 0 ) { + $ns = 0; + } + + # Namespace which no longer exists. Put the page in the main namespace + # since we don't have any idea of the old namespace name. See T70501. + if ( !MWNamespace::exists( $ns ) ) { + $ns = 0; + } + + $clean = 'Broken/' . $prior; + $verified = Title::makeTitleSafe( $ns, $clean ); + if ( !$verified || $verified->exists() ) { + $blah = "Broken/id:" . $row->page_id; + $this->output( "Couldn't legalize; form '$clean' exists; using '$blah'\n" ); + $verified = Title::makeTitleSafe( $ns, $blah ); + } + $title = $verified; + } + if ( is_null( $title ) ) { + $this->error( "Something awry; empty title.", true ); + } + $ns = $title->getNamespace(); + $dest = $title->getDBkey(); + + if ( $this->dryrun ) { + $this->output( "DRY RUN: would rename $row->page_id ($row->page_namespace," . + "'$row->page_title') to ($ns,'$dest')\n" ); + } else { + $this->output( "renaming $row->page_id ($row->page_namespace," . + "'$row->page_title') to ($ns,'$dest')\n" ); + $dbw = $this->getDB( DB_MASTER ); + $dbw->update( 'page', + [ + 'page_namespace' => $ns, + 'page_title' => $dest + ], + [ 'page_id' => $row->page_id ], + __METHOD__ ); + MediaWikiServices::getInstance()->getLinkCache()->clear(); + } + } +} + +$maintClass = "TitleCleanup"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/cleanupUploadStash.php b/www/wiki/maintenance/cleanupUploadStash.php new file mode 100644 index 00000000..95bbe3d3 --- /dev/null +++ b/www/wiki/maintenance/cleanupUploadStash.php @@ -0,0 +1,156 @@ +<?php +/** + * Remove old or broken uploads from temporary uploaded file storage, + * clean up associated database records + * + * Copyright © 2011, Wikimedia Foundation + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @author Ian Baker <ibaker@wikimedia.org> + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to remove old or broken uploads from temporary uploaded + * file storage and clean up associated database records. + * + * @ingroup Maintenance + */ +class UploadStashCleanup extends Maintenance { + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Clean up abandoned files in temporary uploaded file stash' ); + $this->setBatchSize( 50 ); + } + + public function execute() { + global $wgUploadStashMaxAge; + + $repo = RepoGroup::singleton()->getLocalRepo(); + $tempRepo = $repo->getTempRepo(); + + $dbr = $repo->getReplicaDB(); + + // how far back should this look for files to delete? + $cutoff = time() - $wgUploadStashMaxAge; + + $this->output( "Getting list of files to clean up...\n" ); + $res = $dbr->select( + 'uploadstash', + 'us_key', + 'us_timestamp < ' . $dbr->addQuotes( $dbr->timestamp( $cutoff ) ), + __METHOD__ + ); + + // Delete all registered stash files... + if ( $res->numRows() == 0 ) { + $this->output( "No stashed files to cleanup according to the DB.\n" ); + } else { + // finish the read before starting writes. + $keys = []; + foreach ( $res as $row ) { + array_push( $keys, $row->us_key ); + } + + $this->output( 'Removing ' . count( $keys ) . " file(s)...\n" ); + // this could be done some other, more direct/efficient way, but using + // UploadStash's own methods means it's less likely to fall accidentally + // out-of-date someday + $stash = new UploadStash( $repo ); + + $i = 0; + foreach ( $keys as $key ) { + $i++; + try { + $stash->getFile( $key, true ); + $stash->removeFileNoAuth( $key ); + } catch ( UploadStashException $ex ) { + $type = get_class( $ex ); + $this->output( "Failed removing stashed upload with key: $key ($type)\n" ); + } + if ( $i % 100 == 0 ) { + wfWaitForSlaves(); + $this->output( "$i\n" ); + } + } + $this->output( "$i done\n" ); + } + + // Delete all the corresponding thumbnails... + $dir = $tempRepo->getZonePath( 'thumb' ); + $iterator = $tempRepo->getBackend()->getFileList( [ 'dir' => $dir, 'adviseStat' => 1 ] ); + $this->output( "Deleting old thumbnails...\n" ); + $i = 0; + $batch = []; // operation batch + foreach ( $iterator as $file ) { + if ( wfTimestamp( TS_UNIX, $tempRepo->getFileTimestamp( "$dir/$file" ) ) < $cutoff ) { + $batch[] = [ 'op' => 'delete', 'src' => "$dir/$file" ]; + if ( count( $batch ) >= $this->mBatchSize ) { + $this->doOperations( $tempRepo, $batch ); + $i += count( $batch ); + $batch = []; + $this->output( "$i\n" ); + } + } + } + if ( count( $batch ) ) { + $this->doOperations( $tempRepo, $batch ); + $i += count( $batch ); + } + $this->output( "$i done\n" ); + + // Apparently lots of stash files are not registered in the DB... + $dir = $tempRepo->getZonePath( 'public' ); + $iterator = $tempRepo->getBackend()->getFileList( [ 'dir' => $dir, 'adviseStat' => 1 ] ); + $this->output( "Deleting orphaned temp files...\n" ); + if ( strpos( $dir, '/local-temp' ) === false ) { // sanity check + $this->error( "Temp repo is not using the temp container.", 1 ); // die + } + $i = 0; + $batch = []; // operation batch + foreach ( $iterator as $file ) { + if ( wfTimestamp( TS_UNIX, $tempRepo->getFileTimestamp( "$dir/$file" ) ) < $cutoff ) { + $batch[] = [ 'op' => 'delete', 'src' => "$dir/$file" ]; + if ( count( $batch ) >= $this->mBatchSize ) { + $this->doOperations( $tempRepo, $batch ); + $i += count( $batch ); + $batch = []; + $this->output( "$i\n" ); + } + } + } + if ( count( $batch ) ) { + $this->doOperations( $tempRepo, $batch ); + $i += count( $batch ); + } + $this->output( "$i done\n" ); + } + + protected function doOperations( FileRepo $tempRepo, array $ops ) { + $status = $tempRepo->getBackend()->doQuickOperations( $ops ); + if ( !$status->isOK() ) { + $this->error( print_r( $status->getErrorsArray(), true ) ); + } + } +} + +$maintClass = "UploadStashCleanup"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/cleanupWatchlist.php b/www/wiki/maintenance/cleanupWatchlist.php new file mode 100644 index 00000000..9728ac5d --- /dev/null +++ b/www/wiki/maintenance/cleanupWatchlist.php @@ -0,0 +1,99 @@ +<?php +/** + * Remove broken, unparseable titles in the watchlist table. + * + * Usage: php cleanupWatchlist.php [--fix] + * Options: + * --fix Actually remove entries; without will only report. + * + * Copyright © 2005,2006 Brion Vibber <brion@pobox.com> + * https://www.mediawiki.org/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @author Brion Vibber <brion at pobox.com> + * @ingroup Maintenance + */ + +require_once __DIR__ . '/cleanupTable.inc'; + +/** + * Maintenance script to remove broken, unparseable titles in the watchlist table. + * + * @ingroup Maintenance + */ +class WatchlistCleanup extends TableCleanup { + protected $defaultParams = [ + 'table' => 'watchlist', + 'index' => [ 'wl_user', 'wl_namespace', 'wl_title' ], + 'conds' => [], + 'callback' => 'processRow' + ]; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Script to remove broken, unparseable titles in the Watchlist' ); + $this->addOption( 'fix', 'Actually remove entries; without will only report.' ); + } + + function execute() { + if ( !$this->hasOption( 'fix' ) ) { + $this->output( "Dry run only: use --fix to enable updates\n" ); + } + parent::execute(); + } + + protected function processRow( $row ) { + global $wgContLang; + $current = Title::makeTitle( $row->wl_namespace, $row->wl_title ); + $display = $current->getPrefixedText(); + $verified = $wgContLang->normalize( $display ); + $title = Title::newFromText( $verified ); + + if ( $row->wl_user == 0 || is_null( $title ) || !$title->equals( $current ) ) { + $this->output( "invalid watch by {$row->wl_user} for " + . "({$row->wl_namespace}, \"{$row->wl_title}\")\n" ); + $updated = $this->removeWatch( $row ); + $this->progress( $updated ); + + return; + } + $this->progress( 0 ); + } + + private function removeWatch( $row ) { + if ( !$this->dryrun && $this->hasOption( 'fix' ) ) { + $dbw = $this->getDB( DB_MASTER ); + $dbw->delete( + 'watchlist', [ + 'wl_user' => $row->wl_user, + 'wl_namespace' => $row->wl_namespace, + 'wl_title' => $row->wl_title ], + __METHOD__ + ); + + $this->output( "- removed\n" ); + + return 1; + } else { + return 0; + } + } +} + +$maintClass = "WatchlistCleanup"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/clearInterwikiCache.php b/www/wiki/maintenance/clearInterwikiCache.php new file mode 100644 index 00000000..ce19974e --- /dev/null +++ b/www/wiki/maintenance/clearInterwikiCache.php @@ -0,0 +1,58 @@ +<?php +/** + * Clear the cache of interwiki prefixes for all local wikis. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to clear the cache of interwiki prefixes for all local wikis. + * + * @ingroup Maintenance + */ +class ClearInterwikiCache extends Maintenance { + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Clear all interwiki links for all languages from the cache' ); + } + + public function execute() { + global $wgLocalDatabases, $wgMemc; + $dbr = $this->getDB( DB_REPLICA ); + $res = $dbr->select( 'interwiki', [ 'iw_prefix' ], false ); + $prefixes = []; + foreach ( $res as $row ) { + $prefixes[] = $row->iw_prefix; + } + + foreach ( $wgLocalDatabases as $db ) { + $this->output( "$db..." ); + foreach ( $prefixes as $prefix ) { + $wgMemc->delete( "$db:interwiki:$prefix" ); + } + $this->output( "done\n" ); + } + } +} + +$maintClass = "ClearInterwikiCache"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/commandLine.inc b/www/wiki/maintenance/commandLine.inc new file mode 100644 index 00000000..206e0467 --- /dev/null +++ b/www/wiki/maintenance/commandLine.inc @@ -0,0 +1,72 @@ +<?php +/** + * Backwards-compatibility wrapper for old-style maintenance scripts. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +// @codingStandardsIgnoreStart MediaWiki.NamingConventions.ValidGlobalName.wgPrefix +global $optionsWithArgs; +global $optionsWithoutArgs; +// @codingStandardsIgnoreEnd +if ( !isset( $optionsWithArgs ) ) { + $optionsWithArgs = []; +} +if ( !isset( $optionsWithoutArgs ) ) { + $optionsWithoutArgs = []; +} + +class CommandLineInc extends Maintenance { + public function __construct() { + // @codingStandardsIgnoreStart MediaWiki.NamingConventions.ValidGlobalName.wgPrefix + global $optionsWithArgs, $optionsWithoutArgs; + // @codingStandardsIgnoreEnd + parent::__construct(); + foreach ( $optionsWithArgs as $name ) { + $this->addOption( $name, '', false, true ); + } + foreach ( $optionsWithoutArgs as $name ) { + $this->addOption( $name, '', false, false ); + } + } + + /** + * No help, it would just be misleading since it misses custom options + * @param bool $force + */ + protected function maybeHelp( $force = false ) { + if ( !$force ) { + return; + } + parent::maybeHelp( true ); + } + + public function execute() { + // @codingStandardsIgnoreStart MediaWiki.NamingConventions.ValidGlobalName.wgPrefix + global $args, $options; + // @codingStandardsIgnoreEnd + $args = $this->mArgs; + $options = $this->mOptions; + } +} + +$maintClass = 'CommandLineInc'; +require RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/compareParserCache.php b/www/wiki/maintenance/compareParserCache.php new file mode 100644 index 00000000..504c7d7a --- /dev/null +++ b/www/wiki/maintenance/compareParserCache.php @@ -0,0 +1,107 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +use MediaWiki\MediaWikiServices; + +/** + * @ingroup Maintenance + */ +class CompareParserCache extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Parse random pages and compare output to cache.' ); + $this->addOption( 'namespace', 'Page namespace number', true, true ); + $this->addOption( 'maxpages', 'Number of pages to try', true, true ); + } + + public function execute() { + $pages = $this->getOption( 'maxpages' ); + + $dbr = $this->getDB( DB_REPLICA ); + + $totalsec = 0.0; + $scanned = 0; + $withcache = 0; + $withdiff = 0; + $parserCache = MediaWikiServices::getInstance()->getParserCache(); + while ( $pages-- > 0 ) { + $row = $dbr->selectRow( 'page', '*', + [ + 'page_namespace' => $this->getOption( 'namespace' ), + 'page_is_redirect' => 0, + 'page_random >= ' . wfRandom() + ], + __METHOD__, + [ + 'ORDER BY' => 'page_random', + ] + ); + + if ( !$row ) { + continue; + } + ++$scanned; + + $title = Title::newFromRow( $row ); + $page = WikiPage::factory( $title ); + $revision = $page->getRevision(); + $content = $revision->getContent( Revision::RAW ); + + $parserOptions = $page->makeParserOptions( 'canonical' ); + + $parserOutputOld = $parserCache->get( $page, $parserOptions ); + + if ( $parserOutputOld ) { + $t1 = microtime( true ); + $parserOutputNew = $content->getParserOutput( + $title, $revision->getId(), $parserOptions, false ); + $sec = microtime( true ) - $t1; + $totalsec += $sec; + + $this->output( "Parsed '{$title->getPrefixedText()}' in $sec seconds.\n" ); + + $this->output( "Found cache entry found for '{$title->getPrefixedText()}'..." ); + $oldHtml = trim( preg_replace( '#<!-- .+-->#Us', '', $parserOutputOld->getText() ) ); + $newHtml = trim( preg_replace( '#<!-- .+-->#Us', '', $parserOutputNew->getText() ) ); + $diff = wfDiff( $oldHtml, $newHtml ); + if ( strlen( $diff ) ) { + $this->output( "differences found:\n\n$diff\n\n" ); + ++$withdiff; + } else { + $this->output( "No differences found.\n" ); + } + ++$withcache; + } else { + $this->output( "No parser cache entry found for '{$title->getPrefixedText()}'.\n" ); + } + } + + $ave = $totalsec ? $totalsec / $scanned : 0; + $this->output( "Checked $scanned pages; $withcache had prior cache entries.\n" ); + $this->output( "Pages with differences found: $withdiff\n" ); + $this->output( "Average parse time: $ave sec\n" ); + } +} + +$maintClass = "CompareParserCache"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/compareParsers.php b/www/wiki/maintenance/compareParsers.php new file mode 100644 index 00000000..f2540c7a --- /dev/null +++ b/www/wiki/maintenance/compareParsers.php @@ -0,0 +1,189 @@ +<?php +/** + * Take page text out of an XML dump file and render basic HTML out to files. + * This is *NOT* suitable for publishing or offline use; it's intended for + * running comparative tests of parsing behavior using real-world data. + * + * Templates etc are pulled from the local wiki database, not from the dump. + * + * Copyright © 2011 Platonides + * https://www.mediawiki.org/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/dumpIterator.php'; + +/** + * Maintenance script to take page text out of an XML dump file and render + * basic HTML out to files. + * + * @ingroup Maintenance + */ +class CompareParsers extends DumpIterator { + + private $count = 0; + + public function __construct() { + parent::__construct(); + $this->saveFailed = false; + $this->addDescription( 'Run a file or dump with several parsers' ); + $this->addOption( 'parser1', 'The first parser to compare.', true, true ); + $this->addOption( 'parser2', 'The second parser to compare.', true, true ); + $this->addOption( 'tidy', 'Run tidy on the articles.', false, false ); + $this->addOption( + 'save-failed', + 'Folder in which articles which differ will be stored.', + false, + true + ); + $this->addOption( 'show-diff', 'Show a diff of the two renderings.', false, false ); + $this->addOption( + 'diff-bin', + 'Binary to use for diffing (can also be provided by DIFF env var).', + false, + false + ); + $this->addOption( + 'strip-parameters', + 'Remove parameters of html tags to increase readability.', + false, + false + ); + $this->addOption( + 'show-parsed-output', + 'Show the parsed html if both Parsers give the same output.', + false, + false + ); + } + + public function checkOptions() { + if ( $this->hasOption( 'save-failed' ) ) { + $this->saveFailed = $this->getOption( 'save-failed' ); + } + + $this->stripParametersEnabled = $this->hasOption( 'strip-parameters' ); + $this->showParsedOutput = $this->hasOption( 'show-parsed-output' ); + + $this->showDiff = $this->hasOption( 'show-diff' ); + if ( $this->showDiff ) { + $bin = $this->getOption( 'diff-bin', getenv( 'DIFF' ) ); + if ( $bin != '' ) { + global $wgDiff; + $wgDiff = $bin; + } + } + + $user = new User(); + $this->options = ParserOptions::newFromUser( $user ); + + if ( $this->hasOption( 'tidy' ) ) { + global $wgUseTidy; + if ( !$wgUseTidy ) { + $this->error( 'Tidy was requested but $wgUseTidy is not set in LocalSettings.php', true ); + } + $this->options->setTidy( true ); + } + + $this->failed = 0; + } + + public function conclusions() { + $this->error( "{$this->failed} failed revisions out of {$this->count}" ); + if ( $this->count > 0 ) { + $this->output( " (" . ( $this->failed / $this->count ) . "%)\n" ); + } + } + + function stripParameters( $text ) { + if ( !$this->stripParametersEnabled ) { + return $text; + } + + return preg_replace( '/(<a) [^>]+>/', '$1>', $text ); + } + + /** + * Callback function for each revision, parse with both parsers and compare + * @param Revision $rev + */ + public function processRevision( $rev ) { + $title = $rev->getTitle(); + + $parser1Name = $this->getOption( 'parser1' ); + $parser2Name = $this->getOption( 'parser2' ); + + self::checkParserLocally( $parser1Name ); + self::checkParserLocally( $parser2Name ); + + $parser1 = new $parser1Name(); + $parser2 = new $parser2Name(); + + $content = $rev->getContent(); + + if ( $content->getModel() !== CONTENT_MODEL_WIKITEXT ) { + $this->error( "Page {$title->getPrefixedText()} does not contain wikitext " + . "but {$content->getModel()}\n" ); + + return; + } + + $text = strval( $content->getNativeData() ); + + $output1 = $parser1->parse( $text, $title, $this->options ); + $output2 = $parser2->parse( $text, $title, $this->options ); + + if ( $output1->getText() != $output2->getText() ) { + $this->failed++; + $this->error( "Parsing for {$title->getPrefixedText()} differs\n" ); + + if ( $this->saveFailed ) { + file_put_contents( + $this->saveFailed . '/' . rawurlencode( $title->getPrefixedText() ) . ".txt", + $text + ); + } + if ( $this->showDiff ) { + $this->output( wfDiff( + $this->stripParameters( $output1->getText() ), + $this->stripParameters( $output2->getText() ), + '' + ) ); + } + } else { + $this->output( $title->getPrefixedText() . "\tOK\n" ); + + if ( $this->showParsedOutput ) { + $this->output( $this->stripParameters( $output1->getText() ) ); + } + } + } + + private static function checkParserLocally( $parserName ) { + /* Look for the parser in a file appropiately named in the current folder */ + if ( !class_exists( $parserName ) && file_exists( "$parserName.php" ) ) { + global $wgAutoloadClasses; + $wgAutoloadClasses[$parserName] = realpath( '.' ) . "/$parserName.php"; + } + } +} + +$maintClass = "CompareParsers"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/convertExtensionToRegistration.php b/www/wiki/maintenance/convertExtensionToRegistration.php new file mode 100644 index 00000000..05549495 --- /dev/null +++ b/www/wiki/maintenance/convertExtensionToRegistration.php @@ -0,0 +1,307 @@ +<?php + +require_once __DIR__ . '/Maintenance.php'; + +class ConvertExtensionToRegistration extends Maintenance { + + protected $custom = [ + 'MessagesDirs' => 'handleMessagesDirs', + 'ExtensionMessagesFiles' => 'handleExtensionMessagesFiles', + 'AutoloadClasses' => 'removeAbsolutePath', + 'ExtensionCredits' => 'handleCredits', + 'ResourceModules' => 'handleResourceModules', + 'ResourceModuleSkinStyles' => 'handleResourceModules', + 'Hooks' => 'handleHooks', + 'ExtensionFunctions' => 'handleExtensionFunctions', + 'ParserTestFiles' => 'removeAbsolutePath', + ]; + + /** + * Things that were formerly globals and should still be converted + * + * @var array + */ + protected $formerGlobals = [ + 'TrackingCategories', + ]; + + /** + * No longer supported globals (with reason) should not be converted and emit a warning + * + * @var array + */ + protected $noLongerSupportedGlobals = [ + 'SpecialPageGroups' => 'deprecated', // Deprecated 1.21, removed in 1.26 + ]; + + /** + * Keys that should be put at the top of the generated JSON file (T86608) + * + * @var array + */ + protected $promote = [ + 'name', + 'namemsg', + 'version', + 'author', + 'url', + 'description', + 'descriptionmsg', + 'license-name', + 'type', + ]; + + private $json, $dir, $hasWarning = false; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Converts extension entry points to the new JSON registration format' ); + $this->addArg( 'path', 'Location to the PHP entry point you wish to convert', + /* $required = */ true ); + $this->addOption( 'skin', 'Whether to write to skin.json', false, false ); + $this->addOption( 'config-prefix', 'Custom prefix for configuration settings', false, true ); + } + + protected function getAllGlobals() { + $processor = new ReflectionClass( 'ExtensionProcessor' ); + $settings = $processor->getProperty( 'globalSettings' ); + $settings->setAccessible( true ); + return array_merge( $settings->getValue(), $this->formerGlobals ); + } + + public function execute() { + // Extensions will do stuff like $wgResourceModules += array(...) which is a + // fatal unless an array is already set. So set an empty value. + // And use the weird $__settings name to avoid any conflicts + // with real poorly named settings. + $__settings = array_merge( $this->getAllGlobals(), array_keys( $this->custom ) ); + foreach ( $__settings as $var ) { + $var = 'wg' . $var; + $$var = []; + } + unset( $var ); + $arg = $this->getArg( 0 ); + if ( !is_file( $arg ) ) { + $this->error( "$arg is not a file.", true ); + } + require $arg; + unset( $arg ); + // Try not to create any local variables before this line + $vars = get_defined_vars(); + unset( $vars['this'] ); + unset( $vars['__settings'] ); + $this->dir = dirname( realpath( $this->getArg( 0 ) ) ); + $this->json = []; + $globalSettings = $this->getAllGlobals(); + $configPrefix = $this->getOption( 'config-prefix', 'wg' ); + if ( $configPrefix !== 'wg' ) { + $this->json['config']['_prefix'] = $configPrefix; + } + foreach ( $vars as $name => $value ) { + $realName = substr( $name, 2 ); // Strip 'wg' + if ( $realName === false ) { + continue; + } + + // If it's an empty array that we likely set, skip it + if ( is_array( $value ) && count( $value ) === 0 && in_array( $realName, $__settings ) ) { + continue; + } + + if ( isset( $this->custom[$realName] ) ) { + call_user_func_array( [ $this, $this->custom[$realName] ], + [ $realName, $value, $vars ] ); + } elseif ( in_array( $realName, $globalSettings ) ) { + $this->json[$realName] = $value; + } elseif ( array_key_exists( $realName, $this->noLongerSupportedGlobals ) ) { + $this->output( 'Warning: Skipped global "' . $name . '" (' . + $this->noLongerSupportedGlobals[$realName] . '). ' . + "Please update the entry point before convert to registration.\n" ); + $this->hasWarning = true; + } elseif ( strpos( $name, $configPrefix ) === 0 ) { + // Most likely a config setting + $this->json['config'][substr( $name, strlen( $configPrefix ) )] = [ 'value' => $value ]; + } elseif ( $configPrefix !== 'wg' && strpos( $name, 'wg' ) === 0 ) { + // Warn about this + $this->output( 'Warning: Skipped global "' . $name . '" (' . + 'config prefix is "' . $configPrefix . '"). ' . + "Please check that this setting isn't needed.\n" ); + } + } + + // check, if the extension requires composer libraries + if ( $this->needsComposerAutoloader( dirname( $this->getArg( 0 ) ) ) ) { + // set the load composer autoloader automatically property + $this->output( "Detected composer dependencies, setting 'load_composer_autoloader' to true.\n" ); + $this->json['load_composer_autoloader'] = true; + } + + // Move some keys to the top + $out = []; + foreach ( $this->promote as $key ) { + if ( isset( $this->json[$key] ) ) { + $out[$key] = $this->json[$key]; + unset( $this->json[$key] ); + } + } + $out += $this->json; + // Put this at the bottom + $out['manifest_version'] = ExtensionRegistry::MANIFEST_VERSION; + $type = $this->hasOption( 'skin' ) ? 'skin' : 'extension'; + $fname = "{$this->dir}/$type.json"; + $prettyJSON = FormatJson::encode( $out, "\t", FormatJson::ALL_OK ); + file_put_contents( $fname, $prettyJSON . "\n" ); + $this->output( "Wrote output to $fname.\n" ); + if ( $this->hasWarning ) { + $this->output( "Found warnings! Please resolve the warnings and rerun this script.\n" ); + } + } + + protected function handleExtensionFunctions( $realName, $value ) { + foreach ( $value as $func ) { + if ( $func instanceof Closure ) { + $this->error( "Error: Closures cannot be converted to JSON. " . + "Please move your extension function somewhere else.", 1 + ); + } + // check if $func exists in the global scope + if ( function_exists( $func ) ) { + $this->error( "Error: Global functions cannot be converted to JSON. " . + "Please move your extension function ($func) into a class.", 1 + ); + } + } + + $this->json[$realName] = $value; + } + + protected function handleMessagesDirs( $realName, $value ) { + foreach ( $value as $key => $dirs ) { + foreach ( (array)$dirs as $dir ) { + $this->json[$realName][$key][] = $this->stripPath( $dir, $this->dir ); + } + } + } + + protected function handleExtensionMessagesFiles( $realName, $value, $vars ) { + foreach ( $value as $key => $file ) { + $strippedFile = $this->stripPath( $file, $this->dir ); + if ( isset( $vars['wgMessagesDirs'][$key] ) ) { + $this->output( + "Note: Ignoring PHP shim $strippedFile. " . + "If your extension no longer supports versions of MediaWiki " . + "older than 1.23.0, you can safely delete it.\n" + ); + } else { + $this->json[$realName][$key] = $strippedFile; + } + } + } + + private function stripPath( $val, $dir ) { + if ( $val === $dir ) { + $val = ''; + } elseif ( strpos( $val, $dir ) === 0 ) { + // +1 is for the trailing / that won't be in $this->dir + $val = substr( $val, strlen( $dir ) + 1 ); + } + + return $val; + } + + protected function removeAbsolutePath( $realName, $value ) { + $out = []; + foreach ( $value as $key => $val ) { + $out[$key] = $this->stripPath( $val, $this->dir ); + } + $this->json[$realName] = $out; + } + + protected function handleCredits( $realName, $value ) { + $keys = array_keys( $value ); + $this->json['type'] = $keys[0]; + $values = array_values( $value ); + foreach ( $values[0][0] as $name => $val ) { + if ( $name !== 'path' ) { + $this->json[$name] = $val; + } + } + } + + public function handleHooks( $realName, $value ) { + foreach ( $value as $hookName => &$handlers ) { + if ( $hookName === 'UnitTestsList' ) { + $this->output( "Note: the UnitTestsList hook is no longer necessary as " . + "long as your tests are located in the \"tests/phpunit/\" directory. " . + "Please see <https://www.mediawiki.org/wiki/Manual:PHP_unit_testing/" . + "Writing_unit_tests_for_extensions#Register_your_tests> for more details.\n" + ); + } + foreach ( $handlers as $func ) { + if ( $func instanceof Closure ) { + $this->error( "Error: Closures cannot be converted to JSON. " . + "Please move the handler for $hookName somewhere else.", 1 + ); + } + // Check if $func exists in the global scope + if ( function_exists( $func ) ) { + $this->error( "Error: Global functions cannot be converted to JSON. " . + "Please move the handler for $hookName inside a class.", 1 + ); + } + } + if ( count( $handlers ) === 1 ) { + $handlers = $handlers[0]; + } + } + $this->json[$realName] = $value; + } + + protected function handleResourceModules( $realName, $value ) { + $defaults = []; + $remote = $this->hasOption( 'skin' ) ? 'remoteSkinPath' : 'remoteExtPath'; + foreach ( $value as $name => $data ) { + if ( isset( $data['localBasePath'] ) ) { + $data['localBasePath'] = $this->stripPath( $data['localBasePath'], $this->dir ); + if ( !$defaults ) { + $defaults['localBasePath'] = $data['localBasePath']; + unset( $data['localBasePath'] ); + if ( isset( $data[$remote] ) ) { + $defaults[$remote] = $data[$remote]; + unset( $data[$remote] ); + } + } else { + if ( $data['localBasePath'] === $defaults['localBasePath'] ) { + unset( $data['localBasePath'] ); + } + if ( isset( $data[$remote] ) && isset( $defaults[$remote] ) + && $data[$remote] === $defaults[$remote] + ) { + unset( $data[$remote] ); + } + } + } + + $this->json[$realName][$name] = $data; + } + if ( $defaults ) { + $this->json['ResourceFileModulePaths'] = $defaults; + } + } + + protected function needsComposerAutoloader( $path ) { + $path .= '/composer.json'; + if ( file_exists( $path ) ) { + // assume, that the composer.json file is in the root of the extension path + $composerJson = new ComposerJson( $path ); + // check, if there are some dependencies in the require section + if ( $composerJson->getRequiredDependencies() ) { + return true; + } + } + return false; + } +} + +$maintClass = 'ConvertExtensionToRegistration'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/convertLinks.php b/www/wiki/maintenance/convertLinks.php new file mode 100644 index 00000000..54c0edae --- /dev/null +++ b/www/wiki/maintenance/convertLinks.php @@ -0,0 +1,306 @@ +<?php +/** + * Convert from the old links schema (string->ID) to the new schema (ID->ID). + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to convert from the old links schema (string->ID) + * to the new schema (ID->ID). + * + * The wiki should be put into read-only mode while this script executes. + * + * @ingroup Maintenance + */ +class ConvertLinks extends Maintenance { + private $logPerformance; + + public function __construct() { + parent::__construct(); + $this->addDescription( + 'Convert from the old links schema (string->ID) to the new schema (ID->ID). ' + . 'The wiki should be put into read-only mode while this script executes' ); + + $this->addArg( 'logperformance', "Log performance to perfLogFilename.", false ); + $this->addArg( + 'perfLogFilename', + "Filename where performance is logged if --logperformance was set " + . "(defaults to 'convLinksPerf.txt').", + false + ); + $this->addArg( + 'keep-links-table', + "Don't overwrite the old links table with the new one, leave the new table at links_temp.", + false + ); + $this->addArg( + 'nokeys', + /* (What about InnoDB?) */ + "Don't create keys, and so allow duplicates in the new links table.\n" + . "This gives a huge speed improvement for very large links tables which are MyISAM.", + false + ); + } + + public function getDbType() { + return Maintenance::DB_ADMIN; + } + + public function execute() { + $dbw = $this->getDB( DB_MASTER ); + + $type = $dbw->getType(); + if ( $type != 'mysql' ) { + $this->output( "Link table conversion not necessary for $type\n" ); + + return; + } + + global $wgContLang; + + # counters etc + $numBadLinks = $curRowsRead = 0; + + # total tuples INSERTed into links_temp + $totalTuplesInserted = 0; + + # whether or not to give progress reports while reading IDs from cur table + $reportCurReadProgress = true; + + # number of rows between progress reports + $curReadReportInterval = 1000; + + # whether or not to give progress reports during conversion + $reportLinksConvProgress = true; + + # number of rows per INSERT + $linksConvInsertInterval = 1000; + + $initialRowOffset = 0; + + # not used yet; highest row number from links table to process + # $finalRowOffset = 0; + + $overwriteLinksTable = !$this->hasOption( 'keep-links-table' ); + $noKeys = $this->hasOption( 'noKeys' ); + $this->logPerformance = $this->hasOption( 'logperformance' ); + $perfLogFilename = $this->getArg( 'perfLogFilename', "convLinksPerf.txt" ); + + # -------------------------------------------------------------------- + + list( $cur, $links, $links_temp, $links_backup ) = + $dbw->tableNamesN( 'cur', 'links', 'links_temp', 'links_backup' ); + + if ( $dbw->tableExists( 'pagelinks' ) ) { + $this->output( "...have pagelinks; skipping old links table updates\n" ); + + return; + } + + $res = $dbw->query( "SELECT l_from FROM $links LIMIT 1" ); + if ( $dbw->fieldType( $res, 0 ) == "int" ) { + $this->output( "Schema already converted\n" ); + + return; + } + + $res = $dbw->query( "SELECT COUNT(*) AS count FROM $links" ); + $row = $dbw->fetchObject( $res ); + $numRows = $row->count; + $dbw->freeResult( $res ); + + if ( $numRows == 0 ) { + $this->output( "Updating schema (no rows to convert)...\n" ); + $this->createTempTable(); + } else { + $fh = false; + if ( $this->logPerformance ) { + $fh = fopen( $perfLogFilename, "w" ); + if ( !$fh ) { + $this->error( "Couldn't open $perfLogFilename" ); + $this->logPerformance = false; + } + } + $baseTime = $startTime = microtime( true ); + # Create a title -> cur_id map + $this->output( "Loading IDs from $cur table...\n" ); + $this->performanceLog( $fh, "Reading $numRows rows from cur table...\n" ); + $this->performanceLog( $fh, "rows read vs seconds elapsed:\n" ); + + $dbw->bufferResults( false ); + $res = $dbw->query( "SELECT cur_namespace,cur_title,cur_id FROM $cur" ); + $ids = []; + + foreach ( $res as $row ) { + $title = $row->cur_title; + if ( $row->cur_namespace ) { + $title = $wgContLang->getNsText( $row->cur_namespace ) . ":$title"; + } + $ids[$title] = $row->cur_id; + $curRowsRead++; + if ( $reportCurReadProgress ) { + if ( ( $curRowsRead % $curReadReportInterval ) == 0 ) { + $this->performanceLog( + $fh, + $curRowsRead . " " . ( microtime( true ) - $baseTime ) . "\n" + ); + $this->output( "\t$curRowsRead rows of $cur table read.\n" ); + } + } + } + $dbw->freeResult( $res ); + $dbw->bufferResults( true ); + $this->output( "Finished loading IDs.\n\n" ); + $this->performanceLog( + $fh, + "Took " . ( microtime( true ) - $baseTime ) . " seconds to load IDs.\n\n" + ); + + # -------------------------------------------------------------------- + + # Now, step through the links table (in chunks of $linksConvInsertInterval rows), + # convert, and write to the new table. + $this->createTempTable(); + $this->performanceLog( $fh, "Resetting timer.\n\n" ); + $baseTime = microtime( true ); + $this->output( "Processing $numRows rows from $links table...\n" ); + $this->performanceLog( $fh, "Processing $numRows rows from $links table...\n" ); + $this->performanceLog( $fh, "rows inserted vs seconds elapsed:\n" ); + + for ( $rowOffset = $initialRowOffset; $rowOffset < $numRows; + $rowOffset += $linksConvInsertInterval + ) { + $sqlRead = "SELECT * FROM $links "; + $sqlRead = $dbw->limitResult( $sqlRead, $linksConvInsertInterval, $rowOffset ); + $res = $dbw->query( $sqlRead ); + if ( $noKeys ) { + $sqlWrite = [ "INSERT INTO $links_temp (l_from,l_to) VALUES " ]; + } else { + $sqlWrite = [ "INSERT IGNORE INTO $links_temp (l_from,l_to) VALUES " ]; + } + + $tuplesAdded = 0; # no tuples added to INSERT yet + foreach ( $res as $row ) { + $fromTitle = $row->l_from; + if ( array_key_exists( $fromTitle, $ids ) ) { # valid title + $from = $ids[$fromTitle]; + $to = $row->l_to; + if ( $tuplesAdded != 0 ) { + $sqlWrite[] = ","; + } + $sqlWrite[] = "($from,$to)"; + $tuplesAdded++; + } else { # invalid title + $numBadLinks++; + } + } + $dbw->freeResult( $res ); + # $this->output( "rowOffset: $rowOffset\ttuplesAdded: " + # . "$tuplesAdded\tnumBadLinks: $numBadLinks\n" ); + if ( $tuplesAdded != 0 ) { + if ( $reportLinksConvProgress ) { + $this->output( "Inserting $tuplesAdded tuples into $links_temp..." ); + } + $dbw->query( implode( "", $sqlWrite ) ); + $totalTuplesInserted += $tuplesAdded; + if ( $reportLinksConvProgress ) { + $this->output( " done. Total $totalTuplesInserted tuples inserted.\n" ); + $this->performanceLog( + $fh, + $totalTuplesInserted . " " . ( microtime( true ) - $baseTime ) . "\n" + ); + } + } + } + $this->output( "$totalTuplesInserted valid titles and " + . "$numBadLinks invalid titles were processed.\n\n" ); + $this->performanceLog( + $fh, + "$totalTuplesInserted valid titles and $numBadLinks invalid titles were processed.\n" + ); + $this->performanceLog( + $fh, + "Total execution time: " . ( microtime( true ) - $startTime ) . " seconds.\n" + ); + if ( $this->logPerformance ) { + fclose( $fh ); + } + } + # -------------------------------------------------------------------- + + if ( $overwriteLinksTable ) { + # Check for existing links_backup, and delete it if it exists. + $this->output( "Dropping backup links table if it exists..." ); + $dbw->query( "DROP TABLE IF EXISTS $links_backup", __METHOD__ ); + $this->output( " done.\n" ); + + # Swap in the new table, and move old links table to links_backup + $this->output( "Swapping tables '$links' to '$links_backup'; '$links_temp' to '$links'..." ); + $dbw->query( "RENAME TABLE links TO $links_backup, $links_temp TO $links", __METHOD__ ); + $this->output( " done.\n\n" ); + + $this->output( "Conversion complete. The old table remains at $links_backup;\n" ); + $this->output( "delete at your leisure.\n" ); + } else { + $this->output( "Conversion complete. The converted table is at $links_temp;\n" ); + $this->output( "the original links table is unchanged.\n" ); + } + } + + private function createTempTable() { + $dbConn = $this->getDB( DB_MASTER ); + + if ( !( $dbConn->isOpen() ) ) { + $this->output( "Opening connection to database failed.\n" ); + + return; + } + $links_temp = $dbConn->tableName( 'links_temp' ); + + $this->output( "Dropping temporary links table if it exists..." ); + $dbConn->query( "DROP TABLE IF EXISTS $links_temp" ); + $this->output( " done.\n" ); + + $this->output( "Creating temporary links table..." ); + if ( $this->hasOption( 'noKeys' ) ) { + $dbConn->query( "CREATE TABLE $links_temp ( " . + "l_from int(8) unsigned NOT NULL default '0', " . + "l_to int(8) unsigned NOT NULL default '0')" ); + } else { + $dbConn->query( "CREATE TABLE $links_temp ( " . + "l_from int(8) unsigned NOT NULL default '0', " . + "l_to int(8) unsigned NOT NULL default '0', " . + "UNIQUE KEY l_from(l_from,l_to), " . + "KEY (l_to))" ); + } + $this->output( " done.\n\n" ); + } + + private function performanceLog( $fh, $text ) { + if ( $this->logPerformance ) { + fwrite( $fh, $text ); + } + } +} + +$maintClass = "ConvertLinks"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/convertUserOptions.php b/www/wiki/maintenance/convertUserOptions.php new file mode 100644 index 00000000..675d0695 --- /dev/null +++ b/www/wiki/maintenance/convertUserOptions.php @@ -0,0 +1,124 @@ +<?php +/** + * Convert user options to the new `user_properties` table. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +use Wikimedia\Rdbms\ResultWrapper; +use Wikimedia\Rdbms\IDatabase; + +/** + * Maintenance script to convert user options to the new `user_properties` table. + * + * @ingroup Maintenance + */ +class ConvertUserOptions extends Maintenance { + + private $mConversionCount = 0; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Convert user options from old to new system' ); + $this->setBatchSize( 50 ); + } + + public function execute() { + $this->output( "...batch conversion of user_options: " ); + $id = 0; + $dbw = $this->getDB( DB_MASTER ); + + if ( !$dbw->fieldExists( 'user', 'user_options', __METHOD__ ) ) { + $this->output( "nothing to migrate. " ); + + return; + } + while ( $id !== null ) { + $res = $dbw->select( 'user', + [ 'user_id', 'user_options' ], + [ + 'user_id > ' . $dbw->addQuotes( $id ), + "user_options != " . $dbw->addQuotes( '' ), + ], + __METHOD__, + [ + 'ORDER BY' => 'user_id', + 'LIMIT' => $this->mBatchSize, + ] + ); + $id = $this->convertOptionBatch( $res, $dbw ); + + wfWaitForSlaves(); + + if ( $id ) { + $this->output( "--Converted to ID $id\n" ); + } + } + $this->output( "done. Converted " . $this->mConversionCount . " user records.\n" ); + } + + /** + * @param ResultWrapper $res + * @param IDatabase $dbw + * @return null|int + */ + function convertOptionBatch( $res, $dbw ) { + $id = null; + foreach ( $res as $row ) { + $this->mConversionCount++; + $insertRows = []; + foreach ( explode( "\n", $row->user_options ) as $s ) { + $m = []; + if ( !preg_match( "/^(.[^=]*)=(.*)$/", $s, $m ) ) { + continue; + } + + // MW < 1.16 would save even default values. Filter them out + // here (as in User) to avoid adding many unnecessary rows. + $defaultOption = User::getDefaultOption( $m[1] ); + if ( is_null( $defaultOption ) || $m[2] != $defaultOption ) { + $insertRows[] = [ + 'up_user' => $row->user_id, + 'up_property' => $m[1], + 'up_value' => $m[2], + ]; + } + } + + if ( count( $insertRows ) ) { + $dbw->insert( 'user_properties', $insertRows, __METHOD__, [ 'IGNORE' ] ); + } + + $dbw->update( + 'user', + [ 'user_options' => '' ], + [ 'user_id' => $row->user_id ], + __METHOD__ + ); + $id = $row->user_id; + } + + return $id; + } +} + +$maintClass = "ConvertUserOptions"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/copyFileBackend.php b/www/wiki/maintenance/copyFileBackend.php new file mode 100644 index 00000000..4f625fc6 --- /dev/null +++ b/www/wiki/maintenance/copyFileBackend.php @@ -0,0 +1,378 @@ +<?php +/** + * Copy all files in some containers of one backend to another. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Copy all files in one container of one backend to another. + * + * This can also be used to re-shard the files for one backend using the + * config of second backend. The second backend should have the same config + * as the first, except for it having a different name and different sharding + * configuration. The backend should be made read-only while this runs. + * After this script finishes, the old files in the containers can be deleted. + * + * @ingroup Maintenance + */ +class CopyFileBackend extends Maintenance { + /** @var array|null (path sha1 => stat) Pre-computed dst stat entries from listings */ + protected $statCache = null; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Copy files in one backend to another.' ); + $this->addOption( 'src', 'Backend containing the source files', true, true ); + $this->addOption( 'dst', 'Backend where files should be copied to', true, true ); + $this->addOption( 'containers', 'Pipe separated list of containers', true, true ); + $this->addOption( 'subdir', 'Only do items in this child directory', false, true ); + $this->addOption( 'ratefile', 'File to check periodically for batch size', false, true ); + $this->addOption( 'prestat', 'Stat the destination files first (try to use listings)' ); + $this->addOption( 'skiphash', 'Skip SHA-1 sync checks for files' ); + $this->addOption( 'missingonly', 'Only copy files missing from destination listing' ); + $this->addOption( 'syncviadelete', 'Delete destination files missing from source listing' ); + $this->addOption( 'utf8only', 'Skip source files that do not have valid UTF-8 names' ); + $this->setBatchSize( 50 ); + } + + public function execute() { + $src = FileBackendGroup::singleton()->get( $this->getOption( 'src' ) ); + $dst = FileBackendGroup::singleton()->get( $this->getOption( 'dst' ) ); + $containers = explode( '|', $this->getOption( 'containers' ) ); + $subDir = rtrim( $this->getOption( 'subdir', '' ), '/' ); + + $rateFile = $this->getOption( 'ratefile' ); + + foreach ( $containers as $container ) { + if ( $subDir != '' ) { + $backendRel = "$container/$subDir"; + $this->output( "Doing container '$container', directory '$subDir'...\n" ); + } else { + $backendRel = $container; + $this->output( "Doing container '$container'...\n" ); + } + + if ( $this->hasOption( 'missingonly' ) ) { + $this->output( "\tBuilding list of missing files..." ); + $srcPathsRel = $this->getListingDiffRel( $src, $dst, $backendRel ); + $this->output( count( $srcPathsRel ) . " file(s) need to be copied.\n" ); + } else { + $srcPathsRel = $src->getFileList( [ + 'dir' => $src->getRootStoragePath() . "/$backendRel", + 'adviseStat' => true // avoid HEADs + ] ); + if ( $srcPathsRel === null ) { + $this->error( "Could not list files in $container.", 1 ); // die + } + } + + if ( $this->getOption( 'prestat' ) && !$this->hasOption( 'missingonly' ) ) { + // Build the stat cache for the destination files + $this->output( "\tBuilding destination stat cache..." ); + $dstPathsRel = $dst->getFileList( [ + 'dir' => $dst->getRootStoragePath() . "/$backendRel", + 'adviseStat' => true // avoid HEADs + ] ); + if ( $dstPathsRel === null ) { + $this->error( "Could not list files in $container.", 1 ); // die + } + $this->statCache = []; + foreach ( $dstPathsRel as $dstPathRel ) { + $path = $dst->getRootStoragePath() . "/$backendRel/$dstPathRel"; + $this->statCache[sha1( $path )] = $dst->getFileStat( [ 'src' => $path ] ); + } + $this->output( "done [" . count( $this->statCache ) . " file(s)]\n" ); + } + + $this->output( "\tCopying file(s)...\n" ); + $count = 0; + $batchPaths = []; + foreach ( $srcPathsRel as $srcPathRel ) { + // Check up on the rate file periodically to adjust the concurrency + if ( $rateFile && ( !$count || ( $count % 500 ) == 0 ) ) { + $this->mBatchSize = max( 1, (int)file_get_contents( $rateFile ) ); + $this->output( "\tBatch size is now {$this->mBatchSize}.\n" ); + } + $batchPaths[$srcPathRel] = 1; // remove duplicates + if ( count( $batchPaths ) >= $this->mBatchSize ) { + $this->copyFileBatch( array_keys( $batchPaths ), $backendRel, $src, $dst ); + $batchPaths = []; // done + } + ++$count; + } + if ( count( $batchPaths ) ) { // left-overs + $this->copyFileBatch( array_keys( $batchPaths ), $backendRel, $src, $dst ); + $batchPaths = []; // done + } + $this->output( "\tCopied $count file(s).\n" ); + + if ( $this->hasOption( 'syncviadelete' ) ) { + $this->output( "\tBuilding list of excess destination files..." ); + $delPathsRel = $this->getListingDiffRel( $dst, $src, $backendRel ); + $this->output( count( $delPathsRel ) . " file(s) need to be deleted.\n" ); + + $this->output( "\tDeleting file(s)...\n" ); + $count = 0; + $batchPaths = []; + foreach ( $delPathsRel as $delPathRel ) { + // Check up on the rate file periodically to adjust the concurrency + if ( $rateFile && ( !$count || ( $count % 500 ) == 0 ) ) { + $this->mBatchSize = max( 1, (int)file_get_contents( $rateFile ) ); + $this->output( "\tBatch size is now {$this->mBatchSize}.\n" ); + } + $batchPaths[$delPathRel] = 1; // remove duplicates + if ( count( $batchPaths ) >= $this->mBatchSize ) { + $this->delFileBatch( array_keys( $batchPaths ), $backendRel, $dst ); + $batchPaths = []; // done + } + ++$count; + } + if ( count( $batchPaths ) ) { // left-overs + $this->delFileBatch( array_keys( $batchPaths ), $backendRel, $dst ); + $batchPaths = []; // done + } + + $this->output( "\tDeleted $count file(s).\n" ); + } + + if ( $subDir != '' ) { + $this->output( "Finished container '$container', directory '$subDir'.\n" ); + } else { + $this->output( "Finished container '$container'.\n" ); + } + } + + $this->output( "Done.\n" ); + } + + /** + * @param FileBackend $src + * @param FileBackend $dst + * @param string $backendRel + * @return array (rel paths in $src minus those in $dst) + */ + protected function getListingDiffRel( FileBackend $src, FileBackend $dst, $backendRel ) { + $srcPathsRel = $src->getFileList( [ + 'dir' => $src->getRootStoragePath() . "/$backendRel" ] ); + if ( $srcPathsRel === null ) { + $this->error( "Could not list files in source container.", 1 ); // die + } + $dstPathsRel = $dst->getFileList( [ + 'dir' => $dst->getRootStoragePath() . "/$backendRel" ] ); + if ( $dstPathsRel === null ) { + $this->error( "Could not list files in destination container.", 1 ); // die + } + // Get the list of destination files + $relFilesDstSha1 = []; + foreach ( $dstPathsRel as $dstPathRel ) { + $relFilesDstSha1[sha1( $dstPathRel )] = 1; + } + unset( $dstPathsRel ); // free + // Get the list of missing files + $missingPathsRel = []; + foreach ( $srcPathsRel as $srcPathRel ) { + if ( !isset( $relFilesDstSha1[sha1( $srcPathRel )] ) ) { + $missingPathsRel[] = $srcPathRel; + } + } + unset( $srcPathsRel ); // free + + return $missingPathsRel; + } + + /** + * @param array $srcPathsRel + * @param string $backendRel + * @param FileBackend $src + * @param FileBackend $dst + * @return void + */ + protected function copyFileBatch( + array $srcPathsRel, $backendRel, FileBackend $src, FileBackend $dst + ) { + $ops = []; + $fsFiles = []; + $copiedRel = []; // for output message + $wikiId = $src->getWikiId(); + + // Download the batch of source files into backend cache... + if ( $this->hasOption( 'missingonly' ) ) { + $srcPaths = []; + foreach ( $srcPathsRel as $srcPathRel ) { + $srcPaths[] = $src->getRootStoragePath() . "/$backendRel/$srcPathRel"; + } + $t_start = microtime( true ); + $fsFiles = $src->getLocalReferenceMulti( [ 'srcs' => $srcPaths, 'latest' => 1 ] ); + $elapsed_ms = floor( ( microtime( true ) - $t_start ) * 1000 ); + $this->output( "\n\tDownloaded these file(s) [{$elapsed_ms}ms]:\n\t" . + implode( "\n\t", $srcPaths ) . "\n\n" ); + } + + // Determine what files need to be copied over... + foreach ( $srcPathsRel as $srcPathRel ) { + $srcPath = $src->getRootStoragePath() . "/$backendRel/$srcPathRel"; + $dstPath = $dst->getRootStoragePath() . "/$backendRel/$srcPathRel"; + if ( $this->hasOption( 'utf8only' ) && !mb_check_encoding( $srcPath, 'UTF-8' ) ) { + $this->error( "$wikiId: Detected illegal (non-UTF8) path for $srcPath." ); + continue; + } elseif ( !$this->hasOption( 'missingonly' ) + && $this->filesAreSame( $src, $dst, $srcPath, $dstPath ) + ) { + $this->output( "\tAlready have $srcPathRel.\n" ); + continue; // assume already copied... + } + $fsFile = array_key_exists( $srcPath, $fsFiles ) + ? $fsFiles[$srcPath] + : $src->getLocalReference( [ 'src' => $srcPath, 'latest' => 1 ] ); + if ( !$fsFile ) { + $src->clearCache( [ $srcPath ] ); + if ( $src->fileExists( [ 'src' => $srcPath, 'latest' => 1 ] ) === false ) { + $this->error( "$wikiId: File '$srcPath' was listed but does not exist." ); + } else { + $this->error( "$wikiId: Could not get local copy of $srcPath." ); + } + continue; + } elseif ( !$fsFile->exists() ) { + // FSFileBackends just return the path for getLocalReference() and paths with + // illegal slashes may get normalized to a different path. This can cause the + // local reference to not exist...skip these broken files. + $this->error( "$wikiId: Detected possible illegal path for $srcPath." ); + continue; + } + $fsFiles[] = $fsFile; // keep TempFSFile objects alive as needed + // Note: prepare() is usually fast for key/value backends + $status = $dst->prepare( [ 'dir' => dirname( $dstPath ), 'bypassReadOnly' => 1 ] ); + if ( !$status->isOK() ) { + $this->error( print_r( $status->getErrorsArray(), true ) ); + $this->error( "$wikiId: Could not copy $srcPath to $dstPath.", 1 ); // die + } + $ops[] = [ 'op' => 'store', + 'src' => $fsFile->getPath(), 'dst' => $dstPath, 'overwrite' => 1 ]; + $copiedRel[] = $srcPathRel; + } + + // Copy in the batch of source files... + $t_start = microtime( true ); + $status = $dst->doQuickOperations( $ops, [ 'bypassReadOnly' => 1 ] ); + if ( !$status->isOK() ) { + sleep( 10 ); // wait and retry copy again + $status = $dst->doQuickOperations( $ops, [ 'bypassReadOnly' => 1 ] ); + } + $elapsed_ms = floor( ( microtime( true ) - $t_start ) * 1000 ); + if ( !$status->isOK() ) { + $this->error( print_r( $status->getErrorsArray(), true ) ); + $this->error( "$wikiId: Could not copy file batch.", 1 ); // die + } elseif ( count( $copiedRel ) ) { + $this->output( "\n\tCopied these file(s) [{$elapsed_ms}ms]:\n\t" . + implode( "\n\t", $copiedRel ) . "\n\n" ); + } + } + + /** + * @param array $dstPathsRel + * @param string $backendRel + * @param FileBackend $dst + * @return void + */ + protected function delFileBatch( + array $dstPathsRel, $backendRel, FileBackend $dst + ) { + $ops = []; + $deletedRel = []; // for output message + $wikiId = $dst->getWikiId(); + + // Determine what files need to be copied over... + foreach ( $dstPathsRel as $dstPathRel ) { + $dstPath = $dst->getRootStoragePath() . "/$backendRel/$dstPathRel"; + $ops[] = [ 'op' => 'delete', 'src' => $dstPath ]; + $deletedRel[] = $dstPathRel; + } + + // Delete the batch of source files... + $t_start = microtime( true ); + $status = $dst->doQuickOperations( $ops, [ 'bypassReadOnly' => 1 ] ); + if ( !$status->isOK() ) { + sleep( 10 ); // wait and retry copy again + $status = $dst->doQuickOperations( $ops, [ 'bypassReadOnly' => 1 ] ); + } + $elapsed_ms = floor( ( microtime( true ) - $t_start ) * 1000 ); + if ( !$status->isOK() ) { + $this->error( print_r( $status->getErrorsArray(), true ) ); + $this->error( "$wikiId: Could not delete file batch.", 1 ); // die + } elseif ( count( $deletedRel ) ) { + $this->output( "\n\tDeleted these file(s) [{$elapsed_ms}ms]:\n\t" . + implode( "\n\t", $deletedRel ) . "\n\n" ); + } + } + + /** + * @param FileBackend $src + * @param FileBackend $dst + * @param string $sPath + * @param string $dPath + * @return bool + */ + protected function filesAreSame( FileBackend $src, FileBackend $dst, $sPath, $dPath ) { + $skipHash = $this->hasOption( 'skiphash' ); + $srcStat = $src->getFileStat( [ 'src' => $sPath ] ); + $dPathSha1 = sha1( $dPath ); + if ( $this->statCache !== null ) { + // All dst files are already in stat cache + $dstStat = isset( $this->statCache[$dPathSha1] ) + ? $this->statCache[$dPathSha1] + : false; + } else { + $dstStat = $dst->getFileStat( [ 'src' => $dPath ] ); + } + // Initial fast checks to see if files are obviously different + $sameFast = ( + is_array( $srcStat ) // sanity check that source exists + && is_array( $dstStat ) // dest exists + && $srcStat['size'] === $dstStat['size'] + ); + // More thorough checks against files + if ( !$sameFast ) { + $same = false; // no need to look farther + } elseif ( isset( $srcStat['md5'] ) && isset( $dstStat['md5'] ) ) { + // If MD5 was already in the stat info, just use it. + // This is useful as many objects stores can return this in object listing, + // so we can use it to avoid slow per-file HEADs. + $same = ( $srcStat['md5'] === $dstStat['md5'] ); + } elseif ( $skipHash ) { + // This mode is good for copying to a backup location or resyncing clone + // backends in FileBackendMultiWrite (since they get writes second, they have + // higher timestamps). However, when copying the other way, this hits loads of + // false positives (possibly 100%) and wastes a bunch of time on GETs/PUTs. + $same = ( $srcStat['mtime'] <= $dstStat['mtime'] ); + } else { + // This is the slowest method which does many per-file HEADs (unless an object + // store tracks SHA-1 in listings). + $same = ( $src->getFileSha1Base36( [ 'src' => $sPath, 'latest' => 1 ] ) + === $dst->getFileSha1Base36( [ 'src' => $dPath, 'latest' => 1 ] ) ); + } + + return $same; + } +} + +$maintClass = 'CopyFileBackend'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/copyJobQueue.php b/www/wiki/maintenance/copyJobQueue.php new file mode 100644 index 00000000..e1d697d8 --- /dev/null +++ b/www/wiki/maintenance/copyJobQueue.php @@ -0,0 +1,98 @@ +<?php +/** + * Copy all jobs from one job queue system to another. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Copy all jobs from one job queue system to another. + * This uses an ad-hoc $wgJobQueueMigrationConfig setting, + * which is a map of queue system names to JobQueue::factory() parameters. + * The parameters should not have wiki or type settings and thus partial. + * + * @ingroup Maintenance + */ +class CopyJobQueue extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Copy jobs from one queue system to another.' ); + $this->addOption( 'src', 'Key to $wgJobQueueMigrationConfig for source', true, true ); + $this->addOption( 'dst', 'Key to $wgJobQueueMigrationConfig for destination', true, true ); + $this->addOption( 'type', 'Types of jobs to copy (use "all" for all)', true, true ); + $this->setBatchSize( 500 ); + } + + public function execute() { + global $wgJobQueueMigrationConfig; + + $srcKey = $this->getOption( 'src' ); + $dstKey = $this->getOption( 'dst' ); + + if ( !isset( $wgJobQueueMigrationConfig[$srcKey] ) ) { + $this->error( "\$wgJobQueueMigrationConfig not set for '$srcKey'.", 1 ); + } elseif ( !isset( $wgJobQueueMigrationConfig[$dstKey] ) ) { + $this->error( "\$wgJobQueueMigrationConfig not set for '$dstKey'.", 1 ); + } + + $types = ( $this->getOption( 'type' ) === 'all' ) + ? JobQueueGroup::singleton()->getQueueTypes() + : [ $this->getOption( 'type' ) ]; + + foreach ( $types as $type ) { + $baseConfig = [ 'type' => $type, 'wiki' => wfWikiID() ]; + $src = JobQueue::factory( $baseConfig + $wgJobQueueMigrationConfig[$srcKey] ); + $dst = JobQueue::factory( $baseConfig + $wgJobQueueMigrationConfig[$dstKey] ); + + list( $total, $totalOK ) = $this->copyJobs( $src, $dst, $src->getAllQueuedJobs() ); + $this->output( "Copied $totalOK/$total queued $type jobs.\n" ); + + list( $total, $totalOK ) = $this->copyJobs( $src, $dst, $src->getAllDelayedJobs() ); + $this->output( "Copied $totalOK/$total delayed $type jobs.\n" ); + } + } + + protected function copyJobs( JobQueue $src, JobQueue $dst, $jobs ) { + $total = 0; + $totalOK = 0; + $batch = []; + foreach ( $jobs as $job ) { + ++$total; + $batch[] = $job; + if ( count( $batch ) >= $this->mBatchSize ) { + $dst->push( $batch ); + $totalOK += count( $batch ); + $batch = []; + $dst->waitForBackups(); + } + } + if ( count( $batch ) ) { + $dst->push( $batch ); + $totalOK += count( $batch ); + $dst->waitForBackups(); + } + + return [ $total, $totalOK ]; + } +} + +$maintClass = 'CopyJobQueue'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/createAndPromote.php b/www/wiki/maintenance/createAndPromote.php new file mode 100644 index 00000000..1872716f --- /dev/null +++ b/www/wiki/maintenance/createAndPromote.php @@ -0,0 +1,154 @@ +<?php +/** + * Creates an account and grants it rights. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @author Rob Church <robchur@gmail.com> + * @author Pablo Castellano <pablo@anche.no> + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to create an account and grant it rights. + * + * @ingroup Maintenance + */ +class CreateAndPromote extends Maintenance { + private static $permitRoles = [ 'sysop', 'bureaucrat', 'bot' ]; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Create a new user account and/or grant it additional rights' ); + $this->addOption( + 'force', + 'If acccount exists already, just grant it rights or change password.' + ); + foreach ( self::$permitRoles as $role ) { + $this->addOption( $role, "Add the account to the {$role} group" ); + } + + $this->addOption( + 'custom-groups', + 'Comma-separated list of groups to add the user to', + false, + true + ); + + $this->addArg( "username", "Username of new user" ); + $this->addArg( "password", "Password to set (not required if --force is used)", false ); + } + + public function execute() { + $username = $this->getArg( 0 ); + $password = $this->getArg( 1 ); + $force = $this->hasOption( 'force' ); + $inGroups = []; + + $user = User::newFromName( $username ); + if ( !is_object( $user ) ) { + $this->error( "invalid username.", true ); + } + + $exists = ( 0 !== $user->idForName() ); + + if ( $exists && !$force ) { + $this->error( "Account exists. Perhaps you want the --force option?", true ); + } elseif ( !$exists && !$password ) { + $this->error( "Argument <password> required!", false ); + $this->maybeHelp( true ); + } elseif ( $exists ) { + $inGroups = $user->getGroups(); + } + + $groups = array_filter( self::$permitRoles, [ $this, 'hasOption' ] ); + if ( $this->hasOption( 'custom-groups' ) ) { + $allGroups = array_flip( User::getAllGroups() ); + $customGroupsText = $this->getOption( 'custom-groups' ); + if ( $customGroupsText !== '' ) { + $customGroups = explode( ',', $customGroupsText ); + foreach ( $customGroups as $customGroup ) { + if ( isset( $allGroups[$customGroup] ) ) { + $groups[] = trim( $customGroup ); + } else { + $this->output( "$customGroup is not a valid group, ignoring!\n" ); + } + } + } + } + + $promotions = array_diff( + $groups, + $inGroups + ); + + if ( $exists && !$password && count( $promotions ) === 0 ) { + $this->output( "Account exists and nothing to do.\n" ); + + return; + } elseif ( count( $promotions ) !== 0 ) { + $promoText = "User:{$username} into " . implode( ', ', $promotions ) . "...\n"; + if ( $exists ) { + $this->output( wfWikiID() . ": Promoting $promoText" ); + } else { + $this->output( wfWikiID() . ": Creating and promoting $promoText" ); + } + } + + if ( !$exists ) { + # Insert the account into the database + $user->addToDatabase(); + $user->saveSettings(); + } + + if ( $password ) { + # Try to set the password + try { + $status = $user->changeAuthenticationData( [ + 'username' => $user->getName(), + 'password' => $password, + 'retype' => $password, + ] ); + if ( !$status->isGood() ) { + throw new PasswordError( $status->getWikiText( null, null, 'en' ) ); + } + if ( $exists ) { + $this->output( "Password set.\n" ); + $user->saveSettings(); + } + } catch ( PasswordError $pwe ) { + $this->error( $pwe->getText(), true ); + } + } + + # Promote user + array_map( [ $user, 'addGroup' ], $promotions ); + + if ( !$exists ) { + # Increment site_stats.ss_users + $ssu = new SiteStatsUpdate( 0, 0, 0, 0, 1 ); + $ssu->doUpdate(); + } + + $this->output( "done.\n" ); + } +} + +$maintClass = "CreateAndPromote"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/createCommonPasswordCdb.php b/www/wiki/maintenance/createCommonPasswordCdb.php new file mode 100644 index 00000000..f7e0c0fb --- /dev/null +++ b/www/wiki/maintenance/createCommonPasswordCdb.php @@ -0,0 +1,118 @@ +<?php +/** + * Create serialized/commonpasswords.cdb + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to create common password cdb database. + * + * Meant to take a file like those from + * https://github.com/danielmiessler/SecLists + * For example: + * https://github.com/danielmiessler/SecLists/blob/fe2b40dd84/Passwords/rockyou.txt?raw=true + * + * @see serialized/commonpasswords.cdb and PasswordPolicyChecks::checkPopularPasswordBlacklist + * @since 1.27 + * @ingroup Maintenance + */ +class GenerateCommonPassword extends Maintenance { + public function __construct() { + global $IP; + parent::__construct(); + $this->addDescription( 'Generate CDB file of common passwords' ); + $this->addOption( 'limit', "Max number of passwords to write", false, true, 'l' ); + $this->addArg( 'inputfile', 'List of passwords (one per line) to use or - for stdin', true ); + $this->addArg( + 'output', + "Location to write CDB file to (Try $IP/serialized/commonpasswords.cdb)", + true + ); + } + + public function execute() { + $limit = (int)$this->getOption( 'limit', PHP_INT_MAX ); + $langEn = Language::factory( 'en' ); + + $infile = $this->getArg( 0 ); + if ( $infile === '-' ) { + $infile = 'php://stdin'; + } + $outfile = $this->getArg( 1 ); + + if ( !is_readable( $infile ) && $infile !== 'php://stdin' ) { + $this->error( "Cannot open input file $infile for reading", 1 ); + } + + $file = fopen( $infile, 'r' ); + if ( $file === false ) { + $this->error( "Cannot read input file $infile", 1 ); + } + + try { + $db = \Cdb\Writer::open( $outfile ); + + $alreadyWritten = []; + $skipped = 0; + for ( $i = 0; ( $i - $skipped ) < $limit; $i++ ) { + if ( feof( $file ) ) { + break; + } + $rawLine = fgets( $file ); + + if ( $rawLine === false ) { + $this->error( "Error reading input file" ); + break; + } + if ( substr( $rawLine, -1 ) !== "\n" && !feof( $file ) ) { + // We're assuming that this just won't happen. + $this->error( "fgets did not return whole line at $i??" ); + } + $line = $langEn->lc( trim( $rawLine ) ); + if ( $line === '' ) { + $this->error( "Line number " . ( $i + 1 ) . " is blank?" ); + $skipped++; + continue; + } + if ( isset( $alreadyWritten[$line] ) ) { + $this->output( "Password '$line' already written (line " . ( $i + 1 ) .")\n" ); + $skipped++; + continue; + } + $alreadyWritten[$line] = true; + $db->set( $line, $i + 1 - $skipped ); + } + // All caps, so cannot conflict with potential password + $db->set( '_TOTALENTRIES', $i - $skipped ); + $db->close(); + + $this->output( "Successfully wrote " . ( $i - $skipped ) . + " (out of $i) passwords to $outfile\n" + ); + } catch ( \Cdb\Exception $e ) { + $this->error( "Error writing cdb file: " . $e->getMessage(), 2 ); + } + } +} + +$maintClass = "GenerateCommonPassword"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/deleteArchivedFiles.php b/www/wiki/maintenance/deleteArchivedFiles.php new file mode 100644 index 00000000..0f33a141 --- /dev/null +++ b/www/wiki/maintenance/deleteArchivedFiles.php @@ -0,0 +1,134 @@ +<?php +/** + * Delete archived (non-current) files from the database + * + * Based on deleteOldRevisions.php by Rob Church. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to delete archived (non-current) files from the database. + * + * @ingroup Maintenance + */ +class DeleteArchivedFiles extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Deletes all archived images.' ); + $this->addOption( 'delete', 'Perform the deletion' ); + $this->addOption( 'force', 'Force deletion of rows from filearchive' ); + } + + public function execute() { + if ( !$this->hasOption( 'delete' ) ) { + $this->output( "Use --delete to actually confirm this script\n" ); + return; + } + + # Data should come off the master, wrapped in a transaction + $dbw = $this->getDB( DB_MASTER ); + $this->beginTransaction( $dbw, __METHOD__ ); + $repo = RepoGroup::singleton()->getLocalRepo(); + + # Get "active" revisions from the filearchive table + $this->output( "Searching for and deleting archived files...\n" ); + $res = $dbw->select( + 'filearchive', + [ 'fa_id', 'fa_storage_group', 'fa_storage_key', 'fa_sha1', 'fa_name' ], + '', + __METHOD__ + ); + + $count = 0; + foreach ( $res as $row ) { + $key = $row->fa_storage_key; + if ( !strlen( $key ) ) { + $this->output( "Entry with ID {$row->fa_id} has empty key, skipping\n" ); + continue; + } + + /** @var LocalFile $file */ + $file = $repo->newFile( $row->fa_name ); + try { + $file->lock(); + } catch ( LocalFileLockError $e ) { + $this->error( "Could not acquire lock on '{$row->fa_name}', skipping\n" ); + continue; + } + + $group = $row->fa_storage_group; + $id = $row->fa_id; + $path = $repo->getZonePath( 'deleted' ) . + '/' . $repo->getDeletedHashPath( $key ) . $key; + if ( isset( $row->fa_sha1 ) ) { + $sha1 = $row->fa_sha1; + } else { + // old row, populate from key + $sha1 = LocalRepo::getHashFromKey( $key ); + } + + // Check if the file is used anywhere... + $inuse = $dbw->selectField( + 'oldimage', + '1', + [ + 'oi_sha1' => $sha1, + $dbw->bitAnd( 'oi_deleted', File::DELETED_FILE ) => File::DELETED_FILE + ], + __METHOD__, + [ 'FOR UPDATE' ] + ); + + $needForce = true; + if ( !$repo->fileExists( $path ) ) { + $this->output( "Notice - file '$key' not found in group '$group'\n" ); + } elseif ( $inuse ) { + $this->output( "Notice - file '$key' is still in use\n" ); + } elseif ( !$repo->quickPurge( $path ) ) { + $this->output( "Unable to remove file $path, skipping\n" ); + $file->unlock(); + continue; // don't delete even with --force + } else { + $needForce = false; + } + + if ( $needForce ) { + if ( $this->hasOption( 'force' ) ) { + $this->output( "Got --force, deleting DB entry\n" ); + } else { + $file->unlock(); + continue; + } + } + + $count++; + $dbw->delete( 'filearchive', [ 'fa_id' => $id ], __METHOD__ ); + $file->unlock(); + } + + $this->commitTransaction( $dbw, __METHOD__ ); + $this->output( "Done! [$count file(s)]\n" ); + } +} + +$maintClass = "DeleteArchivedFiles"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/deleteArchivedRevisions.php b/www/wiki/maintenance/deleteArchivedRevisions.php new file mode 100644 index 00000000..905b5d9c --- /dev/null +++ b/www/wiki/maintenance/deleteArchivedRevisions.php @@ -0,0 +1,65 @@ +<?php +/** + * Delete archived (deleted from public) revisions from the database + * + * Shamelessly stolen from deleteOldRevisions.php by Rob Church :) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to delete archived (deleted from public) revisions + * from the database. + * + * @ingroup Maintenance + */ +class DeleteArchivedRevisions extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( + "Deletes all archived revisions\nThese revisions will no longer be restorable" ); + $this->addOption( 'delete', 'Performs the deletion' ); + } + + public function execute() { + $dbw = $this->getDB( DB_MASTER ); + + if ( !$this->hasOption( 'delete' ) ) { + $count = $dbw->selectField( 'archive', 'COUNT(*)', '', __METHOD__ ); + $this->output( "Found $count revisions to delete.\n" ); + $this->output( "Please run the script again with the --delete option " + . "to really delete the revisions.\n" ); + return; + } + + $this->output( "Deleting archived revisions... " ); + $dbw->delete( 'archive', '*', __METHOD__ ); + $count = $dbw->affectedRows(); + $this->output( "done. $count revisions deleted.\n" ); + + if ( $count ) { + $this->purgeRedundantText( true ); + } + } +} + +$maintClass = "DeleteArchivedRevisions"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/deleteBatch.php b/www/wiki/maintenance/deleteBatch.php new file mode 100644 index 00000000..0020446b --- /dev/null +++ b/www/wiki/maintenance/deleteBatch.php @@ -0,0 +1,128 @@ +<?php +/** + * Deletes a batch of pages. + * Usage: php deleteBatch.php [-u <user>] [-r <reason>] [-i <interval>] [listfile] + * where + * [listfile] is a file where each line contains the title of a page to be + * deleted, standard input is used if listfile is not given. + * <user> is the username + * <reason> is the delete reason + * <interval> is the number of seconds to sleep for after each delete + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to delete a batch of pages. + * + * @ingroup Maintenance + */ +class DeleteBatch extends Maintenance { + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Deletes a batch of pages' ); + $this->addOption( 'u', "User to perform deletion", false, true ); + $this->addOption( 'r', "Reason to delete page", false, true ); + $this->addOption( 'i', "Interval to sleep between deletions" ); + $this->addArg( 'listfile', 'File with titles to delete, separated by newlines. ' . + 'If not given, stdin will be used.', false ); + } + + public function execute() { + global $wgUser; + + # Change to current working directory + $oldCwd = getcwd(); + chdir( $oldCwd ); + + # Options processing + $username = $this->getOption( 'u', false ); + $reason = $this->getOption( 'r', '' ); + $interval = $this->getOption( 'i', 0 ); + + if ( $username === false ) { + $user = User::newSystemUser( 'Delete page script', [ 'steal' => true ] ); + } else { + $user = User::newFromName( $username ); + } + if ( !$user ) { + $this->error( "Invalid username", true ); + } + $wgUser = $user; + + if ( $this->hasArg() ) { + $file = fopen( $this->getArg(), 'r' ); + } else { + $file = $this->getStdin(); + } + + # Setup + if ( !$file ) { + $this->error( "Unable to read file, exiting", true ); + } + + $dbw = $this->getDB( DB_MASTER ); + + # Handle each entry + // @codingStandardsIgnoreStart Ignore Generic.CodeAnalysis.ForLoopWithTestFunctionCall.NotAllowed + for ( $linenum = 1; !feof( $file ); $linenum++ ) { + // @codingStandardsIgnoreEnd + $line = trim( fgets( $file ) ); + if ( $line == '' ) { + continue; + } + $title = Title::newFromText( $line ); + if ( is_null( $title ) ) { + $this->output( "Invalid title '$line' on line $linenum\n" ); + continue; + } + if ( !$title->exists() ) { + $this->output( "Skipping nonexistent page '$line'\n" ); + continue; + } + + $this->output( $title->getPrefixedText() ); + if ( $title->getNamespace() == NS_FILE ) { + $img = wfFindFile( $title, [ 'ignoreRedirect' => true ] ); + if ( $img && $img->isLocal() && !$img->delete( $reason ) ) { + $this->output( " FAILED to delete associated file... " ); + } + } + $page = WikiPage::factory( $title ); + $error = ''; + $success = $page->doDeleteArticle( $reason, false, 0, true, $error, $user ); + if ( $success ) { + $this->output( " Deleted!\n" ); + } else { + $this->output( " FAILED to delete article\n" ); + } + + if ( $interval ) { + sleep( $interval ); + } + wfWaitForSlaves(); + } + } +} + +$maintClass = "DeleteBatch"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/deleteDefaultMessages.php b/www/wiki/maintenance/deleteDefaultMessages.php new file mode 100644 index 00000000..ba8662ac --- /dev/null +++ b/www/wiki/maintenance/deleteDefaultMessages.php @@ -0,0 +1,99 @@ +<?php +/** + * Deletes all pages in the MediaWiki namespace which were last edited by + * "MediaWiki default". + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that deletes all pages in the MediaWiki namespace + * which were last edited by "MediaWiki default". + * + * @ingroup Maintenance + */ +class DeleteDefaultMessages extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Deletes all pages in the MediaWiki namespace' . + ' which were last edited by "MediaWiki default"' ); + $this->addOption( 'dry-run', 'Perform a dry run, delete nothing' ); + } + + public function execute() { + global $wgUser; + + $this->output( "Checking existence of old default messages..." ); + $dbr = $this->getDB( DB_REPLICA ); + $res = $dbr->select( [ 'page', 'revision' ], + [ 'page_namespace', 'page_title' ], + [ + 'page_namespace' => NS_MEDIAWIKI, + 'page_latest=rev_id', + 'rev_user_text' => 'MediaWiki default', + ] + ); + + if ( $dbr->numRows( $res ) == 0 ) { + // No more messages left + $this->output( "done.\n" ); + return; + } + + $dryrun = $this->hasOption( 'dry-run' ); + if ( $dryrun ) { + foreach ( $res as $row ) { + $title = Title::makeTitle( $row->page_namespace, $row->page_title ); + $this->output( "\n* [[$title]]" ); + } + $this->output( "\n\nRun again without --dry-run to delete these pages.\n" ); + return; + } + + // Deletions will be made by $user temporarly added to the bot group + // in order to hide it in RecentChanges. + $user = User::newFromName( 'MediaWiki default' ); + if ( !$user ) { + $this->error( "Invalid username", true ); + } + $user->addGroup( 'bot' ); + $wgUser = $user; + + // Handle deletion + $this->output( "\n...deleting old default messages (this may take a long time!)...", 'msg' ); + $dbw = $this->getDB( DB_MASTER ); + + foreach ( $res as $row ) { + wfWaitForSlaves(); + $dbw->ping(); + $title = Title::makeTitle( $row->page_namespace, $row->page_title ); + $page = WikiPage::factory( $title ); + $error = ''; // Passed by ref + // FIXME: Deletion failures should be reported, not silently ignored. + $page->doDeleteArticle( 'No longer required', false, 0, true, $error, $user ); + } + + $this->output( "done!\n", 'msg' ); + } +} + +$maintClass = "DeleteDefaultMessages"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/deleteEqualMessages.php b/www/wiki/maintenance/deleteEqualMessages.php new file mode 100644 index 00000000..5fc7d184 --- /dev/null +++ b/www/wiki/maintenance/deleteEqualMessages.php @@ -0,0 +1,206 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that deletes all pages in the MediaWiki namespace + * of which the content is equal to the system default. + * + * @ingroup Maintenance + */ +class DeleteEqualMessages extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Deletes all pages in the MediaWiki namespace that are equal to ' + . 'the default message' ); + $this->addOption( 'delete', 'Actually delete the pages (default: dry run)' ); + $this->addOption( 'delete-talk', 'Don\'t leave orphaned talk pages behind during deletion' ); + $this->addOption( 'lang-code', 'Check for subpages of this language code (default: root ' + . 'page against content language). Use value "*" to run for all mwfile language code ' + . 'subpages (including the base pages that override content language).', false, true ); + } + + /** + * @param string|bool $langCode See --lang-code option. + * @param array &$messageInfo + */ + protected function fetchMessageInfo( $langCode, array &$messageInfo ) { + global $wgContLang; + + if ( $langCode ) { + $this->output( "\n... fetching message info for language: $langCode" ); + $nonContLang = true; + } else { + $this->output( "\n... fetching message info for content language" ); + $langCode = $wgContLang->getCode(); + $nonContLang = false; + } + + /* Based on SpecialAllmessages::reallyDoQuery #filter=modified */ + + $l10nCache = Language::getLocalisationCache(); + $messageNames = $l10nCache->getSubitemList( 'en', 'messages' ); + // Normalise message names for NS_MEDIAWIKI page_title + $messageNames = array_map( [ $wgContLang, 'ucfirst' ], $messageNames ); + + $statuses = AllMessagesTablePager::getCustomisedStatuses( + $messageNames, $langCode, $nonContLang ); + // getCustomisedStatuses is stripping the sub page from the page titles, add it back + $titleSuffix = $nonContLang ? "/$langCode" : ''; + + foreach ( $messageNames as $key ) { + $customised = isset( $statuses['pages'][$key] ); + if ( $customised ) { + $actual = wfMessage( $key )->inLanguage( $langCode )->plain(); + $default = wfMessage( $key )->inLanguage( $langCode )->useDatabase( false )->plain(); + + $messageInfo['relevantPages']++; + + if ( + // Exclude messages that are empty by default, such as sitenotice, specialpage + // summaries and accesskeys. + $default !== '' && $default !== '-' && + $actual === $default + ) { + $hasTalk = isset( $statuses['talks'][$key] ); + $messageInfo['results'][] = [ + 'title' => $key . $titleSuffix, + 'hasTalk' => $hasTalk, + ]; + $messageInfo['equalPages']++; + if ( $hasTalk ) { + $messageInfo['equalPagesTalks']++; + } + } + } + } + } + + public function execute() { + $doDelete = $this->hasOption( 'delete' ); + $doDeleteTalk = $this->hasOption( 'delete-talk' ); + $langCode = $this->getOption( 'lang-code' ); + + $messageInfo = [ + 'relevantPages' => 0, + 'equalPages' => 0, + 'equalPagesTalks' => 0, + 'results' => [], + ]; + + $this->output( 'Checking for pages with default message...' ); + + // Load message information + if ( $langCode ) { + $langCodes = Language::fetchLanguageNames( null, 'mwfile' ); + if ( $langCode === '*' ) { + // All valid lang-code subpages in NS_MEDIAWIKI that + // override the messsages in that language + foreach ( $langCodes as $key => $value ) { + $this->fetchMessageInfo( $key, $messageInfo ); + } + // Lastly, the base pages in NS_MEDIAWIKI that override + // messages in content language + $this->fetchMessageInfo( false, $messageInfo ); + } else { + if ( !isset( $langCodes[$langCode] ) ) { + $this->error( 'Invalid language code: ' . $langCode, 1 ); + } + $this->fetchMessageInfo( $langCode, $messageInfo ); + } + } else { + $this->fetchMessageInfo( false, $messageInfo ); + } + + if ( $messageInfo['equalPages'] === 0 ) { + // No more equal messages left + $this->output( "\ndone.\n" ); + + return; + } + + $this->output( "\n{$messageInfo['relevantPages']} pages in the MediaWiki namespace " + . "override messages." ); + $this->output( "\n{$messageInfo['equalPages']} pages are equal to the default message " + . "(+ {$messageInfo['equalPagesTalks']} talk pages).\n" ); + + if ( !$doDelete ) { + $list = ''; + foreach ( $messageInfo['results'] as $result ) { + $title = Title::makeTitle( NS_MEDIAWIKI, $result['title'] ); + $list .= "* [[$title]]\n"; + if ( $result['hasTalk'] ) { + $title = Title::makeTitle( NS_MEDIAWIKI_TALK, $result['title'] ); + $list .= "* [[$title]]\n"; + } + } + $this->output( "\nList:\n$list\nRun the script again with --delete to delete these pages" ); + if ( $messageInfo['equalPagesTalks'] !== 0 ) { + $this->output( " (include --delete-talk to also delete the talk pages)" ); + } + $this->output( "\n" ); + + return; + } + + $user = User::newSystemUser( 'MediaWiki default', [ 'steal' => true ] ); + if ( !$user ) { + $this->error( "Invalid username", true ); + } + global $wgUser; + $wgUser = $user; + + // Hide deletions from RecentChanges + $user->addGroup( 'bot' ); + + // Handle deletion + $this->output( "\n...deleting equal messages (this may take a long time!)..." ); + $dbw = $this->getDB( DB_MASTER ); + foreach ( $messageInfo['results'] as $result ) { + wfWaitForSlaves(); + $dbw->ping(); + $title = Title::makeTitle( NS_MEDIAWIKI, $result['title'] ); + $this->output( "\n* [[$title]]" ); + $page = WikiPage::factory( $title ); + $error = ''; // Passed by ref + $success = $page->doDeleteArticle( 'No longer required', false, 0, true, $error, $user ); + if ( !$success ) { + $this->output( " (Failed!)" ); + } + if ( $result['hasTalk'] && $doDeleteTalk ) { + $title = Title::makeTitle( NS_MEDIAWIKI_TALK, $result['title'] ); + $this->output( "\n* [[$title]]" ); + $page = WikiPage::factory( $title ); + $error = ''; // Passed by ref + $success = $page->doDeleteArticle( 'Orphaned talk page of no longer required message', + false, 0, true, $error, $user ); + if ( !$success ) { + $this->output( " (Failed!)" ); + } + } + } + $this->output( "\n\ndone!\n" ); + } +} + +$maintClass = "DeleteEqualMessages"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/deleteOldRevisions.php b/www/wiki/maintenance/deleteOldRevisions.php new file mode 100644 index 00000000..aa11cd96 --- /dev/null +++ b/www/wiki/maintenance/deleteOldRevisions.php @@ -0,0 +1,103 @@ +<?php +/** + * Delete old (non-current) revisions from the database + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @author Rob Church <robchur@gmail.com> + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that deletes old (non-current) revisions from the database. + * + * @ingroup Maintenance + */ +class DeleteOldRevisions extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Delete old (non-current) revisions from the database' ); + $this->addOption( 'delete', 'Actually perform the deletion' ); + $this->addOption( 'page_id', 'List of page ids to work on', false ); + } + + public function execute() { + $this->output( "Delete old revisions\n\n" ); + $this->doDelete( $this->hasOption( 'delete' ), $this->mArgs ); + } + + function doDelete( $delete = false, $args = [] ) { + # Data should come off the master, wrapped in a transaction + $dbw = $this->getDB( DB_MASTER ); + $this->beginTransaction( $dbw, __METHOD__ ); + + $pageConds = []; + $revConds = []; + + # If a list of page_ids was provided, limit results to that set of page_ids + if ( count( $args ) > 0 ) { + $pageConds['page_id'] = $args; + $revConds['rev_page'] = $args; + $this->output( "Limiting to page IDs " . implode( ',', $args ) . "\n" ); + } + + # Get "active" revisions from the page table + $this->output( "Searching for active revisions..." ); + $res = $dbw->select( 'page', 'page_latest', $pageConds, __METHOD__ ); + $latestRevs = []; + foreach ( $res as $row ) { + $latestRevs[] = $row->page_latest; + } + $this->output( "done.\n" ); + + # Get all revisions that aren't in this set + $this->output( "Searching for inactive revisions..." ); + if ( count( $latestRevs ) > 0 ) { + $revConds[] = 'rev_id NOT IN (' . $dbw->makeList( $latestRevs ) . ')'; + } + $res = $dbw->select( 'revision', 'rev_id', $revConds, __METHOD__ ); + $oldRevs = []; + foreach ( $res as $row ) { + $oldRevs[] = $row->rev_id; + } + $this->output( "done.\n" ); + + # Inform the user of what we're going to do + $count = count( $oldRevs ); + $this->output( "$count old revisions found.\n" ); + + # Delete as appropriate + if ( $delete && $count ) { + $this->output( "Deleting..." ); + $dbw->delete( 'revision', [ 'rev_id' => $oldRevs ], __METHOD__ ); + $dbw->delete( 'ip_changes', [ 'ipc_rev_id' => $oldRevs ], __METHOD__ ); + $this->output( "done.\n" ); + } + + # This bit's done + # Purge redundant text records + $this->commitTransaction( $dbw, __METHOD__ ); + if ( $delete ) { + $this->purgeRedundantText( true ); + } + } +} + +$maintClass = "DeleteOldRevisions"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/deleteOrphanedRevisions.php b/www/wiki/maintenance/deleteOrphanedRevisions.php new file mode 100644 index 00000000..4d600706 --- /dev/null +++ b/www/wiki/maintenance/deleteOrphanedRevisions.php @@ -0,0 +1,102 @@ +<?php +/** + * Delete revisions which refer to a nonexisting page. + * Sometimes manual deletion done in a rush leaves crap in the database. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @author Rob Church <robchur@gmail.com> + * @todo More efficient cleanup of text records + */ + +require_once __DIR__ . '/Maintenance.php'; + +use Wikimedia\Rdbms\IDatabase; + +/** + * Maintenance script that deletes revisions which refer to a nonexisting page. + * + * @ingroup Maintenance + */ +class DeleteOrphanedRevisions extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( + 'Maintenance script to delete revisions which refer to a nonexisting page' ); + $this->addOption( 'report', 'Prints out a count of affected revisions but doesn\'t delete them' ); + } + + public function execute() { + $this->output( "Delete Orphaned Revisions\n" ); + + $report = $this->hasOption( 'report' ); + + $dbw = $this->getDB( DB_MASTER ); + $this->beginTransaction( $dbw, __METHOD__ ); + list( $page, $revision ) = $dbw->tableNamesN( 'page', 'revision' ); + + # Find all the orphaned revisions + $this->output( "Checking for orphaned revisions..." ); + $sql = "SELECT rev_id FROM {$revision} LEFT JOIN {$page} ON rev_page = page_id " + . "WHERE page_namespace IS NULL"; + $res = $dbw->query( $sql, 'deleteOrphanedRevisions' ); + + # Stash 'em all up for deletion (if needed) + $revisions = []; + foreach ( $res as $row ) { + $revisions[] = $row->rev_id; + } + $count = count( $revisions ); + $this->output( "found {$count}.\n" ); + + # Nothing to do? + if ( $report || $count == 0 ) { + $this->commitTransaction( $dbw, __METHOD__ ); + exit( 0 ); + } + + # Delete each revision + $this->output( "Deleting..." ); + $this->deleteRevs( $revisions, $dbw ); + $this->output( "done.\n" ); + + # Close the transaction and call the script to purge unused text records + $this->commitTransaction( $dbw, __METHOD__ ); + $this->purgeRedundantText( true ); + } + + /** + * Delete one or more revisions from the database + * Do this inside a transaction + * + * @param array $id Array of revision id values + * @param IDatabase $dbw Master DB handle + */ + private function deleteRevs( $id, &$dbw ) { + if ( !is_array( $id ) ) { + $id = [ $id ]; + } + $dbw->delete( 'revision', [ 'rev_id' => $id ], __METHOD__ ); + + // Delete from ip_changes should a record exist. + $dbw->delete( 'ip_changes', [ 'ipc_rev_id' => $id ], __METHOD__ ); + } +} + +$maintClass = "DeleteOrphanedRevisions"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/deleteRevision.php b/www/wiki/maintenance/deleteRevision.php new file mode 100644 index 00000000..0111ac57 --- /dev/null +++ b/www/wiki/maintenance/deleteRevision.php @@ -0,0 +1,111 @@ +<?php +/** + * Delete one or more revisions by moving them to the archive table. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that deletes one or more revisions by moving them + * to the archive table. + * + * @ingroup Maintenance + */ +class DeleteRevision extends Maintenance { + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Delete one or more revisions by moving them to the archive table' ); + } + + public function execute() { + if ( count( $this->mArgs ) == 0 ) { + $this->error( "No revisions specified", true ); + } + + $this->output( "Deleting revision(s) " . implode( ',', $this->mArgs ) . + " from " . wfWikiID() . "...\n" ); + $dbw = $this->getDB( DB_MASTER ); + + $affected = 0; + foreach ( $this->mArgs as $revID ) { + $dbw->insertSelect( 'archive', [ 'page', 'revision' ], + [ + 'ar_namespace' => 'page_namespace', + 'ar_title' => 'page_title', + 'ar_page_id' => 'page_id', + 'ar_comment' => 'rev_comment', + 'ar_user' => 'rev_user', + 'ar_user_text' => 'rev_user_text', + 'ar_timestamp' => 'rev_timestamp', + 'ar_minor_edit' => 'rev_minor_edit', + 'ar_rev_id' => 'rev_id', + 'ar_text_id' => 'rev_text_id', + 'ar_deleted' => 'rev_deleted', + 'ar_len' => 'rev_len', + ], + [ + 'rev_id' => $revID, + 'page_id = rev_page' + ], + __METHOD__ + ); + if ( !$dbw->affectedRows() ) { + $this->output( "Revision $revID not found\n" ); + } else { + $affected += $dbw->affectedRows(); + $pageID = $dbw->selectField( + 'revision', + 'rev_page', + [ 'rev_id' => $revID ], + __METHOD__ + ); + $pageLatest = $dbw->selectField( + 'page', + 'page_latest', + [ 'page_id' => $pageID ], + __METHOD__ + ); + $dbw->delete( 'revision', [ 'rev_id' => $revID ] ); + if ( $pageLatest == $revID ) { + // Database integrity + $newLatest = $dbw->selectField( + 'revision', + 'rev_id', + [ 'rev_page' => $pageID ], + __METHOD__, + [ 'ORDER BY' => 'rev_timestamp DESC' ] + ); + $dbw->update( + 'page', + [ 'page_latest' => $newLatest ], + [ 'page_id' => $pageID ], + __METHOD__ + ); + } + } + } + $this->output( "Deleted $affected revisions\n" ); + } +} + +$maintClass = "DeleteRevision"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/deleteSelfExternals.php b/www/wiki/maintenance/deleteSelfExternals.php new file mode 100644 index 00000000..ed15fd13 --- /dev/null +++ b/www/wiki/maintenance/deleteSelfExternals.php @@ -0,0 +1,58 @@ +<?php +/** + * Delete self-references to $wgServer from the externallinks table. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that deletes self-references to $wgServer + * from the externallinks table. + * + * @ingroup Maintenance + */ +class DeleteSelfExternals extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Delete self-references to $wgServer from externallinks' ); + $this->mBatchSize = 1000; + } + + public function execute() { + global $wgServer; + $this->output( "Deleting self externals from $wgServer\n" ); + $db = $this->getDB( DB_MASTER ); + while ( 1 ) { + wfWaitForSlaves(); + $this->commitTransaction( $db, __METHOD__ ); + $q = $db->limitResult( "DELETE /* deleteSelfExternals */ FROM externallinks WHERE el_to" + . $db->buildLike( $wgServer . '/', $db->anyString() ), $this->mBatchSize ); + $this->output( "Deleting a batch\n" ); + $db->query( $q ); + if ( !$db->affectedRows() ) { + return; + } + } + } +} + +$maintClass = "DeleteSelfExternals"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/dev/README b/www/wiki/maintenance/dev/README new file mode 100644 index 00000000..ca47d136 --- /dev/null +++ b/www/wiki/maintenance/dev/README @@ -0,0 +1,7 @@ +maintenance/dev/ scripts can help quickly setup a local MediaWiki for development purposes. + +Wikis setup in this way are NOT meant to be publicly available. They use a development database not acceptible for use in production. Place a sqlite database in an unsafe location a real wiki should never place it in. And use predictable default logins for the initial administrator user. + +Running maintenance/dev/install.sh will download and install a local copy of php 5.4, install a sqlite powered instance of MW for development, and then start up a local webserver to view the wiki. + +After installation you can bring the webserver back up at any time you want with maintenance/dev/start.sh diff --git a/www/wiki/maintenance/dev/includes/php.sh b/www/wiki/maintenance/dev/includes/php.sh new file mode 100644 index 00000000..3c5bef0d --- /dev/null +++ b/www/wiki/maintenance/dev/includes/php.sh @@ -0,0 +1,14 @@ +# Include-able script to determine the location of our php if any +# We search for a environment var called PHP, native php, +# a local copy, home directory location used by installphp.sh +# and previous home directory location +# The binary path is returned in $PHP if any + +for binary in $PHP $(which php || true) "$DEV/php/bin/php" "$HOME/.mediawiki/php/bin/php" "$HOME/.mwphp/bin/php" ]; do + if [ -x "$binary" ]; then + if "$binary" -r 'exit((int)!version_compare(PHP_VERSION, "5.4", ">="));'; then + PHP="$binary" + break + fi + fi +done diff --git a/www/wiki/maintenance/dev/includes/require-php.sh b/www/wiki/maintenance/dev/includes/require-php.sh new file mode 100644 index 00000000..470e6eb8 --- /dev/null +++ b/www/wiki/maintenance/dev/includes/require-php.sh @@ -0,0 +1,8 @@ +# Include-able script to require that we have a known php binary we can execute + +. "$DEV/includes/php.sh" + +if [ "x$PHP" == "x" -o ! -x "$PHP" ]; then + echo "Local copy of PHP is not installed" + exit 1 +fi diff --git a/www/wiki/maintenance/dev/includes/router.php b/www/wiki/maintenance/dev/includes/router.php new file mode 100644 index 00000000..9917a4fa --- /dev/null +++ b/www/wiki/maintenance/dev/includes/router.php @@ -0,0 +1,97 @@ +<?php +/** + * Router for the php cli-server built-in webserver. + * https://secure.php.net/manual/en/features.commandline.webserver.php + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +if ( PHP_SAPI != 'cli-server' ) { + die( "This script can only be run by php's cli-server sapi." ); +} + +ini_set( 'display_errors', 1 ); +error_reporting( E_ALL ); + +if ( isset( $_SERVER["SCRIPT_FILENAME"] ) ) { + # Known resource, sometimes a script sometimes a file + $file = $_SERVER["SCRIPT_FILENAME"]; +} elseif ( isset( $_SERVER["SCRIPT_NAME"] ) ) { + # Usually unknown, document root relative rather than absolute + # Happens with some cases like /wiki/File:Image.png + if ( is_readable( $_SERVER['DOCUMENT_ROOT'] . $_SERVER["SCRIPT_NAME"] ) ) { + # Just in case this actually IS a file, set it here + $file = $_SERVER['DOCUMENT_ROOT'] . $_SERVER["SCRIPT_NAME"]; + } else { + # Otherwise let's pretend that this is supposed to go to index.php + $file = $_SERVER['DOCUMENT_ROOT'] . '/index.php'; + } +} else { + # Meh, we'll just give up + return false; +} + +# And now do handling for that $file + +if ( !is_readable( $file ) ) { + # Let the server throw the error if it doesn't exist + return false; +} +$ext = pathinfo( $file, PATHINFO_EXTENSION ); +if ( $ext == 'php' || $ext == 'php5' ) { + return false; +} +$mime = false; +// Borrow mime type file from MimeAnalyzer +$lines = explode( "\n", file_get_contents( "includes/libs/mime/mime.types" ) ); +foreach ( $lines as $line ) { + $exts = explode( " ", $line ); + $mime = array_shift( $exts ); + if ( in_array( $ext, $exts ) ) { + break; # this is the right value for $mime + } + $mime = false; +} +if ( !$mime ) { + $basename = basename( $file ); + if ( $basename == strtoupper( $basename ) ) { + # IF it's something like README serve it as text + $mime = "text/plain"; + } +} +if ( $mime ) { + # Use custom handling to serve files with a known MIME type + # This way we can serve things like .svg files that the built-in + # PHP webserver doesn't understand. + # ;) Nicely enough we just happen to bundle a mime.types file + $f = fopen( $file, 'rb' ); + if ( preg_match( '#^text/#', $mime ) ) { + # Text should have a charset=UTF-8 (php's webserver does this too) + header( "Content-Type: $mime; charset=UTF-8" ); + } else { + header( "Content-Type: $mime" ); + } + header( "Content-Length: " . filesize( $file ) ); + // Stream that out to the browser + fpassthru( $f ); + + return true; +} + +# Let the php server handle things on its own otherwise +return false; diff --git a/www/wiki/maintenance/dev/install.sh b/www/wiki/maintenance/dev/install.sh new file mode 100755 index 00000000..2219894d --- /dev/null +++ b/www/wiki/maintenance/dev/install.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +if [ "x$BASH_SOURCE" == "x" ]; then echo '$BASH_SOURCE not set'; exit 1; fi +DEV=$(cd -P "$(dirname "${BASH_SOURCE[0]}" )" && pwd) + +"$DEV/installphp.sh" +"$DEV/installmw.sh" +"$DEV/start.sh" diff --git a/www/wiki/maintenance/dev/installmw.sh b/www/wiki/maintenance/dev/installmw.sh new file mode 100755 index 00000000..9ae3c593 --- /dev/null +++ b/www/wiki/maintenance/dev/installmw.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +if [ "x$BASH_SOURCE" == "x" ]; then echo '$BASH_SOURCE not set'; exit 1; fi +DEV=$(cd -P "$(dirname "${BASH_SOURCE[0]}" )" && pwd) + +. "$DEV/includes/require-php.sh" + +set -e + +PORT=4881 + +cd "$DEV/../../"; # $IP + +mkdir -p "$DEV/data" +"$PHP" maintenance/install.php --server="http://localhost:$PORT" --scriptpath="" --dbtype=sqlite --dbpath="$DEV/data" --pass=admin "Trunk Test" "$USER" +echo "" +echo "Development wiki created with admin user $USER and password 'admin'." +echo "" diff --git a/www/wiki/maintenance/dev/installphp.sh b/www/wiki/maintenance/dev/installphp.sh new file mode 100755 index 00000000..d26ffa67 --- /dev/null +++ b/www/wiki/maintenance/dev/installphp.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +if [ "x$BASH_SOURCE" == "x" ]; then echo '$BASH_SOURCE not set'; exit 1; fi +DEV=$(cd -P "$(dirname "${BASH_SOURCE[0]}" )" && pwd) + +set -e # DO NOT USE PIPES unless this is rewritten + +. "$DEV/includes/php.sh" + +if [ "x$PHP" != "x" -a -x "$PHP" ]; then + echo "PHP is already installed" + exit 0 +fi + +TAR=php5.4-latest.tar.gz +PHPURL="http://snaps.php.net/$TAR" + +cd "$DEV" + +echo "Preparing to download and install a local copy of PHP 5.4, note that this can take some time to do." +echo "If you wish to avoid re-doing this for uture dev installations of MediaWiki we suggest installing php in ~/.mediawiki/php" +echo -n "Install PHP in ~/.mediawiki/php [y/N]: " +read INSTALLINHOME + +case "$INSTALLINHOME" in + [Yy] | [Yy][Ee][Ss] ) + PREFIX="$HOME/.mediawiki/php" + ;; + *) + PREFIX="$DEV/php/" + ;; +esac + +# Some debain-like systems bundle wget but not curl, some other systems +# like os x bundle curl but not wget... use whatever is available +echo -n "Downloading PHP 5.4" +if command -v wget &>/dev/null; then + echo "- using wget" + wget "$PHPURL" +elif command -v curl &>/dev/null; then + echo "- using curl" + curl -O "$PHPURL" +else + echo "- aborting" + echo "Could not find curl or wget." >&2; + exit 1; +fi + +echo "Extracting php 5.4" +tar -xzf "$TAR" + +cd php5.4-*/ + +echo "Configuring and installing php 5.4 in $PREFIX" +./configure --prefix="$PREFIX" +make +make install diff --git a/www/wiki/maintenance/dev/start.sh b/www/wiki/maintenance/dev/start.sh new file mode 100755 index 00000000..dd7363a8 --- /dev/null +++ b/www/wiki/maintenance/dev/start.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +if [ "x$BASH_SOURCE" == "x" ]; then echo '$BASH_SOURCE not set'; exit 1; fi +DEV=$(cd -P "$(dirname "${BASH_SOURCE[0]}" )" && pwd) + +. "$DEV/includes/require-php.sh" + +PORT=4881 + +echo "Starting up MediaWiki at http://localhost:$PORT/" +echo "" + +cd "$DEV/../../"; # $IP +"$PHP" -S "localhost:$PORT" "$DEV/includes/router.php" diff --git a/www/wiki/maintenance/dictionary/mediawiki.dic b/www/wiki/maintenance/dictionary/mediawiki.dic new file mode 100644 index 00000000..7c3c95d7 --- /dev/null +++ b/www/wiki/maintenance/dictionary/mediawiki.dic @@ -0,0 +1,4666 @@ +&add +& +&bar +&img +&sim +&url +&wap +ABNF +API +Aacute +Aborted +Abuse +Account +Accum +Acirc +Action +Activity +Agrave +All +Allocations +Ancientpages +Anim +Api +Apitestsysop +Apitestuser +Aring +Article +As +Atilde +Auml +Autopromote +BACKCOMPAT +Backlinks +Blacklist +Block +Blocked +Blocks +Bodytext +Broken +COMPUTERNAME +CRLF +CURLOPT +Campaign +Capture +Categories +Category +Ccedil +Central +Changes +Check +Click +Client +Clientfor +Colorer +Compare +Config +Console +Continue +Contribs +Contributions +Conversiontable +Coordinates +Create +Creation +Cview +DDLMODE +DWIM +DWIMD +Daily +Dbkeyform +Deadendpages +Debugtext +Delete +Deletedrevs +Denied +Dfile +Double +Duplicate +EAGAIN +EBML +ECMA +EDITFILTERMERGED +EINPROGRESS +EINTR +EOCDR +ETAG +Eacute +Ecirc +Edit +Editor +Education +Egrave +Elig +Email +Empty +End +English +Enlist +Euml +Eval +Events +Exists +Expand +Expression +Ext +External +Extracts +Extraneous +FFFD +FOLLOWLOCATION +Failure +Featured +Feed +Feedback +Feedbackv +Feeds +Fewestrevisions +Ffile +File +Filearchive +Filedelete +Files +Filter +Filters +Flag +Flagged +GI +GRAPHEME +Gadget +Gadgets +Geo +Get +Global +Groups +HEA +HTM +Hardblock +Help +Helpful +ID +IPTC +IWBacklinks +IWLinks +Iacute +Icirc +Igrave +Illegal +Image +Images +Implict +Import +Info +Invalidateemail +Isarticle +Item +Iuml +LOCALISATIONCACHE +Lang +Lastmod +Links +Linktags +List +Listredirects +Living +Log +Login +Logout +Logs +Lonelypages +Longpages +Love +Ltitle +MSVC +Mark +Match +Matrix +Members +Mesg +Messages +Metatags +Mobile +Mostcategories +Mostimages +Mostinterwikis +Mostlinked +Mostlinkedcategories +Mostlinkedtemplates +Mostrevisions +Move +Mssql +Mwstore +Myuploads +NEWPAGE +NOTIC +Name +Need +No +Noscript +Not +Notalk +Notice +Notification +Ntilde +Oacute +Ocirc +Ograve +Oldreviewedpages +Open +Options +Oslash +Otilde +Ouml +PAGEEDITDATE +PAGEEDITOR +PAGEEDITTIME +PAGEINTRO +PAGEMINOREDIT +PAGESUMMARY +PARSEHUGE +PARSERFIRSTCALLINIT +PHPTAL +PMID +Page +Pages +Param +Parse +Parsers +Pass +Passpass +Patrol +People +Plugin +Possible +Program +Props +Protect +Protected +Protectexpiry +Protectother +Protectreason +Protectreasonother +Purge +Query +Queued +Random +Rapid +Ratings +Raw +Recent +Redirects +Redis +Referer +Refresh +Regexlike +Replacer +Reset +Resursive +Revert +Review +Revisions +Rollback +Rsd +SEGSIZE +STDERR +SYSDBA +Scaron +Scribunto +Search +Section +Set +Shortpages +Site +Siteinfo +Solr +Stabilize +Stash +Stats +Status +Success +Syntax +TMPDIR +TOOLBOXEND +TRANSLIT +Tagging +Tags +Template +Templates +Textform +Tfile +Throttled +Timestamp +Title +Titles +Token +Tokens +Tracking +Transcode +Triage +UNWATCHURL +Uacute +Ucirc +Ugrave +Unblock +Uncategorizedcategories +Uncategorizedimages +Uncategorizedpages +Uncategorizedtemplates +Undelete +Unusedcategories +Unusedimages +Unusedtemplates +Unwatchedpages +Upload +Urlform +Usage +User +Usercreate +Userdir +Userlang +Userrights +Users +Useruser +Ustart +Uuml +Value +Video +View +Visual +WATCHINGUSERNAME +WEBPVP +Wantedcategories +Wantedfiles +Wantedpages +Wantedtemplates +Warning +Watch +Watchingusers +Watchlist +Wiki +Wikibase +Withoutinterwiki +Wrong +XX +Xml +YYYY +YYYYMMDDHHMMSS +Yacute +Yuml +\ +a +aa +aacute +abbrv +abcdefghijklmnopqrstuvwxyz +abf +aboutpage +aboutsite +abusefilter +abusefiltercheckmatch +abusefilterchecksyntax +abusefilterevalexpression +abusefilters +abusefilterunblockautopromote +abuselog +abusive +ac +acad +accel +acceptbilling +acceptlang +accessdenied +accesskey +accesskeycache +accesskeys +accessors +acchits +account +accountcreator +accum +acirc +aclimit +acprefix +action +actioncomplete +actionhidden +actions +actiontext +actionthrottled +actionthrottledtext +actiontoken +activeusers +activity +acuxvalidate +add +addablegroups +addbegin +addedline +addedwatchtext +addergroup +addergroups +addin +adding +additional +addr +address +addresses +addsection +addstudent +admin +administrator +adnum +adrelid +adsrc +advancedediting +advancedrc +advancedrendering +advancedsearchoptions +advancedwatchlist +aelig +af +afl +aft +afttest +afvf +age +aggregators +agrave +ahandler +ahttp +ai +aifc +aiff +aiprop +airtel +aisort +al +alefsym +algo +algos +all +all's +allcategories +alldata +alle +allexamples +allfileusages +allhidden +allimages +allimit +alllinks +alllogstext +allmessages +allmonths +allowedctypes +allowedonly +allowemail +allowsduplicates +allowusertalk +allpages +allpagesbadtitle +allpagesprefix +allpagesredirect +allpagessubmit +allpartners +allredirects +allrev +alltitles +alltransclusions +allusers +aloption +alprefix +alreadyblocked +alreadydone +alreadyexists +alreadyrolled +alunique +am +analyticsconfig +anchor +anchorclose +anchorencode +and +andconvert +andreescu +andtitle +anon +anoneditwarning +anonnotice +anononly +anonpreviewwarning +anontalk +anontalkpagetext +anontoken +anonuserpage +anonymous +anti +antispoof +antivirus +anymap +ap +apcond +apdir +api +api's +apibase +apihelp +apihighlimits +apis +aplimit +apnamespace +apng +apos +appendnotsupported +appendtext +apprefix +approve +aprops +aqbt +aqct +archivename +aren +args +argsarams +aring +arnfjörð +article +articleexists +articlefeedbackv +articleid +articlelink +articlepage +articlepath +articles +aryeh +asc +ascending +asctime +asdf +aspx +assert +asymp +async +at +atend +atext +atid +atilde +atime +atlimit +atoi +atom +atprefix +atthasdef +attibs +attibute +attlen +attname +attnum +attrdef +attrelid +attrib +attribs +attributename +attrs +atttypid +atunique +au +auml +authplugins +autoaccount +autobiography +autoblock +autoblocked +autoblockedtext +autoblocker +autoblockid +autoblocking +autoblockip +autoblocks +autocad +autocomment +autocomments +autocomplete +autoconfirm +autoconfirmed +autocreate +autocreated +autocreation +autodetection +autoflag +autofocus +autogen +autogenerated +autohide +autoload +autoloader +autoloaders +autoloading +automagically +automatic +autonym +autopatrol +autoplay +autopromote +autopromoted +autopromotion +autoreview +autoreviewer +autoreviewrestore +autosumm +autosummaries +autosummary +axto +azərbaycanca +backends +backlink +backlinks +backlinksubtitle +backported +backslashed +backtraces +bad +badaccess +badarticleerror +badcontinue +baddiff +bademail +badfilename +badformat +badgenerator +badhookmsg +badinterwiki +badip +badipaddress +badkey +badmd +badmime +badminpassword +badminuser +badnamespace +badoption +badparams +badport +badretype +badrevids +badsig +badsiglength +badsyntax +badtag +badtimestamp +badtitle +badtitletext +badtoken +badtype +badupload +baduser +badversion +balancer +balancers +banjar +barebone +barstein +base +basefont +basename +basepagename +basepagenamee +basetimestamp +bashkir +bashpid +bcancel +bceffd +bcmath +bcompress +bcpio +bdop +bdquo +becampus +beinstructor +belarusian +beonline +bereviewer +berror +bestq +besttype +bg +bgcolor +bgzip +bidi +bigdelete +bingbot +binhex +bitdepth +bitfield +bitfields +bitmask +bjarmason +bk +bkey +bkinvalidparammix +bkmissingparam +bkusers +bl +blanking +blanknamespace +blankpage +blegh +bleh +blinvalidparammix +blksize +blmissingparam +block +blockable +blocked +blockedasrange +blockedby +blockedbyid +blockedemailuser +blockedexpiry +blockedfrommail +blockednoreason +blockedreason +blockedtext +blockedtitle +blockemail +blockexpiry +blockid +blockinfo +blockip +blocklink +blocklogentry +blocklogpage +blocklogtext +blockme +blockquote +blockreason +blocks +blocktoken +bloggs +blogs +blogspot +bltitle +bluelink +bluelinks +bmwschema +bmysql +bname +bodycontent +bogo +boldening +bolding +booksources +bool +boolean +bordercolor +borderhack +bot +botedit +boteditletter +bots +bottom +bottomscripts +bpassword +bpatch +bpchar +bport +bprefix +broeck +brokenlibxml +brokenredirects +brokenredirectstext +browsearchive +brvbar +bserver +bservers +bssl +btestpassword +btestuser +btype +bucket +bucketcount +bugfix +bugfixes +buglist +bugzilla +buildpath +buildpathentry +bulgakov +bulkdelcourses +bulkdelorgs +bureaucrat +buser +by +byemail +byid +bytea +bytesleft +bytesread +bytevalue +cacheable +cached +cachedcount +cachedsidebar +cachedspecial +cachedtimestamp +calimit +callargs +campaign +campus +cancelto +cannotdelete +cannotundelete +canonicalised +canonicalization +canonicalize +canonicalizes +canonicalizing +canremember +canreset +cansecurelogin +cantblock +cantcreate +cantdelete +cantedit +cantexecute +canthide +cantimport +cantmove +cantmovefile +cantopenfile +cantoverwrite +cantrollback +cantsend +cantunblock +cantundelete +capitalizeallnouns +captchaid +captchas +captchaword +carriersnoips +cascade +cascadeable +cascadeon +cascadeprotected +cascadeprotectedwarning +cascading +cascadinglevels +cascadingness +categories +categories's +categorieshtml +category +categoryfinder +categoryinfo +categorylinks +categorymembers +categorypage +categoryviewer +catids +catlinks +catmsg +catpage +catrope +cattitles +ccedil +ccme +ccmeonemails +cdab +cdel +cdlink +cedil +ceebc +cellpadding +cellspacing +cellulant +central +centralauth +centralnotice +centralnoticeallocations +centralnoticelogs +centralnoticequerycampaign +cgroup +cgroups +change +change's +changeablegroups +changed +changedby +changedorcreated +changeemail +changelog +changeslist +changing +characters +chardiff +charoff +chars +checkfreq +checkmatrix +checkstatus +checkuser +checkuserlog +chgrp +childs +chillu +chmoding +choicesstring +chrs +chunk +chunked +chunking +ci +cidr +cidrtoobroad +circ +citeseer +ckers +ckey +cl +clamav +clamscan +classname +clcategorie +cldir +cldr +clear +clearable +clearyourcache +clfrom +clickjacking +clicktracking +clientfor +clientpool +cllimit +clober +closed +clto +cm +cminvalidparammix +cmmissingparam +cmnamespace +cmtitle +co +code +codemap +codepoint +codestr +coi +colgroup +collapsable +collectionsaveascommunitypage +collectionsaveasuserpage +colname +colonseparator +colorer +colspan +commafy +commafying +comment +commentedit +commenthidden +comments +commitdiff +commoncssjs +compactpro +compare +compat +complete +cond +condcomment +condeferrable +condeferred +conds +config +confirmdeletetext +confirmed +confirmedittext +confirmemail +confirmrecreate +conflimit +confstr +conkey +conname +conrelid +console +content +contentformat +contenthandler +contentlanguage +contentless +contentmodel +contenttoobig +continue +contribs +contribslink +conttitle +contype +conv +converttitles +convmv +cookieprefix +cooltalk +coord +coordinates +copyrightico +copyrightpage +copyrightwarning +copyuploadbaddomain +copyuploaddisabled +copyvio +copywarn +cors +couldn +counter +countmsg +country +course +courseid +cpio +cprefs +cprotected +crarr +crashbug +create +createaccount +createonly +createpage +createtalk +creationsort +creativecommons +creditspage +crocker +cryptrand +csize +csrf +css +cssclass +csslinks +cta +ctime +ctor +ctype +cu +cul +curation +curdiff +curid +curlink +curren +currentarticle +currentbrowser +currentday +currentdayname +currentdow +currenthour +currentmonth +currentmonthabbrev +currentmonthname +currentmonthnamegen +currentrev +currentrevisionlink +currenttime +currenttimestamp +currentversion +currentweek +currentyear +customcssprotected +customised +customjsprotected +cut +cyber +cygwin +cyrl +d'oh +dadedad +dairiki +danga +danielc +darr +datalen +datapath +dataset +datasets +datasize +datatable +datatype +datedefault +dateformat +dateheader +dateopts +daysago +dbcnt +dbconnect +dberrortext +dbg +dbgfm +dbkey +dbkeys +dbks +dbname +dbrepllag +dbsettings +dbtype +dbversion +ddjvu +de +deadend +deadendpagestext +deadenpages +dealies +debughtml +decline +declined +decls +decr +decrease +default +defaultcontentmodel +defaultmessagetext +defaultmissing +defaultns +defaultoptions +defaultsort +defaultval +deferr +definite +deflimit +defs +deja +delete +deleteall +deletecomment +deleteconfirm +deleted +deletedhistory +deletedline +deletedonly +deletedrevision +deletedrevs +deletedtext +deletedwhileediting +deleteeducation +deleteglobalaccount +deletelogentry +deleteone +deleteotherreason +deletepage +deletereason +deletereasonotherlist +deleterevision +deleteset +deletethispage +deletetoken +deletion +deletionlog +delim +dellogpage +dellogpagetext +delundel +deprecated +deps +depth +dequeue +dequeued +dequeueing +dequeues +derivatives +desc +descending +description +descriptionmsg +descriptionmsgparams +descriptionurl +deserialization +deserialize +dest +detail +details +devangari +devel +df +dflt +dflts +dhtml +diams +didn +diff +diff's +diffchange +diffhist +difflink +diffonly +difftext +diffto +difftocontent +difftotext +dim +dimensions +dir +direction +directionmark +directorycreateerror +directorynotreadableerror +directoryreadonlyerror +dirmark +dirname +disabled +disabledtranscode +disablemail +disablepp +disclaimerpage +diskussion +displayname +displayrc +displaysearchoptions +displaytitle +displaytitles +displaywatchlist +distclean +distro +djava +djob +djvu +djvudump +djvulibre +djvutoxml +djvutxt +djvuxml +djvuzone +dkjsagfjsgashfajsh +dlen +dltk +dmoz +dnsbl +dnsblacklist +dnumber +docm +docroot +doctype +doctypes +docx +dodiff +doesn +domain +domainnames +domainpart +domainparts +domas +doms +dont +dotdotcount +dotm +dotsc +dotsi +dotsm +dotso +dotwise +dotx +doubleclick +doublequote +doxygen +dpos +dr +dropdown +dump +dumpfm +dupfunc +dupl +duplicatefiles +duplicatesoffile +dvips +dwfx +dwhitelist +e +eacute +earth +eauth +ecirc +ecmascript +edit +editbutton +editconflict +editconflicts +editcount +editfont +editform +edithelp +edithelppage +edithelpurl +editingcomment +editinginterface +editingold +editingsection +editinterface +editintro +edititis +editlink +editmyoptions +editmyprivateinfo +editmyusercss +editmyuserjs +editmywatchlist +editnotice +editnotsupported +editondblclick +editor +editownusertalk +editpage +editprotected +editreasons +editredlink +editrestriction +edits +editsection +editsectionhint +editsectiononrightclick +editsemiprotected +editsonly +editthispage +edittime +edittoken +edittools +editurl +editusercss +edituserjs +edoe +egrave +ei +eich +eiinvalidparammix +eimissingparam +eititle +el +elapsedreal +elastica +elemname +elems +elink +eltitle +email +emailable +emailaddress +emailauthenticated +emailauthentication +emailauthenticationclass +emailcapture +emailconfirm +emailconfirmed +emailconfirmlink +emaildisabled +emailling +emaillink +emailnotauthenticated +emailtoken +emailuser +embeddedin +empty +emptyfile +emptynewsection +emptypage +emsenhuber +emsp +en +enabled +enabledonly +enableparser +encapsed +enctype +end +endcode +endcond +endian +endid +endl +endsortkey +endsortkeyprefix +endtime +endverbatim +enhancedchanges +enlist +enotif +enotifminoredits +enotifrevealaddr +enotifusertalkpages +enotifwatchlistpages +enqueueing +enroll +ensp +entirewatchlist +entityid +envcmd +enwiki +eocdr +ep +eparticle +epcampus +epcoordinator +epinstructor +eponline +erevoke +errno +error +errorbox +errormessage +errorpagetitle +errors +errorstr +errortext +errorunknown +errstr +es +escapenoentities +escapeshellarg +esearch +español +española +etag +eu +euml +event +eventid +ex +exampleextension +examples +excludegroup +excludepage +excludeuser +executables +exempt +exiftool +existingwiki +exists +exiv +expandtab +expandtemplates +expandurl +experiment +expertise +expiry +expiryarray +explainconflict +export +exportnowrap +exportxml +expression +exptime +extauth +extendwatchlist +extensionname +extensions +extensiontags +external +externaldberror +externaldiff +externaledit +externaleditor +externalimages +externallinks +externalstore +extet +extiw +extlink +extlinks +extracts +extradata +extrafields +extralanglink +extraq +extratags +exturlusage +extuser +exxaammppllee +fa +facto +failback +failover +failsafe +fallbacks +false +falsy +fancysig +fastcgi +faux +favicon +fclose +fdef +fdff +feature +featured +featuredfeed +feed +feed's +feedback +feedbackid +feedcontributions +feedformat +feeditems +feedlink +feedlinks +feedurl +feedwatchlist +feff +female +fetchfileerror +fffe +ffff +fffff +ffffff +fieldname +fieldset +fieldsets +file +filearchive +filebackend +filecache +filecopyerror +filedelete +filedeleteerror +fileexists +fileextensions +filehidden +filehist +filehistory +fileinfo +filejournal +filekey +filelinks +filemissing +filemover +filemtime +filename +filenames +filenotfound +filepage +filepath +filerenameerror +filerepo +filerepoinfo +filerevert +filerevisions +files +filesize +filesort +filesorts +filesystem's +filesystems +filetoc +filetoobig +filetype +filetypemismatch +fileversions +filter +filterbots +filteriw +filterlanglinks +filterlocal +filterredir +filterwatched +findnext +finfo +firefox +firstname +firstrev +firsttime +fishbowl +fixme +fixup +flac +flag +flagconfig +flagged +flags +flagtype +flatlist +flds +float +flrevs +fmttime +fname +fnof +foldmarker +foldmethod +followpolicy +footericon +footericons +footerlinks +fopen +for +forall +forbidden +forcearticlepath +forcebot +forceditsummary +forceeditsummary +forcelinkupdate +forcerecursivelinkupdate +forcetoc +forcontent +formaction +format +formatmodules +formatted +formatters +formatting +formedness +formenctype +formnovalidate +formtype +forupdate +found +founder +fr +frac +frameborder +frameless +framesets +frasl +fread +freedomdefined +freeform +freenode +frickin +from +fromdb +fromdbmaster +fromid +fromrev +fromrevid +fromtitle +frontends +fseek +fsockopen +fsync +ftp +fullhistory +fullpagename +fullpagenamee +fulluri +fullurl +funcname +functionhooks +functionname +futuresplash +fvalue +ga +gack +gadgetcategories +gadgets +gaid +gaifilterredir +gaifrom +gallerybox +gallerycaption +gallerytext +gapdir +gapfilterredir +gapfrom +gaplimit +gapnamespace +gapprefix +garber +gblblock +gblock +gblrights +gc +gcldir +gcllimit +gender +general +generatexml +generator +geocoordinate +geodata +geosearch +gerrit +geshi +getcookie +getenv +getheader +getimagesize +getlink +getmac +getmarkashelpfulitem +getmypid +getrusage +gettimeofday +gettingstarted +gettoken +getuid +gfdl +ggp +ghostscript +gimpbaseenums +git +gitblit +gitdir +github +global +globalauth +globalblock +globalblocks +globalgroupmembership +globalgrouppermissions +globalgroups +globalsettings +globalunblock +globalusage +globaluserinfo +globe +gmail +gmdate +goodtitle +googlebot +gopher +graymap +grayscale +greant +greymap +group +groupcounts +groupless +groupmember +grouppage +groupperms +groupprms +groups +growinglink +grxml +gs +gtar +gu +guesstimezone +gui +guid +gunblock +guser +gwicke +gzcompress +gzdeflate +gzencode +gzhandler +gzip +gzipped +gzipping +hacky +hansm +hant +hardblocks +hardcode +hardcoding +harr +hash +hashar +hashcheckfailed +hashsearchdisabled +hashtable +hashtables +hasmatch +hasmsg +hasn +hasrelated +headelement +headerpos +headhtml +headitems +headlinks +headscripts +height +hellip +help +helpful +helppage +helptext +helpurl +helpurls +helpwindow +hexdump +hexstring +hidden +hiddencat +hiddencategories +hiddencats +hide +hideanons +hidebots +hidediff +hideliu +hideminor +hidemyself +hidename +hidepatrolled +hideredirects +hiderevision +hideuser +hidpi +highlimit +highmax +highuse +hilfe +hiphop +histfirst +histlast +historyempty +historysubmit +historywarning +hit +hitcount +hits +hlist +hmac +hobby +homelink +hookaborted +horohoe +hostnames +hours +hphp +hplist +hpos +hreflang +hslots +htaccess +htcp +html +htmlelements +htmlescaped +htmlform +htmlish +htmllist +htmlnest +htmlpair +htmlpairs +htmlsingle +htmlsingleallowed +htmlsingleonly +htmlspecialchars +htmltidy +http +httpaccept +httpbl +https +i +ia +iabn +iacute +icirc +icononly +iconv +icubench +icutest +id +idanduser +ids +ie's +ieinternals +ietf +iexcl +ifconfig +iframe +igbinary +iges +ignorewarnings +igrave +ii +iicontinue +iiprop +iiurlparam +iiurlwidth +iker +ilfrom +ilto +im +image +imagecolorallocate +imagegetsize +imageinfo +imageinvalidfilename +imagelimits +imagelinks +imagemagick +imagemaxsize +imagenocrossnamespace +imagepage +imagerepository +imagerotate +images +imagesize +imagetype +imagetypemismatch +imageusage +imagewhitelistenabled +imagick +imgmultigo +imgmultigoto +imgmultipagenext +imgmultipageprev +imgs +imgserv +immobilenamespace +implicitgroups +import +importbadinterwiki +importcantopen +importlogpage +importlogpagetext +importnofile +importtoken +importupload +importuploaderrorpartial +importuploaderrorsize +importuploaderrortemp +in +iname +inbound +includable +include +includecomments +includelocal +includeonly +includexmlnamespace +incr +increase +indefinite +index +indexfield +indexpageids +indexpolicy +indstr +infin +infinite +infiniteblock +info +infoaction +infobox +infoline +infomsg +ingroups +injectjs +inkscape +inlanguagecode +inlined +inno +inputneeded +insb +inser +instantcommons +institution +instructor +int +integer +integeroutofrange +intentionallyblankpage +interlang +interlangs +interlanguage +internal +internaledit +internalerror +interwiki +interwikimap +interwikipage +interwikis +interwikisearchinfo +interwikisource +intnull +intoken +intra +intro +intrw +ints +intval +invalid +invalidaction +invalidations +invalidcategory +invaliddomain +invalidemail +invalidemailaddress +invalidexpiry +invalidip +invalidlang +invalidlevel +invalidmode +invalidoldimage +invalidpage +invalidpageid +invalidparameter +invalidparammix +invalidpath +invalidrange +invalidsection +invalidsessiondata +invalidsha +invalidspecialpage +invalidtags +invalidtime +invalidtitle +invalidtoken +invaliduser +invalue +iorm +ip +ipbblocked +ipblock +ipblocks +ipbnounblockself +ipchain +ipedits +iphash +ipinrange +ipset +ipsets +ipusers +iquest +irc +ircs +isam +isapi +isbot +isconnected +iscur +isin +isip +islocal +ismap +isminor +ismodsince +ismulti +isnew +ispermalink +isroot +isself +isset +istainted +istalk +iswatch +it +item +itemid +itemprop +itemref +itemscope +itemtype +iter +iu +iuinvalidparammix +iumissingparam +iuml +iw +iwbacklinks +iwbl +iwlfrom +iwlinks +iwlprefix +iwltitle +iwprefix +iwtitle +iwurl +ized +javascript +javascripttest +jbartsh +jconds +jdk's +jhtml +jimbo +joaat +jobqueue +jointype +jorsch +journaling +jpeg +jpegtran +jslint +jsmimetype +jsminplus +json +jsonconfig +jsonfm +jsparse +jstext +jsvarurl +justthis +kabardian +kangxi +kashubia +kattouw +kblength +kernowek +key +keygen +keylen +keyname +keynames +keytype +khash +kikongo +kludgy +knownnamespace +konqueror +kpos +kuza +labarga +labelmsg +laggedslavemode +laggy +lang +langbacklinks +langcode +langcodes +langconversion +langlinks +langname +langprop +langs +language +languagelinks +languages +languageselection +languageshtml +laquo +large +larr +last +lastdiff +lastdot +lastedit +lasteditor +lastedittime +lastfile +lastlink +lastmod +lastmodifiedat +lastname +lastrevid +lastvisited +latgalian +laxström +lbase +lbl +lcattrib +lceil +lcomments +lcount +lcrocker +ldquo +le +len +length +leprop +lesque +lettercase +level +lfloor +lg +lgname +lgpassword +lgpl +lgtoken +lguserid +lgusername +libcurl +libel +libgimpbase +libketama +libmemcached +libre +libtidy +ligabue +lighttpd +limit +limitable +line +linenumber +linestart +link +linkarr +linkcolour +linkprefix +linkprefixcharset +linkpurge +links +linkstoimage +linktbl +linktext +linktodiffs +linktrail +linktype +linkupdate +list +listable +listadmins +listbots +listfiles +listgrouprights +listinfo +listingcontinuesabbrev +listoutput +listresult +lists +listtags +listuser +listusers +listusersfrom +livepreview +ll +llfrom +lllang +lltitle +lnumber +local +localday +localdayname +localdow +locale +localhour +localinterwiki +localmonth +localmonthabbrev +localmonthname +localmonthnamegen +localname +localonly +localsettings +localtimezone +localweek +localyear +lock +lockandhid +lockdb +lockdir +locked +lockmanager +log +logaction +logentry +logevent +logevents +logextract +loggedin +logid +login +loginerror +loginfo +loginlanguagelinks +loginlink +loginout +loginprompt +loginreqlink +loginreqpagetext +loginreqtitle +logins +logitem +loglink +loglist +logname +logonly +logopath +logourl +logout +logpage +logtext +logtitle +logtype +longpage +longpageerror +lookie +lookups +loopback +lossless +lossy +lowast +lowercaps +lowercased +lowlimit +lsaquo +lsquo +ltags +ltitle +ltrimmed +lurl +lysator +macr +magicarr +magicfile +magick +magicword +magicwordkey +magicwords +magnus +mahaction +mailerror +mailmypassword +mailnologin +mailparts +mailpassword +mailtext +mailto +mainmodule +mainpage +maint +maintainership +makesafe +male +malloc +manske +manualthumb +mark +markashelpful +markaspatrolledlink +markaspatrolledtext +markbot +markbotedits +markedaspatrollederror +markpatrolled +masse +match +matchcount +mathml +mathtt +matrixes +matroska +max +maxage +maxdim +maxlag +maxlength +maxlifetime +maxqueue +maxresults +maxsize +maxuploadsize +maxwidth +mazeland +mbresponse +mbstring +mccmnc +mckey +mcklmqw +mcrypt +mcvalue +md +mdash +mdot +medialink +mediaqueries +mediatype +mediawarning +mediawiki +mediawiki's +mediawikipage +megapixels +member +memberingroups +members +memc +memcache +memcached +memlimit +memoryp +memsw +merge +mergeable +merged +mergehistory +mergelog +mergelogpagetext +message +messagekey +messagename +messagepattern +messages +messagetype +meta +metacharacters +metachars +metadata +metadataversion +metafile +mhash +mhtml +micrblogging +microdata +microsyntaxes +microtime +middot +migurski +millitime +mime +mimer +mimesearchdisabled +mimetype +min +minangkabau +minh +minification +minified +minifier +minifies +minify +minifying +minimal +minor +minordefault +minoredit +minoreditletter +minsize +misconfigured +misermode +mismatch +misresolved +missing +missingcommentheader +missingcommenttext +missingdata +missingparam +missingpermission +missingresult +missingrev +missingsummary +missingtext +missingtitle +missinguser +mituzas +mixedapproval +mkdir +mms +mobile +mobileformat +mobilelanding +mobileview +modified +modifiedarticleprotection +modify +modsecurity +modsince +module +moduledisabled +modulename +modules +monitor +monobook +monospace +monospaced +month +monthsall +moodbar +moredotdotdot +morelinkstoimage +morethan +mouseup +move +movedarticleprotection +moveddeleted +movedto +movefile +movelogpage +movelogpagetext +movenologintext +movenotallowed +movenotallowedfile +moveonly +moveoverredirect +movepage +moves +movestable +movesubpages +movetalk +movethispage +movetoken +mozilla +mpeg +mpegurl +mpga +mplink +mptitle +msdn +msdownload +msec +msexcel +msgid +msgkey +msgs +msgsize +msgsmall +msgtext +msie +msmetafile +msnbot +mssql +msvideo +msword +mtime +mtype +mullane +multi +multiactions +multibyte +multicast +multipage +multipageimage +multipageimagenavbox +multipart +multiselect +multisource +multithreaded +multival +multivalue +multpages +munge +musso +mustbeloggedin +mustbeposted +mutator +mutators +muxers +mwdumper +mwfile +mwstore +mwsuggest +mwuser +mxircecho +mycontributions +mycontris +myext +myextension +myisam +mykey +mypage +mypreferences +mysqldump +mytalk +mytext +mywatchlist +möller +nabla +name +namehidden +nameinlowercase +namelookup +namemsg +names +namespace +namespacealiases +namespacebanner +namespacee +namespacenotice +namespacenumber +namespaceoptions +namespaceprotected +namespaces +namespacesall +namespaceselector +namespacing +nassert +nbase +nbsp +nbytes +nchanges +ncount +ndash +nearmatch +nedersaksies +nedersaksisch +needreblock +needservers +needtoken +netcdf +netware +never +new +newaddr +newarticletext +newarticletextanon +newer +newerthanrevid +newgroups +newheader +newid +newimages +newlen +newmessagesdifflinkplural +newmessageslinkplural +newname +newnames +newnamespace +newpage +newpageletter +newpages +newpageshidepatrolled +newparams +newpass +newpassword +newpos +newquery +newrevid +news +newsectionheaderdefaultlevel +newsectionlink +newsectionsummary +newset +newsfeed +newsize +newtalk +newtalks +newtalkseparator +newtext +newtimestamp +newtitle +newuser +newuserlogpage +newuserlogpagetext +newusers +newwidth +newwindow +nextdiff +nextid +nextlink +nextn +nextpage +nextredirect +nextrevision +nextval +nfkc +nfkd +nginx +nheight +niklas +nlink +nlinks +nmime +nnnn +nntp +no +noanimatethumb +noanontoken +noapiwrite +noarchivename +noarticle +noarticletext +noarticletextanon +noautopatrol +noblock +nobots +nobucket +nobuffer +nochange +nochanges +noclasses +nocode +nocomment +nocomplete +nocontent +nocontentconvert +nocontinue +noconvertlink +nocookiesfornew +nocopyright +nocourseid +nocreate +nocreatetext +nocredits +nocta +nodata +nodatabase +nodb +nodefault +nodeid +nodeleteablefile +nodeletion +nodelist +nodename +nodirection +nodotdot +noedit +noeditsection +noemail +noemailprefs +noemailtitle +noeventid +noexec +noexpertise +noexpression +nofeed +nofeedbackid +nofile +nofilekey +nofilename +nofilter +noflagtype +noflip +nofollow +nofound +nogallery +nogomatch +nogroup +noheader +noheadings +nohires +noids +noimage +noimageredirect +noimages +noinclude +noindex +noindexing +nointerwikipage +nointerwikiuserrights +noitem +nojs +nolabel +nolang +nolicense +nolimit +nolink +nolinkstoimage +nologging +nologin +nomahaction +nominornewtalk +nomodule +non +noname +nonamespacenumber +nonascii +noncascading +nondefaults +none +nonewsectionlink +nonexistent +nonfile +nonfilenamespace +nonincludable +noninfringement +noninitial +nonlocal +nonote +nonredirects +nonsense +nonunicodebrowser +noobjective +noofexpiries +noofprotections +noop +nooptions +nooverride +nopaction +nopage +nopageid +nopagetext +nopagetitle +noparser +nopathinfo +nopermission +noport +noprefix +noproject +noprop +noprotections +noquestion +noradius +noratelimit +norating +norcid +noread +noreason +noredir +noredirect +norequest +norestrictiontypes +noresult +noreturnto +norev +norevid +noreviewed +normalizedtitle +norole +norollbackdiff +noscale +noschema +noscript +nosearch +nosectiontitle +nosession +noshade +noskipnotif +noslash +nosniff +nosort +nosortdirection +nosource +nospecialpagetext +nost +nosubaction +nosubject +nosubpage +nosubpages +nosuccess +nosuchaction +nosuchactiontext +nosuchdatabase +nosuchlogid +nosuchpageid +nosuchrcid +nosuchrevid +nosuchsection +nosuchsectiontext +nosuchsectiontitle +nosuchspecialpage +nosuchuser +nosuchusershort +nosummary +notacceptable +notag +notaglist +notalk +notallowed +notanarticle +notarget +notcached +notdeleted +note +notempdir +notemplate +notext +nothumb +notif +notificationtimestamp +notificationtimestamps +notin +notitle +notitleconvert +notloggedin +notminor +noto +notoc +notoggle +notoken +notpatrollable +notransform +notreviewable +notrustworthy +notspecialpage +notsuspended +notvisiblerev +notwatched +notwikitext +notype +noudp +noupdates +nouploadmodule +nouser +nouserid +nousername +nouserspecified +novalues +noview +nowatchlist +nowellwritten +nowiki +nowlocal +nowserver +nparsing +ns +nsassociated +nsfrom +nsinvert +nslinks +nslist +nsname +nsnum +nspname +nsselect +nstab +nsub +ntfs +ntilde +ntitle +nuke +null +nullable +numauthors +number +numberheadings +numberingroup +numberof +numberofactiveusers +numberofadmins +numberofarticles +numberofedits +numberoffiles +numberofpages +numberofusers +numberofwatchingusers +numedits +numentries +numericized +numgroups +numtalkauthors +numtalkedits +numwatchers +nwidth +oacute +objectcache +objective +ocirc +ocount +oelig +of +officedocument +offset +offsite +ofname +ogevents +ogghandler +ograve +old +oldaddr +oldcountable +older +olderror +oldfile +oldgroups +oldid +oldimage +oldlen +oldnamespace +oldquery +oldrev +oldrevid +oldreviewedpages +oldshared +oldsig +oldsize +oldtext +oldtitle +oldtitlemsg +oline +oname +onerror +onkeyup +online +onload +onlyauthor +onlyinclude +onlypst +onlyquery +onsubmit +onthisday +ontop +onuser +openbasedir +opendoc +opendocument +opensearch +opensearchdescription +openssl's +openxml +openxmlformats +operamini +oplus +oppositedm +optgroup +optgroups +optionname +options +optionstoken +optionvalue +optstack +or +ordertype +ordf +ordm +org +orghttp +origcategory +ortime +oslash +other +otherlanguages +otherlist +otheroption +otherreason +othertime +otilde +otimes +otitle +ouml +outparam +outputter +outputtype +outreachwiki +over +overridable +override +overwrite +overwroteimage +own +owner +paction +page +pagecannotexist +pagecategories +pagecategorieslink +pageclass +pagecontent +pagecount +pagecss +pagedeleted +pagedlinks +pageid +pageids +pageimages +pageinfo +pagelink +pagelinks +pagemerge +pagename +pagenamee +pagenames +pagenum +pageoffset +pagepropnames +pageprops +pagerestrictions +pages +pageselector +pageset +pagesetmodule +pagesincategory +pagesinnamespace +pageswithprop +pagetextmsg +pagetitle +pagetools +pagetriage +pagetriageaction +pagetriagelist +pagetriagestats +pagetriagetagging +pagetriagetemplate +pageurl +pageview +pango +param +parameters +paraminfo +paramlist +paramname +params +paren +parens +parentid +parenttree +parms +parse +parsedcomment +parseddescription +parsedsummary +parseerror +parseinline +parsemag +parser +parsercache +parserfuncs +parserfunctions +parserhook +parserrender +parsetree +parsevalue +parsoid +partialupload +partname +pass's +passthru +password +passwordfor +passwordreset +passwordtooshort +paste +pastexpiry +pathchar +pathinfo +pathname +patrol +patroldisabled +patrolled +patrollink +patrolmarks +patroltoken +pattern +pcache +pcntl +pcomment +pdbk +pdf's +pendingdelta +perc +perfcached +perfcachedts +perm +perma +permalink +permdenied +permil +permissiondenied +permissionerror +permissionserrors +permissionserrorstext +permissiontype +perp +perrow +pgsql +photoshop +php +php's +phpfm +phps +phpsapi +phpunit +phpversion +phpwiki +phrasewise +phtml +pi +pipermail +pixmap +pkey +pkuk +pl +plain +plainlink +plainlinks +plaintext +plfrom +plink +pllimit +plns +plpgsql +pltitle +pltitles +plusminus +plusmn +pname +png'd +pnmtojpeg +pnmtopng +pointsize +poolcounter +popts +portlet +portlets +posplus +possible +postcomment +postgre +postsep +potd +potm +potx +poweredby +poweredbyico +powersearch +pp +ppam +ppsm +ppsx +pptm +pptx +precaching +precompiled +preemptively +preferences +preferencestoken +prefill +prefilled +prefix +prefixindex +prefixsearch +prefixsearchdisabled +prefs +prefsection +prefsnologintext2 +prefcontrol +preload +preloads +preloadtitle +prepending +prependtext +preprocess +preprocessing +preprocessors +presentationml +presep +pretransfer +prevchar +prevdiff +previd +previewconflict +previewhead +previewheader +previewnote +previewonfirst +previewontop +previewtext +previousrevision +prevlink +prevn +prexpiry +prfiltercascade +prfx +primary +printableversion +printfooter +printurl +privacypage +private +privs +prlevel +probabalistically +probs +proc +processings +procs +prodromou +profession +profileinfo +programmatically +project +projectpage +promotion +prop +properties +property +propname +props +prot +protect +protectcomment +protectedarticle +protectedinterface +protectednamespace +protectedpage +protectedpages +protectedpagetext +protectedpagewarning +protectedtitle +protectedtitles +protection +protections +protectlevel +protectlogpage +protectlogtext +protectthispage +protecttoken +proto +protocol +protocols +protorel +protos +proxied +proxyblocker +proxyblockreason +proxyunbannable +prtype +psir +pst +psttext +psychedelix +pt +ptext +ptool +pubdate +publicsuffix +publishfailed +punycode +purge +purged +qabardjajəbza +qbar +qbsettings +qlow +qmoicj +qp +quasit +query +querycache +querycachetwo +querycur +querydiff +querykey +querymodule +querymodules +querypage +querypages +querystring +querytype +question +queuefull +quickbar +quicksorts +quicktemplate +quicktime +qunit +quux +qvalues +rabdiff +radic +radius +raggett +raii +raimond +random +randompage +randomredirect +randstr +range +rangeblock +rangeblocks +rangedisabled +rangeend +rangestart +raquo +rarr +rarticle +rasterizations +rasterize +rasterized +rasterizer +ratelimited +ratelimits +rating +ratings +raw +rawfm +rawrow +rbspan +rc +rcdays +rceil +rcfeed +rcid +rcids +rclimit +rcoptions +rcpatroldisabled +rctitle +rctoken +rdev +rdfa +rdfrom +rdftype +rdquo +read +readable +readapidenied +readarray +reader +readline +readonlyreason +readonlytext +readonlywarning +readrequired +readrights +realaudio +realllly +realname +realpath +reason +reasonlist +reasonstr +reblock +rebuildtextindex +recache +recached +recaching +recalc +recentchange +recentchanges +recentchangescount +recentchangesdays +recentchangeslinked +recentchangestext +recenteditcount +recentedits +recip +recips +recreate +recurse +recurses +redir +redirect +redirectable +redirectcreated +redirectedfrom +redirections +redirector +redirectpagesub +redirectparams +redirects +redirectsnippet +redirectstofile +redirecttitle +redirectto +redirid +redirlinks +redirs +redis +redlink +redlinks +redocument +redux +reedyboy +reenables +reencode +reference +refetch +refresheducation +refreshlinks +regexes +regexlike +region +registered +registration +registrationdate +reimport +reindexation +reindexed +releasenotes +relevance +relevant +relicense +relimit +relkind +relname +relnamespace +remarticle +remembermypassword +removablegroups +removal +remove +removed +removedwatchtext +removetags +remreviewer +remstudent +renameuser +renaming +renderable +renderesibanner +renderwarning +renormalized +repeating +repl +replaceafter +replacer +replacers +replag +replyto +reporttime +repos +request +requested +requestid +requeue +required +rerender +rerendered +rescnt +researcher +resends +reset +resetkinds +resetlink +resetpass +resized +resolutioninfo +resolutionunit +resolve +resolved +resourceloader +responsecode +restore +restorelink +restoreprefs +restricted +result +resultset +resultsperpage +retrievedfrom +returnto +returntoquery +retval +reupload +revalidate +revalidation +revdel +revdelete +revdelete'd +revdelundel +revert +reverting +revertpage +reverts +revid +revids +review +reviewactivity +reviewed +reviewer +reviewing +revision +revisionasof +revisionday +revisiondelete +revisionid +revisionmonth +revisions +revisiontext +revisiontimestamp +revisionuser +revisionyear +revlink +revwrongpage +rfloor +rgba +richtext +rights +rightscode +rightsinfo +rightslog +rightslogtext +rked +rmdir +rn +rnlimit +robotstxt +roff +role +rollback +rollbacker +rollbacklink +rollbacklinkcount +rollbacktoken +rootpage +rootuserpages +rowcount +rown +rownum +rowsarr +rowset +rowspan +rowspans +rsaquo +rsargs +rsd +rsdf +rsquo +rss +rsvg +ruleset +rulesets +rusyn +rv +rvcontinue +rvdiffto +rvlimit +rvparse +rvprop +rvstart +rvstartid +rvtoken +sabino +safemode +safesubst +sais +sameorigin +samp +sansserif +save +savearticle +savedprefs +saveprefs +saveusergroups +sawfish +sbin +sbquo +scaler +scalers +scaron +score +screensize +scribunto +scriptable +scriptbuilder +scriptpath +scrolltop +sdot +search +search's +searchaction +searcharticle +searchboxes +searchbutton +searcheverything +searchform +searchindex +searchinfo +searchlimit +searchmenu +searchnamespaces +searchoptions +searchresulttext +searchstring +searchtitle +secondary +section +sectionanchor +sectionedit +sectioneditnotsupported +sectionformat +sectionnumber +sectionprop +sections +sectionsnippet +sectionsnotsupported +sectiontitle +securelogin +seiten +selectandother +selectorother +self +selflink +selfmove +semiglobal +semiprotected +semiprotectedlevels +semiprotectedpagewarning +sendemail +sendmail +sentences +serialize +servedby +servername +servertime +serverurl +sess +session +sessionfailure +sessionid +sessionkey +setchange +setcookie +setemail +setext +setglobalaccountstatus +setnewtype +setnotificationtimestamp +setopt +setrename +setrlimit +setstatus +sha +shar +sharding +shared +shareddescriptionfollows +sharedfile +sharedrepo +sharedupload +shellscript +shiftwidth +shockwave +short +shorturl +shouldn +shouting +show +showalldb +showbots +showdeleted +showdiff +showdifflinks +showfilename +showhiddencats +showhideminor +showhooks +showingresults +showinitializer +showjumplinks +showlinkedto +showme +showmeta +shownavigation +shownumberswatching +showpreview +showredirs +showreviewed +showsizediff +showtoc +showtoolbar +showunreviewed +shtml +si +siebrand +sighhhh +sigkill +sigmaf +signup +sigsegv +sigterm +sii +siit +siiurlwidth +simplesearch +singlegroup +singularthey +sinumberingroup +siprop +site +siteadmin +sitecsspreview +sitedir +siteinfo +sitejspreview +sitemap +sitemaps +sitematrix +sitename +sitenotice +siteprop +sitesearch +sitestats +sitestatsupdate +siteuser +sitewide +size +sizediff +sizediffdisabled +sizes +skey +skinclass +skinkey +skinname +skinnameclass +skins +skipcache +skipcaptcha +skipnotif +skname +sktemplate +slideshow +sm +smaxage +smil +smpp +sms's +smscontent +smslogs +smtp +snippet +sodipodi +softredirect +softtabstop +solaris +somecontent +somefeed +someuser +sorani +sorbs +sorbsreason +sort +sortdirection +sortkey +sortkeyprefix +sortkeys +source +soxred +spam +spamdetected +spamprotected +spamprotectionmatch +spamprotectiontext +spamprotectiontitle +spcontent +special +specialpage +specialpagealiases +specialpageattributes +specialpagegroup +specialpages +specialprotected +speedtip +speedy +speex +spekking +spellcheck +spezial +spoofable +spreadsheetml +sprefs +sprintf +sprotected +sql's +sqlite +sqltotal +sr +srchres +srcset +srgs +srprop +srwhat +stabilize +stable +stablesettings +stansvik +starcode +start +startid +startime +startsortkey +startsortkeyprefix +starttime +starttimestamp +starttransfer +stash +stashfailed +stashimageinfo +state +staticredirect +statistics +statline +status +statuskey +stdclass +stdout +steward +stopwords +storedversion +strcasecmp +strcmp +strftime +string +stripos +stripslashes +strlen +strpos +strrpos +strtime +strtok +strtolower +strtotime +strtr +struct +strval +stubthreshold +student +studies +stuffit +stxt +stylename +stylepath +styleversion +subaction +subarray +subcat +subcats +subclassing +subcond +subconds +subdir +subdomain +subdomains +sube +subelement +subelements +subfunction +subfunctions +subimages +subitem +subitems +subject +subjectid +subjectids +subjectpagename +subjectpagenamee +subjectspace +subjectspacee +subkey +subkeys +sublevels +submatch +submodule +submodule's +submodules +subnet +subpage +subpagename +subpagenamee +subpages +subpagestr +subparents +subprocesses +subsql +substr +succ +success +successbox +suckage +suggest +suggestion +suhosin +suhosin's +summ +summary +summarymissed +summaryrequired +supe +superdomain +superglobals +superset +suppress +suppressed +suppressedredirect +suppressionlog +suppressionlogtext +suppressredirect +suppressrevision +svgs +svn +svnroot +sybase +symlinked +syms +sysinfo +sysop +system +systemnachrichten +szdiff +szlig +szymon +t +tabindex +tablealign +tablecell +tablename +tablesorter +tablestack +tabletags +tabletype +tabstop +tag +tagfilter +tagline +taglist +tags +tagset +tagstack +tahoma +tailorings +talk +talkable +talkfrom +talkid +talkids +talkmove +talkmoveoverredirect +talkpage +talkpageheader +talkpagelinktext +talkpagename +talkpagenamee +talkpagetext +talkspace +talkspacee +talkto +tarask +taraškievica +target +tb +tbase +tbody +tboverride +tcount +tcsh +tddate +tdtime +teardown +telnet +temp +tempdir +template +templatelinks +templatepage +templates +templatesused +templatesusedpreview +templatesusedsection +tempname +tempout +test +testclean +testdata +testmailuser +teston +testpass +testrunner +testswarm +testuser +testutf +texi +texinfo +text +textarea +textareas +textares +textbox +textboxsize +texthidden +textid +textlink +textmissing +textoverride +textsf +textsize +textvector +texvc +tfoot +tful +tg +that'll +thead +thelink +theora +thetasym +thinsp +thisisdeleted +thispage +thumbborder +thumbcaption +thumberror +thumbheight +thumbhtml +thumbimage +thumbinner +thumblimits +thumbmime +thumbnail +thumbnailing +thumbnailsize +thumbname +thumbsize +thumbtext +thumburl +thumbwidth +timeago +timeanddate +timecond +timecorrection +timeframe +timekey +timeoffset +timep +timespans +timestamp +timestamps +timestamptz +timezonelegend +timezoneregion +timezoneuseoffset +timezoneuseserverdefault +tino +title +titleblacklist +titleconversion +titleexists +titlemsg +titleprefixeddbkey +titleprotected +titleprotectedwarning +titles +titlesnippet +titletext +titlevector +tl +tllimit +tltemplates +tmpfile +to +toclevel +tocline +tocnumber +tocsection +toctext +toctitle +tofragment +toggle +toid +token +tokenname +tokens +tolang +tongminh +toobig +toofewexpiries +toohigh +toolarray +toolbarparent +toolboxend +toolboxlink +toolong +toolow +tooltiponly +tooshort +top +toparse +topbar +toplevel +toplinks +topojson +toponly +torev +torevid +tornevall +torunblocked +totalcnt +totalcount +totalhits +totalmemory +totaltime +totitle +touched +tplarg +transcludable +transclude +transcluded +transcluding +transclusion +transclusions +transcode +transcodekey +transcoder +transcodereset +transcodestatus +transcoding +translatewiki +transstat +transwiki +troff +true +truespeed +truncatedtext +trustworthy +truteq +truthy +tsearch +tsquery +tuple +tweakblogs +tweakers +txt +txtfm +type +typemustmatch +typeof +typname +tzstring +uacute +uarr +uc +ucfirst +ucirc +udpprofile +ufffd +ugrave +ui +uids +uint +ulimit +ulink +ulinks +uname +unanchored +unapprove +unary +unattached +unauthenticate +unavailable +unblock +unblocklogentry +unblockself +unblocktoken +unbuffered +uncacheable +uncached +uncategorized +unclosable +uncompress +undel +undelete +undeleted +undeletion +undismissable +undismissible +undo +undoafter +undofailure +undorev +unescape +unescaped +unfeature +unfeatured +unflag +ungrouped +unhelpful +unhidden +unhide +unidata +unidecode +unindent +unindexed +uniq +unique +universaleditbutton +unixtime +unknown +unknownerror +unknownnamespace +unlock +unlockdb +unlogged +unmakesafe +unmark +unmerge +unmodified +unpadded +unpatrolled +unpatrolledletter +unprefixed +unprintables +unprotect +unprotectedarticle +unprotection +unprotectthispage +unreadcount +unredacted +unrequest +unrequested +unresolve +unresolved +unreviewed +unreviewedpages +unsanitized +unseed +unserialization +unserialize +unserialized +unserializes +unserializing +unsetting +unstub +unstubbed +unstubbing +unstubs +unsupportednamespace +unsupportedrepo +untaint +untracked +untrustworthiness +unused +unusual +unversioned +unviewable +unviewed +unwatch +unwatched +unwatchedpages +unwatching +unwatchthispage +unwikified +unwritable +upconvert +updateddate +updatedtime +updatelog +upgradedoc +upgrader +upload +upload's +uploaddisabled +uploadedimage +uploadjava +uploadlogpage +uploadlogpagetext +uploadnewversion +uploadnologintext +uploadpage +uploadscripted +uploadsource +uploadstash +uploadvirus +uploadwarning +uppercased +upsih +urandom +url +url's +urlaction +urldecode +urldecoded +urlencode +urlencoded +urlheight +urlparam +urlparm +urlpath +urlvar +urlwidth +ursh +us +usedomain +useemail +uselang +uselivepreview +usemod +usemsgcache +usenewrc +user +useragent +useragents +userblock +usercan +usercontribs +usercreate +usercreated +usercss +usercsspreview +usercssyoucanpreview +userdailycontribs +userdir +userdoesnotexist +usereditcount +useredits +useremail +userexists +usergroup +usergroups +userhidden +userid +userinfo +userinvalidcssjstitle +userips +userjs +userjsprev +userjspreview +userjsyoucanpreview +userlang +userlangattributes +userlink +userlinks +userlogin +userloginlink +userloginprompt +userlogout +usermaildisabled +usermessage +username +usernameless +usernames +userpage +userpages +userpageurl +userprefix +userrights +userrightstoken +users +usersbody +userspace +usertalk +usertalklink +usertext +usertoollinks +useskin +useto +usort +usrmonth +ussd +ussdcontent +ustar +ustoken +utfnormal +uuml +validate +validationbuilder +valign +vals +value +values +vandal +vandalism +variables +variant +variantarticlepath +varlang +varname +vars +varval +vasiliev +vasilvv +vbase +vbscript +vcount +vcsize +venema's +verbosify +version +versioning +versionlink +versionlog +versionrequired +versionrequiredtext +very +vhost +vi +vibber +videoinfo +view +viewcount +viewdeleted +viewhelppage +viewmyprivateinfo +viewmywatchlist +viewport +viewprevnext +viewsource +viewsourcelink +viewsourcetext +viewvc +viewyourtext +visible +visualeditor +viurlwidth +voff +vofp +voicexml +vorbis +vpad +vrml +vslow +vumi +vvcv +vxml +wais +wait +wakeup +walltime +warmup +warning +wasdeleted +wasn +watch +watchcreations +watchdefault +watchdeletion +watched +watchlist +watchlistdays +watchlisthideanons +watchlisthidebots +watchlisthideliu +watchlisthideminor +watchlisthideown +watchlisthidepatrolled +watchlistraw +watchlists +watchlisttoken +watchmoves +watchthis +watchthispage +watchtoken +watchuser +wb +wbmp +wbxml +wddx +wddxfm +weblog +weblogs +webm +webp +webrequest +webserver +weeks +weierp +weight +wellwritten +werdna +wget +what +whatlinkshere +whatwg +wheely +whether +whitelist +whitelisted +whitelistedittext +whitelisting +whois +wicke +width +widthx +wierkosz +wietse +wiki +wiki'd +wiki's +wikia +wikiadmin +wikibase +wikibits +wikibooks +wikidb +wikifarm +wikiid +wikilink +wikilinks +wikilove +wikiloveimagelog +wikimedia +wikimediacommons +wikimediafoundation +wikinews +wikipage +wikipedia +wikipedian +wikipedias +wikiquote +wikis +wikisource +wikisyntax +wikitable +wikitables +wikitech +wikitext +wikiuser +wikiversity +wikivoyage +wiktionary +wincache +wininet +withaccess +withaction +witheditsonly +withlanglinks +withoutlanglinks +wl +wlallrev +wldir +wlend +wlexcludeuser +wllimit +wlowner +wlprop +wltoken +wmf's +wml +wmlc +wmls +wmlsc +wmlscript +wmlscriptc +wordcount +wordprocessingml +wordwg +workalike +worldwind +wouldn +wr +writeapi +writeapidenied +writedisabled +writerequired +writerights +wrongpassword +x +xanalytics +xbitmap +xcache +xcancel +xdebug +xdiff +xdomain +xdomains +xff +xhtmldefaultnamespace +xhtmlnamespaces +xiff +xlam +xlsb +xlsm +xlsx +xltm +xltx +xml +xmldoublequote +xmlfm +xmlimport +xmlmeta +xmlns +xmlselect +xor +xpinstall +xpixmap +xpsdocument +xtended +xwindowdump +xxxx +xxxxx +yacute +yaml +yamlfm +yandex +year +yes +youhavenewmessages +youhavenewmessagesfromusers +youhavenewmessagesmanyusers +youhavenewmessagesmulti +yourdiff +yourdomainname +youremail +yourgender +yourinternal +yourlanguage +yourname +yournick +yourpassword +yourrealname +yourtext +yourvariant +yourwiki +yuml +yyyymmddhhiiss +zcmd +zerobanner +zerobar +zerobutton +zeroconfig +zerodontask +zerodot +zeroinfo +zeronet +zeroportal +zfile +zhdaemon +zhengzhu +zhtable +zijdel +zlang +zlib +zoffset +zrma +zwnj +ænglisc +ævar +świerkosz diff --git a/www/wiki/maintenance/doMaintenance.php b/www/wiki/maintenance/doMaintenance.php new file mode 100644 index 00000000..e87e0249 --- /dev/null +++ b/www/wiki/maintenance/doMaintenance.php @@ -0,0 +1,111 @@ +<?php +/** + * We want to make this whole thing as seamless as possible to the + * end-user. Unfortunately, we can't do _all_ of the work in the class + * because A) included files are not in global scope, but in the scope + * of their caller, and B) MediaWiki has way too many globals. So instead + * we'll kinda fake it, and do the requires() inline. <3 PHP + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @author Chad Horohoe <chad@anyonecanedit.org> + * @file + * @ingroup Maintenance + */ +use MediaWiki\MediaWikiServices; + +if ( !defined( 'RUN_MAINTENANCE_IF_MAIN' ) ) { + echo "This file must be included after Maintenance.php\n"; + exit( 1 ); +} + +// Wasn't included from the file scope, halt execution (probably wanted the class) +// If a class is using commandLine.inc (old school maintenance), they definitely +// cannot be included and will proceed with execution +if ( !Maintenance::shouldExecute() && $maintClass != 'CommandLineInc' ) { + return; +} + +if ( !$maintClass || !class_exists( $maintClass ) ) { + echo "\$maintClass is not set or is set to a non-existent class.\n"; + exit( 1 ); +} + +// Get an object to start us off +/** @var Maintenance $maintenance */ +$maintenance = new $maintClass(); + +// Basic sanity checks and such +$maintenance->setup(); + +// We used to call this variable $self, but it was moved +// to $maintenance->mSelf. Keep that here for b/c +$self = $maintenance->getName(); + +require_once "$IP/includes/PreConfigSetup.php"; + +if ( defined( 'MW_CONFIG_CALLBACK' ) ) { + # Use a callback function to configure MediaWiki + call_user_func( MW_CONFIG_CALLBACK ); +} else { + // Require the configuration (probably LocalSettings.php) + require $maintenance->loadSettings(); +} + +if ( $maintenance->getDbType() === Maintenance::DB_NONE ) { + if ( $wgLocalisationCacheConf['storeClass'] === false + && ( $wgLocalisationCacheConf['store'] == 'db' + || ( $wgLocalisationCacheConf['store'] == 'detect' && !$wgCacheDirectory ) ) + ) { + $wgLocalisationCacheConf['storeClass'] = 'LCStoreNull'; + } +} + +$maintenance->finalSetup(); +// Some last includes +require_once "$IP/includes/Setup.php"; + +// Initialize main config instance +$maintenance->setConfig( MediaWikiServices::getInstance()->getMainConfig() ); + +// Sanity-check required extensions are installed +$maintenance->checkRequiredExtensions(); + +// A good time when no DBs have writes pending is around lag checks. +// This avoids having long running scripts just OOM and lose all the updates. +$maintenance->setAgentAndTriggers(); + +// Do the work +$maintenance->execute(); + +// Potentially debug globals +$maintenance->globals(); + +if ( $maintenance->getDbType() !== Maintenance::DB_NONE ) { + // Perform deferred updates. + $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory(); + $lbFactory->commitMasterChanges( $maintClass ); + DeferredUpdates::doUpdates(); +} + +// log profiling info +wfLogProfilingData(); + +if ( isset( $lbFactory ) ) { + // Commit and close up! + $lbFactory->commitMasterChanges( 'doMaintenance' ); + $lbFactory->shutdown( $lbFactory::SHUTDOWN_NO_CHRONPROT ); +} diff --git a/www/wiki/maintenance/dumpBackup.php b/www/wiki/maintenance/dumpBackup.php new file mode 100644 index 00000000..9bf12221 --- /dev/null +++ b/www/wiki/maintenance/dumpBackup.php @@ -0,0 +1,137 @@ +<?php +/** + * Script that dumps wiki pages or logging database into an XML interchange + * wrapper format for export or backup + * + * Copyright © 2005 Brion Vibber <brion@pobox.com> + * https://www.mediawiki.org/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Dump Maintenance + */ + +require_once __DIR__ . '/backup.inc'; + +class DumpBackup extends BackupDumper { + function __construct( $args = null ) { + parent::__construct(); + + $this->addDescription( <<<TEXT +This script dumps the wiki page or logging database into an +XML interchange wrapper format for export or backup. + +XML output is sent to stdout; progress reports are sent to stderr. + +WARNING: this is not a full database dump! It is merely for public export + of your wiki. For full backup, see our online help at: + https://www.mediawiki.org/wiki/Backup +TEXT + ); + $this->stderr = fopen( "php://stderr", "wt" ); + // Actions + $this->addOption( 'full', 'Dump all revisions of every page' ); + $this->addOption( 'current', 'Dump only the latest revision of every page.' ); + $this->addOption( 'logs', 'Dump all log events' ); + $this->addOption( 'stable', 'Dump stable versions of pages' ); + $this->addOption( 'revrange', 'Dump range of revisions specified by revstart and ' . + 'revend parameters' ); + $this->addOption( 'orderrevs', 'Dump revisions in ascending revision order ' . + '(implies dump of a range of pages)' ); + $this->addOption( 'pagelist', + 'Dump only pages included in the file', false, true ); + // Options + $this->addOption( 'start', 'Start from page_id or log_id', false, true ); + $this->addOption( 'end', 'Stop before page_id or log_id n (exclusive)', false, true ); + $this->addOption( 'revstart', 'Start from rev_id', false, true ); + $this->addOption( 'revend', 'Stop before rev_id n (exclusive)', false, true ); + $this->addOption( 'skip-header', 'Don\'t output the <mediawiki> header' ); + $this->addOption( 'skip-footer', 'Don\'t output the </mediawiki> footer' ); + $this->addOption( 'stub', 'Don\'t perform old_text lookups; for 2-pass dump' ); + $this->addOption( 'uploads', 'Include upload records without files' ); + $this->addOption( 'include-files', 'Include files within the XML stream' ); + + if ( $args ) { + $this->loadWithArgv( $args ); + $this->processOptions(); + } + } + + function execute() { + $this->processOptions(); + + $textMode = $this->hasOption( 'stub' ) ? WikiExporter::STUB : WikiExporter::TEXT; + + if ( $this->hasOption( 'full' ) ) { + $this->dump( WikiExporter::FULL, $textMode ); + } elseif ( $this->hasOption( 'current' ) ) { + $this->dump( WikiExporter::CURRENT, $textMode ); + } elseif ( $this->hasOption( 'stable' ) ) { + $this->dump( WikiExporter::STABLE, $textMode ); + } elseif ( $this->hasOption( 'logs' ) ) { + $this->dump( WikiExporter::LOGS ); + } elseif ( $this->hasOption( 'revrange' ) ) { + $this->dump( WikiExporter::RANGE, $textMode ); + } else { + $this->error( 'No valid action specified.', 1 ); + } + } + + function processOptions() { + parent::processOptions(); + + // Evaluate options specific to this class + $this->reporting = !$this->hasOption( 'quiet' ); + + if ( $this->hasOption( 'pagelist' ) ) { + $filename = $this->getOption( 'pagelist' ); + $pages = file( $filename ); + if ( $pages === false ) { + $this->fatalError( "Unable to open file {$filename}\n" ); + } + $pages = array_map( 'trim', $pages ); + $this->pages = array_filter( $pages, function ( $x ) { + return $x !== ''; + } ); + } + + if ( $this->hasOption( 'start' ) ) { + $this->startId = intval( $this->getOption( 'start' ) ); + } + + if ( $this->hasOption( 'end' ) ) { + $this->endId = intval( $this->getOption( 'end' ) ); + } + + if ( $this->hasOption( 'revstart' ) ) { + $this->revStartId = intval( $this->getOption( 'revstart' ) ); + } + + if ( $this->hasOption( 'revend' ) ) { + $this->revEndId = intval( $this->getOption( 'revend' ) ); + } + + $this->skipHeader = $this->hasOption( 'skip-header' ); + $this->skipFooter = $this->hasOption( 'skip-footer' ); + $this->dumpUploads = $this->hasOption( 'uploads' ); + $this->dumpUploadFileContents = $this->hasOption( 'include-files' ); + $this->orderRevs = $this->hasOption( 'orderrevs' ); + } +} + +$maintClass = 'DumpBackup'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/dumpCategoriesAsRdf.php b/www/wiki/maintenance/dumpCategoriesAsRdf.php new file mode 100644 index 00000000..ff50498f --- /dev/null +++ b/www/wiki/maintenance/dumpCategoriesAsRdf.php @@ -0,0 +1,158 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + */ +use Wikimedia\Purtle\RdfWriter; +use Wikimedia\Purtle\RdfWriterFactory; +use Wikimedia\Rdbms\IDatabase; + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to provide RDF representation of the category tree. + * + * @ingroup Maintenance + * @since 1.30 + */ +class DumpCategoriesAsRdf extends Maintenance { + /** + * @var RdfWriter + */ + private $rdfWriter; + /** + * Categories RDF helper. + * @var CategoriesRdf + */ + private $categoriesRdf; + + public function __construct() { + parent::__construct(); + + $this->addDescription( "Generate RDF dump of categories in a wiki." ); + + $this->setBatchSize( 200 ); + $this->addOption( 'output', "Output file (default is stdout). Will be overwritten.", + false, true ); + $this->addOption( 'format', "Set the dump format.", false, true ); + } + + /** + * Produce row iterator for categories. + * @param IDatabase $dbr Database connection + * @return RecursiveIterator + */ + public function getCategoryIterator( IDatabase $dbr ) { + $it = new BatchRowIterator( + $dbr, + 'page', + [ 'page_title' ], + $this->mBatchSize + ); + $it->addConditions( [ + 'page_namespace' => NS_CATEGORY, + ] ); + $it->setFetchColumns( [ 'page_title', 'page_id' ] ); + return $it; + } + + /** + * Get iterator for links for categories. + * @param IDatabase $dbr + * @param array $ids List of page IDs + * @return Traversable + */ + public function getCategoryLinksIterator( IDatabase $dbr, array $ids ) { + $it = new BatchRowIterator( + $dbr, + 'categorylinks', + [ 'cl_from', 'cl_to' ], + $this->mBatchSize + ); + $it->addConditions( [ + 'cl_type' => 'subcat', + 'cl_from' => $ids + ] ); + $it->setFetchColumns( [ 'cl_from', 'cl_to' ] ); + return new RecursiveIteratorIterator( $it ); + } + + public function addDumpHeader( $timestamp ) { + global $wgRightsUrl; + $licenseUrl = $wgRightsUrl; + if ( substr( $licenseUrl, 0, 2 ) == '//' ) { + $licenseUrl = 'https:' . $licenseUrl; + } + $this->rdfWriter->about( wfExpandUrl( '/categoriesDump', PROTO_CANONICAL ) ) + ->a( 'schema', 'Dataset' ) + ->a( 'owl', 'Ontology' ) + ->say( 'cc', 'license' )->is( $licenseUrl ) + ->say( 'schema', 'softwareVersion' )->value( CategoriesRdf::FORMAT_VERSION ) + ->say( 'schema', 'dateModified' ) + ->value( wfTimestamp( TS_ISO_8601, $timestamp ), 'xsd', 'dateTime' ) + ->say( 'schema', 'isPartOf' )->is( wfExpandUrl( '/', PROTO_CANONICAL ) ) + ->say( 'owl', 'imports' )->is( CategoriesRdf::OWL_URL ); + } + + public function execute() { + $outFile = $this->getOption( 'output', 'php://stdout' ); + + if ( $outFile === '-' ) { + $outFile = 'php://stdout'; + } + + $output = fopen( $outFile, 'w' ); + $this->rdfWriter = $this->createRdfWriter( $this->getOption( 'format', 'ttl' ) ); + $this->categoriesRdf = new CategoriesRdf( $this->rdfWriter ); + + $this->categoriesRdf->setupPrefixes(); + $this->rdfWriter->start(); + + $this->addDumpHeader( time() ); + fwrite( $output, $this->rdfWriter->drain() ); + + $dbr = $this->getDB( DB_REPLICA, [ 'vslow' ] ); + + foreach ( $this->getCategoryIterator( $dbr ) as $batch ) { + $pages = []; + foreach ( $batch as $row ) { + $this->categoriesRdf->writeCategoryData( $row->page_title ); + $pages[$row->page_id] = $row->page_title; + } + + foreach ( $this->getCategoryLinksIterator( $dbr, array_keys( $pages ) ) as $row ) { + $this->categoriesRdf->writeCategoryLinkData( $pages[$row->cl_from], $row->cl_to ); + } + fwrite( $output, $this->rdfWriter->drain() ); + } + fflush( $output ); + if ( $outFile !== '-' ) { + fclose( $output ); + } + } + + /** + * @param string $format Writer format + * @return RdfWriter + */ + private function createRdfWriter( $format ) { + $factory = new RdfWriterFactory(); + return $factory->getWriter( $factory->getFormatName( $format ) ); + } +} + +$maintClass = "DumpCategoriesAsRdf"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/dumpIterator.php b/www/wiki/maintenance/dumpIterator.php new file mode 100644 index 00000000..6dbad949 --- /dev/null +++ b/www/wiki/maintenance/dumpIterator.php @@ -0,0 +1,186 @@ +<?php +/** + * Take page text out of an XML dump file and perform some operation on it. + * Used as a base class for CompareParsers and PreprocessDump. + * We implement below the simple task of searching inside a dump. + * + * Copyright © 2011 Platonides + * https://www.mediawiki.org/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Base class for interating over a dump. + * + * @ingroup Maintenance + */ +abstract class DumpIterator extends Maintenance { + + private $count = 0; + private $startTime; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Does something with a dump' ); + $this->addOption( 'file', 'File with text to run.', false, true ); + $this->addOption( 'dump', 'XML dump to execute all revisions.', false, true ); + $this->addOption( 'from', 'Article from XML dump to start from.', false, true ); + } + + public function execute() { + if ( !( $this->hasOption( 'file' ) ^ $this->hasOption( 'dump' ) ) ) { + $this->error( "You must provide a file or dump", true ); + } + + $this->checkOptions(); + + if ( $this->hasOption( 'file' ) ) { + $revision = new WikiRevision( $this->getConfig() ); + + $revision->setText( file_get_contents( $this->getOption( 'file' ) ) ); + $revision->setTitle( Title::newFromText( + rawurldecode( basename( $this->getOption( 'file' ), '.txt' ) ) + ) ); + $this->handleRevision( $revision ); + + return; + } + + $this->startTime = microtime( true ); + + if ( $this->getOption( 'dump' ) == '-' ) { + $source = new ImportStreamSource( $this->getStdin() ); + } else { + $this->error( "Sorry, I don't support dump filenames yet. " + . "Use - and provide it on stdin on the meantime.", true ); + } + $importer = new WikiImporter( $source, $this->getConfig() ); + + $importer->setRevisionCallback( + [ $this, 'handleRevision' ] ); + + $this->from = $this->getOption( 'from', null ); + $this->count = 0; + $importer->doImport(); + + $this->conclusions(); + + $delta = microtime( true ) - $this->startTime; + $this->error( "Done {$this->count} revisions in " . round( $delta, 2 ) . " seconds " ); + if ( $delta > 0 ) { + $this->error( round( $this->count / $delta, 2 ) . " pages/sec" ); + } + + # Perform the memory_get_peak_usage() when all the other data has been + # output so there's no damage if it dies. It is only available since + # 5.2.0 (since 5.2.1 if you haven't compiled with --enable-memory-limit) + $this->error( "Memory peak usage of " . memory_get_peak_usage() . " bytes\n" ); + } + + public function finalSetup() { + parent::finalSetup(); + + if ( $this->getDbType() == Maintenance::DB_NONE ) { + global $wgUseDatabaseMessages, $wgLocalisationCacheConf, $wgHooks; + $wgUseDatabaseMessages = false; + $wgLocalisationCacheConf['storeClass'] = 'LCStoreNull'; + $wgHooks['InterwikiLoadPrefix'][] = 'DumpIterator::disableInterwikis'; + } + } + + static function disableInterwikis( $prefix, &$data ) { + # Title::newFromText will check on each namespaced article if it's an interwiki. + # We always answer that it is not. + + return false; + } + + /** + * Callback function for each revision, child classes should override + * processRevision instead. + * @param WikiRevision $rev + */ + public function handleRevision( $rev ) { + $title = $rev->getTitle(); + if ( !$title ) { + $this->error( "Got bogus revision with null title!" ); + + return; + } + + $this->count++; + if ( isset( $this->from ) ) { + if ( $this->from != $title ) { + return; + } + $this->output( "Skipped " . ( $this->count - 1 ) . " pages\n" ); + + $this->count = 1; + $this->from = null; + } + + $this->processRevision( $rev ); + } + + /* Stub function for processing additional options */ + public function checkOptions() { + return; + } + + /* Stub function for giving data about what was computed */ + public function conclusions() { + return; + } + + /* Core function which does whatever the maintenance script is designed to do */ + abstract public function processRevision( $rev ); +} + +/** + * Maintenance script that runs a regex in the revisions from a dump. + * + * @ingroup Maintenance + */ +class SearchDump extends DumpIterator { + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Runs a regex in the revisions from a dump' ); + $this->addOption( 'regex', 'Searching regex', true, true ); + } + + public function getDbType() { + return Maintenance::DB_NONE; + } + + /** + * @param Revision $rev + */ + public function processRevision( $rev ) { + if ( preg_match( $this->getOption( 'regex' ), $rev->getContent()->getTextForSearchIndex() ) ) { + $this->output( $rev->getTitle() . " matches at edit from " . $rev->getTimestamp() . "\n" ); + } + } +} + +$maintClass = "SearchDump"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/dumpLinks.php b/www/wiki/maintenance/dumpLinks.php new file mode 100644 index 00000000..ff4e8945 --- /dev/null +++ b/www/wiki/maintenance/dumpLinks.php @@ -0,0 +1,79 @@ +<?php +/** + * Quick demo hack to generate a plaintext link dump, + * per the proposed wiki link database standard: + * http://www.usemod.com/cgi-bin/mb.pl?LinkDatabase + * + * Includes all (live and broken) intra-wiki links. + * Does not include interwiki or URL links. + * Dumps ASCII text to stdout; command-line. + * + * Copyright © 2005 Brion Vibber <brion@pobox.com> + * https://www.mediawiki.org/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that generates a plaintext link dump. + * + * @ingroup Maintenance + */ +class DumpLinks extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Quick demo hack to generate a plaintext link dump' ); + } + + public function execute() { + $dbr = $this->getDB( DB_REPLICA ); + $result = $dbr->select( [ 'pagelinks', 'page' ], + [ + 'page_id', + 'page_namespace', + 'page_title', + 'pl_namespace', + 'pl_title' ], + [ 'page_id=pl_from' ], + __METHOD__, + [ 'ORDER BY' => 'page_id' ] ); + + $lastPage = null; + foreach ( $result as $row ) { + if ( $lastPage != $row->page_id ) { + if ( $lastPage !== null ) { + $this->output( "\n" ); + } + $page = Title::makeTitle( $row->page_namespace, $row->page_title ); + $this->output( $page->getPrefixedURL() ); + $lastPage = $row->page_id; + } + $link = Title::makeTitle( $row->pl_namespace, $row->pl_title ); + $this->output( " " . $link->getPrefixedURL() ); + } + if ( $lastPage !== null ) { + $this->output( "\n" ); + } + } +} + +$maintClass = "DumpLinks"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/dumpTextPass.php b/www/wiki/maintenance/dumpTextPass.php new file mode 100644 index 00000000..2b79b546 --- /dev/null +++ b/www/wiki/maintenance/dumpTextPass.php @@ -0,0 +1,992 @@ +<?php +/** + * BackupDumper that postprocesses XML dumps from dumpBackup.php to add page text + * + * Copyright (C) 2005 Brion Vibber <brion@pobox.com> + * https://www.mediawiki.org/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/backup.inc'; +require_once __DIR__ . '/../includes/export/WikiExporter.php'; + +use Wikimedia\Rdbms\IMaintainableDatabase; + +/** + * @ingroup Maintenance + */ +class TextPassDumper extends BackupDumper { + /** @var BaseDump */ + public $prefetch = null; + /** @var string|bool */ + private $thisPage; + /** @var string|bool */ + private $thisRev; + + // when we spend more than maxTimeAllowed seconds on this run, we continue + // processing until we write out the next complete page, then save output file(s), + // rename it/them and open new one(s) + public $maxTimeAllowed = 0; // 0 = no limit + + protected $input = "php://stdin"; + protected $history = WikiExporter::FULL; + protected $fetchCount = 0; + protected $prefetchCount = 0; + protected $prefetchCountLast = 0; + protected $fetchCountLast = 0; + + protected $maxFailures = 5; + protected $maxConsecutiveFailedTextRetrievals = 200; + protected $failureTimeout = 5; // Seconds to sleep after db failure + + protected $bufferSize = 524288; // In bytes. Maximum size to read from the stub in on go. + + protected $php = "php"; + protected $spawn = false; + + /** + * @var bool|resource + */ + protected $spawnProc = false; + + /** + * @var bool|resource + */ + protected $spawnWrite = false; + + /** + * @var bool|resource + */ + protected $spawnRead = false; + + /** + * @var bool|resource + */ + protected $spawnErr = false; + + /** + * @var bool|XmlDumpWriter + */ + protected $xmlwriterobj = false; + + protected $timeExceeded = false; + protected $firstPageWritten = false; + protected $lastPageWritten = false; + protected $checkpointJustWritten = false; + protected $checkpointFiles = []; + + /** + * @var IMaintainableDatabase + */ + protected $db; + + /** + * @param array $args For backward compatibility + */ + function __construct( $args = null ) { + parent::__construct(); + + $this->addDescription( <<<TEXT +This script postprocesses XML dumps from dumpBackup.php to add +page text which was stubbed out (using --stub). + +XML input is accepted on stdin. +XML output is sent to stdout; progress reports are sent to stderr. +TEXT + ); + $this->stderr = fopen( "php://stderr", "wt" ); + + $this->addOption( 'stub', 'To load a compressed stub dump instead of stdin. ' . + 'Specify as --stub=<type>:<file>.', false, true ); + $this->addOption( 'prefetch', 'Use a prior dump file as a text source, to savepressure on the ' . + 'database. (Requires the XMLReader extension). Specify as --prefetch=<type>:<file>', + false, true ); + $this->addOption( 'maxtime', 'Write out checkpoint file after this many minutes (writing' . + 'out complete page, closing xml file properly, and opening new one' . + 'with header). This option requires the checkpointfile option.', false, true ); + $this->addOption( 'checkpointfile', 'Use this string for checkpoint filenames,substituting ' . + 'first pageid written for the first %s (required) and the last pageid written for the ' . + 'second %s if it exists.', false, true, false, true ); // This can be specified multiple times + $this->addOption( 'quiet', 'Don\'t dump status reports to stderr.' ); + $this->addOption( 'current', 'Base ETA on number of pages in database instead of all revisions' ); + $this->addOption( 'spawn', 'Spawn a subprocess for loading text records' ); + $this->addOption( 'buffersize', 'Buffer size in bytes to use for reading the stub. ' . + '(Default: 512KB, Minimum: 4KB)', false, true ); + + if ( $args ) { + $this->loadWithArgv( $args ); + $this->processOptions(); + } + } + + function execute() { + $this->processOptions(); + $this->dump( true ); + } + + function processOptions() { + global $IP; + + parent::processOptions(); + + if ( $this->hasOption( 'buffersize' ) ) { + $this->bufferSize = max( intval( $this->getOption( 'buffersize' ) ), 4 * 1024 ); + } + + if ( $this->hasOption( 'prefetch' ) ) { + require_once "$IP/maintenance/backupPrefetch.inc"; + $url = $this->processFileOpt( $this->getOption( 'prefetch' ) ); + $this->prefetch = new BaseDump( $url ); + } + + if ( $this->hasOption( 'stub' ) ) { + $this->input = $this->processFileOpt( $this->getOption( 'stub' ) ); + } + + if ( $this->hasOption( 'maxtime' ) ) { + $this->maxTimeAllowed = intval( $this->getOption( 'maxtime' ) ) * 60; + } + + if ( $this->hasOption( 'checkpointfile' ) ) { + $this->checkpointFiles = $this->getOption( 'checkpointfile' ); + } + + if ( $this->hasOption( 'current' ) ) { + $this->history = WikiExporter::CURRENT; + } + + if ( $this->hasOption( 'full' ) ) { + $this->history = WikiExporter::FULL; + } + + if ( $this->hasOption( 'spawn' ) ) { + $this->spawn = true; + $val = $this->getOption( 'spawn' ); + if ( $val !== 1 ) { + $this->php = $val; + } + } + } + + /** + * Drop the database connection $this->db and try to get a new one. + * + * This function tries to get a /different/ connection if this is + * possible. Hence, (if this is possible) it switches to a different + * failover upon each call. + * + * This function resets $this->lb and closes all connections on it. + * + * @throws MWException + */ + function rotateDb() { + // Cleaning up old connections + if ( isset( $this->lb ) ) { + $this->lb->closeAll(); + unset( $this->lb ); + } + + if ( $this->forcedDb !== null ) { + $this->db = $this->forcedDb; + + return; + } + + if ( isset( $this->db ) && $this->db->isOpen() ) { + throw new MWException( 'DB is set and has not been closed by the Load Balancer' ); + } + + unset( $this->db ); + + // Trying to set up new connection. + // We do /not/ retry upon failure, but delegate to encapsulating logic, to avoid + // individually retrying at different layers of code. + + try { + $this->lb = wfGetLBFactory()->newMainLB(); + } catch ( Exception $e ) { + throw new MWException( __METHOD__ + . " rotating DB failed to obtain new load balancer (" . $e->getMessage() . ")" ); + } + + try { + $this->db = $this->lb->getConnection( DB_REPLICA, 'dump' ); + } catch ( Exception $e ) { + throw new MWException( __METHOD__ + . " rotating DB failed to obtain new database (" . $e->getMessage() . ")" ); + } + } + + function initProgress( $history = WikiExporter::FULL ) { + parent::initProgress(); + $this->timeOfCheckpoint = $this->startTime; + } + + function dump( $history, $text = WikiExporter::TEXT ) { + // Notice messages will foul up your XML output even if they're + // relatively harmless. + if ( ini_get( 'display_errors' ) ) { + ini_set( 'display_errors', 'stderr' ); + } + + $this->initProgress( $this->history ); + + // We are trying to get an initial database connection to avoid that the + // first try of this request's first call to getText fails. However, if + // obtaining a good DB connection fails it's not a serious issue, as + // getText does retry upon failure and can start without having a working + // DB connection. + try { + $this->rotateDb(); + } catch ( Exception $e ) { + // We do not even count this as failure. Just let eventual + // watchdogs know. + $this->progress( "Getting initial DB connection failed (" . + $e->getMessage() . ")" ); + } + + $this->egress = new ExportProgressFilter( $this->sink, $this ); + + // it would be nice to do it in the constructor, oh well. need egress set + $this->finalOptionCheck(); + + // we only want this so we know how to close a stream :-P + $this->xmlwriterobj = new XmlDumpWriter(); + + $input = fopen( $this->input, "rt" ); + $this->readDump( $input ); + + if ( $this->spawnProc ) { + $this->closeSpawn(); + } + + $this->report( true ); + } + + function processFileOpt( $opt ) { + $split = explode( ':', $opt, 2 ); + $val = $split[0]; + $param = ''; + if ( count( $split ) === 2 ) { + $param = $split[1]; + } + $fileURIs = explode( ';', $param ); + foreach ( $fileURIs as $URI ) { + switch ( $val ) { + case "file": + $newURI = $URI; + break; + case "gzip": + $newURI = "compress.zlib://$URI"; + break; + case "bzip2": + $newURI = "compress.bzip2://$URI"; + break; + case "7zip": + $newURI = "mediawiki.compress.7z://$URI"; + break; + default: + $newURI = $URI; + } + $newFileURIs[] = $newURI; + } + $val = implode( ';', $newFileURIs ); + + return $val; + } + + /** + * Overridden to include prefetch ratio if enabled. + */ + function showReport() { + if ( !$this->prefetch ) { + parent::showReport(); + + return; + } + + if ( $this->reporting ) { + $now = wfTimestamp( TS_DB ); + $nowts = microtime( true ); + $deltaAll = $nowts - $this->startTime; + $deltaPart = $nowts - $this->lastTime; + $this->pageCountPart = $this->pageCount - $this->pageCountLast; + $this->revCountPart = $this->revCount - $this->revCountLast; + + if ( $deltaAll ) { + $portion = $this->revCount / $this->maxCount; + $eta = $this->startTime + $deltaAll / $portion; + $etats = wfTimestamp( TS_DB, intval( $eta ) ); + if ( $this->fetchCount ) { + $fetchRate = 100.0 * $this->prefetchCount / $this->fetchCount; + } else { + $fetchRate = '-'; + } + $pageRate = $this->pageCount / $deltaAll; + $revRate = $this->revCount / $deltaAll; + } else { + $pageRate = '-'; + $revRate = '-'; + $etats = '-'; + $fetchRate = '-'; + } + if ( $deltaPart ) { + if ( $this->fetchCountLast ) { + $fetchRatePart = 100.0 * $this->prefetchCountLast / $this->fetchCountLast; + } else { + $fetchRatePart = '-'; + } + $pageRatePart = $this->pageCountPart / $deltaPart; + $revRatePart = $this->revCountPart / $deltaPart; + } else { + $fetchRatePart = '-'; + $pageRatePart = '-'; + $revRatePart = '-'; + } + $this->progress( sprintf( + "%s: %s (ID %d) %d pages (%0.1f|%0.1f/sec all|curr), " + . "%d revs (%0.1f|%0.1f/sec all|curr), %0.1f%%|%0.1f%% " + . "prefetched (all|curr), ETA %s [max %d]", + $now, wfWikiID(), $this->ID, $this->pageCount, $pageRate, + $pageRatePart, $this->revCount, $revRate, $revRatePart, + $fetchRate, $fetchRatePart, $etats, $this->maxCount + ) ); + $this->lastTime = $nowts; + $this->revCountLast = $this->revCount; + $this->prefetchCountLast = $this->prefetchCount; + $this->fetchCountLast = $this->fetchCount; + } + } + + function setTimeExceeded() { + $this->timeExceeded = true; + } + + function checkIfTimeExceeded() { + if ( $this->maxTimeAllowed + && ( $this->lastTime - $this->timeOfCheckpoint > $this->maxTimeAllowed ) + ) { + return true; + } + + return false; + } + + function finalOptionCheck() { + if ( ( $this->checkpointFiles && !$this->maxTimeAllowed ) + || ( $this->maxTimeAllowed && !$this->checkpointFiles ) + ) { + throw new MWException( "Options checkpointfile and maxtime must be specified together.\n" ); + } + foreach ( $this->checkpointFiles as $checkpointFile ) { + $count = substr_count( $checkpointFile, "%s" ); + if ( $count != 2 ) { + throw new MWException( "Option checkpointfile must contain two '%s' " + . "for substitution of first and last pageids, count is $count instead, " + . "file is $checkpointFile.\n" ); + } + } + + if ( $this->checkpointFiles ) { + $filenameList = (array)$this->egress->getFilenames(); + if ( count( $filenameList ) != count( $this->checkpointFiles ) ) { + throw new MWException( "One checkpointfile must be specified " + . "for each output option, if maxtime is used.\n" ); + } + } + } + + /** + * @throws MWException Failure to parse XML input + * @param string $input + * @return bool + */ + function readDump( $input ) { + $this->buffer = ""; + $this->openElement = false; + $this->atStart = true; + $this->state = ""; + $this->lastName = ""; + $this->thisPage = 0; + $this->thisRev = 0; + $this->thisRevModel = null; + $this->thisRevFormat = null; + + $parser = xml_parser_create( "UTF-8" ); + xml_parser_set_option( $parser, XML_OPTION_CASE_FOLDING, false ); + + xml_set_element_handler( + $parser, + [ $this, 'startElement' ], + [ $this, 'endElement' ] + ); + xml_set_character_data_handler( $parser, [ $this, 'characterData' ] ); + + $offset = 0; // for context extraction on error reporting + do { + if ( $this->checkIfTimeExceeded() ) { + $this->setTimeExceeded(); + } + $chunk = fread( $input, $this->bufferSize ); + if ( !xml_parse( $parser, $chunk, feof( $input ) ) ) { + wfDebug( "TextDumpPass::readDump encountered XML parsing error\n" ); + + $byte = xml_get_current_byte_index( $parser ); + $msg = wfMessage( 'xml-error-string', + 'XML import parse failure', + xml_get_current_line_number( $parser ), + xml_get_current_column_number( $parser ), + $byte . ( is_null( $chunk ) ? null : ( '; "' . substr( $chunk, $byte - $offset, 16 ) . '"' ) ), + xml_error_string( xml_get_error_code( $parser ) ) )->escaped(); + + xml_parser_free( $parser ); + + throw new MWException( $msg ); + } + $offset += strlen( $chunk ); + } while ( $chunk !== false && !feof( $input ) ); + if ( $this->maxTimeAllowed ) { + $filenameList = (array)$this->egress->getFilenames(); + // we wrote some stuff after last checkpoint that needs renamed + if ( file_exists( $filenameList[0] ) ) { + $newFilenames = []; + # we might have just written the header and footer and had no + # pages or revisions written... perhaps they were all deleted + # there's no pageID 0 so we use that. the caller is responsible + # for deciding what to do with a file containing only the + # siteinfo information and the mw tags. + if ( !$this->firstPageWritten ) { + $firstPageID = str_pad( 0, 9, "0", STR_PAD_LEFT ); + $lastPageID = str_pad( 0, 9, "0", STR_PAD_LEFT ); + } else { + $firstPageID = str_pad( $this->firstPageWritten, 9, "0", STR_PAD_LEFT ); + $lastPageID = str_pad( $this->lastPageWritten, 9, "0", STR_PAD_LEFT ); + } + + $filenameCount = count( $filenameList ); + for ( $i = 0; $i < $filenameCount; $i++ ) { + $checkpointNameFilledIn = sprintf( $this->checkpointFiles[$i], $firstPageID, $lastPageID ); + $fileinfo = pathinfo( $filenameList[$i] ); + $newFilenames[] = $fileinfo['dirname'] . '/' . $checkpointNameFilledIn; + } + $this->egress->closeAndRename( $newFilenames ); + } + } + xml_parser_free( $parser ); + + return true; + } + + /** + * Applies applicable export transformations to $text. + * + * @param string $text + * @param string $model + * @param string|null $format + * + * @return string + */ + private function exportTransform( $text, $model, $format = null ) { + try { + $handler = ContentHandler::getForModelID( $model ); + $text = $handler->exportTransform( $text, $format ); + } + catch ( MWException $ex ) { + $this->progress( + "Unable to apply export transformation for content model '$model': " . + $ex->getMessage() + ); + } + + return $text; + } + + /** + * Tries to get the revision text for a revision id. + * Export transformations are applied if the content model can is given or can be + * determined from the database. + * + * Upon errors, retries (Up to $this->maxFailures tries each call). + * If still no good revision get could be found even after this retrying, "" is returned. + * If no good revision text could be returned for + * $this->maxConsecutiveFailedTextRetrievals consecutive calls to getText, MWException + * is thrown. + * + * @param string $id The revision id to get the text for + * @param string|bool|null $model The content model used to determine + * applicable export transformations. + * If $model is null, it will be determined from the database. + * @param string|null $format The content format used when applying export transformations. + * + * @throws MWException + * @return string The revision text for $id, or "" + */ + function getText( $id, $model = null, $format = null ) { + global $wgContentHandlerUseDB; + + $prefetchNotTried = true; // Whether or not we already tried to get the text via prefetch. + $text = false; // The candidate for a good text. false if no proper value. + $failures = 0; // The number of times, this invocation of getText already failed. + + // The number of times getText failed without yielding a good text in between. + static $consecutiveFailedTextRetrievals = 0; + + $this->fetchCount++; + + // To allow to simply return on success and do not have to worry about book keeping, + // we assume, this fetch works (possible after some retries). Nevertheless, we koop + // the old value, so we can restore it, if problems occur (See after the while loop). + $oldConsecutiveFailedTextRetrievals = $consecutiveFailedTextRetrievals; + $consecutiveFailedTextRetrievals = 0; + + if ( $model === null && $wgContentHandlerUseDB ) { + $row = $this->db->selectRow( + 'revision', + [ 'rev_content_model', 'rev_content_format' ], + [ 'rev_id' => $this->thisRev ], + __METHOD__ + ); + + if ( $row ) { + $model = $row->rev_content_model; + $format = $row->rev_content_format; + } + } + + if ( $model === null || $model === '' ) { + $model = false; + } + + while ( $failures < $this->maxFailures ) { + // As soon as we found a good text for the $id, we will return immediately. + // Hence, if we make it past the try catch block, we know that we did not + // find a good text. + + try { + // Step 1: Get some text (or reuse from previous iteratuon if checking + // for plausibility failed) + + // Trying to get prefetch, if it has not been tried before + if ( $text === false && isset( $this->prefetch ) && $prefetchNotTried ) { + $prefetchNotTried = false; + $tryIsPrefetch = true; + $text = $this->prefetch->prefetch( (int)$this->thisPage, (int)$this->thisRev ); + + if ( $text === null ) { + $text = false; + } + + if ( is_string( $text ) && $model !== false ) { + // Apply export transformation to text coming from an old dump. + // The purpose of this transformation is to convert up from legacy + // formats, which may still be used in the older dump that is used + // for pre-fetching. Applying the transformation again should not + // interfere with content that is already in the correct form. + $text = $this->exportTransform( $text, $model, $format ); + } + } + + if ( $text === false ) { + // Fallback to asking the database + $tryIsPrefetch = false; + if ( $this->spawn ) { + $text = $this->getTextSpawned( $id ); + } else { + $text = $this->getTextDb( $id ); + } + + if ( $text !== false && $model !== false ) { + // Apply export transformation to text coming from the database. + // Prefetched text should already have transformations applied. + $text = $this->exportTransform( $text, $model, $format ); + } + + // No more checks for texts from DB for now. + // If we received something that is not false, + // We treat it as good text, regardless of whether it actually is or is not + if ( $text !== false ) { + return $text; + } + } + + if ( $text === false ) { + throw new MWException( "Generic error while obtaining text for id " . $id ); + } + + // We received a good candidate for the text of $id via some method + + // Step 2: Checking for plausibility and return the text if it is + // plausible + $revID = intval( $this->thisRev ); + if ( !isset( $this->db ) ) { + throw new MWException( "No database available" ); + } + + if ( $model !== CONTENT_MODEL_WIKITEXT ) { + $revLength = strlen( $text ); + } else { + $revLength = $this->db->selectField( 'revision', 'rev_len', [ 'rev_id' => $revID ] ); + } + + if ( strlen( $text ) == $revLength ) { + if ( $tryIsPrefetch ) { + $this->prefetchCount++; + } + + return $text; + } + + $text = false; + throw new MWException( "Received text is unplausible for id " . $id ); + } catch ( Exception $e ) { + $msg = "getting/checking text " . $id . " failed (" . $e->getMessage() . ")"; + if ( $failures + 1 < $this->maxFailures ) { + $msg .= " (Will retry " . ( $this->maxFailures - $failures - 1 ) . " more times)"; + } + $this->progress( $msg ); + } + + // Something went wrong; we did not a text that was plausible :( + $failures++; + + // A failure in a prefetch hit does not warrant resetting db connection etc. + if ( !$tryIsPrefetch ) { + // After backing off for some time, we try to reboot the whole process as + // much as possible to not carry over failures from one part to the other + // parts + sleep( $this->failureTimeout ); + try { + $this->rotateDb(); + if ( $this->spawn ) { + $this->closeSpawn(); + $this->openSpawn(); + } + } catch ( Exception $e ) { + $this->progress( "Rebooting getText infrastructure failed (" . $e->getMessage() . ")" . + " Trying to continue anyways" ); + } + } + } + + // Retirieving a good text for $id failed (at least) maxFailures times. + // We abort for this $id. + + // Restoring the consecutive failures, and maybe aborting, if the dump + // is too broken. + $consecutiveFailedTextRetrievals = $oldConsecutiveFailedTextRetrievals + 1; + if ( $consecutiveFailedTextRetrievals > $this->maxConsecutiveFailedTextRetrievals ) { + throw new MWException( "Graceful storage failure" ); + } + + return ""; + } + + /** + * May throw a database error if, say, the server dies during query. + * @param int $id + * @return bool|string + * @throws MWException + */ + private function getTextDb( $id ) { + global $wgContLang; + if ( !isset( $this->db ) ) { + throw new MWException( __METHOD__ . "No database available" ); + } + $row = $this->db->selectRow( 'text', + [ 'old_text', 'old_flags' ], + [ 'old_id' => $id ], + __METHOD__ ); + $text = Revision::getRevisionText( $row ); + if ( $text === false ) { + return false; + } + $stripped = str_replace( "\r", "", $text ); + $normalized = $wgContLang->normalize( $stripped ); + + return $normalized; + } + + private function getTextSpawned( $id ) { + MediaWiki\suppressWarnings(); + if ( !$this->spawnProc ) { + // First time? + $this->openSpawn(); + } + $text = $this->getTextSpawnedOnce( $id ); + MediaWiki\restoreWarnings(); + + return $text; + } + + function openSpawn() { + global $IP; + + if ( file_exists( "$IP/../multiversion/MWScript.php" ) ) { + $cmd = implode( " ", + array_map( 'wfEscapeShellArg', + [ + $this->php, + "$IP/../multiversion/MWScript.php", + "fetchText.php", + '--wiki', wfWikiID() ] ) ); + } else { + $cmd = implode( " ", + array_map( 'wfEscapeShellArg', + [ + $this->php, + "$IP/maintenance/fetchText.php", + '--wiki', wfWikiID() ] ) ); + } + $spec = [ + 0 => [ "pipe", "r" ], + 1 => [ "pipe", "w" ], + 2 => [ "file", "/dev/null", "a" ] ]; + $pipes = []; + + $this->progress( "Spawning database subprocess: $cmd" ); + $this->spawnProc = proc_open( $cmd, $spec, $pipes ); + if ( !$this->spawnProc ) { + $this->progress( "Subprocess spawn failed." ); + + return false; + } + list( + $this->spawnWrite, // -> stdin + $this->spawnRead, // <- stdout + ) = $pipes; + + return true; + } + + private function closeSpawn() { + MediaWiki\suppressWarnings(); + if ( $this->spawnRead ) { + fclose( $this->spawnRead ); + } + $this->spawnRead = false; + if ( $this->spawnWrite ) { + fclose( $this->spawnWrite ); + } + $this->spawnWrite = false; + if ( $this->spawnErr ) { + fclose( $this->spawnErr ); + } + $this->spawnErr = false; + if ( $this->spawnProc ) { + pclose( $this->spawnProc ); + } + $this->spawnProc = false; + MediaWiki\restoreWarnings(); + } + + private function getTextSpawnedOnce( $id ) { + global $wgContLang; + + $ok = fwrite( $this->spawnWrite, "$id\n" ); + // $this->progress( ">> $id" ); + if ( !$ok ) { + return false; + } + + $ok = fflush( $this->spawnWrite ); + // $this->progress( ">> [flush]" ); + if ( !$ok ) { + return false; + } + + // check that the text id they are sending is the one we asked for + // this avoids out of sync revision text errors we have encountered in the past + $newId = fgets( $this->spawnRead ); + if ( $newId === false ) { + return false; + } + if ( $id != intval( $newId ) ) { + return false; + } + + $len = fgets( $this->spawnRead ); + // $this->progress( "<< " . trim( $len ) ); + if ( $len === false ) { + return false; + } + + $nbytes = intval( $len ); + // actual error, not zero-length text + if ( $nbytes < 0 ) { + return false; + } + + $text = ""; + + // Subprocess may not send everything at once, we have to loop. + while ( $nbytes > strlen( $text ) ) { + $buffer = fread( $this->spawnRead, $nbytes - strlen( $text ) ); + if ( $buffer === false ) { + break; + } + $text .= $buffer; + } + + $gotbytes = strlen( $text ); + if ( $gotbytes != $nbytes ) { + $this->progress( "Expected $nbytes bytes from database subprocess, got $gotbytes " ); + + return false; + } + + // Do normalization in the dump thread... + $stripped = str_replace( "\r", "", $text ); + $normalized = $wgContLang->normalize( $stripped ); + + return $normalized; + } + + function startElement( $parser, $name, $attribs ) { + $this->checkpointJustWritten = false; + + $this->clearOpenElement( null ); + $this->lastName = $name; + + if ( $name == 'revision' ) { + $this->state = $name; + $this->egress->writeOpenPage( null, $this->buffer ); + $this->buffer = ""; + } elseif ( $name == 'page' ) { + $this->state = $name; + if ( $this->atStart ) { + $this->egress->writeOpenStream( $this->buffer ); + $this->buffer = ""; + $this->atStart = false; + } + } + + if ( $name == "text" && isset( $attribs['id'] ) ) { + $id = $attribs['id']; + $model = trim( $this->thisRevModel ); + $format = trim( $this->thisRevFormat ); + + $model = $model === '' ? null : $model; + $format = $format === '' ? null : $format; + + $text = $this->getText( $id, $model, $format ); + $this->openElement = [ $name, [ 'xml:space' => 'preserve' ] ]; + if ( strlen( $text ) > 0 ) { + $this->characterData( $parser, $text ); + } + } else { + $this->openElement = [ $name, $attribs ]; + } + } + + function endElement( $parser, $name ) { + $this->checkpointJustWritten = false; + + if ( $this->openElement ) { + $this->clearOpenElement( "" ); + } else { + $this->buffer .= "</$name>"; + } + + if ( $name == 'revision' ) { + $this->egress->writeRevision( null, $this->buffer ); + $this->buffer = ""; + $this->thisRev = ""; + $this->thisRevModel = null; + $this->thisRevFormat = null; + } elseif ( $name == 'page' ) { + if ( !$this->firstPageWritten ) { + $this->firstPageWritten = trim( $this->thisPage ); + } + $this->lastPageWritten = trim( $this->thisPage ); + if ( $this->timeExceeded ) { + $this->egress->writeClosePage( $this->buffer ); + // nasty hack, we can't just write the chardata after the + // page tag, it will include leading blanks from the next line + $this->egress->sink->write( "\n" ); + + $this->buffer = $this->xmlwriterobj->closeStream(); + $this->egress->writeCloseStream( $this->buffer ); + + $this->buffer = ""; + $this->thisPage = ""; + // this could be more than one file if we had more than one output arg + + $filenameList = (array)$this->egress->getFilenames(); + $newFilenames = []; + $firstPageID = str_pad( $this->firstPageWritten, 9, "0", STR_PAD_LEFT ); + $lastPageID = str_pad( $this->lastPageWritten, 9, "0", STR_PAD_LEFT ); + $filenamesCount = count( $filenameList ); + for ( $i = 0; $i < $filenamesCount; $i++ ) { + $checkpointNameFilledIn = sprintf( $this->checkpointFiles[$i], $firstPageID, $lastPageID ); + $fileinfo = pathinfo( $filenameList[$i] ); + $newFilenames[] = $fileinfo['dirname'] . '/' . $checkpointNameFilledIn; + } + $this->egress->closeRenameAndReopen( $newFilenames ); + $this->buffer = $this->xmlwriterobj->openStream(); + $this->timeExceeded = false; + $this->timeOfCheckpoint = $this->lastTime; + $this->firstPageWritten = false; + $this->checkpointJustWritten = true; + } else { + $this->egress->writeClosePage( $this->buffer ); + $this->buffer = ""; + $this->thisPage = ""; + } + } elseif ( $name == 'mediawiki' ) { + $this->egress->writeCloseStream( $this->buffer ); + $this->buffer = ""; + } + } + + function characterData( $parser, $data ) { + $this->clearOpenElement( null ); + if ( $this->lastName == "id" ) { + if ( $this->state == "revision" ) { + $this->thisRev .= $data; + } elseif ( $this->state == "page" ) { + $this->thisPage .= $data; + } + } elseif ( $this->lastName == "model" ) { + $this->thisRevModel .= $data; + } elseif ( $this->lastName == "format" ) { + $this->thisRevFormat .= $data; + } + + // have to skip the newline left over from closepagetag line of + // end of checkpoint files. nasty hack!! + if ( $this->checkpointJustWritten ) { + if ( $data[0] == "\n" ) { + $data = substr( $data, 1 ); + } + $this->checkpointJustWritten = false; + } + $this->buffer .= htmlspecialchars( $data ); + } + + function clearOpenElement( $style ) { + if ( $this->openElement ) { + $this->buffer .= Xml::element( $this->openElement[0], $this->openElement[1], $style ); + $this->openElement = false; + } + } +} + +$maintClass = 'TextPassDumper'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/dumpUploads.php b/www/wiki/maintenance/dumpUploads.php new file mode 100644 index 00000000..8d63fe55 --- /dev/null +++ b/www/wiki/maintenance/dumpUploads.php @@ -0,0 +1,128 @@ +<?php +/** + * Dump a the list of files uploaded, for feeding to tar or similar. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to dump a the list of files uploaded, + * for feeding to tar or similar. + * + * @ingroup Maintenance + */ +class UploadDumper extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Generates list of uploaded files which can be fed to tar or similar. +By default, outputs relative paths against the parent directory of $wgUploadDirectory.' ); + $this->addOption( 'base', 'Set base relative path instead of wiki include root', false, true ); + $this->addOption( 'local', 'List all local files, used or not. No shared files included' ); + $this->addOption( 'used', 'Skip local images that are not used' ); + $this->addOption( 'shared', 'Include images used from shared repository' ); + } + + public function execute() { + global $IP; + $this->mAction = 'fetchLocal'; + $this->mBasePath = $this->getOption( 'base', $IP ); + $this->mShared = false; + $this->mSharedSupplement = false; + + if ( $this->hasOption( 'local' ) ) { + $this->mAction = 'fetchLocal'; + } + + if ( $this->hasOption( 'used' ) ) { + $this->mAction = 'fetchUsed'; + } + + if ( $this->hasOption( 'shared' ) ) { + if ( $this->hasOption( 'used' ) ) { + // Include shared-repo files in the used check + $this->mShared = true; + } else { + // Grab all local *plus* used shared + $this->mSharedSupplement = true; + } + } + $this->{$this->mAction} ( $this->mShared ); + if ( $this->mSharedSupplement ) { + $this->fetchUsed( true ); + } + } + + /** + * Fetch a list of used images from a particular image source. + * + * @param bool $shared True to pass shared-dir settings to hash func + */ + function fetchUsed( $shared ) { + $dbr = $this->getDB( DB_REPLICA ); + $image = $dbr->tableName( 'image' ); + $imagelinks = $dbr->tableName( 'imagelinks' ); + + $sql = "SELECT DISTINCT il_to, img_name + FROM $imagelinks + LEFT OUTER JOIN $image + ON il_to=img_name"; + $result = $dbr->query( $sql ); + + foreach ( $result as $row ) { + $this->outputItem( $row->il_to, $shared ); + } + } + + /** + * Fetch a list of all images from a particular image source. + * + * @param bool $shared True to pass shared-dir settings to hash func + */ + function fetchLocal( $shared ) { + $dbr = $this->getDB( DB_REPLICA ); + $result = $dbr->select( 'image', + [ 'img_name' ], + '', + __METHOD__ ); + + foreach ( $result as $row ) { + $this->outputItem( $row->img_name, $shared ); + } + } + + function outputItem( $name, $shared ) { + $file = wfFindFile( $name ); + if ( $file && $this->filterItem( $file, $shared ) ) { + $filename = $file->getLocalRefPath(); + $rel = wfRelativePath( $filename, $this->mBasePath ); + $this->output( "$rel\n" ); + } else { + wfDebug( __METHOD__ . ": base file? $name\n" ); + } + } + + function filterItem( $file, $shared ) { + return $shared || $file->isLocal(); + } +} + +$maintClass = "UploadDumper"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/edit.php b/www/wiki/maintenance/edit.php new file mode 100644 index 00000000..4219ed05 --- /dev/null +++ b/www/wiki/maintenance/edit.php @@ -0,0 +1,107 @@ +<?php +/** + * Make a page edit. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to make a page edit. + * + * @ingroup Maintenance + */ +class EditCLI extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Edit an article from the command line, text is from stdin' ); + $this->addOption( 'user', 'Username', false, true, 'u' ); + $this->addOption( 'summary', 'Edit summary', false, true, 's' ); + $this->addOption( 'minor', 'Minor edit', false, false, 'm' ); + $this->addOption( 'bot', 'Bot edit', false, false, 'b' ); + $this->addOption( 'autosummary', 'Enable autosummary', false, false, 'a' ); + $this->addOption( 'no-rc', 'Do not show the change in recent changes', false, false, 'r' ); + $this->addOption( 'nocreate', 'Don\'t create new pages', false, false ); + $this->addOption( 'createonly', 'Only create new pages', false, false ); + $this->addArg( 'title', 'Title of article to edit' ); + } + + public function execute() { + global $wgUser; + + $userName = $this->getOption( 'user', false ); + $summary = $this->getOption( 'summary', '' ); + $minor = $this->hasOption( 'minor' ); + $bot = $this->hasOption( 'bot' ); + $autoSummary = $this->hasOption( 'autosummary' ); + $noRC = $this->hasOption( 'no-rc' ); + + if ( $userName === false ) { + $wgUser = User::newSystemUser( 'Maintenance script', [ 'steal' => true ] ); + } else { + $wgUser = User::newFromName( $userName ); + } + if ( !$wgUser ) { + $this->error( "Invalid username", true ); + } + if ( $wgUser->isAnon() ) { + $wgUser->addToDatabase(); + } + + $title = Title::newFromText( $this->getArg() ); + if ( !$title ) { + $this->error( "Invalid title", true ); + } + + if ( $this->hasOption( 'nocreate' ) && !$title->exists() ) { + $this->error( "Page does not exist", true ); + } elseif ( $this->hasOption( 'createonly' ) && $title->exists() ) { + $this->error( "Page already exists", true ); + } + + $page = WikiPage::factory( $title ); + + # Read the text + $text = $this->getStdin( Maintenance::STDIN_ALL ); + $content = ContentHandler::makeContent( $text, $title ); + + # Do the edit + $this->output( "Saving... " ); + $status = $page->doEditContent( $content, $summary, + ( $minor ? EDIT_MINOR : 0 ) | + ( $bot ? EDIT_FORCE_BOT : 0 ) | + ( $autoSummary ? EDIT_AUTOSUMMARY : 0 ) | + ( $noRC ? EDIT_SUPPRESS_RC : 0 ) ); + if ( $status->isOK() ) { + $this->output( "done\n" ); + $exit = 0; + } else { + $this->output( "failed\n" ); + $exit = 1; + } + if ( !$status->isGood() ) { + $this->output( $status->getWikiText( false, false, 'en' ) . "\n" ); + } + exit( $exit ); + } +} + +$maintClass = "EditCLI"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/eraseArchivedFile.php b/www/wiki/maintenance/eraseArchivedFile.php new file mode 100644 index 00000000..c90056db --- /dev/null +++ b/www/wiki/maintenance/eraseArchivedFile.php @@ -0,0 +1,117 @@ +<?php +/** + * Delete archived (non-current) files from storage + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to delete archived (non-current) files from storage. + * + * @todo Maybe add some simple logging + * + * @ingroup Maintenance + * @since 1.22 + */ +class EraseArchivedFile extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Erases traces of deleted files.' ); + $this->addOption( 'delete', 'Perform the deletion' ); + $this->addOption( 'filename', 'File name', false, true ); + $this->addOption( 'filekey', 'File storage key (with extension) or "*"', true, true ); + } + + public function execute() { + if ( !$this->hasOption( 'delete' ) ) { + $this->output( "Use --delete to actually confirm this script\n" ); + } + + $filekey = $this->getOption( 'filekey' ); + $filename = $this->getOption( 'filename' ); + + if ( $filekey === '*' ) { // all versions by name + if ( !strlen( $filename ) ) { + $this->error( "Missing --filename parameter.", 1 ); + } + $afile = false; + } else { // specified version + $dbw = $this->getDB( DB_MASTER ); + $row = $dbw->selectRow( 'filearchive', '*', + [ 'fa_storage_group' => 'deleted', 'fa_storage_key' => $filekey ], + __METHOD__ ); + if ( !$row ) { + $this->error( "No deleted file exists with key '$filekey'.", 1 ); + } + $filename = $row->fa_name; + $afile = ArchivedFile::newFromRow( $row ); + } + + $file = wfLocalFile( $filename ); + if ( $file->exists() ) { + $this->error( "File '$filename' is still a public file, use the delete form.\n", 1 ); + } + + $this->output( "Purging all thumbnails for file '$filename'..." ); + $file->purgeCache(); + $this->output( "done.\n" ); + + if ( $afile instanceof ArchivedFile ) { + $this->scrubVersion( $afile ); + } else { + $this->output( "Finding deleted versions of file '$filename'...\n" ); + $this->scrubAllVersions( $filename ); + $this->output( "Done\n" ); + } + } + + protected function scrubAllVersions( $name ) { + $dbw = $this->getDB( DB_MASTER ); + $res = $dbw->select( 'filearchive', '*', + [ 'fa_name' => $name, 'fa_storage_group' => 'deleted' ], + __METHOD__ ); + foreach ( $res as $row ) { + $this->scrubVersion( ArchivedFile::newFromRow( $row ) ); + } + } + + protected function scrubVersion( ArchivedFile $archivedFile ) { + $key = $archivedFile->getStorageKey(); + $name = $archivedFile->getName(); + $ts = $archivedFile->getTimestamp(); + $repo = RepoGroup::singleton()->getLocalRepo(); + $path = $repo->getZonePath( 'deleted' ) . '/' . $repo->getDeletedHashPath( $key ) . $key; + if ( $this->hasOption( 'delete' ) ) { + $status = $repo->getBackend()->delete( [ 'src' => $path ] ); + if ( $status->isOK() ) { + $this->output( "Deleted version '$key' ($ts) of file '$name'\n" ); + } else { + $this->output( "Failed to delete version '$key' ($ts) of file '$name'\n" ); + $this->output( print_r( $status->getErrorsArray(), true ) ); + } + } else { + $this->output( "Would delete version '{$key}' ({$ts}) of file '$name'\n" ); + } + } +} + +$maintClass = "EraseArchivedFile"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/eval.php b/www/wiki/maintenance/eval.php new file mode 100644 index 00000000..40d29ef8 --- /dev/null +++ b/www/wiki/maintenance/eval.php @@ -0,0 +1,93 @@ +<?php +/** + * This script lets a command-line user start up the wiki engine and then poke + * about by issuing PHP commands directly. + * + * Unlike eg Python, you need to use a 'return' statement explicitly for the + * interactive shell to print out the value of the expression. Multiple lines + * are evaluated separately, so blocks need to be input without a line break. + * Fatal errors such as use of undeclared functions can kill the shell. + * + * To get decent line editing behavior, you should compile PHP with support + * for GNU readline (pass --with-readline to configure). + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +use MediaWiki\Logger\LoggerFactory; +use MediaWiki\Logger\ConsoleSpi; +use MediaWiki\MediaWikiServices; + +$optionsWithArgs = [ 'd' ]; + +require_once __DIR__ . "/commandLine.inc"; + +if ( isset( $options['d'] ) ) { + $d = $options['d']; + if ( $d > 0 ) { + LoggerFactory::registerProvider( new ConsoleSpi ); + // Some services hold Logger instances in object properties + MediaWikiServices::resetGlobalInstance(); + } + if ( $d > 1 ) { + wfGetDB( DB_MASTER )->setFlag( DBO_DEBUG ); + wfGetDB( DB_REPLICA )->setFlag( DBO_DEBUG ); + } +} + +$__useReadline = function_exists( 'readline_add_history' ) + && Maintenance::posix_isatty( 0 /*STDIN*/ ); + +if ( $__useReadline ) { + $__historyFile = isset( $_ENV['HOME'] ) ? + "{$_ENV['HOME']}/.mweval_history" : "$IP/maintenance/.mweval_history"; + readline_read_history( $__historyFile ); +} + +$__e = null; // PHP exception +while ( ( $__line = Maintenance::readconsole() ) !== false ) { + if ( $__e && !preg_match( '/^(exit|die);?$/', $__line ) ) { + // Internal state may be corrupted or fatals may occur later due + // to some object not being set. Don't drop out of eval in case + // lines were being pasted in (which would then get dumped to the shell). + // Instead, just absorb the remaning commands. Let "exit" through per DWIM. + echo "Exception was thrown before; please restart eval.php\n"; + continue; + } + if ( $__useReadline ) { + readline_add_history( $__line ); + readline_write_history( $__historyFile ); + } + try { + $__val = eval( $__line . ";" ); + } catch ( Exception $__e ) { + echo "Caught exception " . get_class( $__e ) . + ": {$__e->getMessage()}\n" . $__e->getTraceAsString() . "\n"; + continue; + } + if ( wfIsHHVM() || is_null( $__val ) ) { + echo "\n"; + } elseif ( is_string( $__val ) || is_numeric( $__val ) ) { + echo "$__val\n"; + } else { + var_dump( $__val ); + } +} + +print "\n"; diff --git a/www/wiki/maintenance/exportSites.php b/www/wiki/maintenance/exportSites.php new file mode 100644 index 00000000..b1e4fa94 --- /dev/null +++ b/www/wiki/maintenance/exportSites.php @@ -0,0 +1,56 @@ +<?php + +$basePath = getenv( 'MW_INSTALL_PATH' ) !== false ? getenv( 'MW_INSTALL_PATH' ) : __DIR__ . '/..'; + +require_once $basePath . '/maintenance/Maintenance.php'; + +/** + * Maintenance script for exporting site definitions from XML into the sites table. + * + * @since 1.25 + * + * @licence GNU GPL v2+ + * @author Daniel Kinzler + */ +class ExportSites extends Maintenance { + + public function __construct() { + $this->addDescription( 'Exports site definitions the sites table to XML file' ); + + $this->addArg( 'file', 'A file to write the XML to (see docs/sitelist.txt). ' . + 'Use "php://stdout" to write to stdout.', true + ); + + parent::__construct(); + } + + /** + * Do the actual work. All child classes will need to implement this + */ + public function execute() { + $file = $this->getArg( 0 ); + + if ( $file === 'php://output' || $file === 'php://stdout' ) { + $this->mQuiet = true; + } + + $handle = fopen( $file, 'w' ); + + if ( !$handle ) { + $this->error( "Failed to open $file for writing.\n", 1 ); + } + + $exporter = new SiteExporter( $handle ); + + $siteLookup = \MediaWiki\MediaWikiServices::getInstance()->getSiteLookup(); + $exporter->exportSites( $siteLookup->getSites() ); + + fclose( $handle ); + + $this->output( "Exported sites to " . realpath( $file ) . ".\n" ); + } + +} + +$maintClass = 'ExportSites'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/fetchText.php b/www/wiki/maintenance/fetchText.php new file mode 100644 index 00000000..9c5a3751 --- /dev/null +++ b/www/wiki/maintenance/fetchText.php @@ -0,0 +1,96 @@ +<?php +/** + * Communications protocol. + * This is used by dumpTextPass.php when the --spawn option is present. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +use Wikimedia\Rdbms\IDatabase; + +/** + * Maintenance script used to fetch page text in a subprocess. + * + * @ingroup Maintenance + */ +class FetchText extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( "Fetch the raw revision blob from an old_id.\n" . + "NOTE: Export transformations are NOT applied. " . + "This is left to backupTextPass.php" + ); + } + + /** + * returns a string containing the following in order: + * textid + * \n + * length of text (-1 on error = failure to retrieve/unserialize/gunzip/etc) + * \n + * text (may be empty) + * + * note that the text string itself is *not* followed by newline + */ + public function execute() { + $db = $this->getDB( DB_REPLICA ); + $stdin = $this->getStdin(); + while ( !feof( $stdin ) ) { + $line = fgets( $stdin ); + if ( $line === false ) { + // We appear to have lost contact... + break; + } + $textId = intval( $line ); + $text = $this->doGetText( $db, $textId ); + if ( $text === false ) { + # actual error, not zero-length text + $textLen = "-1"; + } else { + $textLen = strlen( $text ); + } + $this->output( $textId . "\n" . $textLen . "\n" . $text ); + } + } + + /** + * May throw a database error if, say, the server dies during query. + * @param IDatabase $db + * @param int $id The old_id + * @return string + */ + private function doGetText( $db, $id ) { + $id = intval( $id ); + $row = $db->selectRow( 'text', + [ 'old_text', 'old_flags' ], + [ 'old_id' => $id ], + __METHOD__ ); + $text = Revision::getRevisionText( $row ); + if ( $text === false ) { + return false; + } + + return $text; + } +} + +$maintClass = "FetchText"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/fileOpPerfTest.php b/www/wiki/maintenance/fileOpPerfTest.php new file mode 100644 index 00000000..4b6c6194 --- /dev/null +++ b/www/wiki/maintenance/fileOpPerfTest.php @@ -0,0 +1,145 @@ +<?php +/** + * Test for fileop performance. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +error_reporting( E_ALL ); +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to test fileop performance. + * + * @ingroup Maintenance + */ +class TestFileOpPerformance extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Test fileop performance' ); + $this->addOption( 'b1', 'Backend 1', true, true ); + $this->addOption( 'b2', 'Backend 2', false, true ); + $this->addOption( 'srcdir', 'File source directory', true, true ); + $this->addOption( 'maxfiles', 'Max files', false, true ); + $this->addOption( 'quick', 'Avoid operation pre-checks (use doQuickOperations())' ); + $this->addOption( 'parallelize', '"parallelize" flag for doOperations()', false, true ); + } + + public function execute() { + $backend = FileBackendGroup::singleton()->get( $this->getOption( 'b1' ) ); + $this->doPerfTest( $backend ); + + if ( $this->getOption( 'b2' ) ) { + $backend = FileBackendGroup::singleton()->get( $this->getOption( 'b2' ) ); + $this->doPerfTest( $backend ); + } + } + + protected function doPerfTest( FileBackend $backend ) { + $ops1 = []; + $ops2 = []; + $ops3 = []; + $ops4 = []; + $ops5 = []; + + $baseDir = 'mwstore://' . $backend->getName() . '/testing-cont1'; + $backend->prepare( [ 'dir' => $baseDir ] ); + + $dirname = $this->getOption( 'srcdir' ); + $dir = opendir( $dirname ); + if ( !$dir ) { + return; + } + + while ( $dir && ( $file = readdir( $dir ) ) !== false ) { + if ( $file[0] != '.' ) { + $this->output( "Using '$dirname/$file' in operations.\n" ); + $dst = $baseDir . '/' . wfBaseName( $file ); + $ops1[] = [ 'op' => 'store', + 'src' => "$dirname/$file", 'dst' => $dst, 'overwrite' => 1 ]; + $ops2[] = [ 'op' => 'copy', + 'src' => "$dst", 'dst' => "$dst-1", 'overwrite' => 1 ]; + $ops3[] = [ 'op' => 'move', + 'src' => $dst, 'dst' => "$dst-2", 'overwrite' => 1 ]; + $ops4[] = [ 'op' => 'delete', 'src' => "$dst-1" ]; + $ops5[] = [ 'op' => 'delete', 'src' => "$dst-2" ]; + } + if ( count( $ops1 ) >= $this->getOption( 'maxfiles', 20 ) ) { + break; // enough + } + } + closedir( $dir ); + $this->output( "\n" ); + + $method = $this->hasOption( 'quick' ) ? 'doQuickOperations' : 'doOperations'; + + $opts = [ 'force' => 1 ]; + if ( $this->hasOption( 'parallelize' ) ) { + $opts['parallelize'] = ( $this->getOption( 'parallelize' ) === 'true' ); + } + + $start = microtime( true ); + $status = $backend->$method( $ops1, $opts ); + $e = ( microtime( true ) - $start ) * 1000; + if ( $status->getErrorsArray() ) { + print_r( $status->getErrorsArray() ); + exit( 0 ); + } + $this->output( $backend->getName() . ": Stored " . count( $ops1 ) . " files in $e ms.\n" ); + + $start = microtime( true ); + $backend->$method( $ops2, $opts ); + $e = ( microtime( true ) - $start ) * 1000; + if ( $status->getErrorsArray() ) { + print_r( $status->getErrorsArray() ); + exit( 0 ); + } + $this->output( $backend->getName() . ": Copied " . count( $ops2 ) . " files in $e ms.\n" ); + + $start = microtime( true ); + $backend->$method( $ops3, $opts ); + $e = ( microtime( true ) - $start ) * 1000; + if ( $status->getErrorsArray() ) { + print_r( $status->getErrorsArray() ); + exit( 0 ); + } + $this->output( $backend->getName() . ": Moved " . count( $ops3 ) . " files in $e ms.\n" ); + + $start = microtime( true ); + $backend->$method( $ops4, $opts ); + $e = ( microtime( true ) - $start ) * 1000; + if ( $status->getErrorsArray() ) { + print_r( $status->getErrorsArray() ); + exit( 0 ); + } + $this->output( $backend->getName() . ": Deleted " . count( $ops4 ) . " files in $e ms.\n" ); + + $start = microtime( true ); + $backend->$method( $ops5, $opts ); + $e = ( microtime( true ) - $start ) * 1000; + if ( $status->getErrorsArray() ) { + print_r( $status->getErrorsArray() ); + exit( 0 ); + } + $this->output( $backend->getName() . ": Deleted " . count( $ops5 ) . " files in $e ms.\n" ); + } +} + +$maintClass = "TestFileOpPerformance"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/findDeprecated.php b/www/wiki/maintenance/findDeprecated.php new file mode 100644 index 00000000..6128d238 --- /dev/null +++ b/www/wiki/maintenance/findDeprecated.php @@ -0,0 +1,203 @@ +<?php +/** + * Maintenance script that recursively scans MediaWiki's PHP source tree + * for deprecated functions and methods and pretty-prints the results. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; +require_once __DIR__ . '/../vendor/autoload.php'; + +/** + * A PHPParser node visitor that associates each node with its file name. + */ +class FileAwareNodeVisitor extends PhpParser\NodeVisitorAbstract { + private $currentFile = null; + + public function enterNode( PhpParser\Node $node ) { + $retVal = parent::enterNode( $node ); + $node->filename = $this->currentFile; + return $retVal; + } + + public function setCurrentFile( $filename ) { + $this->currentFile = $filename; + } + + public function getCurrentFile() { + return $this->currentFile; + } +} + +/** + * A PHPParser node visitor that finds deprecated functions and methods. + */ +class DeprecatedInterfaceFinder extends FileAwareNodeVisitor { + + private $currentClass = null; + + private $foundNodes = []; + + public function getFoundNodes() { + // Sort results by version, then by filename, then by name. + foreach ( $this->foundNodes as $version => &$nodes ) { + uasort( $nodes, function ( $a, $b ) { + return ( $a['filename'] . $a['name'] ) < ( $b['filename'] . $b['name'] ) ? -1 : 1; + } ); + } + ksort( $this->foundNodes ); + return $this->foundNodes; + } + + /** + * Check whether a function or method includes a call to wfDeprecated(), + * indicating that it is a hard-deprecated interface. + * @param PhpParser\Node $node + * @return bool + */ + public function isHardDeprecated( PhpParser\Node $node ) { + if ( !$node->stmts ) { + return false; + } + foreach ( $node->stmts as $stmt ) { + if ( + $stmt instanceof PhpParser\Node\Expr\FuncCall + && $stmt->name->toString() === 'wfDeprecated' + ) { + return true; + } + return false; + } + } + + public function enterNode( PhpParser\Node $node ) { + $retVal = parent::enterNode( $node ); + + if ( $node instanceof PhpParser\Node\Stmt\ClassLike ) { + $this->currentClass = $node->name; + } + + if ( $node instanceof PhpParser\Node\FunctionLike ) { + $docComment = $node->getDocComment(); + if ( !$docComment ) { + return; + } + if ( !preg_match( '/@deprecated.*(\d+\.\d+)/', $docComment->getText(), $matches ) ) { + return; + } + $version = $matches[1]; + + if ( $node instanceof PhpParser\Node\Stmt\ClassMethod ) { + $name = $this->currentClass . '::' . $node->name; + } else { + $name = $node->name; + } + + $this->foundNodes[ $version ][] = [ + 'filename' => $node->filename, + 'line' => $node->getLine(), + 'name' => $name, + 'hard' => $this->isHardDeprecated( $node ), + ]; + } + + return $retVal; + } +} + +/** + * Maintenance task that recursively scans MediaWiki PHP files for deprecated + * functions and interfaces and produces a report. + */ +class FindDeprecated extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Find deprecated interfaces' ); + } + + public function getFiles() { + global $IP; + + $files = new RecursiveDirectoryIterator( $IP . '/includes' ); + $files = new RecursiveIteratorIterator( $files ); + $files = new RegexIterator( $files, '/\.php$/' ); + return iterator_to_array( $files, false ); + } + + public function execute() { + global $IP; + + $files = $this->getFiles(); + $chunkSize = ceil( count( $files ) / 72 ); + + $parser = ( new PhpParser\ParserFactory )->create( PhpParser\ParserFactory::PREFER_PHP7 ); + $traverser = new PhpParser\NodeTraverser; + $finder = new DeprecatedInterfaceFinder; + $traverser->addVisitor( $finder ); + + $fileCount = count( $files ); + + for ( $i = 0; $i < $fileCount; $i++ ) { + $file = $files[$i]; + $code = file_get_contents( $file ); + + if ( strpos( $code, '@deprecated' ) === -1 ) { + continue; + } + + $finder->setCurrentFile( substr( $file->getPathname(), strlen( $IP ) + 1 ) ); + $nodes = $parser->parse( $code, [ 'throwOnError' => false ] ); + $traverser->traverse( $nodes ); + + if ( $i % $chunkSize === 0 ) { + $percentDone = 100 * $i / $fileCount; + fprintf( STDERR, "\r[%-72s] %d%%", str_repeat( '#', $i / $chunkSize ), $percentDone ); + } + } + + fprintf( STDERR, "\r[%'#-72s] 100%%\n", '' ); + + // Colorize output if STDOUT is an interactive terminal. + if ( posix_isatty( STDOUT ) ) { + $versionFmt = "\n* Deprecated since \033[37;1m%s\033[0m:\n"; + $entryFmt = " %s \033[33;1m%s\033[0m (%s:%d)\n"; + } else { + $versionFmt = "\n* Deprecated since %s:\n"; + $entryFmt = " %s %s (%s:%d)\n"; + } + + foreach ( $finder->getFoundNodes() as $version => $nodes ) { + printf( $versionFmt, $version ); + foreach ( $nodes as $node ) { + printf( + $entryFmt, + $node['hard'] ? '+' : '-', + $node['name'], + $node['filename'], + $node['line'] + ); + } + } + printf( "\nlegend:\n -: soft-deprecated\n +: hard-deprecated (via wfDeprecated())\n" ); + } +} + +$maintClass = 'FindDeprecated'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/findHooks.php b/www/wiki/maintenance/findHooks.php new file mode 100644 index 00000000..fd36db1d --- /dev/null +++ b/www/wiki/maintenance/findHooks.php @@ -0,0 +1,353 @@ +<?php +/** + * Simple script that try to find documented hook and hooks actually + * in the code and show what's missing. + * + * This script assumes that: + * - hooks names in hooks.txt are at the beginning of a line and single quoted. + * - hooks names in code are the first parameter of wfRunHooks. + * + * if --online option is passed, the script will compare the hooks in the code + * with the ones at https://www.mediawiki.org/wiki/Manual:Hooks + * + * Any instance of wfRunHooks that doesn't meet these parameters will be noted. + * + * Copyright © Antoine Musso + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @author Antoine Musso <hashar at free dot fr> + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that compares documented and actually present mismatches. + * + * @ingroup Maintenance + */ +class FindHooks extends Maintenance { + const FIND_NON_RECURSIVE = 0; + const FIND_RECURSIVE = 1; + + /* + * Hooks that are ignored + */ + protected static $ignore = [ 'Test' ]; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Find hooks that are undocumented, missing, or just plain wrong' ); + $this->addOption( 'online', 'Check against MediaWiki.org hook documentation' ); + } + + public function getDbType() { + return Maintenance::DB_NONE; + } + + public function execute() { + global $IP; + + $documentedHooks = $this->getHooksFromDoc( $IP . '/docs/hooks.txt' ); + $potentialHooks = []; + $badHooks = []; + + $recurseDirs = [ + "$IP/includes/", + "$IP/mw-config/", + "$IP/languages/", + "$IP/maintenance/", + // Omit $IP/tests/phpunit as it contains hook tests that shouldn't be documented + "$IP/tests/parser", + "$IP/tests/phpunit/suites", + ]; + $nonRecurseDirs = [ + "$IP/", + ]; + $extraFiles = [ + "$IP/tests/phpunit/MediaWikiTestCase.php", + ]; + + foreach ( $recurseDirs as $dir ) { + $ret = $this->getHooksFromDir( $dir, self::FIND_RECURSIVE ); + $potentialHooks = array_merge( $potentialHooks, $ret['good'] ); + $badHooks = array_merge( $badHooks, $ret['bad'] ); + } + foreach ( $nonRecurseDirs as $dir ) { + $ret = $this->getHooksFromDir( $dir ); + $potentialHooks = array_merge( $potentialHooks, $ret['good'] ); + $badHooks = array_merge( $badHooks, $ret['bad'] ); + } + foreach ( $extraFiles as $file ) { + $potentialHooks = array_merge( $potentialHooks, $this->getHooksFromFile( $file ) ); + $badHooks = array_merge( $badHooks, $this->getBadHooksFromFile( $file ) ); + } + + $documented = array_keys( $documentedHooks ); + $potential = array_keys( $potentialHooks ); + $potential = array_unique( $potential ); + $badHooks = array_diff( array_unique( $badHooks ), self::$ignore ); + $todo = array_diff( $potential, $documented, self::$ignore ); + $deprecated = array_diff( $documented, $potential, self::$ignore ); + + // Check parameter count and references + $badParameterCount = $badParameterReference = []; + foreach ( $potentialHooks as $hook => $args ) { + if ( !isset( $documentedHooks[$hook] ) ) { + // Not documented, but that will also be in $todo + continue; + } + $argsDoc = $documentedHooks[$hook]; + if ( $args === 'unknown' || $argsDoc === 'unknown' ) { + // Could not get parameter information + continue; + } + if ( count( $argsDoc ) !== count( $args ) ) { + $badParameterCount[] = $hook . ': Doc: ' . count( $argsDoc ) . ' vs. Code: ' . count( $args ); + } else { + // Check if & is equal + foreach ( $argsDoc as $index => $argDoc ) { + $arg = $args[$index]; + if ( ( $arg[0] === '&' ) !== ( $argDoc[0] === '&' ) ) { + $badParameterReference[] = $hook . ': References different: Doc: ' . $argDoc . + ' vs. Code: ' . $arg; + } + } + } + } + + // Print the results + $this->printArray( 'Undocumented', $todo ); + $this->printArray( 'Documented and not found', $deprecated ); + $this->printArray( 'Unclear hook calls', $badHooks ); + $this->printArray( 'Different parameter count', $badParameterCount ); + $this->printArray( 'Different parameter reference', $badParameterReference ); + + if ( !$todo && !$deprecated && !$badHooks + && !$badParameterCount && !$badParameterReference + ) { + $this->output( "Looks good!\n" ); + } else { + $this->error( 'The script finished with errors.', 1 ); + } + } + + /** + * Get the hook documentation, either locally or from MediaWiki.org + * @param string $doc + * @return array Array: key => hook name; value => array of arguments or string 'unknown' + */ + private function getHooksFromDoc( $doc ) { + if ( $this->hasOption( 'online' ) ) { + return $this->getHooksFromOnlineDoc(); + } else { + return $this->getHooksFromLocalDoc( $doc ); + } + } + + /** + * Get hooks from a local file (for example docs/hooks.txt) + * @param string $doc Filename to look in + * @return array Array: key => hook name; value => array of arguments or string 'unknown' + */ + private function getHooksFromLocalDoc( $doc ) { + $m = []; + $content = file_get_contents( $doc ); + preg_match_all( + "/\n'(.*?)':.*((?:\n.+)*)/", + $content, + $m, + PREG_SET_ORDER + ); + + // Extract the documented parameter + $hooks = []; + foreach ( $m as $match ) { + $args = []; + if ( isset( $match[2] ) ) { + $n = []; + if ( preg_match_all( "/\n(&?\\$\w+):.+/", $match[2], $n ) ) { + $args = $n[1]; + } + } + $hooks[$match[1]] = $args; + } + return $hooks; + } + + /** + * Get hooks from www.mediawiki.org using the API + * @return array Array: key => hook name; value => string 'unknown' + */ + private function getHooksFromOnlineDoc() { + $allhooks = $this->getHooksFromOnlineDocCategory( 'MediaWiki_hooks' ); + $removed = $this->getHooksFromOnlineDocCategory( 'Removed_hooks' ); + return array_diff_key( $allhooks, $removed ); + } + + /** + * @param string $title + * @return array + */ + private function getHooksFromOnlineDocCategory( $title ) { + $params = [ + 'action' => 'query', + 'list' => 'categorymembers', + 'cmtitle' => "Category:$title", + 'cmlimit' => 500, + 'format' => 'json', + 'continue' => '', + ]; + + $retval = []; + while ( true ) { + $json = Http::get( + wfAppendQuery( 'http://www.mediawiki.org/w/api.php', $params ), + [], + __METHOD__ + ); + $data = FormatJson::decode( $json, true ); + foreach ( $data['query']['categorymembers'] as $page ) { + if ( preg_match( '/Manual\:Hooks\/([a-zA-Z0-9- :]+)/', $page['title'], $m ) ) { + // parameters are unknown, because that needs parsing of wikitext + $retval[str_replace( ' ', '_', $m[1] )] = 'unknown'; + } + } + if ( !isset( $data['continue'] ) ) { + return $retval; + } + $params = array_replace( $params, $data['continue'] ); + } + } + + /** + * Get hooks from a PHP file + * @param string $filePath Full file path to the PHP file. + * @return array Array: key => hook name; value => array of arguments or string 'unknown' + */ + private function getHooksFromFile( $filePath ) { + $content = file_get_contents( $filePath ); + $m = []; + preg_match_all( + // All functions which runs hooks + '/(?:wfRunHooks|Hooks\:\:run)\s*\(\s*' . + // First argument is the hook name as string + '([\'"])(.*?)\1' . + // Comma for second argument + '(?:\s*(,))?' . + // Second argument must start with array to be processed + '(?:\s*(?:array\s*\(|\[)' . + // Matching inside array - allows one deep of brackets + '((?:[^\(\)\[\]]|\((?-1)\)|\[(?-1)\])*)' . + // End + '[\)\]])?/', + $content, + $m, + PREG_SET_ORDER + ); + + // Extract parameter + $hooks = []; + foreach ( $m as $match ) { + $args = []; + if ( isset( $match[4] ) ) { + $n = []; + if ( preg_match_all( '/((?:[^,\(\)]|\([^\(\)]*\))+)/', $match[4], $n ) ) { + $args = array_map( 'trim', $n[1] ); + // remove empty entries from trailing spaces + $args = array_filter( $args ); + } + } elseif ( isset( $match[3] ) ) { + // Found a parameter for Hooks::run, + // but could not extract the hooks argument, + // because there are given by a variable + $args = 'unknown'; + } + $hooks[$match[2]] = $args; + } + + return $hooks; + } + + /** + * Get bad hooks (where the hook name could not be determined) from a PHP file + * @param string $filePath Full filename to the PHP file. + * @return array Array of bad wfRunHooks() lines + */ + private function getBadHooksFromFile( $filePath ) { + $content = file_get_contents( $filePath ); + $m = []; + // We want to skip the "function wfRunHooks()" one. :) + preg_match_all( '/(?<!function )wfRunHooks\(\s*[^\s\'"].*/', $content, $m ); + $list = []; + foreach ( $m[0] as $match ) { + $list[] = $match . "(" . $filePath . ")"; + } + + return $list; + } + + /** + * Get hooks from a directory of PHP files. + * @param string $dir Directory path to start at + * @param int $recursive Pass self::FIND_RECURSIVE + * @return array Array: key => hook name; value => array of arguments or string 'unknown' + */ + private function getHooksFromDir( $dir, $recurse = 0 ) { + $good = []; + $bad = []; + + if ( $recurse === self::FIND_RECURSIVE ) { + $iterator = new RecursiveIteratorIterator( + new RecursiveDirectoryIterator( $dir, RecursiveDirectoryIterator::SKIP_DOTS ), + RecursiveIteratorIterator::SELF_FIRST + ); + } else { + $iterator = new DirectoryIterator( $dir ); + } + + foreach ( $iterator as $info ) { + // Ignore directories, work only on php files, + if ( $info->isFile() && in_array( $info->getExtension(), [ 'php', 'inc' ] ) + // Skip this file as it contains text that looks like a bad wfRunHooks() call + && $info->getRealPath() !== __FILE__ + ) { + $good = array_merge( $good, $this->getHooksFromFile( $info->getRealPath() ) ); + $bad = array_merge( $bad, $this->getBadHooksFromFile( $info->getRealPath() ) ); + } + } + + return [ 'good' => $good, 'bad' => $bad ]; + } + + /** + * Nicely sort an print an array + * @param string $msg A message to show before the value + * @param array $arr + */ + private function printArray( $msg, $arr ) { + asort( $arr ); + + foreach ( $arr as $v ) { + $this->output( "$msg: $v\n" ); + } + } +} + +$maintClass = 'FindHooks'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/findMissingFiles.php b/www/wiki/maintenance/findMissingFiles.php new file mode 100644 index 00000000..4ce7ca68 --- /dev/null +++ b/www/wiki/maintenance/findMissingFiles.php @@ -0,0 +1,118 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +require_once __DIR__ . '/Maintenance.php'; + +class FindMissingFiles extends Maintenance { + function __construct() { + parent::__construct(); + + $this->addDescription( 'Find registered files with no corresponding file.' ); + $this->addOption( 'start', 'Start after this file name', false, true ); + $this->addOption( 'mtimeafter', 'Only include files changed since this time', false, true ); + $this->addOption( 'mtimebefore', 'Only includes files changed before this time', false, true ); + $this->setBatchSize( 300 ); + } + + function execute() { + $lastName = $this->getOption( 'start', '' ); + + $repo = RepoGroup::singleton()->getLocalRepo(); + $dbr = $repo->getReplicaDB(); + $be = $repo->getBackend(); + + $mtime1 = $dbr->timestampOrNull( $this->getOption( 'mtimeafter', null ) ); + $mtime2 = $dbr->timestampOrNull( $this->getOption( 'mtimebefore', null ) ); + + $joinTables = []; + $joinConds = []; + if ( $mtime1 || $mtime2 ) { + $joinTables[] = 'page'; + $joinConds['page'] = [ 'INNER JOIN', + [ 'page_title = img_name', 'page_namespace' => NS_FILE ] ]; + $joinTables[] = 'logging'; + $on = [ 'log_page = page_id', 'log_type' => [ 'upload', 'move', 'delete' ] ]; + if ( $mtime1 ) { + $on[] = "log_timestamp > {$dbr->addQuotes($mtime1)}"; + } + if ( $mtime2 ) { + $on[] = "log_timestamp < {$dbr->addQuotes($mtime2)}"; + } + $joinConds['logging'] = [ 'INNER JOIN', $on ]; + } + + do { + $res = $dbr->select( + array_merge( [ 'image' ], $joinTables ), + [ 'name' => 'img_name' ], + [ "img_name > " . $dbr->addQuotes( $lastName ) ], + __METHOD__, + // DISTINCT causes a pointless filesort + [ 'ORDER BY' => 'name', 'GROUP BY' => 'name', + 'LIMIT' => $this->mBatchSize ], + $joinConds + ); + + // Check if any of these files are missing... + $pathsByName = []; + foreach ( $res as $row ) { + $file = $repo->newFile( $row->name ); + $pathsByName[$row->name] = $file->getPath(); + $lastName = $row->name; + } + $be->preloadFileStat( [ 'srcs' => $pathsByName ] ); + foreach ( $pathsByName as $path ) { + if ( $be->fileExists( [ 'src' => $path ] ) === false ) { + $this->output( "$path\n" ); + } + } + + // Find all missing old versions of any of the files in this batch... + if ( count( $pathsByName ) ) { + $ores = $dbr->select( 'oldimage', + [ 'oi_name', 'oi_archive_name' ], + [ 'oi_name' => array_keys( $pathsByName ) ], + __METHOD__ + ); + + $checkPaths = []; + foreach ( $ores as $row ) { + if ( !strlen( $row->oi_archive_name ) ) { + continue; // broken row + } + $file = $repo->newFromArchiveName( $row->oi_name, $row->oi_archive_name ); + $checkPaths[] = $file->getPath(); + } + + foreach ( array_chunk( $checkPaths, $this->mBatchSize ) as $paths ) { + $be->preloadFileStat( [ 'srcs' => $paths ] ); + foreach ( $paths as $path ) { + if ( $be->fileExists( [ 'src' => $path ] ) === false ) { + $this->output( "$path\n" ); + } + } + } + } + } while ( $res->numRows() >= $this->mBatchSize ); + } +} + +$maintClass = 'FindMissingFiles'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/findOrphanedFiles.php b/www/wiki/maintenance/findOrphanedFiles.php new file mode 100644 index 00000000..765fbe4a --- /dev/null +++ b/www/wiki/maintenance/findOrphanedFiles.php @@ -0,0 +1,155 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +require_once __DIR__ . '/Maintenance.php'; + +class FindOrphanedFiles extends Maintenance { + function __construct() { + parent::__construct(); + + $this->addDescription( "Find unregistered files in the 'public' repo zone." ); + $this->addOption( 'subdir', + 'Only scan files in this subdirectory (e.g. "a/a0")', false, true ); + $this->addOption( 'verbose', "Mention file paths checked" ); + $this->setBatchSize( 500 ); + } + + function execute() { + $subdir = $this->getOption( 'subdir', '' ); + $verbose = $this->hasOption( 'verbose' ); + + $repo = RepoGroup::singleton()->getLocalRepo(); + if ( $repo->hasSha1Storage() ) { + $this->error( "Local repo uses SHA-1 file storage names; aborting.", 1 ); + } + + $directory = $repo->getZonePath( 'public' ); + if ( $subdir != '' ) { + $directory .= "/$subdir/"; + } + + if ( $verbose ) { + $this->output( "Scanning files under $directory:\n" ); + } + + $list = $repo->getBackend()->getFileList( [ 'dir' => $directory ] ); + if ( $list === null ) { + $this->error( "Could not get file listing.", 1 ); + } + + $pathBatch = []; + foreach ( $list as $path ) { + if ( preg_match( '#^(thumb|deleted)/#', $path ) ) { + continue; // handle ugly nested containers on stock installs + } + + $pathBatch[] = $path; + if ( count( $pathBatch ) >= $this->mBatchSize ) { + $this->checkFiles( $repo, $pathBatch, $verbose ); + $pathBatch = []; + } + } + $this->checkFiles( $repo, $pathBatch, $verbose ); + } + + protected function checkFiles( LocalRepo $repo, array $paths, $verbose ) { + if ( !count( $paths ) ) { + return; + } + + $dbr = $repo->getReplicaDB(); + + $curNames = []; + $oldNames = []; + $imgIN = []; + $oiWheres = []; + foreach ( $paths as $path ) { + $name = basename( $path ); + if ( preg_match( '#^archive/#', $path ) ) { + if ( $verbose ) { + $this->output( "Checking old file $name\n" ); + } + + $oldNames[] = $name; + list( , $base ) = explode( '!', $name, 2 ); // <TS_MW>!<img_name> + $oiWheres[] = $dbr->makeList( + [ 'oi_name' => $base, 'oi_archive_name' => $name ], + LIST_AND + ); + } else { + if ( $verbose ) { + $this->output( "Checking current file $name\n" ); + } + + $curNames[] = $name; + $imgIN[] = $name; + } + } + + $res = $dbr->query( + $dbr->unionQueries( + [ + $dbr->selectSQLText( + 'image', + [ 'name' => 'img_name', 'old' => 0 ], + $imgIN ? [ 'img_name' => $imgIN ] : '1=0' + ), + $dbr->selectSQLText( + 'oldimage', + [ 'name' => 'oi_archive_name', 'old' => 1 ], + $oiWheres ? $dbr->makeList( $oiWheres, LIST_OR ) : '1=0' + ) + ], + true // UNION ALL (performance) + ), + __METHOD__ + ); + + $curNamesFound = []; + $oldNamesFound = []; + foreach ( $res as $row ) { + if ( $row->old ) { + $oldNamesFound[] = $row->name; + } else { + $curNamesFound[] = $row->name; + } + } + + foreach ( array_diff( $curNames, $curNamesFound ) as $name ) { + $file = $repo->newFile( $name ); + // Print name and public URL to ease recovery + if ( $file ) { + $this->output( $name . "\n" . $file->getCanonicalUrl() . "\n\n" ); + } else { + $this->error( "Cannot get URL for bad file title '$name'" ); + } + } + + foreach ( array_diff( $oldNames, $oldNamesFound ) as $name ) { + list( , $base ) = explode( '!', $name, 2 ); // <TS_MW>!<img_name> + $file = $repo->newFromArchiveName( Title::makeTitle( NS_FILE, $base ), $name ); + // Print name and public URL to ease recovery + $this->output( $name . "\n" . $file->getCanonicalUrl() . "\n\n" ); + } + } +} + +$maintClass = 'FindOrphanedFiles'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/fixDefaultJsonContentPages.php b/www/wiki/maintenance/fixDefaultJsonContentPages.php new file mode 100644 index 00000000..460b5534 --- /dev/null +++ b/www/wiki/maintenance/fixDefaultJsonContentPages.php @@ -0,0 +1,128 @@ +<?php +/** + * Fix instances of pre-existing JSON pages + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Usage: + * fixDefaultJsonContentPages.php + * + * It is automatically run by update.php + */ +class FixDefaultJsonContentPages extends LoggedUpdateMaintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( + 'Fix instances of JSON pages prior to them being the ContentHandler default' ); + $this->setBatchSize( 100 ); + } + + protected function getUpdateKey() { + return __CLASS__; + } + + protected function doDBUpdates() { + if ( !$this->getConfig()->get( 'ContentHandlerUseDB' ) ) { + $this->output( "\$wgContentHandlerUseDB is not enabled, nothing to do.\n" ); + return true; + } + + $dbr = $this->getDB( DB_REPLICA ); + $namespaces = [ + NS_MEDIAWIKI => $dbr->buildLike( $dbr->anyString(), '.json' ), + NS_USER => $dbr->buildLike( $dbr->anyString(), '/', $dbr->anyString(), '.json' ), + ]; + foreach ( $namespaces as $ns => $like ) { + $lastPage = 0; + do { + $rows = $dbr->select( + 'page', + [ 'page_id', 'page_title', 'page_namespace', 'page_content_model' ], + [ + 'page_namespace' => $ns, + 'page_title ' . $like, + 'page_id > ' . $dbr->addQuotes( $lastPage ) + ], + __METHOD__, + [ 'ORDER BY' => 'page_id', 'LIMIT' => $this->mBatchSize ] + ); + foreach ( $rows as $row ) { + $this->handleRow( $row ); + } + } while ( $rows->numRows() >= $this->mBatchSize ); + } + + return true; + } + + protected function handleRow( stdClass $row ) { + $title = Title::makeTitle( $row->page_namespace, $row->page_title ); + $this->output( "Processing {$title} ({$row->page_id})...\n" ); + $rev = Revision::newFromTitle( $title ); + $content = $rev->getContent( Revision::RAW ); + $dbw = $this->getDB( DB_MASTER ); + if ( $content instanceof JsonContent ) { + if ( $content->isValid() ) { + // Yay, actually JSON. We need to just change the + // page_content_model because revision will automatically + // use the default, which is *now* JSON. + $this->output( "Setting page_content_model to json..." ); + $dbw->update( + 'page', + [ 'page_content_model' => CONTENT_MODEL_JSON ], + [ 'page_id' => $row->page_id ], + __METHOD__ + ); + $this->output( "done.\n" ); + wfWaitForSlaves(); + } else { + // Not JSON...force it to wikitext. We need to update the + // revision table so that these revisions are always processed + // as wikitext in the future. page_content_model is already + // set to "wikitext". + $this->output( "Setting rev_content_model to wikitext..." ); + // Grab all the ids for batching + $ids = $dbw->selectFieldValues( + 'revision', + 'rev_id', + [ 'rev_page' => $row->page_id ], + __METHOD__ + ); + foreach ( array_chunk( $ids, 50 ) as $chunk ) { + $dbw->update( + 'revision', + [ 'rev_content_model' => CONTENT_MODEL_WIKITEXT ], + [ 'rev_page' => $row->page_id, 'rev_id' => $chunk ] + ); + wfWaitForSlaves(); + } + $this->output( "done.\n" ); + } + } else { + $this->output( "not a JSON page? Skipping\n" ); + } + } +} + +$maintClass = 'FixDefaultJsonContentPages'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/fixDoubleRedirects.php b/www/wiki/maintenance/fixDoubleRedirects.php new file mode 100644 index 00000000..8c9faca2 --- /dev/null +++ b/www/wiki/maintenance/fixDoubleRedirects.php @@ -0,0 +1,140 @@ +<?php +/** + * Fix double redirects. + * + * Copyright © 2011 Ilmari Karonen <nospam@vyznev.net> + * https://www.mediawiki.org/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @author Ilmari Karonen <nospam@vyznev.net> + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that fixes double redirects. + * + * @ingroup Maintenance + */ +class FixDoubleRedirects extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Script to fix double redirects' ); + $this->addOption( 'async', 'Don\'t fix anything directly, just queue the jobs' ); + $this->addOption( 'title', 'Fix only redirects pointing to this page', false, true ); + $this->addOption( 'dry-run', 'Perform a dry run, fix nothing' ); + } + + public function execute() { + $async = $this->hasOption( 'async' ); + $dryrun = $this->hasOption( 'dry-run' ); + + if ( $this->hasOption( 'title' ) ) { + $title = Title::newFromText( $this->getOption( 'title' ) ); + if ( !$title || !$title->isRedirect() ) { + $this->error( $title->getPrefixedText() . " is not a redirect!\n", true ); + } + } else { + $title = null; + } + + $dbr = $this->getDB( DB_REPLICA ); + + // See also SpecialDoubleRedirects + $tables = [ + 'redirect', + 'pa' => 'page', + 'pb' => 'page', + ]; + $fields = [ + 'pa.page_namespace AS pa_namespace', + 'pa.page_title AS pa_title', + 'pb.page_namespace AS pb_namespace', + 'pb.page_title AS pb_title', + ]; + $conds = [ + 'rd_from = pa.page_id', + 'rd_namespace = pb.page_namespace', + 'rd_title = pb.page_title', + 'rd_interwiki IS NULL OR rd_interwiki = ' . $dbr->addQuotes( '' ), // T42352 + 'pb.page_is_redirect' => 1, + ]; + + if ( $title != null ) { + $conds['pb.page_namespace'] = $title->getNamespace(); + $conds['pb.page_title'] = $title->getDBkey(); + } + // TODO: support batch querying + + $res = $dbr->select( $tables, $fields, $conds, __METHOD__ ); + + if ( !$res->numRows() ) { + $this->output( "No double redirects found.\n" ); + + return; + } + + $jobs = []; + $processedTitles = "\n"; + $n = 0; + foreach ( $res as $row ) { + $titleA = Title::makeTitle( $row->pa_namespace, $row->pa_title ); + $titleB = Title::makeTitle( $row->pb_namespace, $row->pb_title ); + + $processedTitles .= "* [[$titleA]]\n"; + + $job = new DoubleRedirectJob( $titleA, [ + 'reason' => 'maintenance', + 'redirTitle' => $titleB->getPrefixedDBkey() + ] ); + + if ( !$async ) { + $success = ( $dryrun ? true : $job->run() ); + if ( !$success ) { + $this->error( "Error fixing " . $titleA->getPrefixedText() + . ": " . $job->getLastError() . "\n" ); + } + } else { + $jobs[] = $job; + // @todo FIXME: Hardcoded constant 10000 copied from DoubleRedirectJob class + if ( count( $jobs ) > 10000 ) { + $this->queueJobs( $jobs, $dryrun ); + $jobs = []; + } + } + + if ( ++$n % 100 == 0 ) { + $this->output( "$n...\n" ); + } + } + + if ( count( $jobs ) ) { + $this->queueJobs( $jobs, $dryrun ); + } + $this->output( "$n double redirects processed" . $processedTitles . "\n" ); + } + + protected function queueJobs( $jobs, $dryrun = false ) { + $this->output( "Queuing batch of " . count( $jobs ) . " double redirects.\n" ); + JobQueueGroup::singleton()->push( $dryrun ? [] : $jobs ); + } +} + +$maintClass = "FixDoubleRedirects"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/fixExtLinksProtocolRelative.php b/www/wiki/maintenance/fixExtLinksProtocolRelative.php new file mode 100644 index 00000000..97cd37e0 --- /dev/null +++ b/www/wiki/maintenance/fixExtLinksProtocolRelative.php @@ -0,0 +1,99 @@ +<?php +/** + * Fixes any entries for protocol-relative URLs in the externallinks table, + * replacing each protocol-relative entry with two entries, one for http + * and one for https. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that fixes any entriy for protocol-relative URLs + * in the externallinks table. + * + * @ingroup Maintenance + */ +class FixExtLinksProtocolRelative extends LoggedUpdateMaintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( + 'Fixes any entries in the externallinks table containing protocol-relative URLs' ); + } + + protected function getUpdateKey() { + return 'fix protocol-relative URLs in externallinks'; + } + + protected function updateSkippedMessage() { + return 'protocol-relative URLs in externallinks table already fixed.'; + } + + protected function doDBUpdates() { + $db = $this->getDB( DB_MASTER ); + if ( !$db->tableExists( 'externallinks' ) ) { + $this->error( "externallinks table does not exist" ); + + return false; + } + $this->output( "Fixing protocol-relative entries in the externallinks table...\n" ); + $res = $db->select( 'externallinks', [ 'el_from', 'el_to', 'el_index' ], + [ 'el_index' . $db->buildLike( '//', $db->anyString() ) ], + __METHOD__ + ); + $count = 0; + foreach ( $res as $row ) { + $count++; + if ( $count % 100 == 0 ) { + $this->output( $count . "\n" ); + wfWaitForSlaves(); + } + $db->insert( 'externallinks', + [ + [ + 'el_from' => $row->el_from, + 'el_to' => $row->el_to, + 'el_index' => "http:{$row->el_index}", + ], + [ + 'el_from' => $row->el_from, + 'el_to' => $row->el_to, + 'el_index' => "https:{$row->el_index}", + ] + ], __METHOD__, [ 'IGNORE' ] + ); + $db->delete( + 'externallinks', + [ + 'el_index' => $row->el_index, + 'el_from' => $row->el_from, + 'el_to' => $row->el_to + ], + __METHOD__ + ); + } + $this->output( "Done, $count rows updated.\n" ); + + return true; + } +} + +$maintClass = "FixExtLinksProtocolRelative"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/fixTimestamps.php b/www/wiki/maintenance/fixTimestamps.php new file mode 100644 index 00000000..796ec265 --- /dev/null +++ b/www/wiki/maintenance/fixTimestamps.php @@ -0,0 +1,129 @@ +<?php +/** + * Fixes timestamp corruption caused by one or more webservers temporarily + * being set to the wrong time. + * The time offset must be known and consistent. Start and end times + * (in 14-character format) restrict the search, and must bracket the damage. + * There must be a majority of good timestamps in the search period. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that fixes timestamp corruption caused by one or + * more webservers temporarily being set to the wrong time. + * + * @ingroup Maintenance + */ +class FixTimestamps extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( '' ); + $this->addArg( 'offset', '' ); + $this->addArg( 'start', 'Starting timestamp' ); + $this->addArg( 'end', 'Ending timestamp' ); + } + + public function execute() { + $offset = $this->getArg( 0 ) * 3600; + $start = $this->getArg( 1 ); + $end = $this->getArg( 2 ); + $grace = 60; // maximum normal clock offset + + # Find bounding revision IDs + $dbw = $this->getDB( DB_MASTER ); + $revisionTable = $dbw->tableName( 'revision' ); + $res = $dbw->query( "SELECT MIN(rev_id) as minrev, MAX(rev_id) as maxrev FROM $revisionTable " . + "WHERE rev_timestamp BETWEEN '{$start}' AND '{$end}'", __METHOD__ ); + $row = $dbw->fetchObject( $res ); + + if ( is_null( $row->minrev ) ) { + $this->error( "No revisions in search period.", true ); + } + + $minRev = $row->minrev; + $maxRev = $row->maxrev; + + # Select all timestamps and IDs + $sql = "SELECT rev_id, rev_timestamp FROM $revisionTable " . + "WHERE rev_id BETWEEN $minRev AND $maxRev"; + if ( $offset > 0 ) { + $sql .= " ORDER BY rev_id DESC"; + $expectedSign = -1; + } else { + $expectedSign = 1; + } + + $res = $dbw->query( $sql, __METHOD__ ); + + $lastNormal = 0; + $badRevs = []; + $numGoodRevs = 0; + + foreach ( $res as $row ) { + $timestamp = wfTimestamp( TS_UNIX, $row->rev_timestamp ); + $delta = $timestamp - $lastNormal; + $sign = $delta == 0 ? 0 : $delta / abs( $delta ); + if ( $sign == 0 || $sign == $expectedSign ) { + // Monotonic change + $lastNormal = $timestamp; + ++$numGoodRevs; + continue; + } elseif ( abs( $delta ) <= $grace ) { + // Non-monotonic change within grace interval + ++$numGoodRevs; + continue; + } else { + // Non-monotonic change larger than grace interval + $badRevs[] = $row->rev_id; + } + } + + $numBadRevs = count( $badRevs ); + if ( $numBadRevs > $numGoodRevs ) { + $this->error( + "The majority of revisions in the search interval are marked as bad. + + Are you sure the offset ($offset) has the right sign? Positive means the clock + was incorrectly set forward, negative means the clock was incorrectly set back. + + If the offset is right, then increase the search interval until there are enough + good revisions to provide a majority reference.", true ); + } elseif ( $numBadRevs == 0 ) { + $this->output( "No bad revisions found.\n" ); + exit( 0 ); + } + + $this->output( sprintf( "Fixing %d revisions (%.2f%% of revisions in search interval)\n", + $numBadRevs, $numBadRevs / ( $numGoodRevs + $numBadRevs ) * 100 ) ); + + $fixup = -$offset; + $sql = "UPDATE $revisionTable " . + "SET rev_timestamp=" + . "DATE_FORMAT(DATE_ADD(rev_timestamp, INTERVAL $fixup SECOND), '%Y%m%d%H%i%s') " . + "WHERE rev_id IN (" . $dbw->makeList( $badRevs ) . ')'; + $dbw->query( $sql, __METHOD__ ); + $this->output( "Done\n" ); + } +} + +$maintClass = "FixTimestamps"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/fixUserRegistration.php b/www/wiki/maintenance/fixUserRegistration.php new file mode 100644 index 00000000..37fd44fb --- /dev/null +++ b/www/wiki/maintenance/fixUserRegistration.php @@ -0,0 +1,91 @@ +<?php +/** + * Fix the user_registration field. + * In particular, for values which are NULL, set them to the date of the first edit + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that fixes the user_registration field. + * + * @ingroup Maintenance + */ +class FixUserRegistration extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Fix the user_registration field' ); + $this->setBatchSize( 1000 ); + } + + public function execute() { + $dbw = $this->getDB( DB_MASTER ); + + $lastId = 0; + do { + // Get user IDs which need fixing + $res = $dbw->select( + 'user', + 'user_id', + [ + 'user_id > ' . $dbw->addQuotes( $lastId ), + 'user_registration IS NULL' + ], + __METHOD__, + [ + 'LIMIT' => $this->mBatchSize, + 'ORDER BY' => 'user_id', + ] + ); + foreach ( $res as $row ) { + $id = $row->user_id; + $lastId = $id; + // Get first edit time + $timestamp = $dbw->selectField( + 'revision', + 'MIN(rev_timestamp)', + [ 'rev_user' => $id ], + __METHOD__ + ); + // Update + if ( $timestamp !== null ) { + $dbw->update( + 'user', + [ 'user_registration' => $timestamp ], + [ 'user_id' => $id ], + __METHOD__ + ); + $user = User::newFromId( $id ); + $user->invalidateCache(); + $this->output( "Set registration for #$id to $timestamp\n" ); + } else { + $this->output( "Could not find registration for #$id NULL\n" ); + } + } + $this->output( "Waiting for replica DBs..." ); + wfWaitForSlaves(); + $this->output( " done.\n" ); + } while ( $res->numRows() >= $this->mBatchSize ); + } +} + +$maintClass = "FixUserRegistration"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/formatInstallDoc.php b/www/wiki/maintenance/formatInstallDoc.php new file mode 100644 index 00000000..e2b3c419 --- /dev/null +++ b/www/wiki/maintenance/formatInstallDoc.php @@ -0,0 +1,78 @@ +<?php +/** + * Format RELEASE-NOTE file to wiki text or HTML markup. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that formats RELEASE-NOTE file to wiki text or HTML markup. + * + * @ingroup Maintenance + */ +class MaintenanceFormatInstallDoc extends Maintenance { + function __construct() { + parent::__construct(); + $this->addArg( 'path', 'The file name to format', false ); + $this->addOption( 'outfile', 'The output file name', false, true ); + $this->addOption( 'html', 'Use HTML output format. By default, wikitext is used.' ); + } + + function execute() { + if ( $this->hasArg( 0 ) ) { + $fileName = $this->getArg( 0 ); + $inFile = fopen( $fileName, 'r' ); + if ( !$inFile ) { + $this->error( "Unable to open input file \"$fileName\"" ); + exit( 1 ); + } + } else { + $inFile = STDIN; + } + + if ( $this->hasOption( 'outfile' ) ) { + $fileName = $this->getOption( 'outfile' ); + $outFile = fopen( $fileName, 'w' ); + if ( !$outFile ) { + $this->error( "Unable to open output file \"$fileName\"" ); + exit( 1 ); + } + } else { + $outFile = STDOUT; + } + + $inText = stream_get_contents( $inFile ); + $outText = InstallDocFormatter::format( $inText ); + + if ( $this->hasOption( 'html' ) ) { + global $wgParser; + $opt = new ParserOptions; + $title = Title::newFromText( 'Text file' ); + $out = $wgParser->parse( $outText, $title, $opt ); + $outText = "<html><body>\n" . $out->getText() . "\n</body></html>\n"; + } + + fwrite( $outFile, $outText ); + } +} + +$maintClass = 'MaintenanceFormatInstallDoc'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/generateJsonI18n.php b/www/wiki/maintenance/generateJsonI18n.php new file mode 100644 index 00000000..a84f2ae5 --- /dev/null +++ b/www/wiki/maintenance/generateJsonI18n.php @@ -0,0 +1,196 @@ +<?php + +/** + * Convert a PHP messages file to a set of JSON messages files. + * + * Usage: + * php generateJsonI18n.php ExtensionName.i18n.php i18n/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to generate JSON i18n files from a PHP i18n file. + * + * @ingroup Maintenance + */ +class GenerateJsonI18n extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Build JSON messages files from a PHP messages file' ); + + $this->addArg( 'phpfile', 'PHP file defining a $messages array', false ); + $this->addArg( 'jsondir', 'Directory to write JSON files to', false ); + $this->addOption( 'extension', 'Perform default conversion on an extension', + false, true ); + $this->addOption( 'supplementary', 'Find supplementary i18n files in subdirs and convert those', + false, false ); + } + + public function execute() { + global $IP; + + $phpfile = $this->getArg( 0 ); + $jsondir = $this->getArg( 1 ); + $extension = $this->getOption( 'extension' ); + $convertSupplementaryI18nFiles = $this->hasOption( 'supplementary' ); + + if ( $extension ) { + if ( $phpfile ) { + $this->error( "The phpfile is already specified, conflicts with --extension.", 1 ); + } + $phpfile = "$IP/extensions/$extension/$extension.i18n.php"; + } + + if ( !$phpfile ) { + $this->error( "I'm here for an argument!" ); + $this->maybeHelp( true ); + // dies. + } + + if ( $convertSupplementaryI18nFiles ) { + if ( is_readable( $phpfile ) ) { + $this->transformI18nFile( $phpfile, $jsondir ); + } else { + // This is non-fatal because we might want to continue searching for + // i18n files in subdirs even if the extension does not include a + // primary i18n.php. + $this->error( "Warning: no primary i18n file was found." ); + } + $this->output( "Searching for supplementary i18n files...\n" ); + $dir_iterator = new RecursiveDirectoryIterator( dirname( $phpfile ) ); + $iterator = new RecursiveIteratorIterator( + $dir_iterator, RecursiveIteratorIterator::LEAVES_ONLY ); + foreach ( $iterator as $path => $fileObject ) { + if ( fnmatch( "*.i18n.php", $fileObject->getFilename() ) ) { + $this->output( "Converting $path.\n" ); + $this->transformI18nFile( $path ); + } + } + } else { + // Just convert the primary i18n file. + $this->transformI18nFile( $phpfile, $jsondir ); + } + } + + public function transformI18nFile( $phpfile, $jsondir = null ) { + if ( !$jsondir ) { + // Assume the json directory should be in the same directory as the + // .i18n.php file. + $jsondir = dirname( $phpfile ) . "/i18n"; + } + if ( !is_dir( $jsondir ) ) { + $this->output( "Creating directory $jsondir.\n" ); + $success = mkdir( $jsondir ); + if ( !$success ) { + $this->error( "Could not create directory $jsondir", 1 ); + } + } + + if ( !is_readable( $phpfile ) ) { + $this->error( "Error reading $phpfile", 1 ); + } + $messages = null; + include $phpfile; + $phpfileContents = file_get_contents( $phpfile ); + + if ( !isset( $messages ) ) { + $this->error( "PHP file $phpfile does not define \$messages array", 1 ); + } + + if ( !$messages ) { + $this->error( "PHP file $phpfile contains an empty \$messages array. " . + "Maybe it was already converted?", 1 ); + } + + if ( !isset( $messages['en'] ) || !is_array( $messages['en'] ) ) { + $this->error( "PHP file $phpfile does not set language codes", 1 ); + } + + foreach ( $messages as $langcode => $langmsgs ) { + $authors = $this->getAuthorsFromComment( $this->findCommentBefore( + "\$messages['$langcode'] =", + $phpfileContents + ) ); + // Make sure the @metadata key is the first key in the output + $langmsgs = array_merge( + [ '@metadata' => [ 'authors' => $authors ] ], + $langmsgs + ); + + $jsonfile = "$jsondir/$langcode.json"; + $success = file_put_contents( + $jsonfile, + FormatJson::encode( $langmsgs, "\t", FormatJson::ALL_OK ) . "\n" + ); + if ( $success === false ) { + $this->error( "FAILED to write $jsonfile", 1 ); + } + $this->output( "$jsonfile\n" ); + } + + $this->output( + "All done. To complete the conversion, please do the following:\n" . + "* Add \$wgMessagesDirs['YourExtension'] = __DIR__ . '/i18n';\n" . + "* Remove \$wgExtensionMessagesFiles['YourExtension']\n" . + "* Delete the old PHP message file\n" . + "This script no longer generates backward compatibility shims! If you need\n" . + "compatibility with MediaWiki 1.22 and older, use the MediaWiki 1.23 version\n" . + "of this script instead, or create a shim manually.\n" + ); + } + + /** + * Find the documentation comment immediately before a given search string + * @param string $needle String to search for + * @param string $haystack String to search in + * @return string Substring of $haystack starting at '/**' ending right before $needle, or empty + */ + protected function findCommentBefore( $needle, $haystack ) { + $needlePos = strpos( $haystack, $needle ); + if ( $needlePos === false ) { + return ''; + } + // Need to pass a negative offset to strrpos() so it'll search backwards from the + // offset + $startPos = strrpos( $haystack, '/**', $needlePos - strlen( $haystack ) ); + if ( $startPos === false ) { + return ''; + } + + return substr( $haystack, $startPos, $needlePos - $startPos ); + } + + /** + * Get an array of author names from a documentation comment containing @author declarations. + * @param string $comment Documentation comment + * @return array Array of author names (strings) + */ + protected function getAuthorsFromComment( $comment ) { + $matches = null; + preg_match_all( '/@author (.*?)$/m', $comment, $matches ); + + return $matches && $matches[1] ? $matches[1] : []; + } +} + +$maintClass = "GenerateJsonI18n"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/generateLocalAutoload.php b/www/wiki/maintenance/generateLocalAutoload.php new file mode 100644 index 00000000..0c278bc1 --- /dev/null +++ b/www/wiki/maintenance/generateLocalAutoload.php @@ -0,0 +1,20 @@ +<?php + +if ( PHP_SAPI != 'cli' ) { + die( "This script can only be run from the command line.\n" ); +} + +require_once __DIR__ . '/../includes/utils/AutoloadGenerator.php'; + +// Mediawiki installation directory +$base = dirname( __DIR__ ); + +$generator = new AutoloadGenerator( $base, 'local' ); +$generator->initMediaWikiDefault(); + +// Write out the autoload +$fileinfo = $generator->getTargetFileinfo(); +file_put_contents( + $fileinfo['filename'], + $generator->getAutoload( 'maintenance/generateLocalAutoload.php' ) +); diff --git a/www/wiki/maintenance/generateSitemap.php b/www/wiki/maintenance/generateSitemap.php new file mode 100644 index 00000000..26a9c399 --- /dev/null +++ b/www/wiki/maintenance/generateSitemap.php @@ -0,0 +1,561 @@ +<?php +/** + * Creates a sitemap for the site. + * + * Copyright © 2005, Ævar Arnfjörð Bjarmason, Jens Frank <jeluf@gmx.de> and + * Brion Vibber <brion@pobox.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @see http://www.sitemaps.org/ + * @see http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that generates a sitemap for the site. + * + * @ingroup Maintenance + */ +class GenerateSitemap extends Maintenance { + const GS_MAIN = -2; + const GS_TALK = -1; + + /** + * The maximum amount of urls in a sitemap file + * + * @link http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd + * + * @var int + */ + public $url_limit; + + /** + * The maximum size of a sitemap file + * + * @link http://www.sitemaps.org/faq.php#faq_sitemap_size + * + * @var int + */ + public $size_limit; + + /** + * The path to prepend to the filename + * + * @var string + */ + public $fspath; + + /** + * The URL path to prepend to filenames in the index; + * should resolve to the same directory as $fspath. + * + * @var string + */ + public $urlpath; + + /** + * Whether or not to use compression + * + * @var bool + */ + public $compress; + + /** + * Whether or not to include redirection pages + * + * @var bool + */ + public $skipRedirects; + + /** + * The number of entries to save in each sitemap file + * + * @var array + */ + public $limit = []; + + /** + * Key => value entries of namespaces and their priorities + * + * @var array + */ + public $priorities = []; + + /** + * A one-dimensional array of namespaces in the wiki + * + * @var array + */ + public $namespaces = []; + + /** + * When this sitemap batch was generated + * + * @var string + */ + public $timestamp; + + /** + * A database replica DB object + * + * @var object + */ + public $dbr; + + /** + * A resource pointing to the sitemap index file + * + * @var resource + */ + public $findex; + + /** + * A resource pointing to a sitemap file + * + * @var resource + */ + public $file; + + /** + * Identifier to use in filenames, default $wgDBname + * + * @var string + */ + private $identifier; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Creates a sitemap for the site' ); + $this->addOption( + 'fspath', + 'The file system path to save to, e.g. /tmp/sitemap; defaults to current directory', + false, + true + ); + $this->addOption( + 'urlpath', + 'The URL path corresponding to --fspath, prepended to filenames in the index; ' + . 'defaults to an empty string', + false, + true + ); + $this->addOption( + 'compress', + 'Compress the sitemap files, can take value yes|no, default yes', + false, + true + ); + $this->addOption( 'skip-redirects', 'Do not include redirecting articles in the sitemap' ); + $this->addOption( + 'identifier', + 'What site identifier to use for the wiki, defaults to $wgDBname', + false, + true + ); + } + + /** + * Execute + */ + public function execute() { + $this->setNamespacePriorities(); + $this->url_limit = 50000; + $this->size_limit = pow( 2, 20 ) * 10; + + # Create directory if needed + $fspath = $this->getOption( 'fspath', getcwd() ); + if ( !wfMkdirParents( $fspath, null, __METHOD__ ) ) { + $this->error( "Can not create directory $fspath.", 1 ); + } + + $this->fspath = realpath( $fspath ) . DIRECTORY_SEPARATOR; + $this->urlpath = $this->getOption( 'urlpath', "" ); + if ( $this->urlpath !== "" && substr( $this->urlpath, -1 ) !== '/' ) { + $this->urlpath .= '/'; + } + $this->identifier = $this->getOption( 'identifier', wfWikiID() ); + $this->compress = $this->getOption( 'compress', 'yes' ) !== 'no'; + $this->skipRedirects = $this->hasOption( 'skip-redirects' ); + $this->dbr = $this->getDB( DB_REPLICA ); + $this->generateNamespaces(); + $this->timestamp = wfTimestamp( TS_ISO_8601, wfTimestampNow() ); + $this->findex = fopen( "{$this->fspath}sitemap-index-{$this->identifier}.xml", 'wb' ); + $this->main(); + } + + private function setNamespacePriorities() { + global $wgSitemapNamespacesPriorities; + + // Custom main namespaces + $this->priorities[self::GS_MAIN] = '0.5'; + // Custom talk namesspaces + $this->priorities[self::GS_TALK] = '0.1'; + // MediaWiki standard namespaces + $this->priorities[NS_MAIN] = '1.0'; + $this->priorities[NS_TALK] = '0.1'; + $this->priorities[NS_USER] = '0.5'; + $this->priorities[NS_USER_TALK] = '0.1'; + $this->priorities[NS_PROJECT] = '0.5'; + $this->priorities[NS_PROJECT_TALK] = '0.1'; + $this->priorities[NS_FILE] = '0.5'; + $this->priorities[NS_FILE_TALK] = '0.1'; + $this->priorities[NS_MEDIAWIKI] = '0.0'; + $this->priorities[NS_MEDIAWIKI_TALK] = '0.1'; + $this->priorities[NS_TEMPLATE] = '0.0'; + $this->priorities[NS_TEMPLATE_TALK] = '0.1'; + $this->priorities[NS_HELP] = '0.5'; + $this->priorities[NS_HELP_TALK] = '0.1'; + $this->priorities[NS_CATEGORY] = '0.5'; + $this->priorities[NS_CATEGORY_TALK] = '0.1'; + + // Custom priorities + if ( $wgSitemapNamespacesPriorities !== false ) { + /** + * @var $wgSitemapNamespacesPriorities array + */ + foreach ( $wgSitemapNamespacesPriorities as $namespace => $priority ) { + $float = floatval( $priority ); + if ( $float > 1.0 ) { + $priority = '1.0'; + } elseif ( $float < 0.0 ) { + $priority = '0.0'; + } + $this->priorities[$namespace] = $priority; + } + } + } + + /** + * Generate a one-dimensional array of existing namespaces + */ + function generateNamespaces() { + // Only generate for specific namespaces if $wgSitemapNamespaces is an array. + global $wgSitemapNamespaces; + if ( is_array( $wgSitemapNamespaces ) ) { + $this->namespaces = $wgSitemapNamespaces; + + return; + } + + $res = $this->dbr->select( 'page', + [ 'page_namespace' ], + [], + __METHOD__, + [ + 'GROUP BY' => 'page_namespace', + 'ORDER BY' => 'page_namespace', + ] + ); + + foreach ( $res as $row ) { + $this->namespaces[] = $row->page_namespace; + } + } + + /** + * Get the priority of a given namespace + * + * @param int $namespace The namespace to get the priority for + * @return string + */ + function priority( $namespace ) { + return isset( $this->priorities[$namespace] ) + ? $this->priorities[$namespace] + : $this->guessPriority( $namespace ); + } + + /** + * If the namespace isn't listed on the priority list return the + * default priority for the namespace, varies depending on whether it's + * a talkpage or not. + * + * @param int $namespace The namespace to get the priority for + * @return string + */ + function guessPriority( $namespace ) { + return MWNamespace::isSubject( $namespace ) + ? $this->priorities[self::GS_MAIN] + : $this->priorities[self::GS_TALK]; + } + + /** + * Return a database resolution of all the pages in a given namespace + * + * @param int $namespace Limit the query to this namespace + * @return Resource + */ + function getPageRes( $namespace ) { + return $this->dbr->select( 'page', + [ + 'page_namespace', + 'page_title', + 'page_touched', + 'page_is_redirect' + ], + [ 'page_namespace' => $namespace ], + __METHOD__ + ); + } + + /** + * Main loop + */ + public function main() { + global $wgContLang; + + fwrite( $this->findex, $this->openIndex() ); + + foreach ( $this->namespaces as $namespace ) { + $res = $this->getPageRes( $namespace ); + $this->file = false; + $this->generateLimit( $namespace ); + $length = $this->limit[0]; + $i = $smcount = 0; + + $fns = $wgContLang->getFormattedNsText( $namespace ); + $this->output( "$namespace ($fns)\n" ); + $skippedRedirects = 0; // Number of redirects skipped for that namespace + foreach ( $res as $row ) { + if ( $this->skipRedirects && $row->page_is_redirect ) { + $skippedRedirects++; + continue; + } + + if ( $i++ === 0 + || $i === $this->url_limit + 1 + || $length + $this->limit[1] + $this->limit[2] > $this->size_limit + ) { + if ( $this->file !== false ) { + $this->write( $this->file, $this->closeFile() ); + $this->close( $this->file ); + } + $filename = $this->sitemapFilename( $namespace, $smcount++ ); + $this->file = $this->open( $this->fspath . $filename, 'wb' ); + $this->write( $this->file, $this->openFile() ); + fwrite( $this->findex, $this->indexEntry( $filename ) ); + $this->output( "\t$this->fspath$filename\n" ); + $length = $this->limit[0]; + $i = 1; + } + $title = Title::makeTitle( $row->page_namespace, $row->page_title ); + $date = wfTimestamp( TS_ISO_8601, $row->page_touched ); + $entry = $this->fileEntry( $title->getCanonicalURL(), $date, $this->priority( $namespace ) ); + $length += strlen( $entry ); + $this->write( $this->file, $entry ); + // generate pages for language variants + if ( $wgContLang->hasVariants() ) { + $variants = $wgContLang->getVariants(); + foreach ( $variants as $vCode ) { + if ( $vCode == $wgContLang->getCode() ) { + continue; // we don't want default variant + } + $entry = $this->fileEntry( + $title->getCanonicalURL( '', $vCode ), + $date, + $this->priority( $namespace ) + ); + $length += strlen( $entry ); + $this->write( $this->file, $entry ); + } + } + } + + if ( $this->skipRedirects && $skippedRedirects > 0 ) { + $this->output( " skipped $skippedRedirects redirect(s)\n" ); + } + + if ( $this->file ) { + $this->write( $this->file, $this->closeFile() ); + $this->close( $this->file ); + } + } + fwrite( $this->findex, $this->closeIndex() ); + fclose( $this->findex ); + } + + /** + * gzopen() / fopen() wrapper + * + * @param string $file + * @param string $flags + * @return resource + */ + function open( $file, $flags ) { + $resource = $this->compress ? gzopen( $file, $flags ) : fopen( $file, $flags ); + if ( $resource === false ) { + throw new MWException( __METHOD__ + . " error opening file $file with flags $flags. Check permissions?" ); + } + + return $resource; + } + + /** + * gzwrite() / fwrite() wrapper + * + * @param resource &$handle + * @param string $str + */ + function write( &$handle, $str ) { + if ( $handle === true || $handle === false ) { + throw new MWException( __METHOD__ . " was passed a boolean as a file handle.\n" ); + } + if ( $this->compress ) { + gzwrite( $handle, $str ); + } else { + fwrite( $handle, $str ); + } + } + + /** + * gzclose() / fclose() wrapper + * + * @param resource &$handle + */ + function close( &$handle ) { + if ( $this->compress ) { + gzclose( $handle ); + } else { + fclose( $handle ); + } + } + + /** + * Get a sitemap filename + * + * @param int $namespace The namespace + * @param int $count The count + * @return string + */ + function sitemapFilename( $namespace, $count ) { + $ext = $this->compress ? '.gz' : ''; + + return "sitemap-{$this->identifier}-NS_$namespace-$count.xml$ext"; + } + + /** + * Return the XML required to open an XML file + * + * @return string + */ + function xmlHead() { + return '<?xml version="1.0" encoding="UTF-8"?>' . "\n"; + } + + /** + * Return the XML schema being used + * + * @return string + */ + function xmlSchema() { + return 'http://www.sitemaps.org/schemas/sitemap/0.9'; + } + + /** + * Return the XML required to open a sitemap index file + * + * @return string + */ + function openIndex() { + return $this->xmlHead() . '<sitemapindex xmlns="' . $this->xmlSchema() . '">' . "\n"; + } + + /** + * Return the XML for a single sitemap indexfile entry + * + * @param string $filename The filename of the sitemap file + * @return string + */ + function indexEntry( $filename ) { + return + "\t<sitemap>\n" . + "\t\t<loc>{$this->urlpath}$filename</loc>\n" . + "\t\t<lastmod>{$this->timestamp}</lastmod>\n" . + "\t</sitemap>\n"; + } + + /** + * Return the XML required to close a sitemap index file + * + * @return string + */ + function closeIndex() { + return "</sitemapindex>\n"; + } + + /** + * Return the XML required to open a sitemap file + * + * @return string + */ + function openFile() { + return $this->xmlHead() . '<urlset xmlns="' . $this->xmlSchema() . '">' . "\n"; + } + + /** + * Return the XML for a single sitemap entry + * + * @param string $url An RFC 2396 compliant URL + * @param string $date A ISO 8601 date + * @param string $priority A priority indicator, 0.0 - 1.0 inclusive with a 0.1 stepsize + * @return string + */ + function fileEntry( $url, $date, $priority ) { + return + "\t<url>\n" . + // T36666: $url may contain bad characters such as ampersands. + "\t\t<loc>" . htmlspecialchars( $url ) . "</loc>\n" . + "\t\t<lastmod>$date</lastmod>\n" . + "\t\t<priority>$priority</priority>\n" . + "\t</url>\n"; + } + + /** + * Return the XML required to close sitemap file + * + * @return string + */ + function closeFile() { + return "</urlset>\n"; + } + + /** + * Populate $this->limit + * + * @param int $namespace + */ + function generateLimit( $namespace ) { + // T19961: make a title with the longest possible URL in this namespace + $title = Title::makeTitle( $namespace, str_repeat( "\xf0\xa8\xae\x81", 63 ) . "\xe5\x96\x83" ); + + $this->limit = [ + strlen( $this->openFile() ), + strlen( $this->fileEntry( + $title->getCanonicalURL(), + wfTimestamp( TS_ISO_8601, wfTimestamp() ), + $this->priority( $namespace ) + ) ), + strlen( $this->closeFile() ) + ]; + } +} + +$maintClass = "GenerateSitemap"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/getConfiguration.php b/www/wiki/maintenance/getConfiguration.php new file mode 100644 index 00000000..3c679e6e --- /dev/null +++ b/www/wiki/maintenance/getConfiguration.php @@ -0,0 +1,196 @@ +<?php +/** + * Print serialized output of MediaWiki config vars. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @author Tim Starling + * @author Antoine Musso <hashar@free.fr> + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Print serialized output of MediaWiki config vars + * + * @ingroup Maintenance + */ +class GetConfiguration extends Maintenance { + + protected $regex = null; + + protected $settings_list = []; + + /** + * List of format output internally supported. + * Each item MUST be lower case. + */ + protected static $outFormats = [ + 'json', + 'php', + 'serialize', + 'vardump', + ]; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Get serialized MediaWiki site configuration' ); + $this->addOption( 'regex', 'regex to filter variables with', false, true ); + $this->addOption( 'iregex', 'same as --regex but case insensitive', false, true ); + $this->addOption( 'settings', 'Space-separated list of wg* variables', false, true ); + $this->addOption( 'format', implode( ', ', self::$outFormats ), false, true ); + } + + protected function validateParamsAndArgs() { + $error_out = false; + + # Get the format and make sure it is set to a valid default value + $format = strtolower( $this->getOption( 'format', 'PHP' ) ); + + $validFormat = in_array( $format, self::$outFormats ); + if ( !$validFormat ) { + $this->error( "--format set to an unrecognized format", 0 ); + $error_out = true; + } + + if ( $this->getOption( 'regex' ) && $this->getOption( 'iregex' ) ) { + $this->error( "Can only use either --regex or --iregex" ); + $error_out = true; + } + + parent::validateParamsAndArgs(); + + if ( $error_out ) { + # Force help and quit + $this->maybeHelp( true ); + } + } + + /** + * finalSetup() since we need MWException + */ + public function finalSetup() { + parent::finalSetup(); + + $this->regex = $this->getOption( 'regex' ) ?: $this->getOption( 'iregex' ); + if ( $this->regex ) { + $this->regex = '/' . $this->regex . '/'; + if ( $this->hasOption( 'iregex' ) ) { + $this->regex .= 'i'; # case insensitive regex + } + } + + if ( $this->hasOption( 'settings' ) ) { + $this->settings_list = explode( ' ', $this->getOption( 'settings' ) ); + # Values validation + foreach ( $this->settings_list as $name ) { + if ( !preg_match( '/^wg[A-Z]/', $name ) ) { + throw new MWException( "Variable '$name' does start with 'wg'." ); + } elseif ( !isset( $GLOBALS[$name] ) ) { + throw new MWException( "Variable '$name' is not set." ); + } elseif ( !$this->isAllowedVariable( $GLOBALS[$name] ) ) { + throw new MWException( "Variable '$name' includes non-array, non-scalar, items." ); + } + } + } + } + + public function execute() { + // Settings we will display + $res = []; + + # Sane default: dump any wg / wmg variable + if ( !$this->regex && !$this->getOption( 'settings' ) ) { + $this->regex = '/^wm?g/'; + } + + # Filter out globals based on the regex + if ( $this->regex ) { + $res = []; + foreach ( $GLOBALS as $name => $value ) { + if ( preg_match( $this->regex, $name ) ) { + $res[$name] = $value; + } + } + } + + # Explicitly dumps a list of provided global names + if ( $this->settings_list ) { + foreach ( $this->settings_list as $name ) { + $res[$name] = $GLOBALS[$name]; + } + } + + ksort( $res ); + + $out = null; + switch ( strtolower( $this->getOption( 'format' ) ) ) { + case 'serialize': + case 'php': + $out = serialize( $res ); + break; + case 'vardump': + $out = $this->formatVarDump( $res ); + break; + case 'json': + $out = FormatJson::encode( $res ); + break; + default: + throw new MWException( "Invalid serialization format given." ); + } + if ( !is_string( $out ) ) { + throw new MWException( "Failed to serialize the requested settings." ); + } + + if ( $out ) { + $this->output( $out . "\n" ); + } + } + + protected function formatVarDump( $res ) { + $ret = ''; + foreach ( $res as $key => $value ) { + ob_start(); # intercept var_dump() output + print "\${$key} = "; + var_dump( $value ); + # grab var_dump() output and discard it from the output buffer + $ret .= trim( ob_get_clean() ) . ";\n"; + } + + return trim( $ret, "\n" ); + } + + private function isAllowedVariable( $value ) { + if ( is_array( $value ) ) { + foreach ( $value as $k => $v ) { + if ( !$this->isAllowedVariable( $v ) ) { + return false; + } + } + + return true; + } elseif ( is_scalar( $value ) || $value === null ) { + return true; + } + + return false; + } +} + +$maintClass = "GetConfiguration"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/getLagTimes.php b/www/wiki/maintenance/getLagTimes.php new file mode 100644 index 00000000..ad2fdf88 --- /dev/null +++ b/www/wiki/maintenance/getLagTimes.php @@ -0,0 +1,79 @@ +<?php +/** + * Display replication lag times. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +use MediaWiki\MediaWikiServices; + +/** + * Maintenance script that displays replication lag times. + * + * @ingroup Maintenance + */ +class GetLagTimes extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Dump replication lag times' ); + $this->addOption( 'report', "Report the lag values to StatsD" ); + } + + public function execute() { + $services = MediaWikiServices::getInstance(); + $lbFactory = $services->getDBLoadBalancerFactory(); + $stats = $services->getStatsdDataFactory(); + $lbsByType = [ + 'main' => $lbFactory->getAllMainLBs(), + 'external' => $lbFactory->getAllExternalLBs() + ]; + + foreach ( $lbsByType as $type => $lbs ) { + foreach ( $lbs as $cluster => $lb ) { + if ( $lb->getServerCount() <= 1 ) { + continue; + } + $lags = $lb->getLagTimes(); + foreach ( $lags as $serverIndex => $lag ) { + $host = $lb->getServerName( $serverIndex ); + if ( IP::isValid( $host ) ) { + $ip = $host; + $host = gethostbyaddr( $host ); + } else { + $ip = gethostbyname( $host ); + } + + $starLen = min( intval( $lag ), 40 ); + $stars = str_repeat( '*', $starLen ); + $this->output( sprintf( "%10s %20s %3d %s\n", $ip, $host, $lag, $stars ) ); + + if ( $this->hasOption( 'report' ) ) { + $group = ( $type === 'external' ) ? 'external' : $cluster; + $stats->gauge( "loadbalancer.lag.$group.$host", intval( $lag * 1e3 ) ); + } + } + } + } + } +} + +$maintClass = "GetLagTimes"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/getReplicaServer.php b/www/wiki/maintenance/getReplicaServer.php new file mode 100644 index 00000000..6e0a1fec --- /dev/null +++ b/www/wiki/maintenance/getReplicaServer.php @@ -0,0 +1,55 @@ +<?php +/** + * Reports the hostname of a replica DB server. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that reports the hostname of a replica DB server. + * + * @ingroup Maintenance + */ +class GetSlaveServer extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addOption( "group", "Query group to check specifically" ); + $this->addDescription( 'Report the hostname of a replica DB server' ); + } + + public function execute() { + global $wgAllDBsAreLocalhost; + if ( $wgAllDBsAreLocalhost ) { + $host = 'localhost'; + } elseif ( $this->hasOption( 'group' ) ) { + $db = $this->getDB( DB_REPLICA, $this->getOption( 'group' ) ); + $host = $db->getServer(); + } else { + $lb = wfGetLB(); + $i = $lb->getReaderIndex(); + $host = $lb->getServerName( $i ); + } + $this->output( "$host\n" ); + } +} + +$maintClass = "GetSlaveServer"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/getSlaveServer.php b/www/wiki/maintenance/getSlaveServer.php new file mode 100644 index 00000000..2fa2d7f6 --- /dev/null +++ b/www/wiki/maintenance/getSlaveServer.php @@ -0,0 +1,3 @@ +<?php +// B/C alias +require_once __DIR__ . '/getReplicaServer.php'; diff --git a/www/wiki/maintenance/getText.php b/www/wiki/maintenance/getText.php new file mode 100644 index 00000000..f519a790 --- /dev/null +++ b/www/wiki/maintenance/getText.php @@ -0,0 +1,66 @@ +<?php +/** + * Outputs page text to stdout. + * Useful for command-line editing automation. + * Example: php getText.php "page title" | sed -e '...' | php edit.php "page title" + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that outputs page text to stdout. + * + * @ingroup Maintenance + */ +class GetTextMaint extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Outputs page text to stdout' ); + $this->addOption( 'show-private', 'Show the text even if it\'s not available to the public' ); + $this->addArg( 'title', 'Page title' ); + } + + public function execute() { + $titleText = $this->getArg( 0 ); + $title = Title::newFromText( $titleText ); + if ( !$title ) { + $this->error( "$titleText is not a valid title.\n", true ); + } + + $rev = Revision::newFromTitle( $title ); + if ( !$rev ) { + $titleText = $title->getPrefixedText(); + $this->error( "Page $titleText does not exist.\n", true ); + } + $content = $rev->getContent( $this->hasOption( 'show-private' ) + ? Revision::RAW + : Revision::FOR_PUBLIC ); + + if ( $content === false ) { + $titleText = $title->getPrefixedText(); + $this->error( "Couldn't extract the text from $titleText.\n", true ); + } + $this->output( $content->serialize() ); + } +} + +$maintClass = "GetTextMaint"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/hhvm/makeRepo.php b/www/wiki/maintenance/hhvm/makeRepo.php new file mode 100644 index 00000000..c1aa0820 --- /dev/null +++ b/www/wiki/maintenance/hhvm/makeRepo.php @@ -0,0 +1,161 @@ +<?php + +require __DIR__ . '/../Maintenance.php'; + +class HHVMMakeRepo extends Maintenance { + function __construct() { + parent::__construct(); + $this->addDescription( 'Compile PHP sources for this MediaWiki instance, ' . + 'and generate an HHVM bytecode file to be used with HHVM\'s ' . + 'RepoAuthoritative mode. The MediaWiki core installation path and ' . + 'all registered extensions are automatically searched for the file ' . + 'extensions *.php, *.inc, *.php5 and *.phtml.' ); + $this->addOption( 'output', 'Output filename', true, true, 'o' ); + $this->addOption( 'input-dir', 'Add an input directory. ' . + 'This can be specified multiple times.', false, true, 'd', true ); + $this->addOption( 'exclude-dir', 'Directory to exclude. ' . + 'This can be specified multiple times.', false, true, false, true ); + $this->addOption( 'extension', 'Extra file extension', false, true, false, true ); + $this->addOption( 'hhvm', 'Location of HHVM binary', false, true ); + $this->addOption( 'base-dir', 'The root of all source files. ' . + 'This must match hhvm.server.source_root in the server\'s configuration file. ' . + 'By default, the MW core install path will be used.', + false, true ); + $this->addOption( 'verbose', 'Log level 0-3', false, true, 'v' ); + } + + private static function startsWith( $subject, $search ) { + return substr( $subject, 0, strlen( $search ) === $search ); + } + + function execute() { + global $wgExtensionCredits, $IP; + + $dirs = [ $IP ]; + + foreach ( $wgExtensionCredits as $type => $extensions ) { + foreach ( $extensions as $extension ) { + if ( isset( $extension['path'] ) + && !self::startsWith( $extension['path'], $IP ) + ) { + $dirs[] = dirname( $extension['path'] ); + } + } + } + + $dirs = array_merge( $dirs, $this->getOption( 'input-dir', [] ) ); + $fileExts = + [ + 'php' => true, + 'inc' => true, + 'php5' => true, + 'phtml' => true + ] + + array_flip( $this->getOption( 'extension', [] ) ); + + $dirs = array_unique( $dirs ); + + $baseDir = $this->getOption( 'base-dir', $IP ); + $excludeDirs = array_map( 'realpath', $this->getOption( 'exclude-dir', [] ) ); + + if ( $baseDir !== '' && substr( $baseDir, -1 ) !== '/' ) { + $baseDir .= '/'; + } + + $unfilteredFiles = [ "$IP/LocalSettings.php" ]; + foreach ( $dirs as $dir ) { + $this->appendDir( $unfilteredFiles, $dir ); + } + + $files = []; + foreach ( $unfilteredFiles as $file ) { + $dotPos = strrpos( $file, '.' ); + $slashPos = strrpos( $file, '/' ); + if ( $dotPos === false || $slashPos === false || $dotPos < $slashPos ) { + continue; + } + $extension = substr( $file, $dotPos + 1 ); + if ( !isset( $fileExts[$extension] ) ) { + continue; + } + $canonical = realpath( $file ); + foreach ( $excludeDirs as $excluded ) { + if ( self::startsWith( $canonical, $excluded ) ) { + continue 2; + } + } + if ( self::startsWith( $file, $baseDir ) ) { + $file = substr( $file, strlen( $baseDir ) ); + } + $files[] = $file; + } + + $files = array_unique( $files ); + + print "Found " . count( $files ) . " files in " . + count( $dirs ) . " directories\n"; + + $tmpDir = wfTempDir() . '/mw-make-repo' . mt_rand( 0, 1 << 31 ); + if ( !mkdir( $tmpDir ) ) { + $this->error( 'Unable to create temporary directory', 1 ); + } + file_put_contents( "$tmpDir/file-list", implode( "\n", $files ) ); + + $hhvm = $this->getOption( 'hhvm', 'hhvm' ); + $verbose = $this->getOption( 'verbose', 3 ); + $cmd = wfEscapeShellArg( + $hhvm, + '--hphp', + '--target', 'hhbc', + '--format', 'binary', + '--force', '1', + '--keep-tempdir', '1', + '--log', $verbose, + '-v', 'AllVolatile=true', + '--input-dir', $baseDir, + '--input-list', "$tmpDir/file-list", + '--output-dir', $tmpDir ); + print "$cmd\n"; + passthru( $cmd, $ret ); + if ( $ret ) { + $this->cleanupTemp( $tmpDir ); + $this->error( "Error: HHVM returned error code $ret", 1 ); + } + if ( !rename( "$tmpDir/hhvm.hhbc", $this->getOption( 'output' ) ) ) { + $this->cleanupTemp( $tmpDir ); + $this->error( "Error: unable to rename output file", 1 ); + } + $this->cleanupTemp( $tmpDir ); + return 0; + } + + private function cleanupTemp( $tmpDir ) { + if ( file_exists( "$tmpDir/hhvm.hhbc" ) ) { + unlink( "$tmpDir/hhvm.hhbc" ); + } + if ( file_exists( "$tmpDir/Stats.js" ) ) { + unlink( "$tmpDir/Stats.js" ); + } + + unlink( "$tmpDir/file-list" ); + rmdir( $tmpDir ); + } + + private function appendDir( &$files, $dir ) { + $iter = new RecursiveIteratorIterator( + new RecursiveDirectoryIterator( + $dir, + FilesystemIterator::UNIX_PATHS + ), + RecursiveIteratorIterator::LEAVES_ONLY + ); + foreach ( $iter as $file => $fileInfo ) { + if ( $fileInfo->isFile() ) { + $files[] = $file; + } + } + } +} + +$maintClass = 'HHVMMakeRepo'; +require RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/hhvm/run-server b/www/wiki/maintenance/hhvm/run-server new file mode 100755 index 00000000..2d71b871 --- /dev/null +++ b/www/wiki/maintenance/hhvm/run-server @@ -0,0 +1,28 @@ +#!/usr/bin/hhvm -f +<?php + +require __DIR__ . '/../Maintenance.php'; + +class RunHipHopServer extends Maintenance { + function __construct() { + parent::__construct(); + } + + function execute() { + global $IP; + + passthru( + 'cd ' . wfEscapeShellArg( $IP ) . " && " . + wfEscapeShellArg( + 'hhvm', + '-c', __DIR__."/server.conf", + '--mode=server', + '--port=8080' + ), + $ret + ); + exit( $ret ); + } +} +$maintClass = 'RunHipHopServer'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/hhvm/server.conf b/www/wiki/maintenance/hhvm/server.conf new file mode 100644 index 00000000..558bdad8 --- /dev/null +++ b/www/wiki/maintenance/hhvm/server.conf @@ -0,0 +1,30 @@ +Log { + Level = Warning + UseLogFile = true + NativeStackTrace = true + InjectedStackTrace = true +} +Debug { + FullBacktrace = true + ServerStackTrace = true + ServerErrorMessage = true + TranslateSource = true +} +Server { + EnableStaticContentCache = false + EnableStaticContentFromDisk = true + AlwaysUseRelativePath = true +} +VirtualHost { + * { + ServerName = localhost + Pattern = . + RewriteRules { + * { + pattern = ^/wiki/(.*)$ + to = /index.php?title=$1 + qsa = true + } + } + } +} diff --git a/www/wiki/maintenance/hiphop/run-server b/www/wiki/maintenance/hiphop/run-server new file mode 100755 index 00000000..2d71b871 --- /dev/null +++ b/www/wiki/maintenance/hiphop/run-server @@ -0,0 +1,28 @@ +#!/usr/bin/hhvm -f +<?php + +require __DIR__ . '/../Maintenance.php'; + +class RunHipHopServer extends Maintenance { + function __construct() { + parent::__construct(); + } + + function execute() { + global $IP; + + passthru( + 'cd ' . wfEscapeShellArg( $IP ) . " && " . + wfEscapeShellArg( + 'hhvm', + '-c', __DIR__."/server.conf", + '--mode=server', + '--port=8080' + ), + $ret + ); + exit( $ret ); + } +} +$maintClass = 'RunHipHopServer'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/hiphop/server.conf b/www/wiki/maintenance/hiphop/server.conf new file mode 100644 index 00000000..558bdad8 --- /dev/null +++ b/www/wiki/maintenance/hiphop/server.conf @@ -0,0 +1,30 @@ +Log { + Level = Warning + UseLogFile = true + NativeStackTrace = true + InjectedStackTrace = true +} +Debug { + FullBacktrace = true + ServerStackTrace = true + ServerErrorMessage = true + TranslateSource = true +} +Server { + EnableStaticContentCache = false + EnableStaticContentFromDisk = true + AlwaysUseRelativePath = true +} +VirtualHost { + * { + ServerName = localhost + Pattern = . + RewriteRules { + * { + pattern = ^/wiki/(.*)$ + to = /index.php?title=$1 + qsa = true + } + } + } +} diff --git a/www/wiki/maintenance/importDump.php b/www/wiki/maintenance/importDump.php new file mode 100644 index 00000000..206c7ee6 --- /dev/null +++ b/www/wiki/maintenance/importDump.php @@ -0,0 +1,334 @@ +<?php +/** + * Import XML dump files into the current wiki. + * + * Copyright © 2005 Brion Vibber <brion@pobox.com> + * https://www.mediawiki.org/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that imports XML dump files into the current wiki. + * + * @ingroup Maintenance + */ +class BackupReader extends Maintenance { + public $reportingInterval = 100; + public $pageCount = 0; + public $revCount = 0; + public $dryRun = false; + public $uploads = false; + protected $uploadCount = 0; + public $imageBasePath = false; + public $nsFilter = false; + + function __construct() { + parent::__construct(); + $gz = in_array( 'compress.zlib', stream_get_wrappers() ) + ? 'ok' + : '(disabled; requires PHP zlib module)'; + $bz2 = in_array( 'compress.bzip2', stream_get_wrappers() ) + ? 'ok' + : '(disabled; requires PHP bzip2 module)'; + + $this->addDescription( + <<<TEXT +This script reads pages from an XML file as produced from Special:Export or +dumpBackup.php, and saves them into the current wiki. + +Compressed XML files may be read directly: + .gz $gz + .bz2 $bz2 + .7z (if 7za executable is in PATH) + +Note that for very large data sets, importDump.php may be slow; there are +alternate methods which can be much faster for full site restoration: +<https://www.mediawiki.org/wiki/Manual:Importing_XML_dumps> +TEXT + ); + $this->stderr = fopen( "php://stderr", "wt" ); + $this->addOption( 'report', + 'Report position and speed after every n pages processed', false, true ); + $this->addOption( 'namespaces', + 'Import only the pages from namespaces belonging to the list of ' . + 'pipe-separated namespace names or namespace indexes', false, true ); + $this->addOption( 'rootpage', 'Pages will be imported as subpages of the specified page', + false, true ); + $this->addOption( 'dry-run', 'Parse dump without actually importing pages' ); + $this->addOption( 'debug', 'Output extra verbose debug information' ); + $this->addOption( 'uploads', 'Process file upload data if included (experimental)' ); + $this->addOption( + 'no-updates', + 'Disable link table updates. Is faster but leaves the wiki in an inconsistent state' + ); + $this->addOption( 'image-base-path', 'Import files from a specified path', false, true ); + $this->addOption( 'skip-to', 'Start from nth page by skipping first n-1 pages', false, true ); + $this->addArg( 'file', 'Dump file to import [else use stdin]', false ); + } + + public function execute() { + if ( wfReadOnly() ) { + $this->error( "Wiki is in read-only mode; you'll need to disable it for import to work.", true ); + } + + $this->reportingInterval = intval( $this->getOption( 'report', 100 ) ); + if ( !$this->reportingInterval ) { + $this->reportingInterval = 100; // avoid division by zero + } + + $this->dryRun = $this->hasOption( 'dry-run' ); + $this->uploads = $this->hasOption( 'uploads' ); // experimental! + if ( $this->hasOption( 'image-base-path' ) ) { + $this->imageBasePath = $this->getOption( 'image-base-path' ); + } + if ( $this->hasOption( 'namespaces' ) ) { + $this->setNsfilter( explode( '|', $this->getOption( 'namespaces' ) ) ); + } + + if ( $this->hasArg() ) { + $this->importFromFile( $this->getArg() ); + } else { + $this->importFromStdin(); + } + + $this->output( "Done!\n" ); + $this->output( "You might want to run rebuildrecentchanges.php to regenerate RecentChanges,\n" ); + $this->output( "and initSiteStats.php to update page and revision counts\n" ); + } + + function setNsfilter( array $namespaces ) { + if ( count( $namespaces ) == 0 ) { + $this->nsFilter = false; + + return; + } + $this->nsFilter = array_unique( array_map( [ $this, 'getNsIndex' ], $namespaces ) ); + } + + private function getNsIndex( $namespace ) { + global $wgContLang; + $result = $wgContLang->getNsIndex( $namespace ); + if ( $result !== false ) { + return $result; + } + $ns = intval( $namespace ); + if ( strval( $ns ) === $namespace && $wgContLang->getNsText( $ns ) !== false ) { + return $ns; + } + $this->error( "Unknown namespace text / index specified: $namespace", true ); + } + + /** + * @param Title|Revision $obj + * @return bool + */ + private function skippedNamespace( $obj ) { + $title = null; + if ( $obj instanceof Title ) { + $title = $obj; + } elseif ( $obj instanceof Revision ) { + $title = $obj->getTitle(); + } elseif ( $obj instanceof WikiRevision ) { + $title = $obj->title; + } else { + throw new MWException( "Cannot get namespace of object in " . __METHOD__ ); + } + + if ( is_null( $title ) ) { + // Probably a log entry + return false; + } + + $ns = $title->getNamespace(); + + return is_array( $this->nsFilter ) && !in_array( $ns, $this->nsFilter ); + } + + function reportPage( $page ) { + $this->pageCount++; + } + + /** + * @param Revision $rev + */ + function handleRevision( $rev ) { + $title = $rev->getTitle(); + if ( !$title ) { + $this->progress( "Got bogus revision with null title!" ); + + return; + } + + if ( $this->skippedNamespace( $title ) ) { + return; + } + + $this->revCount++; + $this->report(); + + if ( !$this->dryRun ) { + call_user_func( $this->importCallback, $rev ); + } + } + + /** + * @param Revision $revision + * @return bool + */ + function handleUpload( $revision ) { + if ( $this->uploads ) { + if ( $this->skippedNamespace( $revision ) ) { + return false; + } + $this->uploadCount++; + // $this->report(); + $this->progress( "upload: " . $revision->getFilename() ); + + if ( !$this->dryRun ) { + // bluuuh hack + // call_user_func( $this->uploadCallback, $revision ); + $dbw = $this->getDB( DB_MASTER ); + + return $dbw->deadlockLoop( [ $revision, 'importUpload' ] ); + } + } + + return false; + } + + function handleLogItem( $rev ) { + if ( $this->skippedNamespace( $rev ) ) { + return; + } + $this->revCount++; + $this->report(); + + if ( !$this->dryRun ) { + call_user_func( $this->logItemCallback, $rev ); + } + } + + function report( $final = false ) { + if ( $final xor ( $this->pageCount % $this->reportingInterval == 0 ) ) { + $this->showReport(); + } + } + + function showReport() { + if ( !$this->mQuiet ) { + $delta = microtime( true ) - $this->startTime; + if ( $delta ) { + $rate = sprintf( "%.2f", $this->pageCount / $delta ); + $revrate = sprintf( "%.2f", $this->revCount / $delta ); + } else { + $rate = '-'; + $revrate = '-'; + } + # Logs dumps don't have page tallies + if ( $this->pageCount ) { + $this->progress( "$this->pageCount ($rate pages/sec $revrate revs/sec)" ); + } else { + $this->progress( "$this->revCount ($revrate revs/sec)" ); + } + } + wfWaitForSlaves(); + } + + function progress( $string ) { + fwrite( $this->stderr, $string . "\n" ); + } + + function importFromFile( $filename ) { + if ( preg_match( '/\.gz$/', $filename ) ) { + $filename = 'compress.zlib://' . $filename; + } elseif ( preg_match( '/\.bz2$/', $filename ) ) { + $filename = 'compress.bzip2://' . $filename; + } elseif ( preg_match( '/\.7z$/', $filename ) ) { + $filename = 'mediawiki.compress.7z://' . $filename; + } + + $file = fopen( $filename, 'rt' ); + + return $this->importFromHandle( $file ); + } + + function importFromStdin() { + $file = fopen( 'php://stdin', 'rt' ); + if ( self::posix_isatty( $file ) ) { + $this->maybeHelp( true ); + } + + return $this->importFromHandle( $file ); + } + + function importFromHandle( $handle ) { + $this->startTime = microtime( true ); + + $source = new ImportStreamSource( $handle ); + $importer = new WikiImporter( $source, $this->getConfig() ); + + // Updating statistics require a lot of time so disable it + $importer->disableStatisticsUpdate(); + + if ( $this->hasOption( 'debug' ) ) { + $importer->setDebug( true ); + } + if ( $this->hasOption( 'no-updates' ) ) { + $importer->setNoUpdates( true ); + } + if ( $this->hasOption( 'rootpage' ) ) { + $statusRootPage = $importer->setTargetRootPage( $this->getOption( 'rootpage' ) ); + if ( !$statusRootPage->isGood() ) { + // Die here so that it doesn't print "Done!" + $this->error( $statusRootPage->getMessage()->text(), 1 ); + return false; + } + } + if ( $this->hasOption( 'skip-to' ) ) { + $nthPage = (int)$this->getOption( 'skip-to' ); + $importer->setPageOffset( $nthPage ); + $this->pageCount = $nthPage - 1; + } + $importer->setPageCallback( [ $this, 'reportPage' ] ); + $this->importCallback = $importer->setRevisionCallback( + [ $this, 'handleRevision' ] ); + $this->uploadCallback = $importer->setUploadCallback( + [ $this, 'handleUpload' ] ); + $this->logItemCallback = $importer->setLogItemCallback( + [ $this, 'handleLogItem' ] ); + if ( $this->uploads ) { + $importer->setImportUploads( true ); + } + if ( $this->imageBasePath ) { + $importer->setImageBasePath( $this->imageBasePath ); + } + + if ( $this->dryRun ) { + $importer->setPageOutCallback( null ); + } + + return $importer->doImport(); + } +} + +$maintClass = 'BackupReader'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/importImages.inc b/www/wiki/maintenance/importImages.inc new file mode 100644 index 00000000..fc9428d7 --- /dev/null +++ b/www/wiki/maintenance/importImages.inc @@ -0,0 +1,137 @@ +<?php +/** + * Support functions for the importImages.php script + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @author Rob Church <robchur@gmail.com> + * @author Mij <mij@bitchx.it> + */ + +/** + * Search a directory for files with one of a set of extensions + * + * @param string $dir Path to directory to search + * @param array $exts Array of extensions to search for + * @param bool $recurse Search subdirectories recursively + * @return array|bool Array of filenames on success, or false on failure + */ +function findFiles( $dir, $exts, $recurse = false ) { + if ( is_dir( $dir ) ) { + $dhl = opendir( $dir ); + if ( $dhl ) { + $files = []; + while ( ( $file = readdir( $dhl ) ) !== false ) { + if ( is_file( $dir . '/' . $file ) ) { + list( /* $name */, $ext ) = splitFilename( $dir . '/' . $file ); + if ( array_search( strtolower( $ext ), $exts ) !== false ) { + $files[] = $dir . '/' . $file; + } + } elseif ( $recurse && is_dir( $dir . '/' . $file ) && $file !== '..' && $file !== '.' ) { + $files = array_merge( $files, findFiles( $dir . '/' . $file, $exts, true ) ); + } + } + + return $files; + } else { + return []; + } + } else { + return []; + } +} + +/** + * Split a filename into filename and extension + * + * @param string $filename Filename + * @return array + */ +function splitFilename( $filename ) { + $parts = explode( '.', $filename ); + $ext = $parts[count( $parts ) - 1]; + unset( $parts[count( $parts ) - 1] ); + $fname = implode( '.', $parts ); + + return [ $fname, $ext ]; +} + +/** + * Find an auxilliary file with the given extension, matching + * the give base file path. $maxStrip determines how many extensions + * may be stripped from the original file name before appending the + * new extension. For example, with $maxStrip = 1 (the default), + * file files acme.foo.bar.txt and acme.foo.txt would be auxilliary + * files for acme.foo.bar and the extension ".txt". With $maxStrip = 2, + * acme.txt would also be acceptable. + * + * @param string $file Base path + * @param string $auxExtension The extension to be appended to the base path + * @param int $maxStrip The maximum number of extensions to strip from the base path (default: 1) + * @return string|bool + */ +function findAuxFile( $file, $auxExtension, $maxStrip = 1 ) { + if ( strpos( $auxExtension, '.' ) !== 0 ) { + $auxExtension = '.' . $auxExtension; + } + + $d = dirname( $file ); + $n = basename( $file ); + + while ( $maxStrip >= 0 ) { + $f = $d . '/' . $n . $auxExtension; + + if ( file_exists( $f ) ) { + return $f; + } + + $idx = strrpos( $n, '.' ); + if ( !$idx ) { + break; + } + + $n = substr( $n, 0, $idx ); + $maxStrip -= 1; + } + + return false; +} + +# @todo FIXME: Access the api in a saner way and performing just one query +# (preferably batching files too). +function getFileCommentFromSourceWiki( $wiki_host, $file ) { + $url = $wiki_host . '/api.php?action=query&format=xml&titles=File:' + . rawurlencode( $file ) . '&prop=imageinfo&&iiprop=comment'; + $body = Http::get( $url, [], __METHOD__ ); + if ( preg_match( '#<ii comment="([^"]*)" />#', $body, $matches ) == 0 ) { + return false; + } + + return html_entity_decode( $matches[1] ); +} + +function getFileUserFromSourceWiki( $wiki_host, $file ) { + $url = $wiki_host . '/api.php?action=query&format=xml&titles=File:' + . rawurlencode( $file ) . '&prop=imageinfo&&iiprop=user'; + $body = Http::get( $url, [], __METHOD__ ); + if ( preg_match( '#<ii user="([^"]*)" />#', $body, $matches ) == 0 ) { + return false; + } + + return html_entity_decode( $matches[1] ); +} diff --git a/www/wiki/maintenance/importImages.php b/www/wiki/maintenance/importImages.php new file mode 100644 index 00000000..625e1f70 --- /dev/null +++ b/www/wiki/maintenance/importImages.php @@ -0,0 +1,523 @@ +<?php +/** + * Import one or more images from the local file system into the wiki without + * using the web-based interface. + * + * "Smart import" additions: + * - aim: preserve the essential metadata (user, description) when importing media + * files from an existing wiki. + * - process: + * - interface with the source wiki, don't use bare files only (see --source-wiki-url). + * - fetch metadata from source wiki for each file to import. + * - commit the fetched metadata to the destination wiki while submitting. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @author Rob Church <robchur@gmail.com> + * @author Mij <mij@bitchx.it> + */ + +require_once __DIR__ . '/Maintenance.php'; + +class ImportImages extends Maintenance { + + public function __construct() { + parent::__construct(); + + $this->addDescription( 'Imports images and other media files into the wiki' ); + $this->addArg( 'dir', 'Path to the directory containing images to be imported' ); + + $this->addOption( 'extensions', + 'Comma-separated list of allowable extensions, defaults to $wgFileExtensions', + false, + true + ); + $this->addOption( 'overwrite', + 'Overwrite existing images with the same name (default is to skip them)' ); + $this->addOption( 'limit', + 'Limit the number of images to process. Ignored or skipped images are not counted', + false, + true + ); + $this->addOption( 'from', + "Ignore all files until the one with the given name. Useful for resuming aborted " + . "imports. The name should be the file's canonical database form.", + false, + true + ); + $this->addOption( 'skip-dupes', + 'Skip images that were already uploaded under a different name (check SHA1)' ); + $this->addOption( 'search-recursively', 'Search recursively for files in subdirectories' ); + $this->addOption( 'sleep', + 'Sleep between files. Useful mostly for debugging', + false, + true + ); + $this->addOption( 'user', + "Set username of uploader, default 'Maintenance script'", + false, + true + ); + // This parameter can optionally have an argument. If none specified, getOption() + // returns 1 which is precisely what we need. + $this->addOption( 'check-userblock', 'Check if the user got blocked during import' ); + $this->addOption( 'comment', + "Set file description, default 'Importing file'", + false, + true + ); + $this->addOption( 'comment-file', + 'Set description to the content of this file', + false, + true + ); + $this->addOption( 'comment-ext', + 'Causes the description for each file to be loaded from a file with the same name, but ' + . 'the extension provided. If a global description is also given, it is appended.', + false, + true + ); + $this->addOption( 'summary', + 'Upload summary, description will be used if not provided', + false, + true + ); + $this->addOption( 'license', + 'Use an optional license template', + false, + true + ); + $this->addOption( 'timestamp', + 'Override upload time/date, all MediaWiki timestamp formats are accepted', + false, + true + ); + $this->addOption( 'protect', + 'Specify the protect value (autoconfirmed,sysop)', + false, + true + ); + $this->addOption( 'unprotect', 'Unprotects all uploaded images' ); + $this->addOption( 'source-wiki-url', + 'If specified, take User and Comment data for each imported file from this URL. ' + . 'For example, --source-wiki-url="http://en.wikipedia.org/', + false, + true + ); + $this->addOption( 'dry', "Dry run, don't import anything" ); + } + + public function execute() { + global $wgFileExtensions, $wgUser, $wgRestrictionLevels; + + $processed = $added = $ignored = $skipped = $overwritten = $failed = 0; + + $this->output( "Import Images\n\n" ); + + $dir = $this->getArg( 0 ); + + # Check Protection + if ( $this->hasOption( 'protect' ) && $this->hasOption( 'unprotect' ) ) { + $this->error( "Cannot specify both protect and unprotect. Only 1 is allowed.\n", 1 ); + } + + if ( $this->hasOption( 'protect' ) && trim( $this->getOption( 'protect' ) ) ) { + $this->error( "You must specify a protection option.\n", 1 ); + } + + # Prepare the list of allowed extensions + $extensions = $this->hasOption( 'extensions' ) + ? explode( ',', strtolower( $this->getOption( 'extensions' ) ) ) + : $wgFileExtensions; + + # Search the path provided for candidates for import + $files = $this->findFiles( $dir, $extensions, $this->hasOption( 'search-recursively' ) ); + + # Initialise the user for this operation + $user = $this->hasOption( 'user' ) + ? User::newFromName( $this->getOption( 'user' ) ) + : User::newSystemUser( 'Maintenance script', [ 'steal' => true ] ); + if ( !$user instanceof User ) { + $user = User::newSystemUser( 'Maintenance script', [ 'steal' => true ] ); + } + $wgUser = $user; + + # Get block check. If a value is given, this specified how often the check is performed + $checkUserBlock = (int)$this->getOption( 'check-userblock' ); + + $from = $this->getOption( 'from' ); + $sleep = (int)$this->getOption( 'sleep' ); + $limit = (int)$this->getOption( 'limit' ); + $timestamp = $this->getOption( 'timestamp', false ); + + # Get the upload comment. Provide a default one in case there's no comment given. + $commentFile = $this->getOption( 'comment-file' ); + if ( $commentFile !== null ) { + $comment = file_get_contents( $commentFile ); + if ( $comment === false || $comment === null ) { + $this->error( "failed to read comment file: {$commentFile}\n", 1 ); + } + } else { + $comment = $this->getOption( 'comment', 'Importing file' ); + } + $commentExt = $this->getOption( 'comment-ext' ); + $summary = $this->getOption( 'summary', '' ); + + $license = $this->getOption( 'license', '' ); + + $sourceWikiUrl = $this->getOption( 'source-wiki-url' ); + + # Batch "upload" operation + $count = count( $files ); + if ( $count > 0 ) { + foreach ( $files as $file ) { + if ( $sleep && ( $processed > 0 ) ) { + sleep( $sleep ); + } + + $base = UtfNormal\Validator::cleanUp( wfBaseName( $file ) ); + + # Validate a title + $title = Title::makeTitleSafe( NS_FILE, $base ); + if ( !is_object( $title ) ) { + $this->output( + "{$base} could not be imported; a valid title cannot be produced\n" ); + continue; + } + + if ( $from ) { + if ( $from == $title->getDBkey() ) { + $from = null; + } else { + $ignored++; + continue; + } + } + + if ( $checkUserBlock && ( ( $processed % $checkUserBlock ) == 0 ) ) { + $user->clearInstanceCache( 'name' ); // reload from DB! + if ( $user->isBlocked() ) { + $this->output( $user->getName() . " was blocked! Aborting.\n" ); + break; + } + } + + # Check existence + $image = wfLocalFile( $title ); + if ( $image->exists() ) { + if ( $this->hasOption( 'overwrite' ) ) { + $this->output( "{$base} exists, overwriting..." ); + $svar = 'overwritten'; + } else { + $this->output( "{$base} exists, skipping\n" ); + $skipped++; + continue; + } + } else { + if ( $this->hasOption( 'skip-dupes' ) ) { + $repo = $image->getRepo(); + # XXX: we end up calculating this again when actually uploading. that sucks. + $sha1 = FSFile::getSha1Base36FromPath( $file ); + + $dupes = $repo->findBySha1( $sha1 ); + + if ( $dupes ) { + $this->output( + "{$base} already exists as {$dupes[0]->getName()}, skipping\n" ); + $skipped++; + continue; + } + } + + $this->output( "Importing {$base}..." ); + $svar = 'added'; + } + + if ( $sourceWikiUrl ) { + /* find comment text directly from source wiki, through MW's API */ + $real_comment = $this->getFileCommentFromSourceWiki( $sourceWikiUrl, $base ); + if ( $real_comment === false ) { + $commentText = $comment; + } else { + $commentText = $real_comment; + } + + /* find user directly from source wiki, through MW's API */ + $real_user = $this->getFileUserFromSourceWiki( $sourceWikiUrl, $base ); + if ( $real_user === false ) { + $wgUser = $user; + } else { + $wgUser = User::newFromName( $real_user ); + if ( $wgUser === false ) { + # user does not exist in target wiki + $this->output( + "failed: user '$real_user' does not exist in target wiki." ); + continue; + } + } + } else { + # Find comment text + $commentText = false; + + if ( $commentExt ) { + $f = $this->findAuxFile( $file, $commentExt ); + if ( !$f ) { + $this->output( " No comment file with extension {$commentExt} found " + . "for {$file}, using default comment. " ); + } else { + $commentText = file_get_contents( $f ); + if ( !$commentText ) { + $this->output( + " Failed to load comment file {$f}, using default comment. " ); + } + } + } + + if ( !$commentText ) { + $commentText = $comment; + } + } + + # Import the file + if ( $this->hasOption( 'dry' ) ) { + $this->output( + " publishing {$file} by '{$wgUser->getName()}', comment '$commentText'... " + ); + } else { + $mwProps = new MWFileProps( MimeMagic::singleton() ); + $props = $mwProps->getPropsFromPath( $file, true ); + $flags = 0; + $publishOptions = []; + $handler = MediaHandler::getHandler( $props['mime'] ); + if ( $handler ) { + $metadata = MediaWiki\quietCall( 'unserialize', $props['metadata'] ); + + $publishOptions['headers'] = $handler->getContentHeaders( $metadata ); + } else { + $publishOptions['headers'] = []; + } + $archive = $image->publish( $file, $flags, $publishOptions ); + if ( !$archive->isGood() ) { + $this->output( "failed. (" . + $archive->getWikiText( false, false, 'en' ) . + ")\n" ); + $failed++; + continue; + } + } + + $commentText = SpecialUpload::getInitialPageText( $commentText, $license ); + if ( !$this->hasOption( 'summary' ) ) { + $summary = $commentText; + } + + if ( $this->hasOption( 'dry' ) ) { + $this->output( "done.\n" ); + } elseif ( $image->recordUpload2( + $archive->value, + $summary, + $commentText, + $props, + $timestamp + ) ) { + # We're done! + $this->output( "done.\n" ); + + $doProtect = false; + + $protectLevel = $this->getOption( 'protect' ); + + if ( $protectLevel && in_array( $protectLevel, $wgRestrictionLevels ) ) { + $doProtect = true; + } + if ( $this->hasOption( 'unprotect' ) ) { + $protectLevel = ''; + $doProtect = true; + } + + if ( $doProtect ) { + # Protect the file + $this->output( "\nWaiting for replica DBs...\n" ); + // Wait for replica DBs. + sleep( 2.0 ); # Why this sleep? + wfWaitForSlaves(); + + $this->output( "\nSetting image restrictions ... " ); + + $cascade = false; + $restrictions = []; + foreach ( $title->getRestrictionTypes() as $type ) { + $restrictions[$type] = $protectLevel; + } + + $page = WikiPage::factory( $title ); + $status = $page->doUpdateRestrictions( $restrictions, [], $cascade, '', $user ); + $this->output( ( $status->isOK() ? 'done' : 'failed' ) . "\n" ); + } + } else { + $this->output( "failed. (at recordUpload stage)\n" ); + $svar = 'failed'; + } + + $$svar++; + $processed++; + + if ( $limit && $processed >= $limit ) { + break; + } + } + + # Print out some statistics + $this->output( "\n" ); + foreach ( + [ + 'count' => 'Found', + 'limit' => 'Limit', + 'ignored' => 'Ignored', + 'added' => 'Added', + 'skipped' => 'Skipped', + 'overwritten' => 'Overwritten', + 'failed' => 'Failed' + ] as $var => $desc + ) { + if ( $$var > 0 ) { + $this->output( "{$desc}: {$$var}\n" ); + } + } + } else { + $this->output( "No suitable files could be found for import.\n" ); + } + } + + /** + * Search a directory for files with one of a set of extensions + * + * @param string $dir Path to directory to search + * @param array $exts Array of extensions to search for + * @param bool $recurse Search subdirectories recursively + * @return array|bool Array of filenames on success, or false on failure + */ + private function findFiles( $dir, $exts, $recurse = false ) { + if ( is_dir( $dir ) ) { + $dhl = opendir( $dir ); + if ( $dhl ) { + $files = []; + while ( ( $file = readdir( $dhl ) ) !== false ) { + if ( is_file( $dir . '/' . $file ) ) { + list( /* $name */, $ext ) = $this->splitFilename( $dir . '/' . $file ); + if ( array_search( strtolower( $ext ), $exts ) !== false ) { + $files[] = $dir . '/' . $file; + } + } elseif ( $recurse && is_dir( $dir . '/' . $file ) && $file !== '..' && $file !== '.' ) { + $files = array_merge( $files, $this->findFiles( $dir . '/' . $file, $exts, true ) ); + } + } + + return $files; + } else { + return []; + } + } else { + return []; + } + } + + /** + * Split a filename into filename and extension + * + * @param string $filename Filename + * @return array + */ + private function splitFilename( $filename ) { + $parts = explode( '.', $filename ); + $ext = $parts[count( $parts ) - 1]; + unset( $parts[count( $parts ) - 1] ); + $fname = implode( '.', $parts ); + + return [ $fname, $ext ]; + } + + /** + * Find an auxilliary file with the given extension, matching + * the give base file path. $maxStrip determines how many extensions + * may be stripped from the original file name before appending the + * new extension. For example, with $maxStrip = 1 (the default), + * file files acme.foo.bar.txt and acme.foo.txt would be auxilliary + * files for acme.foo.bar and the extension ".txt". With $maxStrip = 2, + * acme.txt would also be acceptable. + * + * @param string $file Base path + * @param string $auxExtension The extension to be appended to the base path + * @param int $maxStrip The maximum number of extensions to strip from the base path (default: 1) + * @return string|bool + */ + private function findAuxFile( $file, $auxExtension, $maxStrip = 1 ) { + if ( strpos( $auxExtension, '.' ) !== 0 ) { + $auxExtension = '.' . $auxExtension; + } + + $d = dirname( $file ); + $n = basename( $file ); + + while ( $maxStrip >= 0 ) { + $f = $d . '/' . $n . $auxExtension; + + if ( file_exists( $f ) ) { + return $f; + } + + $idx = strrpos( $n, '.' ); + if ( !$idx ) { + break; + } + + $n = substr( $n, 0, $idx ); + $maxStrip -= 1; + } + + return false; + } + + # @todo FIXME: Access the api in a saner way and performing just one query + # (preferably batching files too). + private function getFileCommentFromSourceWiki( $wiki_host, $file ) { + $url = $wiki_host . '/api.php?action=query&format=xml&titles=File:' + . rawurlencode( $file ) . '&prop=imageinfo&&iiprop=comment'; + $body = Http::get( $url, [], __METHOD__ ); + if ( preg_match( '#<ii comment="([^"]*)" />#', $body, $matches ) == 0 ) { + return false; + } + + return html_entity_decode( $matches[1] ); + } + + private function getFileUserFromSourceWiki( $wiki_host, $file ) { + $url = $wiki_host . '/api.php?action=query&format=xml&titles=File:' + . rawurlencode( $file ) . '&prop=imageinfo&&iiprop=user'; + $body = Http::get( $url, [], __METHOD__ ); + if ( preg_match( '#<ii user="([^"]*)" />#', $body, $matches ) == 0 ) { + return false; + } + + return html_entity_decode( $matches[1] ); + } + +} + +$maintClass = 'ImportImages'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/importSiteScripts.php b/www/wiki/maintenance/importSiteScripts.php new file mode 100644 index 00000000..7fdb355a --- /dev/null +++ b/www/wiki/maintenance/importSiteScripts.php @@ -0,0 +1,118 @@ +<?php +/** + * Import all scripts in the MediaWiki namespace from a local site. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to import all scripts in the MediaWiki namespace from a + * local site. + * + * @ingroup Maintenance + */ +class ImportSiteScripts extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Import site scripts from a site' ); + $this->addArg( 'api', 'API base url' ); + $this->addArg( 'index', 'index.php base url' ); + $this->addOption( 'username', 'User name of the script importer' ); + } + + public function execute() { + global $wgUser; + + $username = $this->getOption( 'username', false ); + if ( $username === false ) { + $user = User::newSystemUser( 'ScriptImporter', [ 'steal' => true ] ); + } else { + $user = User::newFromName( $username ); + } + $wgUser = $user; + + $baseUrl = $this->getArg( 1 ); + $pageList = $this->fetchScriptList(); + $this->output( 'Importing ' . count( $pageList ) . " pages\n" ); + + foreach ( $pageList as $page ) { + $title = Title::makeTitleSafe( NS_MEDIAWIKI, $page ); + if ( !$title ) { + $this->error( "$page is an invalid title; it will not be imported\n" ); + continue; + } + + $this->output( "Importing $page\n" ); + $url = wfAppendQuery( $baseUrl, [ + 'action' => 'raw', + 'title' => "MediaWiki:{$page}" ] ); + $text = Http::get( $url, [], __METHOD__ ); + + $wikiPage = WikiPage::factory( $title ); + $content = ContentHandler::makeContent( $text, $wikiPage->getTitle() ); + $wikiPage->doEditContent( $content, "Importing from $url", 0, false, $user ); + } + } + + protected function fetchScriptList() { + $data = [ + 'action' => 'query', + 'format' => 'json', + 'list' => 'allpages', + 'apnamespace' => '8', + 'aplimit' => '500', + 'continue' => '', + ]; + $baseUrl = $this->getArg( 0 ); + $pages = []; + + while ( true ) { + $url = wfAppendQuery( $baseUrl, $data ); + $strResult = Http::get( $url, [], __METHOD__ ); + $result = FormatJson::decode( $strResult, true ); + + $page = null; + foreach ( $result['query']['allpages'] as $page ) { + if ( substr( $page['title'], -3 ) === '.js' ) { + strtok( $page['title'], ':' ); + $pages[] = strtok( '' ); + } + } + + if ( $page !== null ) { + $this->output( "Fetched list up to {$page['title']}\n" ); + } + + if ( isset( $result['continue'] ) ) { // >= 1.21 + $data = array_replace( $data, $result['continue'] ); + } elseif ( isset( $result['query-continue']['allpages'] ) ) { // <= 1.20 + $data = array_replace( $data, $result['query-continue']['allpages'] ); + } else { + break; + } + } + + return $pages; + } +} + +$maintClass = 'ImportSiteScripts'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/importSites.php b/www/wiki/maintenance/importSites.php new file mode 100644 index 00000000..57223442 --- /dev/null +++ b/www/wiki/maintenance/importSites.php @@ -0,0 +1,54 @@ +<?php + +$basePath = getenv( 'MW_INSTALL_PATH' ) !== false ? getenv( 'MW_INSTALL_PATH' ) : __DIR__ . '/..'; + +require_once $basePath . '/maintenance/Maintenance.php'; + +/** + * Maintenance script for importing site definitions from XML into the sites table. + * + * @since 1.25 + * + * @license GNU GPL v2+ + * @author Daniel Kinzler + */ +class ImportSites extends Maintenance { + + public function __construct() { + $this->addDescription( 'Imports site definitions from XML into the sites table.' ); + + $this->addArg( 'file', 'An XML file containing site definitions (see docs/sitelist.txt). ' . + 'Use "php://stdin" to read from stdin.', true + ); + + parent::__construct(); + } + + /** + * Do the import. + */ + public function execute() { + $file = $this->getArg( 0 ); + + $siteStore = \MediaWiki\MediaWikiServices::getInstance()->getSiteStore(); + $importer = new SiteImporter( $siteStore ); + $importer->setExceptionCallback( [ $this, 'reportException' ] ); + + $importer->importFromFile( $file ); + + $this->output( "Done.\n" ); + } + + /** + * Outputs a message via the output() method. + * + * @param Exception $ex + */ + public function reportException( Exception $ex ) { + $msg = $ex->getMessage(); + $this->output( "$msg\n" ); + } +} + +$maintClass = 'ImportSites'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/importTextFiles.php b/www/wiki/maintenance/importTextFiles.php new file mode 100644 index 00000000..816e7452 --- /dev/null +++ b/www/wiki/maintenance/importTextFiles.php @@ -0,0 +1,208 @@ +<?php +/** + * Import pages from text files + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +use MediaWiki\MediaWikiServices; + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script which reads in text files + * and imports their content to a page of the wiki. + * + * @ingroup Maintenance + */ +class ImportTextFiles extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Reads in text files and imports their content to pages of the wiki' ); + $this->addOption( 'user', 'Username to which edits should be attributed. ' . + 'Default: "Maintenance script"', false, true, 'u' ); + $this->addOption( 'summary', 'Specify edit summary for the edits', false, true, 's' ); + $this->addOption( 'use-timestamp', 'Use the modification date of the text file ' . + 'as the timestamp for the edit' ); + $this->addOption( 'overwrite', 'Overwrite existing pages. If --use-timestamp is passed, this ' . + 'will only overwrite pages if the file has been modified since the page was last modified.' ); + $this->addOption( 'prefix', 'A string to place in front of the file name', false, true, 'p' ); + $this->addOption( 'bot', 'Mark edits as bot edits in the recent changes list.' ); + $this->addOption( 'rc', 'Place revisions in RecentChanges.' ); + $this->addArg( 'files', 'Files to import' ); + } + + public function execute() { + $userName = $this->getOption( 'user', false ); + $summary = $this->getOption( 'summary', 'Imported from text file' ); + $useTimestamp = $this->hasOption( 'use-timestamp' ); + $rc = $this->hasOption( 'rc' ); + $bot = $this->hasOption( 'bot' ); + $overwrite = $this->hasOption( 'overwrite' ); + $prefix = $this->getOption( 'prefix', '' ); + + // Get all the arguments. A loop is required since Maintenance doesn't + // support an arbitrary number of arguments. + $files = []; + $i = 0; + while ( $arg = $this->getArg( $i++ ) ) { + if ( file_exists( $arg ) ) { + $files[$arg] = file_get_contents( $arg ); + } else { + // use glob to support the Windows shell, which doesn't automatically + // expand wildcards + $found = false; + foreach ( glob( $arg ) as $filename ) { + $found = true; + $files[$filename] = file_get_contents( $filename ); + } + if ( !$found ) { + $this->error( "Fatal error: The file '$arg' does not exist!", 1 ); + } + } + }; + + $count = count( $files ); + $this->output( "Importing $count pages...\n" ); + + if ( $userName === false ) { + $user = User::newSystemUser( 'Maintenance script', [ 'steal' => true ] ); + } else { + $user = User::newFromName( $userName ); + } + + if ( !$user ) { + $this->error( "Invalid username\n", true ); + } + if ( $user->isAnon() ) { + $user->addToDatabase(); + } + + $exit = 0; + + $successCount = 0; + $failCount = 0; + $skipCount = 0; + + foreach ( $files as $file => $text ) { + $pageName = $prefix . pathinfo( $file, PATHINFO_FILENAME ); + $timestamp = $useTimestamp ? wfTimestamp( TS_UNIX, filemtime( $file ) ) : wfTimestampNow(); + + $title = Title::newFromText( $pageName ); + // Have to check for # manually, since it gets interpreted as a fragment + if ( !$title || $title->hasFragment() ) { + $this->error( "Invalid title $pageName. Skipping.\n" ); + $skipCount++; + continue; + } + + $exists = $title->exists(); + $oldRevID = $title->getLatestRevID(); + $oldRev = $oldRevID ? Revision::newFromId( $oldRevID ) : null; + $actualTitle = $title->getPrefixedText(); + + if ( $exists ) { + $touched = wfTimestamp( TS_UNIX, $title->getTouched() ); + if ( !$overwrite ) { + $this->output( "Title $actualTitle already exists. Skipping.\n" ); + $skipCount++; + continue; + } elseif ( $useTimestamp && intval( $touched ) >= intval( $timestamp ) ) { + $this->output( "File for title $actualTitle has not been modified since the " . + "destination page was touched. Skipping.\n" ); + $skipCount++; + continue; + } + } + + $rev = new WikiRevision( MediaWikiServices::getInstance()->getMainConfig() ); + $rev->setText( rtrim( $text ) ); + $rev->setTitle( $title ); + $rev->setUserObj( $user ); + $rev->setComment( $summary ); + $rev->setTimestamp( $timestamp ); + + if ( $exists && $overwrite && $rev->getContent()->equals( $oldRev->getContent() ) ) { + $this->output( "File for title $actualTitle contains no changes from the current " . + "revision. Skipping.\n" ); + $skipCount++; + continue; + } + + $status = $rev->importOldRevision(); + $newId = $title->getLatestRevID(); + + if ( $status ) { + $action = $exists ? 'updated' : 'created'; + $this->output( "Successfully $action $actualTitle\n" ); + $successCount++; + } else { + $action = $exists ? 'update' : 'create'; + $this->output( "Failed to $action $actualTitle\n" ); + $failCount++; + $exit = 1; + } + + // Create the RecentChanges entry if necessary + if ( $rc && $status ) { + if ( $exists ) { + if ( is_object( $oldRev ) ) { + $oldContent = $oldRev->getContent(); + RecentChange::notifyEdit( + $timestamp, + $title, + $rev->getMinor(), + $user, + $summary, + $oldRevID, + $oldRev->getTimestamp(), + $bot, + '', + $oldContent ? $oldContent->getSize() : 0, + $rev->getContent()->getSize(), + $newId, + 1 /* the pages don't need to be patrolled */ + ); + } + } else { + RecentChange::notifyNew( + $timestamp, + $title, + $rev->getMinor(), + $user, + $summary, + $bot, + '', + $rev->getContent()->getSize(), + $newId, + 1 + ); + } + } + } + + $this->output( "Done! $successCount succeeded, $skipCount skipped.\n" ); + if ( $exit ) { + $this->error( "Import failed with $failCount failed pages.\n", $exit ); + } + } +} + +$maintClass = "ImportTextFiles"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/initEditCount.php b/www/wiki/maintenance/initEditCount.php new file mode 100644 index 00000000..96aea034 --- /dev/null +++ b/www/wiki/maintenance/initEditCount.php @@ -0,0 +1,105 @@ +<?php +/** + * Init the user_editcount database field based on the number of rows in the + * revision table. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +class InitEditCount extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addOption( 'quick', 'Force the update to be done in a single query' ); + $this->addOption( 'background', 'Force replication-friendly mode; may be inefficient but + avoids locking tables or lagging replica DBs with large updates; + calculates counts on a replica DB if possible. + +Background mode will be automatically used if multiple servers are listed +in the load balancer, usually indicating a replication environment.' ); + $this->addDescription( 'Batch-recalculate user_editcount fields from the revision table' ); + } + + public function execute() { + $dbw = $this->getDB( DB_MASTER ); + $user = $dbw->tableName( 'user' ); + $revision = $dbw->tableName( 'revision' ); + + // Autodetect mode... + if ( $this->hasOption( 'background' ) ) { + $backgroundMode = true; + } elseif ( $this->hasOption( 'quick' ) ) { + $backgroundMode = false; + } else { + $backgroundMode = wfGetLB()->getServerCount() > 1; + } + + if ( $backgroundMode ) { + $this->output( "Using replication-friendly background mode...\n" ); + + $dbr = $this->getDB( DB_REPLICA ); + $chunkSize = 100; + $lastUser = $dbr->selectField( 'user', 'MAX(user_id)', '', __METHOD__ ); + + $start = microtime( true ); + $migrated = 0; + for ( $min = 0; $min <= $lastUser; $min += $chunkSize ) { + $max = $min + $chunkSize; + $result = $dbr->query( + "SELECT + user_id, + COUNT(rev_user) AS user_editcount + FROM $user + LEFT OUTER JOIN $revision ON user_id=rev_user + WHERE user_id > $min AND user_id <= $max + GROUP BY user_id", + __METHOD__ ); + + foreach ( $result as $row ) { + $dbw->update( 'user', + [ 'user_editcount' => $row->user_editcount ], + [ 'user_id' => $row->user_id ], + __METHOD__ ); + ++$migrated; + } + + $delta = microtime( true ) - $start; + $rate = ( $delta == 0.0 ) ? 0.0 : $migrated / $delta; + $this->output( sprintf( "%s %d (%0.1f%%) done in %0.1f secs (%0.3f accounts/sec).\n", + wfWikiID(), + $migrated, + min( $max, $lastUser ) / $lastUser * 100.0, + $delta, + $rate ) ); + + wfWaitForSlaves(); + } + } else { + $this->output( "Using single-query mode...\n" ); + $sql = "UPDATE $user SET user_editcount=(SELECT COUNT(*) FROM $revision WHERE rev_user=user_id)"; + $dbw->query( $sql ); + } + + $this->output( "Done!\n" ); + } +} + +$maintClass = "InitEditCount"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/initSiteStats.php b/www/wiki/maintenance/initSiteStats.php new file mode 100644 index 00000000..b2530ce6 --- /dev/null +++ b/www/wiki/maintenance/initSiteStats.php @@ -0,0 +1,82 @@ +<?php +/** + * Re-initialise or update the site statistics table. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @author Brion Vibber + * @author Rob Church <robchur@gmail.com> + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to re-initialise or update the site statistics table + * + * @ingroup Maintenance + */ +class InitSiteStats extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Re-initialise the site statistics tables' ); + $this->addOption( 'update', 'Update the existing statistics' ); + $this->addOption( 'active', 'Also update active users count' ); + $this->addOption( 'use-master', 'Count using the master database' ); + } + + public function execute() { + $this->output( "Refresh Site Statistics\n\n" ); + $counter = new SiteStatsInit( $this->hasOption( 'use-master' ) ); + + $this->output( "Counting total edits..." ); + $edits = $counter->edits(); + $this->output( "{$edits}\nCounting number of articles..." ); + + $good = $counter->articles(); + $this->output( "{$good}\nCounting total pages..." ); + + $pages = $counter->pages(); + $this->output( "{$pages}\nCounting number of users..." ); + + $users = $counter->users(); + $this->output( "{$users}\nCounting number of images..." ); + + $image = $counter->files(); + $this->output( "{$image}\n" ); + + if ( $this->hasOption( 'update' ) ) { + $this->output( "\nUpdating site statistics..." ); + $counter->refresh(); + $this->output( "done.\n" ); + } else { + $this->output( "\nTo update the site statistics table, run the script " + . "with the --update option.\n" ); + } + + if ( $this->hasOption( 'active' ) ) { + $this->output( "\nCounting and updating active users..." ); + $active = SiteStatsUpdate::cacheUpdate( $this->getDB( DB_MASTER ) ); + $this->output( "{$active}\n" ); + } + + $this->output( "\nDone.\n" ); + } +} + +$maintClass = "InitSiteStats"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/initUserPreference.php b/www/wiki/maintenance/initUserPreference.php new file mode 100644 index 00000000..f4da570f --- /dev/null +++ b/www/wiki/maintenance/initUserPreference.php @@ -0,0 +1,84 @@ +<?php +/** + * Initialize a user preference based on the value + * of another preference. + * + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that initializes a user preference + * based on the value of another preference. + * + * @ingroup Maintenance + */ +class InitUserPreference extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addOption( + 'target', + 'Name of the user preference to initialize', + true, + true, + 't' + ); + $this->addOption( + 'source', + 'Name of the user preference to take the value from', + true, + true, + 's' + ); + $this->setBatchSize( 300 ); + } + + public function execute() { + $target = $this->getOption( 'target' ); + $source = $this->getOption( 'source' ); + $this->output( "Initializing '$target' based on the value of '$source'\n" ); + + $dbr = $this->getDB( DB_REPLICA ); + $dbw = $this->getDB( DB_MASTER ); + + $iterator = new BatchRowIterator( + $dbr, + 'user_properties', + [ 'up_user', 'up_property' ], + $this->mBatchSize + ); + $iterator->setFetchColumns( [ 'up_user', 'up_value' ] ); + $iterator->addConditions( [ + 'up_property' => $source, + 'up_value IS NOT NULL', + 'up_value != 0', + ] ); + + $processed = 0; + foreach ( $iterator as $batch ) { + foreach ( $batch as $row ) { + $values = [ + 'up_user' => $row->up_user, + 'up_property' => $target, + 'up_value' => $row->up_value, + ]; + $dbw->upsert( + 'user_properties', + $values, + [ 'up_user', 'up_property' ], + $values, + __METHOD__ + ); + + $processed += $dbw->affectedRows(); + } + } + + $this->output( "Processed $processed user(s)\n" ); + $this->output( "Finished!\n" ); + } +} + +$maintClass = 'InitUserPreference'; // Tells it to run the class +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/install.php b/www/wiki/maintenance/install.php new file mode 100644 index 00000000..0b3fc843 --- /dev/null +++ b/www/wiki/maintenance/install.php @@ -0,0 +1,175 @@ +<?php +/** + * CLI-based MediaWiki installation and configuration. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +define( 'MW_CONFIG_CALLBACK', 'Installer::overrideConfig' ); +define( 'MEDIAWIKI_INSTALL', true ); + +/** + * Maintenance script to install and configure MediaWiki + * + * Default values for the options are defined in DefaultSettings.php + * (see the mapping in CliInstaller.php) + * Default for --dbpath (SQLite-specific) is defined in SqliteInstaller::getGlobalDefaults + * + * @ingroup Maintenance + */ +class CommandLineInstaller extends Maintenance { + function __construct() { + parent::__construct(); + global $IP; + + $this->addDescription( "CLI-based MediaWiki installation and configuration.\n" . + "Default options are indicated in parentheses." ); + + $this->addArg( 'name', 'The name of the wiki (MediaWiki)', false ); + + $this->addArg( 'admin', 'The username of the wiki administrator.' ); + $this->addOption( 'pass', 'The password for the wiki administrator.', false, true ); + $this->addOption( + 'passfile', + 'An alternative way to provide pass option, as the contents of this file', + false, + true + ); + /* $this->addOption( 'email', 'The email for the wiki administrator', false, true ); */ + $this->addOption( + 'scriptpath', + 'The relative path of the wiki in the web server (/wiki)', + false, + true + ); + + $this->addOption( 'lang', 'The language to use (en)', false, true ); + /* $this->addOption( 'cont-lang', 'The content language (en)', false, true ); */ + + $this->addOption( 'dbtype', 'The type of database (mysql)', false, true ); + $this->addOption( 'dbserver', 'The database host (localhost)', false, true ); + $this->addOption( 'dbport', 'The database port; only for PostgreSQL (5432)', false, true ); + $this->addOption( 'dbname', 'The database name (my_wiki)', false, true ); + $this->addOption( 'dbpath', 'The path for the SQLite DB ($IP/data)', false, true ); + $this->addOption( 'dbprefix', 'Optional database table name prefix', false, true ); + $this->addOption( 'installdbuser', 'The user to use for installing (root)', false, true ); + $this->addOption( 'installdbpass', 'The password for the DB user to install as.', false, true ); + $this->addOption( 'dbuser', 'The user to use for normal operations (wikiuser)', false, true ); + $this->addOption( 'dbpass', 'The password for the DB user for normal operations', false, true ); + $this->addOption( + 'dbpassfile', + 'An alternative way to provide dbpass option, as the contents of this file', + false, + true + ); + $this->addOption( 'confpath', "Path to write LocalSettings.php to ($IP)", false, true ); + $this->addOption( 'dbschema', 'The schema for the MediaWiki DB in ' + . 'PostgreSQL/Microsoft SQL Server (mediawiki)', false, true ); + /* + $this->addOption( 'namespace', 'The project namespace (same as the "name" argument)', + false, true ); + */ + $this->addOption( 'env-checks', "Run environment checks only, don't change anything" ); + + $this->addOption( 'with-extensions', "Detect and include extensions" ); + } + + public function getDbType() { + if ( $this->hasOption( 'env-checks' ) ) { + return Maintenance::DB_NONE; + } + return parent::getDbType(); + } + + function execute() { + global $IP; + + $siteName = $this->getArg( 0, 'MediaWiki' ); // Will not be set if used with --env-checks + $adminName = $this->getArg( 1 ); + $envChecksOnly = $this->hasOption( 'env-checks' ); + + $this->setDbPassOption(); + if ( !$envChecksOnly ) { + $this->setPassOption(); + } + + $installer = InstallerOverrides::getCliInstaller( $siteName, $adminName, $this->mOptions ); + + $status = $installer->doEnvironmentChecks(); + if ( $status->isGood() ) { + $installer->showMessage( 'config-env-good' ); + } else { + $installer->showStatusMessage( $status ); + + return; + } + if ( !$envChecksOnly ) { + $installer->execute(); + $installer->writeConfigurationFile( $this->getOption( 'confpath', $IP ) ); + } + } + + private function setDbPassOption() { + $dbpassfile = $this->getOption( 'dbpassfile' ); + if ( $dbpassfile !== null ) { + if ( $this->getOption( 'dbpass' ) !== null ) { + $this->error( 'WARNING: You have provided the options "dbpass" and "dbpassfile". ' + . 'The content of "dbpassfile" overrides "dbpass".' ); + } + MediaWiki\suppressWarnings(); + $dbpass = file_get_contents( $dbpassfile ); // returns false on failure + MediaWiki\restoreWarnings(); + if ( $dbpass === false ) { + $this->error( "Couldn't open $dbpassfile", true ); + } + $this->mOptions['dbpass'] = trim( $dbpass, "\r\n" ); + } + } + + private function setPassOption() { + $passfile = $this->getOption( 'passfile' ); + if ( $passfile !== null ) { + if ( $this->getOption( 'pass' ) !== null ) { + $this->error( 'WARNING: You have provided the options "pass" and "passfile". ' + . 'The content of "passfile" overrides "pass".' ); + } + MediaWiki\suppressWarnings(); + $pass = file_get_contents( $passfile ); // returns false on failure + MediaWiki\restoreWarnings(); + if ( $pass === false ) { + $this->error( "Couldn't open $passfile", true ); + } + $this->mOptions['pass'] = trim( $pass, "\r\n" ); + } elseif ( $this->getOption( 'pass' ) === null ) { + $this->error( 'You need to provide the option "pass" or "passfile"', true ); + } + } + + function validateParamsAndArgs() { + if ( !$this->hasOption( 'env-checks' ) ) { + parent::validateParamsAndArgs(); + } + } +} + +$maintClass = 'CommandLineInstaller'; + +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/interwiki.list b/www/wiki/maintenance/interwiki.list new file mode 100644 index 00000000..fa8bf3b8 --- /dev/null +++ b/www/wiki/maintenance/interwiki.list @@ -0,0 +1,68 @@ +# Based more or less on the public interwiki map from MeatballWiki +# Default interwiki prefixes... +acronym|http://www.acronymfinder.com/~/search/af.aspx?string=exact&Acronym=$1|0| +advogato|http://www.advogato.org/$1|0| +arxiv|http://www.arxiv.org/abs/$1|0| +c2find|http://c2.com/cgi/wiki?FindPage&value=$1|0| +cache|http://www.google.com/search?q=cache:$1|0| +commons|https://commons.wikimedia.org/wiki/$1|0|https://commons.wikimedia.org/w/api.php +dictionary|http://www.dict.org/bin/Dict?Database=*&Form=Dict1&Strategy=*&Query=$1|0| +doi|http://dx.doi.org/$1|0| +drumcorpswiki|http://www.drumcorpswiki.com/$1|0|http://drumcorpswiki.com/api.php +dwjwiki|http://www.suberic.net/cgi-bin/dwj/wiki.cgi?$1|0| +elibre|http://enciclopedia.us.es/index.php/$1|0|http://enciclopedia.us.es/api.php +emacswiki|http://www.emacswiki.org/cgi-bin/wiki.pl?$1|0| +foldoc|http://foldoc.org/?$1|0| +foxwiki|http://fox.wikis.com/wc.dll?Wiki~$1|0| +freebsdman|http://www.FreeBSD.org/cgi/man.cgi?apropos=1&query=$1|0| +gentoo-wiki|http://gentoo-wiki.com/$1|0| +google|http://www.google.com/search?q=$1|0| +googlegroups|http://groups.google.com/groups?q=$1|0| +hammondwiki|http://www.dairiki.org/HammondWiki/$1|0| +hrwiki|http://www.hrwiki.org/wiki/$1|0|http://www.hrwiki.org/w/api.php +imdb|http://www.imdb.com/find?q=$1&tt=on|0| +kmwiki|http://kmwiki.wikispaces.com/$1|0| +linuxwiki|http://linuxwiki.de/$1|0| +lojban|http://mw.lojban.org/papri/$1|0| +lqwiki|http://wiki.linuxquestions.org/wiki/$1|0| +meatball|http://www.usemod.com/cgi-bin/mb.pl?$1|0| +mediawikiwiki|https://www.mediawiki.org/wiki/$1|0|https://www.mediawiki.org/w/api.php +memoryalpha|http://en.memory-alpha.org/wiki/$1|0|http://en.memory-alpha.org/api.php +metawiki|http://sunir.org/apps/meta.pl?$1|0| +metawikimedia|https://meta.wikimedia.org/wiki/$1|0|https://meta.wikimedia.org/w/api.php +mozillawiki|http://wiki.mozilla.org/$1|0|https://wiki.mozilla.org/api.php +mw|https://www.mediawiki.org/wiki/$1|0|https://www.mediawiki.org/w/api.php +oeis|http://oeis.org/$1|0| +openwiki|http://openwiki.com/ow.asp?$1|0| +pmid|https://www.ncbi.nlm.nih.gov/pubmed/$1?dopt=Abstract|0| +pythoninfo|http://wiki.python.org/moin/$1|0| +rfc|https://tools.ietf.org/html/rfc$1|0| +s23wiki|http://s23.org/wiki/$1|0|http://s23.org/w/api.php +seattlewireless|http://seattlewireless.net/$1|0| +senseislibrary|http://senseis.xmp.net/?$1|0| +shoutwiki|http://www.shoutwiki.com/wiki/$1|0|http://www.shoutwiki.com/w/api.php +squeak|http://wiki.squeak.org/squeak/$1|0| +tmbw|http://www.tmbw.net/wiki/$1|0|http://tmbw.net/wiki/api.php +tmnet|http://www.technomanifestos.net/?$1|0| +theopedia|http://www.theopedia.com/$1|0| +twiki|http://twiki.org/cgi-bin/view/$1|0| +uncyclopedia|http://en.uncyclopedia.co/wiki/$1|0|http://en.uncyclopedia.co/w/api.php +unreal|http://wiki.beyondunreal.com/$1|0|http://wiki.beyondunreal.com/w/api.php +usemod|http://www.usemod.com/cgi-bin/wiki.pl?$1|0| +wiki|http://c2.com/cgi/wiki?$1|0| +wikia|http://www.wikia.com/wiki/$1|0| +wikibooks|https://en.wikibooks.org/wiki/$1|0|https://en.wikibooks.org/w/api.php +wikidata|https://www.wikidata.org/wiki/$1|0|https://www.wikidata.org/w/api.php +wikif1|http://www.wikif1.org/$1|0| +wikihow|http://www.wikihow.com/$1|0|http://www.wikihow.com/api.php +wikinfo|http://wikinfo.co/English/index.php/$1|0| +wikimedia|https://wikimediafoundation.org/wiki/$1|0|https://wikimediafoundation.org/w/api.php +wikinews|https://en.wikinews.org/wiki/$1|0|https://en.wikinews.org/w/api.php +wikipedia|https://en.wikipedia.org/wiki/$1|0|https://en.wikipedia.org/w/api.php +wikiquote|https://en.wikiquote.org/wiki/$1|0|https://en.wikiquote.org/w/api.php +wikisource|https://wikisource.org/wiki/$1|0|https://wikisource.org/w/api.php +wikispecies|https://species.wikimedia.org/wiki/$1|0|https://species.wikimedia.org/w/api.php +wikiversity|https://en.wikiversity.org/wiki/$1|0|https://en.wikiversity.org/w/api.php +wikivoyage|https://en.wikivoyage.org/wiki/$1|0|https://en.wikivoyage.org/w/api.php +wikt|https://en.wiktionary.org/wiki/$1|0|https://en.wiktionary.org/w/api.php +wiktionary|https://en.wiktionary.org/wiki/$1|0|https://en.wiktionary.org/w/api.php diff --git a/www/wiki/maintenance/interwiki.sql b/www/wiki/maintenance/interwiki.sql new file mode 100644 index 00000000..adb6cd14 --- /dev/null +++ b/www/wiki/maintenance/interwiki.sql @@ -0,0 +1,71 @@ +-- Based more or less on the public interwiki map from MeatballWiki +-- Default interwiki prefixes... + +REPLACE INTO /*$wgDBprefix*/interwiki (iw_prefix,iw_url,iw_local,iw_api) VALUES +('acronym','http://www.acronymfinder.com/~/search/af.aspx?string=exact&Acronym=$1',0,''), +('advogato','http://www.advogato.org/$1',0,''), +('arxiv','http://www.arxiv.org/abs/$1',0,''), +('c2find','http://c2.com/cgi/wiki?FindPage&value=$1',0,''), +('cache','http://www.google.com/search?q=cache:$1',0,''), +('commons','https://commons.wikimedia.org/wiki/$1',0,'https://commons.wikimedia.org/w/api.php'), +('dictionary','http://www.dict.org/bin/Dict?Database=*&Form=Dict1&Strategy=*&Query=$1',0,''), +('doi','http://dx.doi.org/$1',0,''), +('drumcorpswiki','http://www.drumcorpswiki.com/$1',0,'http://drumcorpswiki.com/api.php'), +('dwjwiki','http://www.suberic.net/cgi-bin/dwj/wiki.cgi?$1',0,''), +('elibre','http://enciclopedia.us.es/index.php/$1',0,'http://enciclopedia.us.es/api.php'), +('emacswiki','http://www.emacswiki.org/cgi-bin/wiki.pl?$1',0,''), +('foldoc','http://foldoc.org/?$1',0,''), +('foxwiki','http://fox.wikis.com/wc.dll?Wiki~$1',0,''), +('freebsdman','http://www.FreeBSD.org/cgi/man.cgi?apropos=1&query=$1',0,''), +('gentoo-wiki','http://gentoo-wiki.com/$1',0,''), +('google','http://www.google.com/search?q=$1',0,''), +('googlegroups','http://groups.google.com/groups?q=$1',0,''), +('hammondwiki','http://www.dairiki.org/HammondWiki/$1',0,''), +('hrwiki','http://www.hrwiki.org/wiki/$1',0,'http://www.hrwiki.org/w/api.php'), +('imdb','http://www.imdb.com/find?q=$1&tt=on',0,''), +('kmwiki','http://kmwiki.wikispaces.com/$1',0,''), +('linuxwiki','http://linuxwiki.de/$1',0,''), +('lojban','http://www.lojban.org/tiki/tiki-index.php?page=$1',0,''), +('lqwiki','http://wiki.linuxquestions.org/wiki/$1',0,''), +('meatball','http://www.usemod.com/cgi-bin/mb.pl?$1',0,''), +('mediawikiwiki','https://www.mediawiki.org/wiki/$1',0,'https://www.mediawiki.org/w/api.php'), +('memoryalpha','http://en.memory-alpha.org/wiki/$1',0,'http://en.memory-alpha.org/api.php'), +('metawiki','http://sunir.org/apps/meta.pl?$1',0,''), +('metawikimedia','https://meta.wikimedia.org/wiki/$1',0,'https://meta.wikimedia.org/w/api.php'), +('mozillawiki','http://wiki.mozilla.org/$1',0,'https://wiki.mozilla.org/api.php'), +('mw','https://www.mediawiki.org/wiki/$1',0,'https://www.mediawiki.org/w/api.php'), +('oeis','http://oeis.org/$1',0,''), +('openwiki','http://openwiki.com/ow.asp?$1',0,''), +('pmid', 'https://www.ncbi.nlm.nih.gov/pubmed/$1?dopt=Abstract',0,''), +('pythoninfo','http://wiki.python.org/moin/$1',0,''), +('rfc','https://tools.ietf.org/html/rfc$1',0,''), +('s23wiki','http://s23.org/wiki/$1',0,'http://s23.org/w/api.php'), +('seattlewireless','http://seattlewireless.net/$1',0,''), +('senseislibrary','http://senseis.xmp.net/?$1',0,''), +('shoutwiki','http://www.shoutwiki.com/wiki/$1',0,'http://www.shoutwiki.com/w/api.php'), +('squeak','http://wiki.squeak.org/squeak/$1',0,''), +('tmbw','http://www.tmbw.net/wiki/$1',0,'http://tmbw.net/wiki/api.php'), +('tmnet','http://www.technomanifestos.net/?$1',0,''), +('theopedia','http://www.theopedia.com/$1',0,''), +('twiki','http://twiki.org/cgi-bin/view/$1',0,''), +('uncyclopedia','http://en.uncyclopedia.co/wiki/$1',0,'http://en.uncyclopedia.co/w/api.php'), +('unreal','http://wiki.beyondunreal.com/$1',0,'http://wiki.beyondunreal.com/w/api.php'), +('usemod','http://www.usemod.com/cgi-bin/wiki.pl?$1',0,''), +('wiki','http://c2.com/cgi/wiki?$1',0,''), +('wikia','http://www.wikia.com/wiki/$1',0,''), +('wikibooks','https://en.wikibooks.org/wiki/$1',0,'https://en.wikibooks.org/w/api.php'), +('wikidata','https://www.wikidata.org/wiki/$1',0,'https://www.wikidata.org/w/api.php'), +('wikif1','http://www.wikif1.org/$1',0,''), +('wikihow','http://www.wikihow.com/$1',0,'http://www.wikihow.com/api.php'), +('wikinfo','http://wikinfo.co/English/index.php/$1',0,''), +('wikimedia','https://wikimediafoundation.org/wiki/$1',0,'https://wikimediafoundation.org/w/api.php'), +('wikinews','https://en.wikinews.org/wiki/$1',0,'https://en.wikinews.org/w/api.php'), +('wikipedia','https://en.wikipedia.org/wiki/$1',0,'https://en.wikipedia.org/w/api.php'), +('wikiquote','https://en.wikiquote.org/wiki/$1',0,'https://en.wikiquote.org/w/api.php'), +('wikisource','https://wikisource.org/wiki/$1',0,'https://wikisource.org/w/api.php'), +('wikispecies','https://species.wikimedia.org/wiki/$1',0,'https://species.wikimedia.org/w/api.php'), +('wikiversity','https://en.wikiversity.org/wiki/$1',0,'https://en.wikiversity.org/w/api.php'), +('wikivoyage','https://en.wikivoyage.org/wiki/$1',0,'https://en.wikivoyage.org/w/api.php'), +('wikt','https://en.wiktionary.org/wiki/$1',0,'https://en.wiktionary.org/w/api.php'), +('wiktionary','https://en.wiktionary.org/wiki/$1',0,'https://en.wiktionary.org/w/api.php') +; diff --git a/www/wiki/maintenance/invalidateUserSessions.php b/www/wiki/maintenance/invalidateUserSessions.php new file mode 100644 index 00000000..11e3372c --- /dev/null +++ b/www/wiki/maintenance/invalidateUserSessions.php @@ -0,0 +1,94 @@ +<?php +/** + * Invalidate the sessions of certain users on the wiki. + * If you want to invalidate all sessions, use $wgAuthenticationTokenVersion instead. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +use MediaWiki\MediaWikiServices; +use MediaWiki\Session\SessionManager; + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Invalidate the sessions of certain users on the wiki. + * If you want to invalidate all sessions, use $wgAuthenticationTokenVersion instead. + * + * @ingroup Maintenance + */ +class InvalidateUserSesssions extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( + 'Invalidate the sessions of certain users on the wiki.' + ); + $this->addOption( 'user', 'Username', false, true, 'u' ); + $this->addOption( 'file', 'File with one username per line', false, true, 'f' ); + $this->setBatchSize( 1000 ); + } + + public function execute() { + $username = $this->getOption( 'user' ); + $file = $this->getOption( 'file' ); + + if ( $username === null && $file === null ) { + $this->error( 'Either --user or --file is required', 1 ); + } elseif ( $username !== null && $file !== null ) { + $this->error( 'Cannot use both --user and --file', 1 ); + } + + if ( $username !== null ) { + $usernames = [ $username ]; + } else { + $usernames = is_readable( $file ) ? + file( $file, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES ) : false; + if ( $usernames === false ) { + $this->error( "Could not open $file", 2 ); + } + } + + $i = 0; + $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory(); + $sessionManager = SessionManager::singleton(); + foreach ( $usernames as $username ) { + $i++; + $user = User::newFromName( $username ); + try { + $sessionManager->invalidateSessionsForUser( $user ); + if ( $user->getId() ) { + $this->output( "Invalidated sessions for user $username\n" ); + } else { + # session invalidation might still work if there is a central identity provider + $this->output( "Could not find user $username, tried to invalidate anyway\n" ); + } + } catch ( Exception $e ) { + $this->output( "Failed to invalidate sessions for user $username | " + . str_replace( [ "\r", "\n" ], ' ', $e->getMessage() ) . "\n" ); + } + + if ( $i % $this->mBatchSize ) { + $lbFactory->waitForReplication(); + } + } + } +} + +$maintClass = "InvalidateUserSesssions"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/jsduck/categories.json b/www/wiki/maintenance/jsduck/categories.json new file mode 100644 index 00000000..899d80da --- /dev/null +++ b/www/wiki/maintenance/jsduck/categories.json @@ -0,0 +1,140 @@ +[ + { + "name": "MediaWiki", + "groups": [ + { + "name": "Base", + "classes": [ + "mw", + "mw.Message", + "mw.loader", + "mw.loader.store", + "mw.html", + "mw.html.Cdata", + "mw.html.Raw", + "mw.hook", + "mw.template", + "mw.errorLogger" + ] + }, + { + "name": "General", + "classes": [ + "mw.Title", + "mw.Uri", + "mw.RegExp", + "mw.messagePoster.*", + "mw.notification", + "mw.Notification_", + "mw.storage", + "mw.storage.session", + "mw.user", + "mw.util", + "mw.plugin.*", + "mw.cookie", + "mw.experiments", + "mw.viewport", + "mw.htmlform.*" + ] + }, + { + "name": "Actions", + "classes": ["mw.toolbar"] + }, + { + "name": "API", + "classes": ["mw.Api*", "mw.ForeignApi*"] + }, + { + "name": "Language", + "classes": [ + "mw.language*", + "mw.cldr", + "mw.jqueryMsg" + ] + }, + { + "name": "Page", + "classes": [ + "mw.page*" + ] + }, + { + "name": "Interfaces", + "classes": [ + "mw.Feedback*", + "mw.Upload*", + "mw.ForeignUpload", + "mw.ForeignStructuredUpload*", + "mw.GallerySlideshow", + "mw.rcfilters*" + ] + }, + { + "name": "Widgets", + "classes": [ + "mw.widgets*" + ] + }, + { + "name": "Special", + "classes": [ + "mw.special*" + ] + }, + { + "name": "Development", + "classes": [ + "mw.log", + "mw.inspect", + "mw.inspect.reports", + "mw.Debug" + ] + } + ] + }, + { + "name": "jQuery", + "groups": [ + { + "name": "Plugins", + "classes": [ + "jQuery.client", + "jQuery.colorUtil", + "jQuery.plugin.*" + ] + } + ] + }, + { + "name": "Upstream", + "groups": [ + { + "name": "OOjs", + "classes": [ + "OO", + "OO.EmitterList", + "OO.EventEmitter", + "OO.Factory", + "OO.Registry", + "OO.SortedEmitterList" + ] + }, + { + "name": "OOUI", + "classes": [ + "OO.ui", + "OO.ui.*" + ] + }, + { + "name": "jQuery", + "classes": ["jQuery", "jQuery.Event", "jQuery.Callbacks", "jQuery.Promise", "jQuery.Deferred", "jQuery.jqXHR", "QUnit"] + }, + { + "name": "JavaScript", + "classes": ["Array", "Boolean", "Date", "Function", "Number", "Object", "RegExp", "String"] + } + ] + } +] diff --git a/www/wiki/maintenance/jsduck/custom_tags.rb b/www/wiki/maintenance/jsduck/custom_tags.rb new file mode 100644 index 00000000..21cb658d --- /dev/null +++ b/www/wiki/maintenance/jsduck/custom_tags.rb @@ -0,0 +1,102 @@ +# Custom tags for JSDuck 5.x +# See also: +# - https://github.com/senchalabs/jsduck/wiki/Custom-tags +# - https://github.com/senchalabs/jsduck/wiki/Custom-tags/7f5c32e568eab9edc8e3365e935bcb836cb11f1d +require 'jsduck/tag/tag' + +class CommonTag < JsDuck::Tag::Tag + def initialize + @html_position = POS_DOC + 0.1 + @repeatable = true + end + + def parse_doc(scanner, _position) + if @multiline + return { tagname: @tagname, doc: :multiline } + else + text = scanner.match(/.*$/) + return { tagname: @tagname, doc: text } + end + end + + def process_doc(context, tags, _position) + context[@tagname] = tags + end + + def format(context, formatter) + context[@tagname].each do |tag| + tag[:doc] = formatter.format(tag[:doc]) + end + end +end + +class SeeTag < CommonTag + def initialize + @tagname = :see + @pattern = 'see' + super + end + + def format(context, formatter) + position = context[:files][0] + context[@tagname].each do |tag| + tag[:doc] = '<li>' + render_long_see(tag[:doc], formatter, position) + '</li>' + end + end + + def to_html(context) + <<-EOHTML + <h3 class="pa">Related</h3> + <ul> + #{context[@tagname].map { |tag| tag[:doc] }.join("\n")} + </ul> + EOHTML + end + + def render_long_see(tag, formatter, position) + match = /\A([^\s]+)( .*)?\Z/m.match(tag) + + if match + name = match[1] + doc = match[2] ? ': ' + match[2] : '' + return formatter.format("{@link #{name}} #{doc}") + else + JsDuck::Logger.warn(nil, 'Unexpected @see argument: "' + tag + '"', position) + return tag + end + end +end + +class ContextTag < CommonTag + def initialize + @tagname = :context + @pattern = 'context' + super + end + + def format(context, formatter) + position = context[:files][0] + context[@tagname].each do |tag| + tag[:doc] = render_long_context(tag[:doc], formatter, position) + end + end + + def to_html(context) + <<-EOHTML + <h3 class="pa">Context</h3> + #{context[@tagname].last[:doc]} + EOHTML + end + + def render_long_context(tag, formatter, position) + match = /\A([^\s]+)/m.match(tag) + + if match + name = match[1] + return formatter.format("`context` : {@link #{name}}") + else + JsDuck::Logger.warn(nil, 'Unexpected @context argument: "' + tag + '"', position) + return tag + end + end +end diff --git a/www/wiki/maintenance/jsduck/eg-iframe.html b/www/wiki/maintenance/jsduck/eg-iframe.html new file mode 100644 index 00000000..91e0bc12 --- /dev/null +++ b/www/wiki/maintenance/jsduck/eg-iframe.html @@ -0,0 +1,117 @@ +<!DOCTYPE html> +<html> +<head> + <meta charset="utf-8"> + <title>MediaWiki Code Example</title> + <script> + /** + * Basic log console for the example iframe in documentation pages. + */ + var log = ( function () { + var pre; + return function () { + var str, i, len, line; + if ( !pre ) { + pre = document.createElement( 'pre' ); + pre.className = 'mw-jsduck-log'; + ( document.body || document.documentElement ).appendChild( pre ); + } + str = []; + for ( i = 0, len = arguments.length; i < len; i++ ) { + str.push( String( arguments[ i ] ) ); + } + line = document.createElement( 'div' ); + line.className = 'mw-jsduck-log-line'; + line.appendChild( + document.createTextNode( str.join( ' , ' ) + '\n' ) + ); + pre.appendChild( line ); + }; + }() ); + + window.onerror = function ( error, filePath, linerNr ) { + log( error + '\n' + filePath + ':' + linerNr ); + }; + </script> + <script> + // Mock startup.js + var mwPerformance = { mark: function () {} }, + mwNow = Date.now; + + function startUp() { + mw.config = new mw.Map(); + } + </script> + <script src="modules/lib/jquery/jquery.js"></script> + <script src="modules/src/mediawiki/mediawiki.js"></script> + <script src="modules/src/mediawiki/mediawiki.errorLogger.js"></script> + <script src="modules/lib/oojs/oojs.jquery.js"></script> + <script src="modules/lib/oojs-ui/oojs-ui-core.js"></script> + <script src="modules/lib/oojs-ui/oojs-ui-widgets.js"></script> + <script src="modules/lib/oojs-ui/oojs-ui-toolbars.js"></script> + <script src="modules/lib/oojs-ui/oojs-ui-windows.js"></script> + <script src="modules/lib/oojs-ui/oojs-ui-wikimediaui.js"></script> + <style> + body { + font-size: 0.8em; + font-family: sans-serif; + } + + .mw-jsduck-log { + position: relative; + min-height: 3em; + margin-top: 2em; + background: #f7f7f7; + border: 1px solid #e4e4e4; + } + + .mw-jsduck-log::after { + position: absolute; + bottom: 100%; + right: -1px; + padding: 0.5em; + background: #fff; + border: 1px solid #e4e4e4; + border-bottom: 0; + border-radius: 0.5em 0.5em 0 0; + font: normal 0.5em sans-serif; + content: 'console'; + } + + .mw-jsduck-log-line { + padding: 0.2em 0.5em; + white-space: pre-wrap; + } + + .mw-jsduck-log-line:nth-child(odd) { + background: #fff; + } + </style> + <link rel="stylesheet" href="modules/src/oojs-ui-local.css"> + <link rel="stylesheet" href="modules/lib/oojs-ui/oojs-ui-core-mediawiki.css"> + <link rel="stylesheet" href="modules/lib/oojs-ui/oojs-ui-widgets-mediawiki.css"> + <link rel="stylesheet" href="modules/lib/oojs-ui/oojs-ui-toolbars-mediawiki.css"> + <link rel="stylesheet" href="modules/lib/oojs-ui/oojs-ui-windows-mediawiki.css"> +</head> +<body> +<script> + if ( window.mw ) { + mw.log = log; + } + + /** + * Method called by jsduck to execute the example code. + */ + function loadInlineExample( code, options, callback ) { + try { + eval( code ); + callback && callback( true ); + } catch ( e ) { + log( 'Uncaught ' + e ); + callback && callback( false, e ); + throw e; + } + } +</script> +</body> +</html> diff --git a/www/wiki/maintenance/jsduck/external.js b/www/wiki/maintenance/jsduck/external.js new file mode 100644 index 00000000..85b1f921 --- /dev/null +++ b/www/wiki/maintenance/jsduck/external.js @@ -0,0 +1,43 @@ +/** + * Source: <https://api.jquery.com/> + * @class jQuery + */ + +/** + * Source: <https://api.jquery.com/jQuery.ajax/> + * @method ajax + * @static + * @return {jqXHR} + */ + +/** + * Source: <https://api.jquery.com/Types/#Event> + * @class jQuery.Event + */ + +/** + * Source: <https://api.jquery.com/jQuery.Callbacks/> + * @class jQuery.Callbacks + */ + +/** + * Source: <https://api.jquery.com/Types/#Promise> + * @class jQuery.Promise + */ + +/** + * Source: <https://api.jquery.com/jQuery.Deferred/> + * @class jQuery.Deferred + * @mixins jQuery.Promise + */ + +/** + * Source: <https://api.jquery.com/Types/#jqXHR> + * @class jQuery.jqXHR + * @alternateClassName jqXHR + */ + +/** + * Source: <https://api.qunitjs.com/> + * @class QUnit + */ diff --git a/www/wiki/maintenance/jsparse.php b/www/wiki/maintenance/jsparse.php new file mode 100644 index 00000000..49b945cb --- /dev/null +++ b/www/wiki/maintenance/jsparse.php @@ -0,0 +1,77 @@ +<?php +/** + * Test JavaScript validity parses using jsmin+'s parser + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to test JavaScript validity using JsMinPlus' parser + * + * @ingroup Maintenance + */ +class JSParseHelper extends Maintenance { + public $errs = 0; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Runs parsing/syntax checks on JavaScript files' ); + $this->addArg( 'file(s)', 'JavaScript file to test', false ); + } + + public function execute() { + if ( $this->hasArg() ) { + $files = $this->mArgs; + } else { + $this->maybeHelp( true ); // @todo fixme this is a lame API :) + exit( 1 ); // it should exit from the above first... + } + + $parser = new JSParser(); + foreach ( $files as $filename ) { + MediaWiki\suppressWarnings(); + $js = file_get_contents( $filename ); + MediaWiki\restoreWarnings(); + if ( $js === false ) { + $this->output( "$filename ERROR: could not read file\n" ); + $this->errs++; + continue; + } + + try { + $parser->parse( $js, $filename, 1 ); + } catch ( Exception $e ) { + $this->errs++; + $this->output( "$filename ERROR: " . $e->getMessage() . "\n" ); + continue; + } + + $this->output( "$filename OK\n" ); + } + + if ( $this->errs > 0 ) { + exit( 1 ); + } + } +} + +$maintClass = "JSParseHelper"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/lag.php b/www/wiki/maintenance/lag.php new file mode 100644 index 00000000..fa2bd54e --- /dev/null +++ b/www/wiki/maintenance/lag.php @@ -0,0 +1,72 @@ +<?php +/** + * Shows database lag + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to show database lag. + * + * @ingroup Maintenance + */ +class DatabaseLag extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Shows database lag' ); + $this->addOption( 'r', "Don't exit immediately, but show the lag every 5 seconds" ); + } + + public function execute() { + if ( $this->hasOption( 'r' ) ) { + $lb = wfGetLB(); + echo 'time '; + + $serverCount = $lb->getServerCount(); + for ( $i = 1; $i < $serverCount; $i++ ) { + $hostname = $lb->getServerName( $i ); + printf( "%-12s ", $hostname ); + } + echo "\n"; + + while ( 1 ) { + $lags = $lb->getLagTimes(); + unset( $lags[0] ); + echo gmdate( 'H:i:s' ) . ' '; + foreach ( $lags as $lag ) { + printf( "%-12s ", $lag === false ? 'false' : $lag ); + } + echo "\n"; + sleep( 5 ); + } + } else { + $lb = wfGetLB(); + $lags = $lb->getLagTimes(); + foreach ( $lags as $i => $lag ) { + $name = $lb->getServerName( $i ); + $this->output( sprintf( "%-20s %s\n", $name, $lag === false ? 'false' : $lag ) ); + } + } + } +} + +$maintClass = "DatabaseLag"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/language/StatOutputs.php b/www/wiki/maintenance/language/StatOutputs.php new file mode 100644 index 00000000..15ccb2db --- /dev/null +++ b/www/wiki/maintenance/language/StatOutputs.php @@ -0,0 +1,146 @@ +<?php +/** + * Statistic output classes. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup MaintenanceLanguage + * @author Ævar Arnfjörð Bjarmason <avarab@gmail.com> + * @author Antoine Musso <hashar at free dot fr> + */ + +/** A general output object. Need to be overridden */ +class StatsOutput { + function formatPercent( $subset, $total, $revert = false, $accuracy = 2 ) { + MediaWiki\suppressWarnings(); + $return = sprintf( '%.' . $accuracy . 'f%%', 100 * $subset / $total ); + MediaWiki\restoreWarnings(); + + return $return; + } + + # Override the following methods + function heading() { + } + + function footer() { + } + + function blockstart() { + } + + function blockend() { + } + + function element( $in, $heading = false ) { + } +} + +/** Outputs WikiText */ +class WikiStatsOutput extends StatsOutput { + function heading() { + global $wgDummyLanguageCodes; + $version = SpecialVersion::getVersion( 'nodb' ); + echo "'''Statistics are based on:''' <code>" . $version . "</code>\n\n"; + echo "'''Note:''' These statistics can be generated by running " . + "<code>php maintenance/language/transstat.php</code>.\n\n"; + echo "For additional information on specific languages (the message names, the actual " . + "problems, etc.), run <code>php maintenance/language/checkLanguage.php --lang=foo</code>.\n\n"; + echo 'English (en) is excluded because it is the default localization'; + if ( is_array( $wgDummyLanguageCodes ) ) { + $dummyCodes = []; + foreach ( $wgDummyLanguageCodes as $dummyCode => $correctCode ) { + $dummyCodes[] = Language::fetchLanguageName( $dummyCode ) . ' (' . $dummyCode . ')'; + } + echo ', as well as the following languages that are not intended for ' . + 'system message translations, usually because they redirect to other ' . + 'language codes: ' . implode( ', ', $dummyCodes ); + } + echo ".\n\n"; # dot to end sentence + echo '{| class="sortable wikitable" border="2" style="background-color: #F9F9F9; ' . + 'border: 1px #AAAAAA solid; border-collapse: collapse; clear:both; width:100%;"' . "\n"; + } + + function footer() { + echo "|}\n"; + } + + function blockstart() { + echo "|-\n"; + } + + function blockend() { + echo ''; + } + + function element( $in, $heading = false ) { + echo ( $heading ? '!' : '|' ) . "$in\n"; + } + + function formatPercent( $subset, $total, $revert = false, $accuracy = 2 ) { + MediaWiki\suppressWarnings(); + $v = round( 255 * $subset / $total ); + MediaWiki\restoreWarnings(); + + if ( $revert ) { + # Weigh reverse with factor 20 so coloring takes effect more quickly as + # this option is used solely for reporting 'bad' percentages. + $v = $v * 20; + if ( $v > 255 ) { + $v = 255; + } + $v = 255 - $v; + } + if ( $v < 128 ) { + # Red to Yellow + $red = 'FF'; + $green = sprintf( '%02X', 2 * $v ); + } else { + # Yellow to Green + $red = sprintf( '%02X', 2 * ( 255 - $v ) ); + $green = 'FF'; + } + $blue = '00'; + $color = $red . $green . $blue; + + $percent = parent::formatPercent( $subset, $total, $revert, $accuracy ); + + return 'style="background-color:#' . $color . ';"|' . $percent; + } +} + +/** Output text. To be used on a terminal for example. */ +class TextStatsOutput extends StatsOutput { + function element( $in, $heading = false ) { + echo $in . "\t"; + } + + function blockend() { + echo "\n"; + } +} + +/** csv output. Some people love excel */ +class CsvStatsOutput extends StatsOutput { + function element( $in, $heading = false ) { + echo $in . ";"; + } + + function blockend() { + echo "\n"; + } +} diff --git a/www/wiki/maintenance/language/alltrans.php b/www/wiki/maintenance/language/alltrans.php new file mode 100644 index 00000000..931718f5 --- /dev/null +++ b/www/wiki/maintenance/language/alltrans.php @@ -0,0 +1,47 @@ +<?php +/** + * Get all the translations messages, as defined in the English language file. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup MaintenanceLanguage + */ + +require_once __DIR__ . '/../Maintenance.php'; + +/** + * Maintenance script that gets all messages as defined by the + * English language file. + * + * @ingroup MaintenanceLanguage + */ +class AllTrans extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Get all messages as defined by the English language file' ); + } + + public function execute() { + $englishMessages = array_keys( Language::getMessagesFor( 'en' ) ); + foreach ( $englishMessages as $key ) { + $this->output( "$key\n" ); + } + } +} + +$maintClass = "AllTrans"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/language/checkDupeMessages.php b/www/wiki/maintenance/language/checkDupeMessages.php new file mode 100644 index 00000000..92ddc449 --- /dev/null +++ b/www/wiki/maintenance/language/checkDupeMessages.php @@ -0,0 +1,137 @@ +<?php +/** + * Print out duplicates in message array + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup MaintenanceLanguage + */ + +$optionsWithArgs = [ 'lang', 'clang', 'mode' ]; +require_once __DIR__ . '/../commandLine.inc'; +$messagesDir = __DIR__ . '/../../languages/messages/'; +$runTest = false; +$run = false; +$runMode = 'text'; + +// Check parameters +if ( isset( $options['lang'] ) && isset( $options['clang'] ) ) { + if ( !isset( $options['mode'] ) ) { + $runMode = 'text'; + } else { + if ( !strcmp( $options['mode'], 'wiki' ) ) { + $runMode = 'wiki'; + } elseif ( !strcmp( $options['mode'], 'php' ) ) { + $runMode = 'php'; + } elseif ( !strcmp( $options['mode'], 'raw' ) ) { + $runMode = 'raw'; + } else { + } + } + $runTest = true; +} else { + echo <<<TEXT +Run this script to print out the duplicates against a message array. +Parameters: + * lang: Language code to be checked. + * clang: Language code to be compared. +Options: + * mode: Output format, can be either: + * text: Text output on the console (default) + * wiki: Wiki format, with * at beginning of each line + * php: Output text as PHP syntax in an array named \$dupeMessages + * raw: Raw output for duplicates +TEXT; +} + +// Check file exists +if ( $runTest ) { + $langCode = $options['lang']; + $langCodeC = $options['clang']; + $langCodeF = ucfirst( strtolower( preg_replace( '/-/', '_', $langCode ) ) ); + $langCodeFC = ucfirst( strtolower( preg_replace( '/-/', '_', $langCodeC ) ) ); + $messagesFile = $messagesDir . 'Messages' . $langCodeF . '.php'; + $messagesFileC = $messagesDir . 'Messages' . $langCodeFC . '.php'; + if ( file_exists( $messagesFile ) && file_exists( $messagesFileC ) ) { + $run = true; + } else { + echo "Messages file(s) could not be found.\nMake sure both files are exists.\n"; + } +} + +// Run to check the dupes +if ( $run ) { + if ( !strcmp( $runMode, 'wiki' ) ) { + $runMode = 'wiki'; + } elseif ( !strcmp( $runMode, 'raw' ) ) { + $runMode = 'raw'; + } + include $messagesFile; + $messageExist = isset( $messages ); + if ( $messageExist ) { + $wgMessages[$langCode] = $messages; + } + include $messagesFileC; + $messageCExist = isset( $messages ); + if ( $messageCExist ) { + $wgMessages[$langCodeC] = $messages; + } + $count = 0; + + if ( ( $messageExist ) && ( $messageCExist ) ) { + if ( !strcmp( $runMode, 'php' ) ) { + print "<?php\n"; + print '$dupeMessages = [' . "\n"; + } + foreach ( $wgMessages[$langCodeC] as $key => $value ) { + foreach ( $wgMessages[$langCode] as $ckey => $cvalue ) { + if ( !strcmp( $key, $ckey ) ) { + if ( ( !strcmp( $key, $ckey ) ) && ( !strcmp( $value, $cvalue ) ) ) { + if ( !strcmp( $runMode, 'raw' ) ) { + print "$key\n"; + } elseif ( !strcmp( $runMode, 'php' ) ) { + print "'$key' => '',\n"; + } elseif ( !strcmp( $runMode, 'wiki' ) ) { + $uKey = ucfirst( $key ); + print "* MediaWiki:$uKey/$langCode\n"; + } else { + print "* $key\n"; + } + $count++; + } + } + } + } + if ( !strcmp( $runMode, 'php' ) ) { + print "];\n"; + } + if ( !strcmp( $runMode, 'text' ) ) { + if ( $count == 1 ) { + echo "\nThere are $count duplicated message in $langCode, against to $langCodeC.\n"; + } else { + echo "\nThere are $count duplicated messages in $langCode, against to $langCodeC.\n"; + } + } + } else { + if ( !$messageExist ) { + echo "There are no messages defined in $langCode.\n"; + } + if ( !$messageCExist ) { + echo "There are no messages defined in $langCodeC.\n"; + } + } +} diff --git a/www/wiki/maintenance/language/checkExtensions.php b/www/wiki/maintenance/language/checkExtensions.php new file mode 100644 index 00000000..79a4dd98 --- /dev/null +++ b/www/wiki/maintenance/language/checkExtensions.php @@ -0,0 +1,40 @@ +<?php +/** + * Check the extensions language files. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup MaintenanceLanguage + */ + +require_once __DIR__ . '/../commandLine.inc'; +require_once 'languages.inc'; +require_once 'checkLanguage.inc'; + +if ( !class_exists( 'MessageGroups' ) || !class_exists( 'PremadeMediawikiExtensionGroups' ) ) { + echo <<<TEXT +Please add the Translate extension to LocalSettings.php, and enable the extension groups: + require_once 'extensions/Translate/Translate.php'; + \$wgTranslateEC = array_keys( \$wgTranslateAC ); +If you still get this message, update Translate to its latest version. + +TEXT; + exit( -1 ); +} + +$cli = new CheckExtensionsCLI( $options, $argv[0] ); +$cli->execute(); diff --git a/www/wiki/maintenance/language/checkLanguage.inc b/www/wiki/maintenance/language/checkLanguage.inc new file mode 100644 index 00000000..9e9fd3ee --- /dev/null +++ b/www/wiki/maintenance/language/checkLanguage.inc @@ -0,0 +1,782 @@ +<?php +/** + * Helper class for checkLanguage.php script. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup MaintenanceLanguage + */ + +/** + * @ingroup MaintenanceLanguage + */ +class CheckLanguageCLI { + protected $code = null; + protected $level = 2; + protected $doLinks = false; + protected $linksPrefix = ''; + protected $wikiCode = 'en'; + protected $checkAll = false; + protected $output = 'plain'; + protected $checks = []; + protected $L = null; + + protected $results = []; + + private $includeExif = false; + + /** + * @param array $options Options for script. + */ + public function __construct( array $options ) { + if ( isset( $options['help'] ) ) { + echo $this->help(); + exit( 1 ); + } + + if ( isset( $options['lang'] ) ) { + $this->code = $options['lang']; + } else { + global $wgLanguageCode; + $this->code = $wgLanguageCode; + } + + if ( isset( $options['level'] ) ) { + $this->level = $options['level']; + } + + $this->doLinks = isset( $options['links'] ); + $this->includeExif = !isset( $options['noexif'] ); + $this->checkAll = isset( $options['all'] ); + + if ( isset( $options['prefix'] ) ) { + $this->linksPrefix = $options['prefix']; + } + + if ( isset( $options['wikilang'] ) ) { + $this->wikiCode = $options['wikilang']; + } + + if ( isset( $options['whitelist'] ) ) { + $this->checks = explode( ',', $options['whitelist'] ); + } elseif ( isset( $options['blacklist'] ) ) { + $this->checks = array_diff( + isset( $options['easy'] ) ? $this->easyChecks() : $this->defaultChecks(), + explode( ',', $options['blacklist'] ) + ); + } elseif ( isset( $options['easy'] ) ) { + $this->checks = $this->easyChecks(); + } else { + $this->checks = $this->defaultChecks(); + } + + if ( isset( $options['output'] ) ) { + $this->output = $options['output']; + } + + $this->L = new Languages( $this->includeExif ); + } + + /** + * Get the default checks. + * @return array A list of the default checks. + */ + protected function defaultChecks() { + return [ + 'untranslated', 'duplicate', 'obsolete', 'variables', 'empty', 'plural', + 'whitespace', 'xhtml', 'chars', 'links', 'unbalanced', 'namespace', + 'projecttalk', 'magic', 'magic-old', 'magic-over', 'magic-case', + 'special', 'special-old', + ]; + } + + /** + * Get the checks which check other things than messages. + * @return array A list of the non-message checks. + */ + protected function nonMessageChecks() { + return [ + 'namespace', 'projecttalk', 'magic', 'magic-old', 'magic-over', + 'magic-case', 'special', 'special-old', + ]; + } + + /** + * Get the checks that can easily be treated by non-speakers of the language. + * @return array A list of the easy checks. + */ + protected function easyChecks() { + return [ + 'duplicate', 'obsolete', 'empty', 'whitespace', 'xhtml', 'chars', 'magic-old', + 'magic-over', 'magic-case', 'special-old', + ]; + } + + /** + * Get all checks. + * @return array An array of all check names mapped to their function names. + */ + protected function getChecks() { + return [ + 'untranslated' => 'getUntranslatedMessages', + 'duplicate' => 'getDuplicateMessages', + 'obsolete' => 'getObsoleteMessages', + 'variables' => 'getMessagesWithMismatchVariables', + 'plural' => 'getMessagesWithoutPlural', + 'empty' => 'getEmptyMessages', + 'whitespace' => 'getMessagesWithWhitespace', + 'xhtml' => 'getNonXHTMLMessages', + 'chars' => 'getMessagesWithWrongChars', + 'links' => 'getMessagesWithDubiousLinks', + 'unbalanced' => 'getMessagesWithUnbalanced', + 'namespace' => 'getUntranslatedNamespaces', + 'projecttalk' => 'getProblematicProjectTalks', + 'magic' => 'getUntranslatedMagicWords', + 'magic-old' => 'getObsoleteMagicWords', + 'magic-over' => 'getOverridingMagicWords', + 'magic-case' => 'getCaseMismatchMagicWords', + 'special' => 'getUntraslatedSpecialPages', + 'special-old' => 'getObsoleteSpecialPages', + ]; + } + + /** + * Get total count for each check non-messages check. + * @return array An array of all check names mapped to a two-element array: + * function name to get the total count and language code or null + * for checked code. + */ + protected function getTotalCount() { + return [ + 'namespace' => [ 'getNamespaceNames', 'en' ], + 'projecttalk' => null, + 'magic' => [ 'getMagicWords', 'en' ], + 'magic-old' => [ 'getMagicWords', null ], + 'magic-over' => [ 'getMagicWords', null ], + 'magic-case' => [ 'getMagicWords', null ], + 'special' => [ 'getSpecialPageAliases', 'en' ], + 'special-old' => [ 'getSpecialPageAliases', null ], + ]; + } + + /** + * Get all check descriptions. + * @return array An array of all check names mapped to their descriptions. + */ + protected function getDescriptions() { + return [ + 'untranslated' => '$1 message(s) of $2 are not translated to $3, but exist in en:', + 'duplicate' => '$1 message(s) of $2 are translated the same in en and $3:', + 'obsolete' => + '$1 message(s) of $2 do not exist in en or are in the ignore list, but exist in $3:', + 'variables' => '$1 message(s) of $2 in $3 don\'t match the variables used in en:', + 'plural' => '$1 message(s) of $2 in $3 don\'t use {{plural}} while en uses:', + 'empty' => '$1 message(s) of $2 in $3 are empty or -:', + 'whitespace' => '$1 message(s) of $2 in $3 have trailing whitespace:', + 'xhtml' => '$1 message(s) of $2 in $3 contain illegal XHTML:', + 'chars' => + '$1 message(s) of $2 in $3 include hidden chars which should not be used in the messages:', + 'links' => '$1 message(s) of $2 in $3 have problematic link(s):', + 'unbalanced' => '$1 message(s) of $2 in $3 have unbalanced {[]}:', + 'namespace' => '$1 namespace name(s) of $2 are not translated to $3, but exist in en:', + 'projecttalk' => + '$1 namespace name(s) and alias(es) in $3 are project talk namespaces without the parameter:', + 'magic' => '$1 magic word(s) of $2 are not translated to $3, but exist in en:', + 'magic-old' => '$1 magic word(s) of $2 do not exist in en, but exist in $3:', + 'magic-over' => '$1 magic word(s) of $2 in $3 do not contain the original en word(s):', + 'magic-case' => + '$1 magic word(s) of $2 in $3 change the case-sensitivity of the original en word:', + 'special' => '$1 special page alias(es) of $2 are not translated to $3, but exist in en:', + 'special-old' => '$1 special page alias(es) of $2 do not exist in en, but exist in $3:', + ]; + } + + /** + * Get help. + * @return string The help string. + */ + protected function help() { + return <<<ENDS +Run this script to check a specific language file, or all of them. +Command line settings are in form --parameter[=value]. +Parameters: + --help: Show this help. + --lang: Language code (default: the installation default language). + --all: Check all customized languages. + --level: Show the following display level (default: 2): + * 0: Skip the checks (useful for checking syntax). + * 1: Show only the stub headers and number of wrong messages, without + list of messages. + * 2: Show only the headers and the message keys, without the message + values. + * 3: Show both the headers and the complete messages, with both keys and + values. + --links: Link the message values (default off). + --prefix: prefix to add to links. + --wikilang: For the links, what is the content language of the wiki to + display the output in (default en). + --noexif: Do not check for Exif messages (a bit hard and boring to + translate), if you know what they are currently not translated and want + to focus on other problems (default off). + --whitelist: Do only the following checks (form: code,code). + --blacklist: Do not do the following checks (form: code,code). + --easy: Do only the easy checks, which can be treated by non-speakers of + the language. + +Check codes (ideally, all of them should result 0; all the checks are executed +by default (except language-specific check blacklists in checkLanguage.inc): + * untranslated: Messages which are required to translate, but are not + translated. + * duplicate: Messages which translation equal to fallback. + * obsolete: Messages which are untranslatable or do not exist, but are + translated. + * variables: Messages without variables which should be used, or with + variables which should not be used. + * empty: Empty messages and messages that contain only -. + * whitespace: Messages which have trailing whitespace. + * xhtml: Messages which are not well-formed XHTML (checks only few common + errors). + * chars: Messages with hidden characters. + * links: Messages which contains broken links to pages (does not find all). + * unbalanced: Messages which contains unequal numbers of opening {[ and + closing ]}. + * namespace: Namespace names that were not translated. + * projecttalk: Namespace names and aliases where the project talk does not + contain $1. + * magic: Magic words that were not translated. + * magic-old: Magic words which do not exist. + * magic-over: Magic words that override the original English word. + * magic-case: Magic words whose translation changes the case-sensitivity of + the original English word. + * special: Special page names that were not translated. + * special-old: Special page names which do not exist. + +ENDS; + } + + /** + * Execute the script. + */ + public function execute() { + $this->doChecks(); + if ( $this->level > 0 ) { + switch ( $this->output ) { + case 'plain': + $this->outputText(); + break; + case 'wiki': + $this->outputWiki(); + break; + default: + throw new MWException( "Invalid output type $this->output" ); + } + } + } + + /** + * Execute the checks. + */ + protected function doChecks() { + $ignoredCodes = [ 'en', 'enRTL' ]; + + $this->results = []; + # Check the language + if ( $this->checkAll ) { + foreach ( $this->L->getLanguages() as $language ) { + if ( !in_array( $language, $ignoredCodes ) ) { + $this->results[$language] = $this->checkLanguage( $language ); + } + } + } else { + if ( in_array( $this->code, $ignoredCodes ) ) { + throw new MWException( "Cannot check code $this->code." ); + } else { + $this->results[$this->code] = $this->checkLanguage( $this->code ); + } + } + + $results = $this->results; + foreach ( $results as $code => $checks ) { + foreach ( $checks as $check => $messages ) { + foreach ( $messages as $key => $details ) { + if ( $this->isCheckBlacklisted( $check, $code, $key ) ) { + unset( $this->results[$code][$check][$key] ); + } + } + } + } + } + + /** + * Get the check blacklist. + * @return array The list of checks which should not be executed. + */ + protected function getCheckBlacklist() { + static $blacklist = null; + + if ( $blacklist !== null ) { + return $blacklist; + } + + // @codingStandardsIgnoreStart Ignore that globals should have a "wg" prefix. + global $checkBlacklist; + // @codingStandardsIgnoreEnd + + $blacklist = $checkBlacklist; + + Hooks::run( 'LocalisationChecksBlacklist', [ &$blacklist ] ); + + return $blacklist; + } + + /** + * Verify whether a check is blacklisted. + * + * @param string $check Check name + * @param string $code Language code + * @param string|bool $message Message name, or False for a whole language + * @return bool Whether the check is blacklisted + */ + protected function isCheckBlacklisted( $check, $code, $message ) { + $blacklist = $this->getCheckBlacklist(); + + foreach ( $blacklist as $item ) { + if ( isset( $item['check'] ) && $check !== $item['check'] ) { + continue; + } + + if ( isset( $item['code'] ) && !in_array( $code, $item['code'] ) ) { + continue; + } + + if ( isset( $item['message'] ) && + ( $message === false || !in_array( $message, $item['message'] ) ) + ) { + continue; + } + + return true; + } + + return false; + } + + /** + * Check a language. + * @param string $code The language code. + * @throws MWException + * @return array The results. + */ + protected function checkLanguage( $code ) { + # Syntax check only + $results = []; + if ( $this->level === 0 ) { + $this->L->getMessages( $code ); + + return $results; + } + + $checkFunctions = $this->getChecks(); + foreach ( $this->checks as $check ) { + if ( $this->isCheckBlacklisted( $check, $code, false ) ) { + $results[$check] = []; + continue; + } + + $callback = [ $this->L, $checkFunctions[$check] ]; + if ( !is_callable( $callback ) ) { + throw new MWException( "Unkown check $check." ); + } + $results[$check] = call_user_func( $callback, $code ); + } + + return $results; + } + + /** + * Format a message key. + * @param string $key The message key. + * @param string $code The language code. + * @return string The formatted message key. + */ + protected function formatKey( $key, $code ) { + if ( $this->doLinks ) { + $displayKey = ucfirst( $key ); + if ( $code == $this->wikiCode ) { + return "[[{$this->linksPrefix}MediaWiki:$displayKey|$key]]"; + } else { + return "[[{$this->linksPrefix}MediaWiki:$displayKey/$code|$key]]"; + } + } else { + return $key; + } + } + + /** + * Output the checks results as plain text. + */ + protected function outputText() { + foreach ( $this->results as $code => $results ) { + $translated = $this->L->getMessages( $code ); + $translated = count( $translated['translated'] ); + foreach ( $results as $check => $messages ) { + $count = count( $messages ); + if ( $count ) { + if ( $check == 'untranslated' ) { + $translatable = $this->L->getGeneralMessages(); + $total = count( $translatable['translatable'] ); + } elseif ( in_array( $check, $this->nonMessageChecks() ) ) { + $totalCount = $this->getTotalCount(); + $totalCount = $totalCount[$check]; + $callback = [ $this->L, $totalCount[0] ]; + $callCode = $totalCount[1] ? $totalCount[1] : $code; + $total = count( call_user_func( $callback, $callCode ) ); + } else { + $total = $translated; + } + $search = [ '$1', '$2', '$3' ]; + $replace = [ $count, $total, $code ]; + $descriptions = $this->getDescriptions(); + echo "\n" . str_replace( $search, $replace, $descriptions[$check] ) . "\n"; + if ( $this->level == 1 ) { + echo "[messages are hidden]\n"; + } else { + foreach ( $messages as $key => $value ) { + if ( !in_array( $check, $this->nonMessageChecks() ) ) { + $key = $this->formatKey( $key, $code ); + } + if ( $this->level == 2 || empty( $value ) ) { + echo "* $key\n"; + } else { + echo "* $key: '$value'\n"; + } + } + } + } + } + } + } + + /** + * Output the checks results as wiki text. + */ + function outputWiki() { + $detailText = ''; + $rows[] = '! Language !! Code !! Total !! ' . + implode( ' !! ', array_diff( $this->checks, $this->nonMessageChecks() ) ); + foreach ( $this->results as $code => $results ) { + $detailTextForLang = "==$code==\n"; + $numbers = []; + $problems = 0; + $detailTextForLangChecks = []; + foreach ( $results as $check => $messages ) { + if ( in_array( $check, $this->nonMessageChecks() ) ) { + continue; + } + $count = count( $messages ); + if ( $count ) { + $problems += $count; + $messageDetails = []; + foreach ( $messages as $key => $details ) { + $displayKey = $this->formatKey( $key, $code ); + $messageDetails[] = $displayKey; + } + $detailTextForLangChecks[] = "=== $code-$check ===\n* " . implode( ', ', $messageDetails ); + $numbers[] = "'''[[#$code-$check|$count]]'''"; + } else { + $numbers[] = $count; + } + } + + if ( count( $detailTextForLangChecks ) ) { + $detailText .= $detailTextForLang . implode( "\n", $detailTextForLangChecks ) . "\n"; + } + + if ( !$problems ) { + # Don't list languages without problems + continue; + } + $language = Language::fetchLanguageName( $code ); + $rows[] = "| $language || $code || $problems || " . implode( ' || ', $numbers ); + } + + $tableRows = implode( "\n|-\n", $rows ); + + $version = SpecialVersion::getVersion( 'nodb' ); + // @codingStandardsIgnoreStart Long line. + echo <<<EOL +'''Check results are for:''' <code>$version</code> + + +{| class="sortable wikitable" border="2" cellpadding="4" cellspacing="0" style="background-color: #F9F9F9; border: 1px #AAAAAA solid; border-collapse: collapse; clear: both;" +$tableRows +|} + +$detailText + +EOL; + // @codingStandardsIgnoreEnd + } + + /** + * Check if there are any results for the checks, in any language. + * @return bool True if there are any results, false if not. + */ + protected function isEmpty() { + foreach ( $this->results as $results ) { + foreach ( $results as $messages ) { + if ( !empty( $messages ) ) { + return false; + } + } + } + + return true; + } +} + +/** + * @ingroup MaintenanceLanguage + */ +class CheckExtensionsCLI extends CheckLanguageCLI { + private $extensions; + + /** + * @param array $options Options for script. + * @param string $extension The extension name (or names). + */ + public function __construct( array $options, $extension ) { + if ( isset( $options['help'] ) ) { + echo $this->help(); + exit( 1 ); + } + + if ( isset( $options['lang'] ) ) { + $this->code = $options['lang']; + } else { + global $wgLanguageCode; + $this->code = $wgLanguageCode; + } + + if ( isset( $options['level'] ) ) { + $this->level = $options['level']; + } + + $this->doLinks = isset( $options['links'] ); + + if ( isset( $options['wikilang'] ) ) { + $this->wikiCode = $options['wikilang']; + } + + if ( isset( $options['whitelist'] ) ) { + $this->checks = explode( ',', $options['whitelist'] ); + } elseif ( isset( $options['blacklist'] ) ) { + $this->checks = array_diff( + isset( $options['easy'] ) ? $this->easyChecks() : $this->defaultChecks(), + explode( ',', $options['blacklist'] ) + ); + } elseif ( isset( $options['easy'] ) ) { + $this->checks = $this->easyChecks(); + } else { + $this->checks = $this->defaultChecks(); + } + + if ( isset( $options['output'] ) ) { + $this->output = $options['output']; + } + + # Some additional checks not enabled by default + if ( isset( $options['duplicate'] ) ) { + $this->checks[] = 'duplicate'; + } + + $this->extensions = []; + $extensions = new PremadeMediawikiExtensionGroups(); + $extensions->addAll(); + if ( $extension == 'all' ) { + foreach ( MessageGroups::singleton()->getGroups() as $group ) { + if ( strpos( $group->getId(), 'ext-' ) === 0 && !$group->isMeta() ) { + $this->extensions[] = new ExtensionLanguages( $group ); + } + } + } elseif ( $extension == 'wikimedia' ) { + $wikimedia = MessageGroups::getGroup( 'ext-0-wikimedia' ); + foreach ( $wikimedia->wmfextensions() as $extension ) { + $group = MessageGroups::getGroup( $extension ); + $this->extensions[] = new ExtensionLanguages( $group ); + } + } elseif ( $extension == 'flaggedrevs' ) { + foreach ( MessageGroups::singleton()->getGroups() as $group ) { + if ( strpos( $group->getId(), 'ext-flaggedrevs-' ) === 0 && !$group->isMeta() ) { + $this->extensions[] = new ExtensionLanguages( $group ); + } + } + } else { + $extensions = explode( ',', $extension ); + foreach ( $extensions as $extension ) { + $group = MessageGroups::getGroup( 'ext-' . $extension ); + if ( $group ) { + $extension = new ExtensionLanguages( $group ); + $this->extensions[] = $extension; + } else { + print "No such extension $extension.\n"; + } + } + } + } + + /** + * Get the default checks. + * @return array A list of the default checks. + */ + protected function defaultChecks() { + return [ + 'untranslated', 'duplicate', 'obsolete', 'variables', 'empty', 'plural', + 'whitespace', 'xhtml', 'chars', 'links', 'unbalanced', + ]; + } + + /** + * Get the checks which check other things than messages. + * @return array A list of the non-message checks. + */ + protected function nonMessageChecks() { + return []; + } + + /** + * Get the checks that can easily be treated by non-speakers of the language. + * @return array A list of the easy checks. + */ + protected function easyChecks() { + return [ + 'duplicate', 'obsolete', 'empty', 'whitespace', 'xhtml', 'chars', + ]; + } + + /** + * Get help. + * @return string The help string. + */ + protected function help() { + return <<<ENDS +Run this script to check the status of a specific language in extensions, or +all of them. Command line settings are in form --parameter[=value], except for +the first one. +Parameters: + * First parameter (mandatory): Extension name, multiple extension names + (separated by commas), "all" for all the extensions, "wikimedia" for + extensions used by Wikimedia or "flaggedrevs" for all FLaggedRevs + extension messages. + * lang: Language code (default: the installation default language). + * help: Show this help. + * level: Show the following display level (default: 2). + * links: Link the message values (default off). + * wikilang: For the links, what is the content language of the wiki to + display the output in (default en). + * whitelist: Do only the following checks (form: code,code). + * blacklist: Do not perform the following checks (form: code,code). + * easy: Do only the easy checks, which can be treated by non-speakers of + the language. + +Check codes (ideally, all of them should result 0; all the checks are executed +by default (except language-specific check blacklists in checkLanguage.inc): + * untranslated: Messages which are required to translate, but are not + translated. + * duplicate: Messages which translation equal to fallback. + * obsolete: Messages which are untranslatable, but translated. + * variables: Messages without variables which should be used, or with + variables which should not be used. + * empty: Empty messages. + * whitespace: Messages which have trailing whitespace. + * xhtml: Messages which are not well-formed XHTML (checks only few common + errors). + * chars: Messages with hidden characters. + * links: Messages which contains broken links to pages (does not find all). + * unbalanced: Messages which contains unequal numbers of opening {[ and + closing ]}. + +Display levels (default: 2): + * 0: Skip the checks (useful for checking syntax). + * 1: Show only the stub headers and number of wrong messages, without list + of messages. + * 2: Show only the headers and the message keys, without the message + values. + * 3: Show both the headers and the complete messages, with both keys and + values. + +ENDS; + } + + /** + * Execute the script. + */ + public function execute() { + $this->doChecks(); + } + + /** + * Check a language and show the results. + * @param string $code The language code. + * @throws MWException + */ + protected function checkLanguage( $code ) { + foreach ( $this->extensions as $extension ) { + $this->L = $extension; + $this->results = []; + $this->results[$code] = parent::checkLanguage( $code ); + + if ( !$this->isEmpty() ) { + echo $extension->name() . ":\n"; + + if ( $this->level > 0 ) { + switch ( $this->output ) { + case 'plain': + $this->outputText(); + break; + case 'wiki': + $this->outputWiki(); + break; + default: + throw new MWException( "Invalid output type $this->output" ); + } + } + + echo "\n"; + } + } + } +} + +// Blacklist some checks for some languages or some messages +// Possible keys of the sub arrays are: 'check', 'code' and 'message'. +$checkBlacklist = [ + [ + 'check' => 'plural', + 'code' => [ 'az', 'bo', 'cdo', 'dz', 'id', 'fa', 'gan', 'gan-hans', + 'gan-hant', 'gn', 'hak', 'hu', 'ja', 'jv', 'ka', 'kk-arab', + 'kk-cyrl', 'kk-latn', 'km', 'kn', 'ko', 'lzh', 'mn', 'ms', + 'my', 'sah', 'sq', 'tet', 'th', 'to', 'tr', 'vi', 'wuu', 'xmf', + 'yo', 'yue', 'zh', 'zh-classical', 'zh-cn', 'zh-hans', + 'zh-hant', 'zh-hk', 'zh-sg', 'zh-tw', 'zh-yue' + ], + ], + [ + 'check' => 'chars', + 'code' => [ 'my' ], + ], +]; diff --git a/www/wiki/maintenance/language/checkLanguage.php b/www/wiki/maintenance/language/checkLanguage.php new file mode 100644 index 00000000..a8cbac1c --- /dev/null +++ b/www/wiki/maintenance/language/checkLanguage.php @@ -0,0 +1,40 @@ +<?php +/** + * Check a language file. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup MaintenanceLanguage + */ + +$optionsWithArgs = [ + 'lang', 'level', 'blacklist', 'whitelist', 'wikilang', 'output', 'prefix' +]; +$optionsWithoutArgs = [ + 'help', 'links', 'noexif', 'easy', 'duplicate', 'all' +]; +require_once __DIR__ . '/../commandLine.inc'; +require_once 'checkLanguage.inc'; +require_once 'languages.inc'; + +$cli = new CheckLanguageCLI( $options ); + +try { + $cli->execute(); +} catch ( Exception $e ) { + print 'Error: ' . $e->getMessage() . "\n"; +} diff --git a/www/wiki/maintenance/language/date-formats.php b/www/wiki/maintenance/language/date-formats.php new file mode 100644 index 00000000..2142c245 --- /dev/null +++ b/www/wiki/maintenance/language/date-formats.php @@ -0,0 +1,82 @@ +<?php +/** + * Test various language time and date functions + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup MaintenanceLanguage + */ + +require_once __DIR__ . '/../Maintenance.php'; + +/** + * Maintenance script that tests various language time and date functions. + * + * @ingroup MaintenanceLanguage + */ +class DateFormats extends Maintenance { + + private $ts = '20010115123456'; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Test various language time and date functions' ); + } + + public function execute() { + global $IP; + foreach ( glob( "$IP/languages/messages/Messages*.php" ) as $filename ) { + $base = basename( $filename ); + $m = []; + if ( !preg_match( '/Messages(.*)\.php$/', $base, $m ) ) { + continue; + } + $code = str_replace( '_', '-', strtolower( $m[1] ) ); + $this->output( "$code " ); + $lang = Language::factory( $code ); + $prefs = $lang->getDatePreferences(); + if ( !$prefs ) { + $prefs = [ 'default' ]; + } + $this->output( "date: " ); + foreach ( $prefs as $index => $pref ) { + if ( $index > 0 ) { + $this->output( ' | ' ); + } + $this->output( $lang->date( $this->ts, false, $pref ) ); + } + $this->output( "\n$code time: " ); + foreach ( $prefs as $index => $pref ) { + if ( $index > 0 ) { + $this->output( ' | ' ); + } + $this->output( $lang->time( $this->ts, false, $pref ) ); + } + $this->output( "\n$code both: " ); + foreach ( $prefs as $index => $pref ) { + if ( $index > 0 ) { + $this->output( ' | ' ); + } + $this->output( $lang->timeanddate( $this->ts, false, $pref ) ); + } + $this->output( "\n\n" ); + } + } +} + +$maintClass = "DateFormats"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/language/digit2html.php b/www/wiki/maintenance/language/digit2html.php new file mode 100644 index 00000000..bb1f3d24 --- /dev/null +++ b/www/wiki/maintenance/language/digit2html.php @@ -0,0 +1,69 @@ +<?php +/** + * Check digit transformation + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup MaintenanceLanguage + */ + +require_once __DIR__ . '/../Maintenance.php'; + +/** + * Maintenance script that check digit transformation. + * + * @ingroup MaintenanceLanguage + */ +class Digit2Html extends Maintenance { + + # A list of unicode numerals is available at: + # https://www.fileformat.info/info/unicode/category/Nd/list.htm + private $mLangs = [ + 'Ar', 'As', 'Bh', 'Bo', 'Dz', + 'Fa', 'Gu', 'Hi', 'Km', 'Kn', + 'Ks', 'Lo', 'Ml', 'Mr', 'Ne', + 'New', 'Or', 'Pa', 'Pi', 'Sa' + ]; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Check digit transformation' ); + } + + public function execute() { + foreach ( $this->mLangs as $code ) { + $filename = Language::getMessagesFileName( $code ); + $this->output( "Loading language [$code] ... " ); + unset( $digitTransformTable ); + require_once $filename; + if ( !isset( $digitTransformTable ) ) { + $this->error( "\$digitTransformTable not found for lang: $code" ); + continue; + } + + $this->output( "OK\n\$digitTransformTable = [\n" ); + foreach ( $digitTransformTable as $latin => $translation ) { + $htmlent = utf8ToHexSequence( $translation ); + $this->output( "'$latin' => '$translation', # &#x$htmlent;\n" ); + } + $this->output( "];\n" ); + } + } +} + +$maintClass = "Digit2Html"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/language/dumpMessages.php b/www/wiki/maintenance/language/dumpMessages.php new file mode 100644 index 00000000..37c87a8a --- /dev/null +++ b/www/wiki/maintenance/language/dumpMessages.php @@ -0,0 +1,52 @@ +<?php +/** + * Dump an entire language, using the keys from English + * so we get all the values, not just the customized ones + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup MaintenanceLanguage + * @todo Make this more useful, right now just dumps $wgContLang + */ + +require_once __DIR__ . '/../Maintenance.php'; + +/** + * Maintenance script that dumps an entire language, using the keys from English. + * + * @ingroup MaintenanceLanguage + */ +class DumpMessages extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Dump an entire language, using the keys from English' ); + } + + public function execute() { + global $wgVersion; + + $messages = []; + foreach ( array_keys( Language::getMessagesFor( 'en' ) ) as $key ) { + $messages[$key] = wfMessage( $key )->text(); + } + $this->output( "MediaWiki $wgVersion language file\n" ); + $this->output( serialize( $messages ) ); + } +} + +$maintClass = "DumpMessages"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/language/generateCollationData.php b/www/wiki/maintenance/language/generateCollationData.php new file mode 100644 index 00000000..ccfece01 --- /dev/null +++ b/www/wiki/maintenance/language/generateCollationData.php @@ -0,0 +1,472 @@ +<?php +/** + * Maintenance script to generate first letter data files for Collation.php. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup MaintenanceLanguage + */ + +require_once __DIR__ . '/../Maintenance.php'; + +/** + * Generate first letter data files for Collation.php + * + * @ingroup MaintenanceLanguage + */ +class GenerateCollationData extends Maintenance { + /** The directory with source data files in it */ + public $dataDir; + + /** The primary weights, indexed by codepoint */ + public $weights; + + /** + * A hashtable keyed by codepoint, where presence indicates that a character + * has a decomposition mapping. This makes it non-preferred for group header + * selection. + */ + public $mappedChars; + + public $debugOutFile; + + /** + * Important tertiary weights from UTS #10 section 7.2 + */ + const NORMAL_UPPERCASE = 0x08; + const NORMAL_HIRAGANA = 0x0E; + + public function __construct() { + parent::__construct(); + $this->addOption( 'data-dir', 'A directory on the local filesystem ' . + 'containing allkeys.txt and ucd.all.grouped.xml from unicode.org', + false, true ); + $this->addOption( 'debug-output', 'Filename for sending debug output to', + false, true ); + } + + public function execute() { + $this->dataDir = $this->getOption( 'data-dir', '.' ); + + $allkeysPresent = file_exists( "{$this->dataDir}/allkeys.txt" ); + $ucdallPresent = file_exists( "{$this->dataDir}/ucd.all.grouped.xml" ); + + // As of January 2013, these links work for all versions of Unicode + // between 5.1 and 6.2, inclusive. + $allkeysURL = "http://www.unicode.org/Public/UCA/<Unicode version>/allkeys.txt"; + $ucdallURL = "http://www.unicode.org/Public/<Unicode version>/ucdxml/ucd.all.grouped.zip"; + + if ( !$allkeysPresent || !$ucdallPresent ) { + $icuVersion = IcuCollation::getICUVersion(); + $unicodeVersion = IcuCollation::getUnicodeVersionForICU(); + + $error = ""; + + if ( !$allkeysPresent ) { + $error .= "Unable to find allkeys.txt. " + . "Download it and specify its location with --data-dir=<DIR>. " + . "\n\n"; + } + if ( !$ucdallPresent ) { + $error .= "Unable to find ucd.all.grouped.xml. " + . "Download it, unzip, and specify its location with --data-dir=<DIR>. " + . "\n\n"; + } + + $versionKnown = false; + if ( !$icuVersion ) { + // Unknown version - either very old intl, + // or PHP < 5.3.7 which does not expose this information + $error .= "As MediaWiki could not determine the version of ICU library used by your PHP's " + . "intl extension it can't suggest which file version to download. " + . "This can be caused by running a very old version of intl or PHP < 5.3.7. " + . "If you are sure everything is all right, find out the ICU version " + . "by running phpinfo(), check what is the Unicode version it is using " + . "at http://site.icu-project.org/download, then try finding appropriate data file(s) at:"; + } elseif ( version_compare( $icuVersion, "4.0", "<" ) ) { + // Extra old version + $error .= "You are using outdated version of ICU ($icuVersion), intended for " + . ( $unicodeVersion ? "Unicode $unicodeVersion" : "an unknown version of Unicode" ) + . "; this file might not be avalaible for it, and it's not supported by MediaWiki. " + . " You are on your own; consider upgrading PHP's intl extension or try " + . "one of the files available at:"; + } elseif ( version_compare( $icuVersion, "51.0", ">=" ) ) { + // Extra recent version + $error .= "You are using ICU $icuVersion, released after this script was last updated. " + . "Check what is the Unicode version it is using at http://site.icu-project.org/download . " + . "It can't be guaranteed everything will work, but appropriate file(s) should " + . "be available at:"; + } else { + // ICU 4.0 to 50.x + $versionKnown = true; + $error .= "You are using ICU $icuVersion, intended for " + . ( $unicodeVersion ? "Unicode $unicodeVersion" : "an unknown version of Unicode" ) + . ". Appropriate file(s) should be available at:"; + } + $error .= "\n"; + + if ( $versionKnown && $unicodeVersion ) { + $allkeysURL = str_replace( "<Unicode version>", "$unicodeVersion.0", $allkeysURL ); + $ucdallURL = str_replace( "<Unicode version>", "$unicodeVersion.0", $ucdallURL ); + } + + if ( !$allkeysPresent ) { + $error .= "* $allkeysURL\n"; + } + if ( !$ucdallPresent ) { + $error .= "* $ucdallURL\n"; + } + + $this->error( $error ); + exit( 1 ); + } + + $debugOutFileName = $this->getOption( 'debug-output' ); + if ( $debugOutFileName ) { + $this->debugOutFile = fopen( $debugOutFileName, 'w' ); + if ( !$this->debugOutFile ) { + $this->error( "Unable to open debug output file for writing" ); + exit( 1 ); + } + } + $this->loadUcd(); + $this->generateFirstChars(); + } + + function loadUcd() { + $uxr = new UcdXmlReader( "{$this->dataDir}/ucd.all.grouped.xml" ); + $uxr->readChars( [ $this, 'charCallback' ] ); + } + + function charCallback( $data ) { + // Skip non-printable characters, + // but do not skip a normal space (U+0020) since + // people like to use that as a fake no header symbol. + $category = substr( $data['gc'], 0, 1 ); + if ( strpos( 'LNPS', $category ) === false + && $data['cp'] !== '0020' + ) { + return; + } + $cp = hexdec( $data['cp'] ); + + // Skip the CJK ideograph blocks, as an optimisation measure. + // UCA doesn't sort them properly anyway, without tailoring. + if ( IcuCollation::isCjk( $cp ) ) { + return; + } + + // Skip the composed Hangul syllables, we will use the bare Jamo + // as first letters + if ( $data['block'] == 'Hangul Syllables' ) { + return; + } + + // Calculate implicit weight per UTS #10 v6.0.0, sec 7.1.3 + if ( $data['UIdeo'] === 'Y' ) { + if ( $data['block'] == 'CJK Unified Ideographs' + || $data['block'] == 'CJK Compatibility Ideographs' + ) { + $base = 0xFB40; + } else { + $base = 0xFB80; + } + } else { + $base = 0xFBC0; + } + $a = $base + ( $cp >> 15 ); + $b = ( $cp & 0x7fff ) | 0x8000; + + $this->weights[$cp] = sprintf( ".%04X.%04X", $a, $b ); + + if ( $data['dm'] !== '#' ) { + $this->mappedChars[$cp] = true; + } + + if ( $cp % 4096 == 0 ) { + print "{$data['cp']}\n"; + } + } + + function generateFirstChars() { + $file = fopen( "{$this->dataDir}/allkeys.txt", 'r' ); + if ( !$file ) { + $this->error( "Unable to open allkeys.txt" ); + exit( 1 ); + } + global $IP; + $outFile = fopen( "$IP/serialized/first-letters-root.ser", 'w' ); + if ( !$outFile ) { + $this->error( "Unable to open output file first-letters-root.ser" ); + exit( 1 ); + } + + $goodTertiaryChars = []; + + // For each character with an entry in allkeys.txt, overwrite the implicit + // entry in $this->weights that came from the UCD. + // Also gather a list of tertiary weights, for use in selecting the group header + while ( false !== ( $line = fgets( $file ) ) ) { + // We're only interested in single-character weights, pick them out with a regex + $line = trim( $line ); + if ( !preg_match( '/^([0-9A-F]+)\s*;\s*([^#]*)/', $line, $m ) ) { + continue; + } + + $cp = hexdec( $m[1] ); + $allWeights = trim( $m[2] ); + $primary = ''; + $tertiary = ''; + + if ( !isset( $this->weights[$cp] ) ) { + // Non-printable, ignore + continue; + } + foreach ( StringUtils::explode( '[', $allWeights ) as $weightStr ) { + preg_match_all( '/[*.]([0-9A-F]+)/', $weightStr, $m ); + if ( !empty( $m[1] ) ) { + if ( $m[1][0] !== '0000' ) { + $primary .= '.' . $m[1][0]; + } + if ( $m[1][2] !== '0000' ) { + $tertiary .= '.' . $m[1][2]; + } + } + } + $this->weights[$cp] = $primary; + if ( $tertiary === '.0008' + || $tertiary === '.000E' + ) { + $goodTertiaryChars[$cp] = true; + } + } + fclose( $file ); + + // Identify groups of characters with the same primary weight + $this->groups = []; + asort( $this->weights, SORT_STRING ); + $prevWeight = reset( $this->weights ); + $group = []; + foreach ( $this->weights as $cp => $weight ) { + if ( $weight !== $prevWeight ) { + $this->groups[$prevWeight] = $group; + $prevWeight = $weight; + if ( isset( $this->groups[$weight] ) ) { + $group = $this->groups[$weight]; + } else { + $group = []; + } + } + $group[] = $cp; + } + if ( $group ) { + $this->groups[$prevWeight] = $group; + } + + // If one character has a given primary weight sequence, and a second + // character has a longer primary weight sequence with an initial + // portion equal to the first character, then remove the second + // character. This avoids having characters like U+A732 (double A) + // polluting the basic latin sort area. + + foreach ( $this->groups as $weight => $group ) { + if ( preg_match( '/(\.[0-9A-F]*)\./', $weight, $m ) ) { + if ( isset( $this->groups[$m[1]] ) ) { + unset( $this->groups[$weight] ); + } + } + } + + ksort( $this->groups, SORT_STRING ); + + // Identify the header character in each group + $headerChars = []; + $prevChar = "\000"; + $tertiaryCollator = new Collator( 'root' ); + $primaryCollator = new Collator( 'root' ); + $primaryCollator->setStrength( Collator::PRIMARY ); + $numOutOfOrder = 0; + foreach ( $this->groups as $weight => $group ) { + $uncomposedChars = []; + $goodChars = []; + foreach ( $group as $cp ) { + if ( isset( $goodTertiaryChars[$cp] ) ) { + $goodChars[] = $cp; + } + if ( !isset( $this->mappedChars[$cp] ) ) { + $uncomposedChars[] = $cp; + } + } + $x = array_intersect( $goodChars, $uncomposedChars ); + if ( !$x ) { + $x = $uncomposedChars; + if ( !$x ) { + $x = $group; + } + } + + // Use ICU to pick the lowest sorting character in the selection + $tertiaryCollator->sort( $x ); + $cp = $x[0]; + + $char = UtfNormal\Utils::codepointToUtf8( $cp ); + $headerChars[] = $char; + if ( $primaryCollator->compare( $char, $prevChar ) <= 0 ) { + $numOutOfOrder++; + /* + printf( "Out of order: U+%05X > U+%05X\n", + utf8ToCodepoint( $prevChar ), + utf8ToCodepoint( $char ) ); + */ + } + $prevChar = $char; + + if ( $this->debugOutFile ) { + fwrite( $this->debugOutFile, sprintf( "%05X %s %s (%s)\n", $cp, $weight, $char, + implode( ' ', array_map( 'UtfNormal\Utils::codepointToUtf8', $group ) ) ) ); + } + } + + print "Out of order: $numOutOfOrder / " . count( $headerChars ) . "\n"; + + fwrite( $outFile, serialize( $headerChars ) ); + } +} + +class UcdXmlReader { + public $fileName; + public $callback; + public $groupAttrs; + public $xml; + public $blocks = []; + public $currentBlock; + + function __construct( $fileName ) { + $this->fileName = $fileName; + } + + public function readChars( $callback ) { + $this->getBlocks(); + $this->currentBlock = reset( $this->blocks ); + $xml = $this->open(); + $this->callback = $callback; + + while ( $xml->name !== 'repertoire' && $xml->next() ); + + while ( $xml->read() ) { + if ( $xml->nodeType == XMLReader::ELEMENT ) { + if ( $xml->name === 'group' ) { + $this->groupAttrs = $this->readAttributes(); + } elseif ( $xml->name === 'char' ) { + $this->handleChar(); + } + } elseif ( $xml->nodeType === XMLReader::END_ELEMENT ) { + if ( $xml->name === 'group' ) { + $this->groupAttrs = []; + } + } + } + $xml->close(); + } + + protected function open() { + $this->xml = new XMLReader; + $this->xml->open( $this->fileName ); + if ( !$this->xml ) { + throw new MWException( __METHOD__ . ": unable to open {$this->fileName}" ); + } + while ( $this->xml->name !== 'ucd' && $this->xml->read() ); + $this->xml->read(); + + return $this->xml; + } + + /** + * Read the attributes of the current element node and return them + * as an array + * @return array + */ + protected function readAttributes() { + $attrs = []; + while ( $this->xml->moveToNextAttribute() ) { + $attrs[$this->xml->name] = $this->xml->value; + } + + return $attrs; + } + + protected function handleChar() { + $attrs = $this->readAttributes() + $this->groupAttrs; + if ( isset( $attrs['cp'] ) ) { + $first = $last = hexdec( $attrs['cp'] ); + } else { + $first = hexdec( $attrs['first-cp'] ); + $last = hexdec( $attrs['last-cp'] ); + unset( $attrs['first-cp'] ); + unset( $attrs['last-cp'] ); + } + + for ( $cp = $first; $cp <= $last; $cp++ ) { + $hexCp = sprintf( "%04X", $cp ); + foreach ( [ 'na', 'na1' ] as $nameProp ) { + if ( isset( $attrs[$nameProp] ) ) { + $attrs[$nameProp] = str_replace( '#', $hexCp, $attrs[$nameProp] ); + } + } + + while ( $this->currentBlock ) { + if ( $cp < $this->currentBlock[0] ) { + break; + } elseif ( $cp <= $this->currentBlock[1] ) { + $attrs['block'] = key( $this->blocks ); + break; + } else { + $this->currentBlock = next( $this->blocks ); + } + } + + $attrs['cp'] = $hexCp; + call_user_func( $this->callback, $attrs ); + } + } + + public function getBlocks() { + if ( $this->blocks ) { + return $this->blocks; + } + + $xml = $this->open(); + while ( $xml->name !== 'blocks' && $xml->read() ); + + while ( $xml->read() ) { + if ( $xml->nodeType == XMLReader::ELEMENT ) { + if ( $xml->name === 'block' ) { + $attrs = $this->readAttributes(); + $first = hexdec( $attrs['first-cp'] ); + $last = hexdec( $attrs['last-cp'] ); + $this->blocks[$attrs['name']] = [ $first, $last ]; + } + } + } + $xml->close(); + + return $this->blocks; + } +} + +$maintClass = 'GenerateCollationData'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/language/generateNormalizerDataAr.php b/www/wiki/maintenance/language/generateNormalizerDataAr.php new file mode 100644 index 00000000..34903de1 --- /dev/null +++ b/www/wiki/maintenance/language/generateNormalizerDataAr.php @@ -0,0 +1,134 @@ +<?php +/** + * Generates the normalizer data file for Arabic. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup MaintenanceLanguage + */ + +require_once __DIR__ . '/../Maintenance.php'; + +/** + * Generates the normalizer data file for Arabic. + * + * This data file is used after normalizing to NFC. + * + * @ingroup MaintenanceLanguage + */ +class GenerateNormalizerDataAr extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Generate the normalizer data file for Arabic' ); + $this->addOption( 'unicode-data-file', 'The local location of the data file ' . + 'from http://unicode.org/Public/UNIDATA/UnicodeData.txt', false, true ); + } + + public function getDbType() { + return Maintenance::DB_NONE; + } + + public function execute() { + if ( !$this->hasOption( 'unicode-data-file' ) ) { + $dataFile = 'UnicodeData.txt'; + if ( !file_exists( $dataFile ) ) { + $this->error( "Unable to find UnicodeData.txt. Please specify " . + "its location with --unicode-data-file=<FILE>" ); + exit( 1 ); + } + } else { + $dataFile = $this->getOption( 'unicode-data-file' ); + if ( !file_exists( $dataFile ) ) { + $this->error( 'Unable to find the specified data file.' ); + exit( 1 ); + } + } + + $file = fopen( $dataFile, 'r' ); + if ( !$file ) { + $this->error( 'Unable to open the data file.' ); + exit( 1 ); + } + + // For the file format, see http://www.unicode.org/reports/tr44/ + $fieldNames = [ + 'Code', + 'Name', + 'General_Category', + 'Canonical_Combining_Class', + 'Bidi_Class', + 'Decomposition_Type_Mapping', + 'Numeric_Type_Value_6', + 'Numeric_Type_Value_7', + 'Numeric_Type_Value_8', + 'Bidi_Mirrored', + 'Unicode_1_Name', + 'ISO_Comment', + 'Simple_Uppercase_Mapping', + 'Simple_Lowercase_Mapping', + 'Simple_Titlecase_Mapping' + ]; + + $pairs = []; + + $lineNum = 0; + while ( false !== ( $line = fgets( $file ) ) ) { + ++$lineNum; + + # Strip comments + $line = trim( substr( $line, 0, strcspn( $line, '#' ) ) ); + if ( $line === '' ) { + continue; + } + + # Split fields + $numberedData = explode( ';', $line ); + $data = []; + foreach ( $fieldNames as $number => $name ) { + $data[$name] = $numberedData[$number]; + } + + $code = base_convert( $data['Code'], 16, 10 ); + if ( ( $code >= 0xFB50 && $code <= 0xFDFF ) # Arabic presentation forms A + || ( $code >= 0xFE70 && $code <= 0xFEFF ) # Arabic presentation forms B + ) { + if ( $data['Decomposition_Type_Mapping'] === '' ) { + // No decomposition + continue; + } + if ( !preg_match( '/^ *(<\w*>) +([0-9A-F ]*)$/', + $data['Decomposition_Type_Mapping'], $m ) + ) { + $this->error( "Can't parse Decomposition_Type/Mapping on line $lineNum" ); + $this->error( $line ); + continue; + } + + $source = UtfNormal\Utils::hexSequenceToUtf8( $data['Code'] ); + $dest = UtfNormal\Utils::hexSequenceToUtf8( $m[2] ); + $pairs[$source] = $dest; + } + } + + global $IP; + file_put_contents( "$IP/serialized/normalize-ar.ser", serialize( $pairs ) ); + echo "ar: " . count( $pairs ) . " pairs written.\n"; + } +} + +$maintClass = 'GenerateNormalizerDataAr'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/language/generateNormalizerDataMl.php b/www/wiki/maintenance/language/generateNormalizerDataMl.php new file mode 100644 index 00000000..a84ffb0e --- /dev/null +++ b/www/wiki/maintenance/language/generateNormalizerDataMl.php @@ -0,0 +1,70 @@ +<?php +/** + * Generates the normalizer data file for Malayalam. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup MaintenanceLanguage + */ + +require_once __DIR__ . '/../Maintenance.php'; + +/** + * Generates the normalizer data file for Malayalam. + * + * This data file is used after normalizing to NFC. + * + * @ingroup MaintenanceLanguage + */ +class GenerateNormalizerDataMl extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Generate the normalizer data file for Malayalam' ); + } + + public function getDbType() { + return Maintenance::DB_NONE; + } + + public function execute() { + $hexPairs = [ + # From http://unicode.org/versions/Unicode5.1.0/#Malayalam_Chillu_Characters + '0D23 0D4D 200D' => '0D7A', + '0D28 0D4D 200D' => '0D7B', + '0D30 0D4D 200D' => '0D7C', + '0D32 0D4D 200D' => '0D7D', + '0D33 0D4D 200D' => '0D7E', + + # From http://permalink.gmane.org/gmane.science.linguistics.wikipedia.technical/46413 + '0D15 0D4D 200D' => '0D7F', + ]; + + $pairs = []; + foreach ( $hexPairs as $hexSource => $hexDest ) { + $source = UtfNormal\Utils::hexSequenceToUtf8( $hexSource ); + $dest = UtfNormal\Utils::hexSequenceToUtf8( $hexDest ); + $pairs[$source] = $dest; + } + + global $IP; + file_put_contents( "$IP/serialized/normalize-ml.ser", serialize( $pairs ) ); + echo "ml: " . count( $pairs ) . " pairs written.\n"; + } +} + +$maintClass = 'GenerateNormalizerDataMl'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/language/langmemusage.php b/www/wiki/maintenance/language/langmemusage.php new file mode 100644 index 00000000..7c16602e --- /dev/null +++ b/www/wiki/maintenance/language/langmemusage.php @@ -0,0 +1,65 @@ +<?php +/** + * Dumb program that tries to get the memory usage for each language file. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup MaintenanceLanguage + */ + +/** This is a command line script */ +require_once __DIR__ . '/../Maintenance.php'; +require_once __DIR__ . '/languages.inc'; + +/** + * Maintenance script that tries to get the memory usage for each language file. + * + * @ingroup MaintenanceLanguage + */ +class LangMemUsage extends Maintenance { + + public function __construct() { + parent::__construct(); + $this->addDescription( "Dumb program that tries to get the memory usage\n" . + "for each language file" ); + } + + public function execute() { + if ( !function_exists( 'memory_get_usage' ) ) { + $this->error( "You must compile PHP with --enable-memory-limit", true ); + } + + $langtool = new Languages(); + $memlast = $memstart = memory_get_usage(); + + $this->output( "Base memory usage: $memstart\n" ); + + foreach ( $langtool->getLanguages() as $langcode ) { + Language::factory( $langcode ); + $memstep = memory_get_usage(); + $this->output( sprintf( "%12s: %d\n", $langcode, ( $memstep - $memlast ) ) ); + $memlast = $memstep; + } + + $memend = memory_get_usage(); + + $this->output( ' Total Usage: ' . ( $memend - $memstart ) . "\n" ); + } +} + +$maintClass = "LangMemUsage"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/language/languages.inc b/www/wiki/maintenance/language/languages.inc new file mode 100644 index 00000000..ad80af53 --- /dev/null +++ b/www/wiki/maintenance/language/languages.inc @@ -0,0 +1,827 @@ +<?php +/** + * Handle messages in the language files. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup MaintenanceLanguage + */ + +/** + * @ingroup MaintenanceLanguage + */ +class Languages { + /** @var array List of languages */ + protected $mLanguages; + + /** @var array Raw list of the messages in each language */ + protected $mRawMessages; + + /** @var array Messages in each language (except for English), divided to groups */ + protected $mMessages; + + /** @var array Fallback language in each language */ + protected $mFallback; + + /** @var array General messages in English, divided to groups */ + protected $mGeneralMessages; + + /** @var array All the messages which should be exist only in the English file */ + protected $mIgnoredMessages; + + /** @var array All the messages which may be translated or not, depending on the language */ + protected $mOptionalMessages; + + /** @var array Namespace names */ + protected $mNamespaceNames; + + /** @var array Namespace aliases */ + protected $mNamespaceAliases; + + /** @var array Magic words */ + protected $mMagicWords; + + /** @var array Special page aliases */ + protected $mSpecialPageAliases; + + /** + * Load the list of languages: all the Messages*.php + * files in the languages directory. + */ + function __construct() { + Hooks::run( 'LocalisationIgnoredOptionalMessages', + [ &$this->mIgnoredMessages, &$this->mOptionalMessages ] ); + + $this->mLanguages = array_keys( Language::fetchLanguageNames( null, 'mwfile' ) ); + sort( $this->mLanguages ); + } + + /** + * Get the language list. + * + * @return array The language list. + */ + public function getLanguages() { + return $this->mLanguages; + } + + /** + * Get the ignored messages list. + * + * @return array The ignored messages list. + */ + public function getIgnoredMessages() { + return $this->mIgnoredMessages; + } + + /** + * Get the optional messages list. + * + * @return array The optional messages list. + */ + public function getOptionalMessages() { + return $this->mOptionalMessages; + } + + /** + * Load the language file. + * + * @param string $code The language code. + */ + protected function loadFile( $code ) { + if ( isset( $this->mRawMessages[$code] ) && + isset( $this->mFallback[$code] ) && + isset( $this->mNamespaceNames[$code] ) && + isset( $this->mNamespaceAliases[$code] ) && + isset( $this->mMagicWords[$code] ) && + isset( $this->mSpecialPageAliases[$code] ) + ) { + return; + } + $this->mRawMessages[$code] = []; + $this->mFallback[$code] = ''; + $this->mNamespaceNames[$code] = []; + $this->mNamespaceAliases[$code] = []; + $this->mMagicWords[$code] = []; + $this->mSpecialPageAliases[$code] = []; + + $jsonfilename = Language::getJsonMessagesFileName( $code ); + if ( file_exists( $jsonfilename ) ) { + $json = Language::getLocalisationCache()->readJSONFile( $jsonfilename ); + $this->mRawMessages[$code] = $json['messages']; + } + + $filename = Language::getMessagesFileName( $code ); + if ( file_exists( $filename ) ) { + require $filename; + if ( isset( $fallback ) ) { + $this->mFallback[$code] = $fallback; + } + if ( isset( $namespaceNames ) ) { + $this->mNamespaceNames[$code] = $namespaceNames; + } + if ( isset( $namespaceAliases ) ) { + $this->mNamespaceAliases[$code] = $namespaceAliases; + } + if ( isset( $magicWords ) ) { + $this->mMagicWords[$code] = $magicWords; + } + if ( isset( $specialPageAliases ) ) { + $this->mSpecialPageAliases[$code] = $specialPageAliases; + } + } + } + + /** + * Load the messages for a specific language (which is not English) and divide them to + * groups: + * all - all the messages. + * required - messages which should be translated in order to get a complete translation. + * optional - messages which can be translated, the fallback translation is used if not + * translated. + * obsolete - messages which should not be translated, either because they do not exist, + * or they are ignored messages. + * translated - messages which are either required or optional, but translated from + * English and needed. + * + * @param string $code The language code. + */ + private function loadMessages( $code ) { + if ( isset( $this->mMessages[$code] ) ) { + return; + } + $this->loadFile( $code ); + $this->loadGeneralMessages(); + $this->mMessages[$code]['all'] = $this->mRawMessages[$code]; + $this->mMessages[$code]['required'] = []; + $this->mMessages[$code]['optional'] = []; + $this->mMessages[$code]['obsolete'] = []; + $this->mMessages[$code]['translated'] = []; + foreach ( $this->mMessages[$code]['all'] as $key => $value ) { + if ( isset( $this->mGeneralMessages['required'][$key] ) ) { + $this->mMessages[$code]['required'][$key] = $value; + $this->mMessages[$code]['translated'][$key] = $value; + } elseif ( isset( $this->mGeneralMessages['optional'][$key] ) ) { + $this->mMessages[$code]['optional'][$key] = $value; + $this->mMessages[$code]['translated'][$key] = $value; + } else { + $this->mMessages[$code]['obsolete'][$key] = $value; + } + } + } + + /** + * Load the messages for English and divide them to groups: + * all - all the messages. + * required - messages which should be translated to other languages in order to get a + * complete translation. + * optional - messages which can be translated to other languages, but it's not required + * for a complete translation. + * ignored - messages which should not be translated to other languages. + * translatable - messages which are either required or optional, but can be translated + * from English. + */ + private function loadGeneralMessages() { + if ( isset( $this->mGeneralMessages ) ) { + return; + } + $this->loadFile( 'en' ); + $this->mGeneralMessages['all'] = $this->mRawMessages['en']; + $this->mGeneralMessages['required'] = []; + $this->mGeneralMessages['optional'] = []; + $this->mGeneralMessages['ignored'] = []; + $this->mGeneralMessages['translatable'] = []; + foreach ( $this->mGeneralMessages['all'] as $key => $value ) { + if ( in_array( $key, $this->mIgnoredMessages ) ) { + $this->mGeneralMessages['ignored'][$key] = $value; + } elseif ( in_array( $key, $this->mOptionalMessages ) ) { + $this->mGeneralMessages['optional'][$key] = $value; + $this->mGeneralMessages['translatable'][$key] = $value; + } else { + $this->mGeneralMessages['required'][$key] = $value; + $this->mGeneralMessages['translatable'][$key] = $value; + } + } + } + + /** + * Get all the messages for a specific language (not English), without the + * fallback language messages, divided to groups: + * all - all the messages. + * required - messages which should be translated in order to get a complete translation. + * optional - messages which can be translated, the fallback translation is used if not + * translated. + * obsolete - messages which should not be translated, either because they do not exist, + * or they are ignored messages. + * translated - messages which are either required or optional, but translated from + * English and needed. + * + * @param string $code The language code. + * + * @return string The messages in this language. + */ + public function getMessages( $code ) { + $this->loadMessages( $code ); + + return $this->mMessages[$code]; + } + + /** + * Get all the general English messages, divided to groups: + * all - all the messages. + * required - messages which should be translated to other languages in + * order to get a complete translation. + * optional - messages which can be translated to other languages, but it's + * not required for a complete translation. + * ignored - messages which should not be translated to other languages. + * translatable - messages which are either required or optional, but can be + * translated from English. + * + * @return array The general English messages. + */ + public function getGeneralMessages() { + $this->loadGeneralMessages(); + + return $this->mGeneralMessages; + } + + /** + * Get fallback language code for a specific language. + * + * @param string $code The language code. + * + * @return string Fallback code. + */ + public function getFallback( $code ) { + $this->loadFile( $code ); + + return $this->mFallback[$code]; + } + + /** + * Get namespace names for a specific language. + * + * @param string $code The language code. + * + * @return array Namespace names. + */ + public function getNamespaceNames( $code ) { + $this->loadFile( $code ); + + return $this->mNamespaceNames[$code]; + } + + /** + * Get namespace aliases for a specific language. + * + * @param string $code The language code. + * + * @return array Namespace aliases. + */ + public function getNamespaceAliases( $code ) { + $this->loadFile( $code ); + + return $this->mNamespaceAliases[$code]; + } + + /** + * Get magic words for a specific language. + * + * @param string $code The language code. + * + * @return array Magic words. + */ + public function getMagicWords( $code ) { + $this->loadFile( $code ); + + return $this->mMagicWords[$code]; + } + + /** + * Get special page aliases for a specific language. + * + * @param string $code The language code. + * + * @return array Special page aliases. + */ + public function getSpecialPageAliases( $code ) { + $this->loadFile( $code ); + + return $this->mSpecialPageAliases[$code]; + } + + /** + * Get the untranslated messages for a specific language. + * + * @param string $code The language code. + * + * @return array The untranslated messages for this language. + */ + public function getUntranslatedMessages( $code ) { + $this->loadGeneralMessages(); + $this->loadMessages( $code ); + + return array_diff_key( $this->mGeneralMessages['required'], $this->mMessages[$code]['required'] ); + } + + /** + * Get the duplicate messages for a specific language. + * + * @param string $code The language code. + * + * @return array The duplicate messages for this language. + */ + public function getDuplicateMessages( $code ) { + $this->loadGeneralMessages(); + $this->loadMessages( $code ); + $duplicateMessages = []; + foreach ( $this->mMessages[$code]['translated'] as $key => $value ) { + if ( $this->mGeneralMessages['translatable'][$key] == $value ) { + $duplicateMessages[$key] = $value; + } + } + + return $duplicateMessages; + } + + /** + * Get the obsolete messages for a specific language. + * + * @param string $code The language code. + * + * @return array The obsolete messages for this language. + */ + public function getObsoleteMessages( $code ) { + $this->loadGeneralMessages(); + $this->loadMessages( $code ); + + return $this->mMessages[$code]['obsolete']; + } + + /** + * Get the messages whose variables do not match the original ones. + * + * @param string $code The language code. + * + * @return array The messages whose variables do not match the original ones. + */ + public function getMessagesWithMismatchVariables( $code ) { + $this->loadGeneralMessages(); + $this->loadMessages( $code ); + $variables = [ '\$1', '\$2', '\$3', '\$4', '\$5', '\$6', '\$7', '\$8', '\$9' ]; + $mismatchMessages = []; + foreach ( $this->mMessages[$code]['translated'] as $key => $value ) { + $missing = false; + foreach ( $variables as $var ) { + if ( preg_match( "/$var/sU", $this->mGeneralMessages['translatable'][$key] ) && + !preg_match( "/$var/sU", $value ) + ) { + $missing = true; + } + if ( !preg_match( "/$var/sU", $this->mGeneralMessages['translatable'][$key] ) && + preg_match( "/$var/sU", $value ) + ) { + $missing = true; + } + } + if ( $missing ) { + $mismatchMessages[$key] = $value; + } + } + + return $mismatchMessages; + } + + /** + * Get the messages which do not use plural. + * + * @param string $code The language code. + * + * @return array The messages which do not use plural in this language. + */ + public function getMessagesWithoutPlural( $code ) { + $this->loadGeneralMessages(); + $this->loadMessages( $code ); + $messagesWithoutPlural = []; + foreach ( $this->mMessages[$code]['translated'] as $key => $value ) { + if ( stripos( $this->mGeneralMessages['translatable'][$key], '{{plural:' ) !== false && + stripos( $value, '{{plural:' ) === false + ) { + $messagesWithoutPlural[$key] = $value; + } + } + + return $messagesWithoutPlural; + } + + /** + * Get the empty messages. + * + * @param string $code The language code. + * + * @return array The empty messages for this language. + */ + public function getEmptyMessages( $code ) { + $this->loadGeneralMessages(); + $this->loadMessages( $code ); + $emptyMessages = []; + foreach ( $this->mMessages[$code]['translated'] as $key => $value ) { + if ( $value === '' || $value === '-' ) { + $emptyMessages[$key] = $value; + } + } + + return $emptyMessages; + } + + /** + * Get the messages with trailing whitespace. + * + * @param string $code The language code. + * + * @return array The messages with trailing whitespace in this language. + */ + public function getMessagesWithWhitespace( $code ) { + $this->loadGeneralMessages(); + $this->loadMessages( $code ); + $messagesWithWhitespace = []; + foreach ( $this->mMessages[$code]['translated'] as $key => $value ) { + if ( $this->mGeneralMessages['translatable'][$key] !== '' && $value !== rtrim( $value ) ) { + $messagesWithWhitespace[$key] = $value; + } + } + + return $messagesWithWhitespace; + } + + /** + * Get the non-XHTML messages. + * + * @param string $code The language code. + * + * @return array The non-XHTML messages for this language. + */ + public function getNonXHTMLMessages( $code ) { + $this->loadGeneralMessages(); + $this->loadMessages( $code ); + $wrongPhrases = [ + '<hr *\\?>', + '<br *\\?>', + '<hr/>', + '<br/>', + '<hr>', + '<br>', + ]; + $wrongPhrases = '~(' . implode( '|', $wrongPhrases ) . ')~sDu'; + $nonXHTMLMessages = []; + foreach ( $this->mMessages[$code]['translated'] as $key => $value ) { + if ( preg_match( $wrongPhrases, $value ) ) { + $nonXHTMLMessages[$key] = $value; + } + } + + return $nonXHTMLMessages; + } + + /** + * Get the messages which include wrong characters. + * + * @param string $code The language code. + * + * @return array The messages which include wrong characters in this language. + */ + public function getMessagesWithWrongChars( $code ) { + $this->loadGeneralMessages(); + $this->loadMessages( $code ); + $wrongChars = [ + '[LRM]' => "\xE2\x80\x8E", + '[RLM]' => "\xE2\x80\x8F", + '[LRE]' => "\xE2\x80\xAA", + '[RLE]' => "\xE2\x80\xAB", + '[POP]' => "\xE2\x80\xAC", + '[LRO]' => "\xE2\x80\xAD", + '[RLO]' => "\xE2\x80\xAB", + '[ZWSP]' => "\xE2\x80\x8B", + '[NBSP]' => "\xC2\xA0", + '[WJ]' => "\xE2\x81\xA0", + '[BOM]' => "\xEF\xBB\xBF", + '[FFFD]' => "\xEF\xBF\xBD", + ]; + $wrongRegExp = '/(' . implode( '|', array_values( $wrongChars ) ) . ')/sDu'; + $wrongCharsMessages = []; + foreach ( $this->mMessages[$code]['translated'] as $key => $value ) { + if ( preg_match( $wrongRegExp, $value ) ) { + foreach ( $wrongChars as $viewableChar => $hiddenChar ) { + $value = str_replace( $hiddenChar, $viewableChar, $value ); + } + $wrongCharsMessages[$key] = $value; + } + } + + return $wrongCharsMessages; + } + + /** + * Get the messages which include dubious links. + * + * @param string $code The language code. + * + * @return array The messages which include dubious links in this language. + */ + public function getMessagesWithDubiousLinks( $code ) { + $this->loadGeneralMessages(); + $this->loadMessages( $code ); + $tc = Title::legalChars() . '#%{}'; + $messages = []; + foreach ( $this->mMessages[$code]['translated'] as $key => $value ) { + $matches = []; + preg_match_all( "/\[\[([{$tc}]+)(?:\\|(.+?))?]]/sDu", $value, $matches ); + $numMatches = count( $matches[0] ); + for ( $i = 0; $i < $numMatches; $i++ ) { + if ( preg_match( "/.*project.*/isDu", $matches[1][$i] ) ) { + $messages[$key][] = $matches[0][$i]; + } + } + + if ( isset( $messages[$key] ) ) { + $messages[$key] = implode( $messages[$key], ", " ); + } + } + + return $messages; + } + + /** + * Get the messages which include unbalanced brackets. + * + * @param string $code The language code. + * + * @return array The messages which include unbalanced brackets in this language. + */ + public function getMessagesWithUnbalanced( $code ) { + $this->loadGeneralMessages(); + $this->loadMessages( $code ); + $messages = []; + foreach ( $this->mMessages[$code]['translated'] as $key => $value ) { + $a = $b = $c = $d = 0; + foreach ( preg_split( '//', $value ) as $char ) { + switch ( $char ) { + case '[': + $a++; + break; + case ']': + $b++; + break; + case '{': + $c++; + break; + case '}': + $d++; + break; + } + } + + if ( $a !== $b || $c !== $d ) { + $messages[$key] = "$a, $b, $c, $d"; + } + } + + return $messages; + } + + /** + * Get the untranslated namespace names. + * + * @param string $code The language code. + * + * @return array The untranslated namespace names in this language. + */ + public function getUntranslatedNamespaces( $code ) { + $this->loadFile( 'en' ); + $this->loadFile( $code ); + $namespacesDiff = array_diff_key( $this->mNamespaceNames['en'], $this->mNamespaceNames[$code] ); + if ( isset( $namespacesDiff[NS_MAIN] ) ) { + unset( $namespacesDiff[NS_MAIN] ); + } + + return $namespacesDiff; + } + + /** + * Get the project talk namespace names with no $1. + * + * @param string $code The language code. + * + * @return array The problematic project talk namespaces in this language. + */ + public function getProblematicProjectTalks( $code ) { + $this->loadFile( $code ); + $namespaces = []; + + # Check default namespace name + if ( isset( $this->mNamespaceNames[$code][NS_PROJECT_TALK] ) ) { + $default = $this->mNamespaceNames[$code][NS_PROJECT_TALK]; + if ( strpos( $default, '$1' ) === false ) { + $namespaces[$default] = 'default'; + } + } + + # Check namespace aliases + foreach ( $this->mNamespaceAliases[$code] as $key => $value ) { + if ( $value == NS_PROJECT_TALK && strpos( $key, '$1' ) === false ) { + $namespaces[$key] = ''; + } + } + + return $namespaces; + } + + /** + * Get the untranslated magic words. + * + * @param string $code The language code. + * + * @return array The untranslated magic words in this language. + */ + public function getUntranslatedMagicWords( $code ) { + $this->loadFile( 'en' ); + $this->loadFile( $code ); + $magicWords = []; + foreach ( $this->mMagicWords['en'] as $key => $value ) { + if ( !isset( $this->mMagicWords[$code][$key] ) ) { + $magicWords[$key] = $value[1]; + } + } + + return $magicWords; + } + + /** + * Get the obsolete magic words. + * + * @param string $code The language code. + * + * @return array The obsolete magic words in this language. + */ + public function getObsoleteMagicWords( $code ) { + $this->loadFile( 'en' ); + $this->loadFile( $code ); + $magicWords = []; + foreach ( $this->mMagicWords[$code] as $key => $value ) { + if ( !isset( $this->mMagicWords['en'][$key] ) ) { + $magicWords[$key] = $value[1]; + } + } + + return $magicWords; + } + + /** + * Get the magic words that override the original English magic word. + * + * @param string $code The language code. + * + * @return array The overriding magic words in this language. + */ + public function getOverridingMagicWords( $code ) { + $this->loadFile( 'en' ); + $this->loadFile( $code ); + $magicWords = []; + foreach ( $this->mMagicWords[$code] as $key => $local ) { + if ( !isset( $this->mMagicWords['en'][$key] ) ) { + # Unrecognized magic word + continue; + } + $en = $this->mMagicWords['en'][$key]; + array_shift( $local ); + array_shift( $en ); + foreach ( $en as $word ) { + if ( !in_array( $word, $local ) ) { + $magicWords[$key] = $word; + break; + } + } + } + + return $magicWords; + } + + /** + * Get the magic words which do not match the case-sensitivity of the original words. + * + * @param string $code The language code. + * + * @return array The magic words whose case does not match in this language. + */ + public function getCaseMismatchMagicWords( $code ) { + $this->loadFile( 'en' ); + $this->loadFile( $code ); + $magicWords = []; + foreach ( $this->mMagicWords[$code] as $key => $local ) { + if ( !isset( $this->mMagicWords['en'][$key] ) ) { + # Unrecognized magic word + continue; + } + if ( $local[0] != $this->mMagicWords['en'][$key][0] ) { + $magicWords[$key] = $local[0]; + } + } + + return $magicWords; + } + + /** + * Get the untranslated special page names. + * + * @param string $code The language code. + * + * @return array The untranslated special page names in this language. + */ + public function getUntraslatedSpecialPages( $code ) { + $this->loadFile( 'en' ); + $this->loadFile( $code ); + $specialPageAliases = []; + foreach ( $this->mSpecialPageAliases['en'] as $key => $value ) { + if ( !isset( $this->mSpecialPageAliases[$code][$key] ) ) { + $specialPageAliases[$key] = $value[0]; + } + } + + return $specialPageAliases; + } + + /** + * Get the obsolete special page names. + * + * @param string $code The language code. + * + * @return array The obsolete special page names in this language. + */ + public function getObsoleteSpecialPages( $code ) { + $this->loadFile( 'en' ); + $this->loadFile( $code ); + $specialPageAliases = []; + foreach ( $this->mSpecialPageAliases[$code] as $key => $value ) { + if ( !isset( $this->mSpecialPageAliases['en'][$key] ) ) { + $specialPageAliases[$key] = $value[0]; + } + } + + return $specialPageAliases; + } +} + +class ExtensionLanguages extends Languages { + /** + * @var MessageGroup + */ + private $mMessageGroup; + + /** + * Load the messages group. + * @param MessageGroup $group The messages group. + */ + function __construct( MessageGroup $group ) { + $this->mMessageGroup = $group; + + $this->mIgnoredMessages = $this->mMessageGroup->getIgnored(); + $this->mOptionalMessages = $this->mMessageGroup->getOptional(); + } + + /** + * Get the extension name. + * + * @return string The extension name. + */ + public function name() { + return $this->mMessageGroup->getLabel(); + } + + /** + * Load the language file. + * + * @param string $code The language code. + */ + protected function loadFile( $code ) { + if ( !isset( $this->mRawMessages[$code] ) ) { + $this->mRawMessages[$code] = $this->mMessageGroup->load( $code ); + if ( empty( $this->mRawMessages[$code] ) ) { + $this->mRawMessages[$code] = []; + } + } + } +} diff --git a/www/wiki/maintenance/language/listVariants.php b/www/wiki/maintenance/language/listVariants.php new file mode 100644 index 00000000..e8d0e768 --- /dev/null +++ b/www/wiki/maintenance/language/listVariants.php @@ -0,0 +1,73 @@ +<?php +/** + * Lists all language variants + * + * Copyright © 2014 MediaWiki developers + * https://www.mediawiki.org/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ +require_once dirname( __DIR__ ) . '/Maintenance.php'; + +/** + * @since 1.24 + */ +class ListVariants extends Maintenance { + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Outputs a list of language variants' ); + $this->addOption( 'flat', 'Output variants in a flat list' ); + $this->addOption( 'json', 'Output variants as JSON' ); + } + + public function execute() { + $variantLangs = []; + $variants = []; + foreach ( LanguageConverter::$languagesWithVariants as $langCode ) { + $lang = Language::factory( $langCode ); + if ( count( $lang->getVariants() ) > 1 ) { + $variants += array_flip( $lang->getVariants() ); + $variantLangs[$langCode] = $lang->getVariants(); + } + } + $variants = array_keys( $variants ); + sort( $variants ); + $result = $this->hasOption( 'flat' ) ? $variants : $variantLangs; + + // Not using $this->output() because muting makes no sense here + if ( $this->hasOption( 'json' ) ) { + echo FormatJson::encode( $result, true ) . "\n"; + } else { + foreach ( $result as $key => $value ) { + if ( is_array( $value ) ) { + echo "$key\n"; + foreach ( $value as $variant ) { + echo " $variant\n"; + } + } else { + echo "$value\n"; + } + } + } + } +} + +$maintClass = 'ListVariants'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/language/transstat.php b/www/wiki/maintenance/language/transstat.php new file mode 100644 index 00000000..72029523 --- /dev/null +++ b/www/wiki/maintenance/language/transstat.php @@ -0,0 +1,152 @@ +<?php +/** + * Statistics about the localisation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup MaintenanceLanguage + * + * @author Ævar Arnfjörð Bjarmason <avarab@gmail.com> + * @author Antoine Musso <hashar at free dot fr> + * + * Output is posted from time to time on: + * https://www.mediawiki.org/wiki/Localisation_statistics + */ +$optionsWithArgs = [ 'output' ]; +$optionsWithoutArgs = [ 'help' ]; + +require_once __DIR__ . '/../commandLine.inc'; +require_once 'languages.inc'; +require_once __DIR__ . '/StatOutputs.php'; + +if ( isset( $options['help'] ) ) { + showUsage(); +} + +# Default output is WikiText +if ( !isset( $options['output'] ) ) { + $options['output'] = 'wiki'; +} + +/** Print a usage message*/ +function showUsage() { + print <<<TEXT +Usage: php transstat.php [--help] [--output=csv|text|wiki] + --help : this helpful message + --output : select an output engine one of: + * 'csv' : Comma Separated Values. + * 'wiki' : MediaWiki syntax (default). + * 'text' : Text with tabs. +Example: php maintenance/transstat.php --output=text + +TEXT; + exit( 1 ); +} + +# Select an output engine +switch ( $options['output'] ) { + case 'wiki': + $output = new WikiStatsOutput(); + break; + case 'text': + $output = new TextStatsOutput(); + break; + case 'csv': + $output = new CsvStatsOutput(); + break; + default: + showUsage(); +} + +# Languages +$languages = new Languages(); + +# Header +$output->heading(); +$output->blockstart(); +$output->element( 'Language', true ); +$output->element( 'Code', true ); +$output->element( 'Fallback', true ); +$output->element( 'Translated', true ); +$output->element( '%', true ); +$output->element( 'Obsolete', true ); +$output->element( '%', true ); +$output->element( 'Problematic', true ); +$output->element( '%', true ); +$output->blockend(); + +$wgGeneralMessages = $languages->getGeneralMessages(); +$wgRequiredMessagesNumber = count( $wgGeneralMessages['required'] ); + +foreach ( $languages->getLanguages() as $code ) { + # Don't check English, RTL English or dummy language codes + if ( $code == 'en' || $code == 'enRTL' || ( is_array( $wgDummyLanguageCodes ) && + isset( $wgDummyLanguageCodes[$code] ) ) + ) { + continue; + } + + # Calculate the numbers + $language = Language::fetchLanguageName( $code ); + $fallback = $languages->getFallback( $code ); + $messages = $languages->getMessages( $code ); + $messagesNumber = count( $messages['translated'] ); + $requiredMessagesNumber = count( $messages['required'] ); + $requiredMessagesPercent = $output->formatPercent( + $requiredMessagesNumber, + $wgRequiredMessagesNumber + ); + $obsoleteMessagesNumber = count( $messages['obsolete'] ); + $obsoleteMessagesPercent = $output->formatPercent( + $obsoleteMessagesNumber, + $messagesNumber, + true + ); + $messagesWithMismatchVariables = $languages->getMessagesWithMismatchVariables( $code ); + $emptyMessages = $languages->getEmptyMessages( $code ); + $messagesWithWhitespace = $languages->getMessagesWithWhitespace( $code ); + $nonXHTMLMessages = $languages->getNonXHTMLMessages( $code ); + $messagesWithWrongChars = $languages->getMessagesWithWrongChars( $code ); + $problematicMessagesNumber = count( array_unique( array_merge( + $messagesWithMismatchVariables, + $emptyMessages, + $messagesWithWhitespace, + $nonXHTMLMessages, + $messagesWithWrongChars + ) ) ); + $problematicMessagesPercent = $output->formatPercent( + $problematicMessagesNumber, + $messagesNumber, + true + ); + + # Output them + $output->blockstart(); + $output->element( "$language" ); + $output->element( "$code" ); + $output->element( "$fallback" ); + $output->element( "$requiredMessagesNumber/$wgRequiredMessagesNumber" ); + $output->element( $requiredMessagesPercent ); + $output->element( "$obsoleteMessagesNumber/$messagesNumber" ); + $output->element( $obsoleteMessagesPercent ); + $output->element( "$problematicMessagesNumber/$messagesNumber" ); + $output->element( $problematicMessagesPercent ); + $output->blockend(); +} + +# Footer +$output->footer(); diff --git a/www/wiki/maintenance/language/zhtable/Makefile b/www/wiki/maintenance/language/zhtable/Makefile new file mode 100644 index 00000000..afa71f21 --- /dev/null +++ b/www/wiki/maintenance/language/zhtable/Makefile @@ -0,0 +1,2 @@ +../../../languages/data/ZhConversion.php: Makefile.py $(wildcard *.manual) + ./Makefile.py diff --git a/www/wiki/maintenance/language/zhtable/Makefile.py b/www/wiki/maintenance/language/zhtable/Makefile.py new file mode 100755 index 00000000..abe08e4b --- /dev/null +++ b/www/wiki/maintenance/language/zhtable/Makefile.py @@ -0,0 +1,452 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @author Philip +import os +import platform +import re +import shutil +import sys +import tarfile +import zipfile + +pyversion = platform.python_version() +islinux = platform.system().lower() == 'linux' + +if pyversion[:3] in ['2.6', '2.7']: + import urllib as urllib_request + import codecs + open = codecs.open + _unichr = unichr + if sys.maxunicode < 0x10000: + def unichr(i): + if i < 0x10000: + return _unichr(i) + else: + return _unichr(0xD7C0 + (i >> 10)) + _unichr(0xDC00 + (i & 0x3FF)) +elif pyversion[:2] == '3.': + import urllib.request as urllib_request + unichr = chr + + +def unichr2(*args): + return [unichr(int(i.split('<')[0][2:], 16)) for i in args] + + +def unichr3(*args): + return [unichr(int(i[2:7], 16)) for i in args if i[2:7]] + +# DEFINE +UNIHAN_VER = '6.3.0' +SF_MIRROR = 'dfn' +SCIM_TABLES_VER = '0.5.13' +SCIM_PINYIN_VER = '0.5.92' +LIBTABE_VER = '0.2.3' +# END OF DEFINE + + +def download(url, dest): + if os.path.isfile(dest): + print('File %s is up to date.' % dest) + return + global islinux + if islinux: + # we use wget instead urlretrieve under Linux, + # because wget could display details like download progress + os.system('wget %s -O %s' % (url, dest)) + else: + print('Downloading from [%s] ...' % url) + urllib_request.urlretrieve(url, dest) + print('Download complete.\n') + return + + +def uncompress(fp, member, encoding='U8'): + name = member.rsplit('/', 1)[-1] + print('Extracting %s ...' % name) + fp.extract(member) + shutil.move(member, name) + if '/' in member: + shutil.rmtree(member.split('/', 1)[0]) + if pyversion[:1] in ['2']: + fc = open(name, 'rb', encoding, 'ignore') + else: + fc = open(name, 'r', encoding=encoding, errors='ignore') + return fc + +unzip = lambda path, member, encoding = 'U8': \ + uncompress(zipfile.ZipFile(path), member, encoding) + +untargz = lambda path, member, encoding = 'U8': \ + uncompress(tarfile.open(path, 'r:gz'), member, encoding) + + +def parserCore(fp, pos, beginmark=None, endmark=None): + if beginmark and endmark: + start = False + else: + start = True + mlist = set() + for line in fp: + if beginmark and line.startswith(beginmark): + start = True + continue + elif endmark and line.startswith(endmark): + break + if start and not line.startswith('#'): + elems = line.split() + if len(elems) < 2: + continue + elif len(elems[0]) > 1 and len(elems[pos]) > 1: # words only + mlist.add(elems[pos]) + return mlist + + +def tablesParser(path, name): + """ Read file from scim-tables and parse it. """ + global SCIM_TABLES_VER + src = 'scim-tables-%s/tables/zh/%s' % (SCIM_TABLES_VER, name) + fp = untargz(path, src, 'U8') + return parserCore(fp, 1, 'BEGIN_TABLE', 'END_TABLE') + +ezbigParser = lambda path: tablesParser(path, 'EZ-Big.txt.in') +wubiParser = lambda path: tablesParser(path, 'Wubi.txt.in') +zrmParser = lambda path: tablesParser(path, 'Ziranma.txt.in') + + +def phraseParser(path): + """ Read phrase_lib.txt and parse it. """ + global SCIM_PINYIN_VER + src = 'scim-pinyin-%s/data/phrase_lib.txt' % SCIM_PINYIN_VER + fp = untargz(path, src, 'U8') + return parserCore(fp, 0) + + +def tsiParser(path): + """ Read tsi.src and parse it. """ + src = 'libtabe/tsi-src/tsi.src' + fp = untargz(path, src, 'big5hkscs') + return parserCore(fp, 0) + + +def unihanParser(path): + """ Read Unihan_Variants.txt and parse it. """ + fp = unzip(path, 'Unihan_Variants.txt', 'U8') + t2s = dict() + s2t = dict() + for line in fp: + if line.startswith('#'): + continue + else: + elems = line.split() + if len(elems) < 3: + continue + type = elems.pop(1) + elems = unichr2(*elems) + if type == 'kTraditionalVariant': + s2t[elems[0]] = elems[1:] + elif type == 'kSimplifiedVariant': + t2s[elems[0]] = elems[1:] + fp.close() + return (t2s, s2t) + + +def applyExcludes(mlist, path): + """ Apply exclude rules from path to mlist. """ + if pyversion[:1] in ['2']: + excludes = open(path, 'rb', 'U8').read().split() + else: + excludes = open(path, 'r', encoding='U8').read().split() + excludes = [word.split('#')[0].strip() for word in excludes] + excludes = '|'.join(excludes) + excptn = re.compile('.*(?:%s).*' % excludes) + diff = [mword for mword in mlist if excptn.search(mword)] + mlist.difference_update(diff) + return mlist + + +def charManualTable(path): + fp = open(path, 'r', encoding='U8') + for line in fp: + elems = line.split('#')[0].split('|') + elems = unichr3(*elems) + if len(elems) > 1: + yield elems[0], elems[1:] + + +def toManyRules(src_table): + tomany = set() + if pyversion[:1] in ['2']: + for (f, t) in src_table.iteritems(): + for i in range(1, len(t)): + tomany.add(t[i]) + else: + for (f, t) in src_table.items(): + for i in range(1, len(t)): + tomany.add(t[i]) + return tomany + + +def removeRules(path, table): + fp = open(path, 'r', encoding='U8') + texc = list() + for line in fp: + elems = line.split('=>') + f = t = elems[0].strip() + if len(elems) == 2: + t = elems[1].strip() + f = f.strip('"').strip("'") + t = t.strip('"').strip("'") + if f: + try: + table.pop(f) + except: + pass + if t: + texc.append(t) + texcptn = re.compile('^(?:%s)$' % '|'.join(texc)) + if pyversion[:1] in ['2']: + for (tmp_f, tmp_t) in table.copy().iteritems(): + if texcptn.match(tmp_t): + table.pop(tmp_f) + else: + for (tmp_f, tmp_t) in table.copy().items(): + if texcptn.match(tmp_t): + table.pop(tmp_f) + return table + + +def customRules(path): + fp = open(path, 'r', encoding='U8') + ret = dict() + for line in fp: + line = line.rstrip('\r\n') + if '#' in line: + line = line.split('#')[0].rstrip() + elems = line.split('\t') + if len(elems) > 1: + ret[elems[0]] = elems[1] + return ret + + +def dictToSortedList(src_table, pos): + return sorted(src_table.items(), key=lambda m: (m[pos], m[1 - pos])) + + +def translate(text, conv_table): + i = 0 + while i < len(text): + for j in range(len(text) - i, 0, -1): + f = text[i:][:j] + t = conv_table.get(f) + if t: + text = text[:i] + t + text[i:][j:] + i += len(t) - 1 + break + i += 1 + return text + + +def manualWordsTable(path, conv_table, reconv_table): + fp = open(path, 'r', encoding='U8') + reconv_table = reconv_table.copy() + out_table = {} + wordlist = [line.split('#')[0].strip() for line in fp] + wordlist = list(set(wordlist)) + wordlist.sort(key=lambda w: (len(w), w), reverse=True) + while wordlist: + word = wordlist.pop() + new_word = translate(word, conv_table) + rcv_word = translate(word, reconv_table) + if word != rcv_word: + reconv_table[word] = out_table[word] = word + reconv_table[new_word] = out_table[new_word] = word + return out_table + + +def defaultWordsTable(src_wordlist, src_tomany, char_conv_table, + char_reconv_table): + wordlist = list(src_wordlist) + wordlist.sort(key=lambda w: (len(w), w), reverse=True) + word_conv_table = {} + word_reconv_table = {} + conv_table = char_conv_table.copy() + reconv_table = char_reconv_table.copy() + tomanyptn = re.compile('(?:%s)' % '|'.join(src_tomany)) + while wordlist: + conv_table.update(word_conv_table) + reconv_table.update(word_reconv_table) + word = wordlist.pop() + new_word_len = word_len = len(word) + while new_word_len == word_len: + test_word = translate(word, reconv_table) + new_word = translate(word, conv_table) + if not reconv_table.get(new_word) and \ + (test_word != word or + (tomanyptn.search(word) and + word != translate(new_word, reconv_table))): + word_conv_table[word] = new_word + word_reconv_table[new_word] = word + try: + word = wordlist.pop() + except IndexError: + break + new_word_len = len(word) + return word_reconv_table + + +def PHPArray(table): + lines = ['\'%s\' => \'%s\',' % (f, t) for (f, t) in table if f and t] + return '\n'.join(lines) + + +def main(): + # Get Unihan.zip: + url = 'http://www.unicode.org/Public/%s/ucd/Unihan.zip' % UNIHAN_VER + han_dest = 'Unihan-%s.zip' % UNIHAN_VER + download(url, han_dest) + + sfurlbase = 'http://%s.dl.sourceforge.net/sourceforge/' % SF_MIRROR + + # Get scim-tables-$(SCIM_TABLES_VER).tar.gz: + url = sfurlbase + 'scim/scim-tables-%s.tar.gz' % SCIM_TABLES_VER + tbe_dest = 'scim-tables-%s.tar.gz' % SCIM_TABLES_VER + download(url, tbe_dest) + + # Get scim-pinyin-$(SCIM_PINYIN_VER).tar.gz: + url = sfurlbase + 'scim/scim-pinyin-%s.tar.gz' % SCIM_PINYIN_VER + pyn_dest = 'scim-pinyin-%s.tar.gz' % SCIM_PINYIN_VER + download(url, pyn_dest) + + # Get libtabe-$(LIBTABE_VER).tgz: + url = sfurlbase + 'libtabe/libtabe-%s.tgz' % LIBTABE_VER + lbt_dest = 'libtabe-%s.tgz' % LIBTABE_VER + download(url, lbt_dest) + + # Unihan.txt + (t2s_1tomany, s2t_1tomany) = unihanParser(han_dest) + + t2s_1tomany.update(charManualTable('symme_supp.manual')) + t2s_1tomany.update(charManualTable('trad2simp.manual')) + s2t_1tomany.update((t[0], [f]) for (f, t) in charManualTable('symme_supp.manual')) + s2t_1tomany.update(charManualTable('simp2trad.manual')) + + if pyversion[:1] in ['2']: + t2s_1to1 = dict([(f, t[0]) for (f, t) in t2s_1tomany.iteritems()]) + s2t_1to1 = dict([(f, t[0]) for (f, t) in s2t_1tomany.iteritems()]) + else: + t2s_1to1 = dict([(f, t[0]) for (f, t) in t2s_1tomany.items()]) + s2t_1to1 = dict([(f, t[0]) for (f, t) in s2t_1tomany.items()]) + + s_tomany = toManyRules(t2s_1tomany) + t_tomany = toManyRules(s2t_1tomany) + + # noconvert rules + t2s_1to1 = removeRules('trad2simp_noconvert.manual', t2s_1to1) + s2t_1to1 = removeRules('simp2trad_noconvert.manual', s2t_1to1) + + # the supper set for word to word conversion + t2s_1to1_supp = t2s_1to1.copy() + s2t_1to1_supp = s2t_1to1.copy() + t2s_1to1_supp.update(customRules('trad2simp_supp_set.manual')) + s2t_1to1_supp.update(customRules('simp2trad_supp_set.manual')) + + # word to word manual rules + t2s_word2word_manual = manualWordsTable('simpphrases.manual', + s2t_1to1_supp, t2s_1to1_supp) + t2s_word2word_manual.update(customRules('toSimp.manual')) + s2t_word2word_manual = manualWordsTable('tradphrases.manual', + t2s_1to1_supp, s2t_1to1_supp) + s2t_word2word_manual.update(customRules('toTrad.manual')) + + # word to word rules from input methods + t_wordlist = set() + s_wordlist = set() + t_wordlist.update(ezbigParser(tbe_dest), + tsiParser(lbt_dest)) + s_wordlist.update(wubiParser(tbe_dest), + zrmParser(tbe_dest), + phraseParser(pyn_dest)) + + # exclude + s_wordlist = applyExcludes(s_wordlist, 'simpphrases_exclude.manual') + t_wordlist = applyExcludes(t_wordlist, 'tradphrases_exclude.manual') + + s2t_supp = s2t_1to1_supp.copy() + s2t_supp.update(s2t_word2word_manual) + t2s_supp = t2s_1to1_supp.copy() + t2s_supp.update(t2s_word2word_manual) + + # parse list to dict + t2s_word2word = defaultWordsTable(s_wordlist, s_tomany, + s2t_1to1_supp, t2s_supp) + t2s_word2word.update(t2s_word2word_manual) + s2t_word2word = defaultWordsTable(t_wordlist, t_tomany, + t2s_1to1_supp, s2t_supp) + s2t_word2word.update(s2t_word2word_manual) + + # Final tables + # sorted list toHans + if pyversion[:1] in ['2']: + t2s_1to1 = dict([(f, t) for (f, t) in t2s_1to1.iteritems() if f != t]) + else: + t2s_1to1 = dict([(f, t) for (f, t) in t2s_1to1.items() if f != t]) + toHans = dictToSortedList(t2s_1to1, 0) + dictToSortedList(t2s_word2word, 1) + # sorted list toHant + if pyversion[:1] in ['2']: + s2t_1to1 = dict([(f, t) for (f, t) in s2t_1to1.iteritems() if f != t]) + else: + s2t_1to1 = dict([(f, t) for (f, t) in s2t_1to1.items() if f != t]) + toHant = dictToSortedList(s2t_1to1, 0) + dictToSortedList(s2t_word2word, 1) + # sorted list toCN + toCN = dictToSortedList(customRules('toCN.manual'), 1) + # sorted list toHK + toHK = dictToSortedList(customRules('toHK.manual'), 1) + # sorted list toTW + toTW = dictToSortedList(customRules('toTW.manual'), 1) + + # Get PHP Array + php = '''<?php +/** + * Simplified / Traditional Chinese conversion tables + * + * Automatically generated using code and data in maintenance/language/zhtable/ + * Do not modify directly! + * + * @file + */ + +namespace MediaWiki\Languages\Data; + +class ZhConversion { +public static $zh2Hant = [\n''' + php += PHPArray(toHant) \ + + '\n];\n\npublic static $zh2Hans = [\n' \ + + PHPArray(toHans) \ + + '\n];\n\npublic static $zh2TW = [\n' \ + + PHPArray(toTW) \ + + '\n];\n\npublic static $zh2HK = [\n' \ + + PHPArray(toHK) \ + + '\n];\n\npublic static $zh2CN = [\n' \ + + PHPArray(toCN) \ + + '\n];\n}\n' + + if pyversion[:1] in ['2']: + f = open(os.path.join('..', '..', '..', 'languages', 'data', 'ZhConversion.php'), 'wb', encoding='utf8') + else: + f = open(os.path.join('..', '..', '..', 'languages', 'data', 'ZhConversion.php'), 'w', buffering=4096, encoding='utf8') + print ('Writing ZhConversion.php ... ') + f.write(php) + f.close() + + # Remove temporary files + print ('Deleting temporary files ... ') + os.remove('EZ-Big.txt.in') + os.remove('phrase_lib.txt') + os.remove('tsi.src') + os.remove('Unihan_Variants.txt') + os.remove('Wubi.txt.in') + os.remove('Ziranma.txt.in') + + +if __name__ == '__main__': + main() diff --git a/www/wiki/maintenance/language/zhtable/README b/www/wiki/maintenance/language/zhtable/README new file mode 100644 index 00000000..e183e56c --- /dev/null +++ b/www/wiki/maintenance/language/zhtable/README @@ -0,0 +1,35 @@ +The various .manual files contains special mappings not included in the +unihan database, and phrases not included in the SCIM package. + +- symme_supp.manual: Supplementary character mapping of symmetric conversion + (1 to 1) between Simplified and Traditional Chinese. + +- simp2trad.manual: Simplified to Traditional asymmetric charactermapping. + +- trad2simp.manual: Traditional to Simplified asymmetric character mapping. + +- simp2trad_noconvert.manual: Do not convert the chars as inapporiate. + +- trad2simp_noconvert.manual: Do not convert the chars as inapporiate. + +- tradphrases.manual: Phrases in Traditional Chinese. A portition is obtained + from the TongWen package (http://tongwen.mozdev.org/) + +- simpphrases.manual: Phrases in Simplified Chinese. + +- tradphrases_exclude.manual: Excluding several phrases from + the SCIM phrasesas inappoiated. + +- simpphrases_exclude.manual: Excluding several phrases from + the SCIM phrases as inapporated. + +- toTrad.manual, toSimp.manual: Special phrase mappings that + tradphrases.manual or simphrases.manual cannot be handled. + +- toTW.manual, toCN.manual and toHK.manual: Special phrase mappings. + +* 為方便轉換,以上均含不完整詞組,請勿隨意刪除。 + +zhengzhu at gmail dot com & shinjiman at gmail dot com + +Modified by User:Chiefwei at Chinese Wikipedia in 2015.
\ No newline at end of file diff --git a/www/wiki/maintenance/language/zhtable/simp2trad.manual b/www/wiki/maintenance/language/zhtable/simp2trad.manual new file mode 100644 index 00000000..9fee611c --- /dev/null +++ b/www/wiki/maintenance/language/zhtable/simp2trad.manual @@ -0,0 +1,235 @@ +U+04E07万|U+0842C萬|U+04E07万| +U+04E0E与|U+08207與|U+04E0E与| +U+04E11丑|U+04E11丑|U+0919C醜| +U+04E2A个|U+0500B個|U+07B87箇| +U+04E30丰|U+08C50豐|U+04E30丰| +U+04E3A为|U+070BA為|U+07232爲| +U+04E48么|U+09EBC麼|U+04E48么|U+09EBD麽|U+05E7A幺| +U+04E86了|U+04E86了|U+077AD瞭| +U+04E8E于|U+065BC於|U+04E8E于| +U+04E91云|U+096F2雲|U+04E91云| +U+04EA7产|U+07522產|U+07523産| +U+04EC6仆|U+04EC6仆|U+050D5僕| +U+04EC7仇|U+04EC7仇|U+08B8E讎| +U+04ED1仑|U+04F96侖|U+05D19崙| +U+04EF7价|U+050F9價|U+04EF7价| +U+04F17众|U+0773E眾|U+08846衆| +U+04F19伙|U+04F19伙|U+05925夥| +U+04F2A伪|U+0507D偽|U+050DE僞| +U+04F53体|U+09AD4體|U+04F53体| +U+04F59余|U+04F59余|U+09918餘| +U+04F63佣|U+050AD傭|U+04F63佣| +U+0501F借|U+0501F借|U+085C9藉| +U+0513F儿|U+05152兒|U+0513F儿| +U+0514B克|U+0514B克|U+0524B剋| +U+0515A党|U+09EE8黨|U+0515A党| +U+051AC冬|U+051AC冬|U+09F15鼕| +U+051B2冲|U+06C96沖|U+0885D衝| +U+051C0净|U+06DE8淨| +U+051C4凄|U+06DD2淒|U+060BD悽| +U+051C6准|U+051C6准|U+06E96準| +U+051E0几|U+05E7E幾|U+051E0几| +U+051EB凫|U+09CE7鳧|U+09CEC鳬| +U+051FA出|U+051FA出|U+09F63齣| +U+05212划|U+05283劃|U+05212划| +U+0522B别|U+05225別|U+05F46彆| +U+0522E刮|U+0522E刮|U+098B3颳| +U+05236制|U+05236制|U+088FD製| +U+05343千|U+05343千|U+097C6韆| +U+05347升|U+05347升|U+06607昇|U+0965E陞| +U+0535C卜|U+0535C卜|U+08514蔔| +U+05360占|U+05360占|U+04F54佔| +U+05364卤|U+09E75鹵|U+06EF7滷| +U+05367卧|U+081E5臥| +U+05377卷|U+05377卷|U+06372捲| +U+05382厂|U+05EE0廠|U+05382厂| +U+05386历|U+06B77歷|U+066C6曆|U+053A4厤| +U+05395厕|U+05EC1廁|U+053A0厠| +U+05398厘|U+05398厘|U+091D0釐| +U+053D1发|U+0767C發|U+09AEE髮| +U+053EA只|U+053EA只|U+096BB隻| +U+053F0台|U+053F0台|U+081FA臺|U+06AAF檯|U+098B1颱| +U+053F6叶|U+08449葉|U+053F6叶| +U+05401吁|U+05401吁|U+07C72籲| +U+05408合|U+05408合|U+095A4閤| +U+0540A吊|U+0540A吊|U+05F14弔| +U+0540C同|U+0540C同|U+08855衕| +U+0540E后|U+05F8C後|U+0540E后| +U+05411向|U+05411向|U+056AE嚮| +U+0542F启|U+0555F啟|U+05553啓| +U+05446呆|U+05446呆|U+07343獃| +U+054B8咸|U+054B8咸|U+09E79鹹| +U+054C4哄|U+054C4哄|U+09B28鬨| +U+0556E啮|U+09F67齧|U+056D3囓|U+05699嚙| +U+05582喂|U+09935餵|U+05582喂| +U+056DE回|U+056DE回|U+08FF4迴| +U+056E2团|U+05718團|U+07CF0糰| +U+056F0困|U+056F0困|U+0774F睏| +U+05742坂|U+05742坂|U+0962A阪| +U+0574F坏|U+058DE壞|U+0574F坏| +U+0575B坛|U+058C7壇|U+07F48罈| +U+05899墙|U+07246牆|U+058BB墻| +U+058F3壳|U+06BBC殼|U+06BBB殻| +U+0590D复|U+05FA9復|U+08907複| +U+05938夸|U+05938夸|U+08A87誇| +U+05956奖|U+0734E獎|U+0596C奬| +U+05978奸|U+05978奸|U+059E6姦| +U+059AB妫|U+05AAF媯|U+05B00嬀| +U+059DC姜|U+059DC姜|U+08591薑| +U+05B81宁|U+05BE7寧|U+05B81宁| +U+05BB6家|U+05BB6家|U+050A2傢| +U+05C3D尽|U+076E1盡|U+05118儘| +U+05CB3岳|U+05CB3岳|U+05DBD嶽| +U+05E03布|U+05E03布|U+04F48佈| +U+05E18帘|U+07C3E簾|U+05E18帘| +U+05E2D席|U+05E2D席|U+084C6蓆| +U+05E72干|U+05E72干|U+04E7E乾|U+05E79幹|U+069A6榦| +U+05E76并|U+04E26並|U+04F75併| +U+05E78幸|U+05E78幸|U+05016倖| +U+05E7F广|U+05EE3廣|U+05E7F广| +U+05E84庄|U+0838A莊|U+05E84庄| +U+05EB5庵|U+05EB5庵|U+083F4菴| +U+05F25弥|U+05F4C彌|U+07030瀰| +U+05F53当|U+07576當|U+05679噹| +U+05F55录|U+09304錄|U+09332録| +U+05F69彩|U+05F69彩|U+07DB5綵| +U+05F81征|U+05F81征|U+05FB5徵| +U+05FA1御|U+05FA1御|U+079A6禦| +U+05FD7志|U+05FD7志|U+08A8C誌| +U+06076恶|U+060E1惡|U+05641噁| +U+060AB悫|U+06128愨|U+06164慤| +U+0613F愿|U+09858願|U+0613F愿| +U+0621A戚|U+0621A戚|U+0617C慼|U+093DA鏚| +U+0624D才|U+0624D才|U+07E94纔| +U+0624E扎|U+0624E扎|U+07D2E紮| +U+06258托|U+06258托|U+08A17託| +U+06298折|U+06298折|U+0647A摺| +U+062C5担|U+064D4擔|U+062C5担| +U+062FC拼|U+062FC拼|U+062DA拚| +U+06328挨|U+06328挨|U+06371捱| +U+0633D挽|U+0633D挽|U+08F13輓| +U+0636E据|U+064DA據|U+0636E据| +U+06597斗|U+06597斗|U+09B25鬥| +U+065CB旋|U+065CB旋|U+0955F镟| +U+065D7旗|U+065D7旗|U+065C2旂| +U+066F2曲|U+066F2曲|U+09EAF麯|U+09EB4麯| +U+0672F术|U+08853術|U+0672E朮| +U+06731朱|U+06731朱|U+07843硃| +U+06734朴|U+06734朴|U+06A38樸| +U+06760杠|U+069D3槓|U+06760杠| +U+0676F杯|U+0676F杯|U+076C3盃| +U+0677E松|U+0677E松|U+09B06鬆| +U+0677F板|U+0677F板|U+095C6闆| +U+06781极|U+06975極|U+06781极| +U+067DC柜|U+06AC3櫃|U+067DC柜| +U+06817栗|U+06817栗|U+06144慄| +U+06881梁|U+06881梁|U+06A11樑| +U+068F1棱|U+068F1棱|U+07A1C稜| +U+06B32欲|U+06B32欲|U+0617E慾| +U+06C47汇|U+0532F匯|U+06ED9滙|U+05F59彙| +U+06C84沄|U+06C84沄|U+06F90澐| +U+06C88沈|U+06C88沈|U+0700B瀋| +U+06CA9沩|U+06E88溈|U+06F59潙| +U+06CE8注|U+06CE8注|U+08A3B註| +U+06D82涂|U+05857塗|U+06D82涂| +U+06D8C涌|U+06D8C涌|U+06E67湧| +U+06DC0淀|U+06DC0淀|U+06FB1澱| +U+06E16渖|U+0700B瀋| +U+06E38游|U+06E38游|U+0904A遊| +U+06EAF溯|U+06EAF溯|U+06CDD泝| +U+06F13漓|U+06F13漓|U+07055灕| +U+070BC炼|U+07149煉|U+0934A鍊| +U+07096炖|U+071C9燉| +U+0753B画|U+0756B畫|U+07575畵| +U+075C7症|U+075C7症|U+07665癥| +U+07618瘘|U+0763A瘺|U+0763B瘻| +U+0786E确|U+078BA確|U+0786E确| +U+07877硷|U+09E7C鹼|U+07906礆| +U+078B1碱|U+09E7C鹼| +U+079CB秋|U+079CB秋|U+097A6鞦| +U+079CD种|U+07A2E種|U+079CD种| +U+07A57穗|U+07A57穗|U+07E50繐| +U+07AD6竖|U+08C4E豎|U+07AEA竪| +U+07B51筑|U+07BC9築|U+07B51筑| +U+07B7E签|U+07C3D簽|U+07C64籤| +U+07CFB系|U+07CFB系|U+07E6B繫|U+04FC2係| +U+07D2F累|U+07D2F累|U+07E8D纍| +U+07EA4纤|U+07E96纖|U+07E34縴| +U+07EBF线|U+07DDA線|U+07DAB綫| +U+07EDD绝|U+07D55絕|U+07D76絶| +U+07EE3绣|U+07E61繡|U+07D89綉| +U+07EE6绦|U+07D5B絛|U+07E27縧| +U+07EF1绱|U+0979D鞝|U+07DD4緔| +U+07EF7绷|U+07E43繃|U+07DB3綳| +U+07EFF绿|U+07DA0綠|U+07DD1緑| +U+07F10缐|U+07DDA線| +U+07F30缰|U+097C1韁|U+07E6E繮| +U+07FA1羡|U+07FA8羨| +U+080DC胜|U+052DD勝|U+080DC胜| +U+080E1胡|U+080E1胡|U+09B0D鬍|U+0885A衚| +U+0810F脏|U+09AD2髒|U+081DF臟| +U+0814A腊|U+081D8臘|U+0814A腊| +U+0814C腌|U+09183醃| +U+081F4致|U+081F4致|U+07DFB緻| +U+0820D舍|U+0820D舍|U+06368捨| +U+082B8芸|U+082B8芸|U+08553蕓| +U+082CF苏|U+08607蘇|U+056CC囌|U+07C64甦| +U+08303范|U+08303范|U+07BC4範| +U+0836F药|U+085E5藥|U+0846F葯| +U+083B7获|U+07372獲|U+07A6B穫| +U+083BC莼|U+084F4蓴|U+08493蒓| +U+08499蒙|U+08499蒙|U+077C7矇|U+06FDB濛|U+061DE懞| +U+084D1蓑|U+084D1蓑|U+07C11簑| +U+08511蔑|U+08511蔑|U+0884A衊| +U+08574蕴|U+0860A蘊|U+085F4藴| +U+0866B虫|U+087F2蟲|U+0866B虫| +U+08721蜡|U+0881F蠟|U+08721蜡| +U+0874E蝎|U+0880D蠍|U+0874E蝎| +U+08868表|U+08868表|U+09336錶| +U+08BF4说|U+08AAA說|U+08AAC説| +U+08C23谣|U+08B20謠|U+08B21謡| +U+08C2B谫|U+08B7E譾|U+08B2D謭| +U+08C37谷|U+08C37谷|U+07A40穀| +U+08D43赃|U+08D13贓|U+08D1C贜| +U+08D4D赍|U+09F4E齎|U+08CEB賫| +U+08D5D赝|U+08D17贗|U+08D0B贋| +U+08D5E赞|U+08D0A贊|U+08B9A讚| +U+08F9F辟|U+08F9F辟|U+095E2闢| +U+09002适|U+09069適|U+09002适| +U+090C1郁|U+090C1郁|U+09B31鬱| +U+0915D酝|U+0919E醞|U+09196醖| +U+09170酰|U+09170酰|U+091AF醯| +U+09178酸|U+09178酸|U+075E0痠| +U+091C7采|U+091C7采|U+063A1採|U+05BC0寀| +U+091CC里|U+091CC里|U+088E1裡|U+088CF裏| +U+0949F钟|U+0937E鍾|U+09418鐘| +U+094A9钩|U+0920E鈎|U+09264鉤| +U+094B5钵|U+07F3D缽|U+09262鉢| +U+094F2铲|U+093DF鏟|U+05277剷| +U+09508锈|U+092B9銹|U+093FD鏽| +U+09510锐|U+092B3銳|U+092ED鋭| +U+09528锨|U+06774杴|U+09341鍁| +U+0954B镋|U+09482钂|U+093B2鎲| +U+0954C镌|U+0942B鐫|U+093B8鎸| +U+09562镢|U+09481钁|U+0941D鐝| +U+09605阅|U+095B1閱|U+095B2閲| +U+096C7雇|U+096C7雇|U+050F1僱| +U+096D5雕|U+096D5雕|U+09D70鵰| +U+095F2闲|U+09592閒|U+09591閑| +U+09709霉|U+09709霉|U+09EF4黴| +U+09762面|U+09762面|U+09EB5麵|U+09EAA麪|U+09EAB麫| +U+0987B须|U+09808須|U+09B1A鬚| +U+09893颓|U+09839頹|U+0983D頽| +U+0989C颜|U+0984F顏|U+09854顔| +U+09965饥|U+098E2飢|U+09951饑| +U+09980馀|U+09918餘| +U+09986馆|U+09928館|U+08218舘| +U+09A82骂|U+07F75罵|U+099E1駡| +U+09CC1鳁|U+09C2E鰮| +U+09C87鲇|U+09BF0鯰|U+09B8E鮎| +U+09C9E鲞|U+09BD7鯗|U+09B9D鮝| +U+09CC4鳄|U+09C77鱷|U+09C10鰐| +U+09E21鸡|U+096DE雞|U+09DC4鷄| +U+09E5A鹚|U+09DBF鶿|U+09DC0鷀| +U+09EB9麹|U+09EB4麴| +U+080C4胄|U+080C4胄|U+05191冑| diff --git a/www/wiki/maintenance/language/zhtable/simp2trad_noconvert.manual b/www/wiki/maintenance/language/zhtable/simp2trad_noconvert.manual new file mode 100644 index 00000000..77ad2434 --- /dev/null +++ b/www/wiki/maintenance/language/zhtable/simp2trad_noconvert.manual @@ -0,0 +1,18 @@ +著 +竈 +彞 +咤 +吒 +疴 +桿 +錶 +蘋 +詑 +堖 +嶴 +灡 +薳 +虯 +凈 +垵 +獃 diff --git a/www/wiki/maintenance/language/zhtable/simp2trad_supp_set.manual b/www/wiki/maintenance/language/zhtable/simp2trad_supp_set.manual new file mode 100644 index 00000000..a5038a5d --- /dev/null +++ b/www/wiki/maintenance/language/zhtable/simp2trad_supp_set.manual @@ -0,0 +1,2 @@ +余 餘 +着 著
\ No newline at end of file diff --git a/www/wiki/maintenance/language/zhtable/simpphrases.manual b/www/wiki/maintenance/language/zhtable/simpphrases.manual new file mode 100644 index 00000000..19ec7b15 --- /dev/null +++ b/www/wiki/maintenance/language/zhtable/simpphrases.manual @@ -0,0 +1,266 @@ +乾上乾下 +乾为天 +乾为阳 +乾九 +乾乾 +乾亨 +乾仪 +乾位 +乾健 +乾元 +乾光 +乾兴 +乾冈 +乾刘 +乾刚 +乾化 +乾卦 +乾县 +乾台 +乾吉 +乾启 +乾命 +乾和 +乾嘉 +乾图 +乾坤 +乾城 +乾基 +乾始 +乾姓 +乾宁 +乾宅 +乾宇 +乾安 +乾定 +乾封 +乾居 +乾岗 +乾巛 +乾州 +乾录 +乾律 +乾德 +乾心 +乾文 +乾断 +乾方 +乾施 +乾旦 +乾明 +乾昧 +乾晖 +乾景 +乾晷 +乾曜 +乾构 +乾枢 +乾栋 +乾步 +乾氏 +乾泉 +乾清 +乾渥 +乾灵 +乾男 +乾皋 +乾盛世 +乾矢 +乾祐 +乾穹 +乾窦 +乾竺 +乾笃 +乾符 +乾策 +乾精 +乾红 +乾纲 +乾纽 +乾络 +乾统 +乾维 +乾罗 +乾花 +乾荫 +乾行 +乾衡 +乾覆 +乾象 +乾象历 +乾贞 +乾贶 +乾车 +乾轴 +乾造 +乾道 +乾鉴 +乾钧 +乾闼 +乾陀 +乾陵 +乾隆 +乾音 +乾顾 +乾风 +乾首 +乾马 +乾鹄 +乾鹊 +乾龙 +乾,健也 +乾,天也 +乾健也 +乾天也 +坤乾 +天道为乾 +尼乾陀 +康乾 +张法乾 +旋乾转坤 +易·乾 +《易乾 +周易乾 +易经·乾 +易经乾 +李乾德 +萧乾 +郭子乾 +雍乾 +乾务 +乾沓和 +乾沓婆 +乾通 +乾忠 +乾淳 +李乾顺 +黄润乾 +男性为乾 +男为乾 +阳为乾 +乾一组 +乾一坛 +陈乾生 +陈公乾生 +字乾生 +乾神 +乾西 +乾东 +象乾 +陈遇乾 +曾运乾 +王道乾 +孙乾 +乾潭 +乾贵士 +承乾 +乾生元 +蔡孝乾 +於乎 +於戏 +魏徵 +柳诒徵 +於姓 +於氏 +於夫罗 +於梨华 +樊於期 +於菟 +於潜县 +石碁镇 +李泽钜 +於祥玉 +於崇文 +於世成 +於乙宇同 +於宇同 +朴於宇同 +於哲 +於除鞬 +於志贺 +覆盖 +五箇山 +阿部正瞭 +醯酱 +醯鸡 +醯醋 +醯醢 +醯壶 +苧烯 +後姓 +先名后姓 +矇眬 +朱有燉 +缐姓 +缐国安 +仇雠 +雠校 +雠定 +校雠 +雠夷 +雠问 +雠正 +施雠 +无言不雠 +甚夥 +吴克羣 +宏碁 +石碁 +碁圣 +暗闇 +闇公 +山崎闇斋 +繙㠾 +惏慄 +惏悷 +目劄 +谢肇淛 +朱淛 +諲譔 +李譔 +扞格 +陈元扞 +祕宜 +李祕 +剋了 +挨剋 +剋架 +皁保 +爨翫 +碁所 +於之莹 +陆徵祥 +瞭台 +文徵明 +博和讬 +楈枒 +米渖 +白渖 +拾渖 +渖液 +醉渖 +墨渖 +如渖 +残渖 +馀渖 +庆馀 +馀庆 +子馀 +行馀 +王馀鱼 +傒倖 +倖田 +倖一郎 +兒宽 +穀旦 +不穀 +穀水 +穀阳 +岳讬 +硕讬 +讬庸 +讬恩多 +博和讬 +讬麻 +饱讬 +蔡絛 diff --git a/www/wiki/maintenance/language/zhtable/simpphrases_exclude.manual b/www/wiki/maintenance/language/zhtable/simpphrases_exclude.manual new file mode 100644 index 00000000..b47d3b79 --- /dev/null +++ b/www/wiki/maintenance/language/zhtable/simpphrases_exclude.manual @@ -0,0 +1,33 @@ +整飭 +後 +谘 +彷佛 +三番四复 +三复 +藉 +关於 +对於 +属於 +至於 +夥计 +薹 +嚇 +醣 +捱 +簑 +樑 +摺叠 +餗 +安甯 +傢俬 +癥瘕 +存摺 +着录 +硷淡 +悽恻 +鲇鱼 +钟 +余 +么 +麽 + diff --git a/www/wiki/maintenance/language/zhtable/symme_supp.manual b/www/wiki/maintenance/language/zhtable/symme_supp.manual new file mode 100644 index 00000000..7470a381 --- /dev/null +++ b/www/wiki/maintenance/language/zhtable/symme_supp.manual @@ -0,0 +1,27 @@ +U+03476㑶|U+03439㐹| +U+042F9䋹|U+0433F䌿| +U+043B1䎱|U+043AC䎬| +U+04C98䲘|U+09CE4鳤| +U+0508C傌|U+03437㐷| +U+05DA8嶨|U+05CC3峃| +U+05ECE廎|U+05EBC庼| +U+069EE槮|U+0692E椮| +U+06EAE溮|U+06D49浉| +U+07069灩|U+06EDF滟| +U+074A1璡|U+0740E琎| +U+074B5璵|U+07399玙| +U+074B8璸|U+07478瑸| +U+075F2痲|U+075F3痳| +U+0819E膞|U+043DD䏝| +U+085ED藭|U+044D6䓖| +U+08600蘀|U+0841A萚| +U+08AE1諡|U+08C25谥| +U+09746靆|U+053C7叇| +U+09749靉|U+053C6叆| +U+09A44驄|U+09AA2骢| +U+09C1B鰛|U+09CC1鳁| +U+09EB3麳|U+2A38C𪎌| +U+295E1𩗡|U+29667𩙧| +U+298F5𩣵|U+299FB𩧻| +U+29F47𩽇|U+29F8E𩾎| +U+2A23C𪈼|U+2A253𪉓| diff --git a/www/wiki/maintenance/language/zhtable/toCN.manual b/www/wiki/maintenance/language/zhtable/toCN.manual new file mode 100644 index 00000000..a63149e6 --- /dev/null +++ b/www/wiki/maintenance/language/zhtable/toCN.manual @@ -0,0 +1,2684 @@ +餘 余 +諮 咨 +鍅 钫 +鉳 锫 +鑀 锿 +錼 镎 +鋂 镅 +鈽 钚 +鎝 锝 +鉲 锎 +矽 硅 +矽肺 矽肺 +矽塵 矽尘 +矽尘 矽尘 +矽鋼 矽钢 +矽钢 矽钢 +侏儸紀 侏罗纪 +甚麽 什么 +甚麼 什么 +胺基酸 氨基酸 +水氣 水汽 +計畫 计划 +規畫 规划 +天份 天分 +名份 名分 +職份 职分 +份外 分外 +份內 分内 +部份 部分 +知識份子 知识分子 +積極份子 积极分子 +投機份子 投机分子 +一份子 一分子 +水份 水分 +氧份 氧分 +糖份 糖分 +鹽份 盐分 +組份 组分 +成份 成分 +成份股 成份股 +本份 本分 +本本份份 本本分分 +恰如其份 恰如其分 +非份 非分 +過份 过分 +份量 分量 +緣份 缘分 +身分 身份 +煞車 刹车 +疊代 迭代 +叱吒 叱咤 +啸吒 啸咤 +姊姊 姐姐 +姊弟 姐弟 +姊夫 姐夫 +大姊 大姐 +大姊姊 大姐姐 +御姊 御姐 +表姊 表姐 +堂姊 堂姐 +學姊 学姐 +乾姊 干姐 +清澈 清澈 #分詞用 +澈底 彻底 +仲介 中介 +卯足 铆足 +保鑣 保镖 +逕庭 径庭 +逕到 径到 +逕取 径取 +逕入 径入 +逕行 径行 +逕自 径自 +逕往 径往 +逕寄 径寄 +逕啟 径启 +逕迎 径迎 +徵狀 症状 +報帳 报账 +本帳 本账 +筆帳 笔账 +查帳 查账 +沖帳 冲账 +呆帳 呆账 +倒帳 倒账 +到帳 到账 +對帳 对账 +放帳 放账 +付帳 付账 +公帳 公账 +關帳 关账 +管帳 管账 +還帳 还账 +糊塗帳 糊涂账 +混帳 混账 +記帳 记账 +假帳 假账 +建帳 建账 +交帳 交账 +結帳 结账 +進帳 进账 +經常帳 经常账 +經濟帳 经济账 +舊帳 旧账 +開帳 开账 +賴帳 赖账 +爛帳 烂账 +流水帳 流水账 +買帳 买账 +明白帳 明白账 +簽帳 签账 +欠帳 欠账 +清帳 清账 +認帳 认账 +入帳 入账 +賒帳 赊账 +收帳 收账 +私帳 私账 +死帳 死账 +算帳 算账 +台帳 台账 +銷帳 销账 +要帳 要账 +轉帳 转账 +總帳 总账 +帳本 账本 +帳簿 账簿 +帳冊 账册 +帳單 账单 +帳房 账房 +帳號 账号 +帳戶 账户 +帳款 账款 +帳面 账面 +帳目 账目 +帳上 账上 +帳外 账外 +帳務 账务 +螢光棒 荧光棒 +著業 着业 +著絲 着丝 +著麼 着么 +著人 着人 +著什 着什 +著他 着他 +著令 着令 +著位 着位 +著體 着体 +著你 着你 +著便 着便 +著涼 着凉 +著力 着力 +著勁 着劲 +著號 着号 +著呢 着呢 +著哩 着哩 +著地 着地 +著墨 着墨 +著聲 着声 +著處 着处 +著她 着她 +著妳 着妳 +著它 着它 +著定 着定 +著實 着实 +著己 着己 +著帳 着帐 +著床 着床 +著庸 着庸 +著式 着式 +著錄 着录 +著心 着心 +著志 着志 +著忙 着忙 +著急 着急 +著惱 着恼 +著驚 着惊 +著想 着想 +著意 着意 +著慌 着慌 +著我 着我 +著手 着手 +著抹 着抹 +著摸 着摸 +著撰 着撰 +著數 着数 +著明 着明 +著末 着末 +著極 着极 +著格 着格 +著棋 着棋 +著槁 着槁 +著氣 着气 +著法 着法 +著淺 着浅 +著火 着火 +著然 着然 +著甚 着甚 +著生 着生 +著疑 着疑 +著白 着白 +著相 着相 +著眼 着眼 +著著 着着 +著祂 着祂 +著積 着积 +著稿 着稿 +著筆 着笔 +著籍 着籍 +著緊 着紧 +著緑 着緑 +著絆 着绊 +著績 着绩 +著緋 着绯 +著綠 着绿 +著肉 着肉 +著腳 着脚 +著艦 着舰 +著色 着色 +著節 着节 +著花 着花 +著莫 着莫 +著落 着落 +著槁 着藁 +著衣 着衣 +著裝 着装 +著要 着要 +著警 着警 +著趣 着趣 +著邊 着边 +著迷 着迷 +著跡 着迹 +著重 着重 +著録 着録 +著聞 着闻 +著陸 着陆 +著雝 着雝 +著鞭 着鞭 +著題 着题 +著魔 着魔 +不著 不着 +不著書 不著书 +不著名 不著名 +不著錄 不著录 +不著稱 不著称 +不著述 不著述 +與著 与着 +與著書 与著书 +與著作 与著作 +與著名 与著名 +與著錄 与著录 +與著稱 与著称 +與著者 与著者 +與著述 与著述 +醜著 丑着 +醜著書 丑著书 +醜著作 丑著作 +醜著名 丑著名 +醜著錄 丑著录 +醜著稱 丑著称 +醜著者 丑著者 +醜著述 丑著述 +臨著 临着 +臨著書 临著书 +臨著作 临著作 +臨著名 临著名 +臨著錄 临著录 +臨著稱 临著称 +臨著者 临著者 +臨著述 临著述 +麗著 丽着 +麗著書 丽著书 +麗著作 丽著作 +麗著名 丽著名 +麗著錄 丽著录 +麗著稱 丽著称 +麗著者 丽著者 +麗著述 丽著述 +樂著 乐着 +樂著書 乐著书 +樂著作 乐著作 +樂著名 乐著名 +樂著錄 乐著录 +樂著稱 乐著称 +樂著者 乐著者 +樂著述 乐著述 +乘著 乘着 +乘著書 乘著书 +乘著作 乘著作 +乘著名 乘著名 +乘著錄 乘著录 +乘著稱 乘著称 +乘著称 乘著称 +乘著者 乘著者 +乘著述 乘著述 +爭著 争着 +爭著書 争著书 +爭著作 争著作 +爭著名 争著名 +爭著錄 争著录 +爭著稱 争著称 +爭著者 争著者 +爭著述 争著述 +亮著 亮着 +亮著書 亮著书 +亮著作 亮著作 +亮著名 亮著名 +亮著錄 亮著录 +亮著稱 亮著称 +亮著称 亮著称 +亮著者 亮著者 +亮著述 亮著述 +仗著 仗着 +仗著書 仗著书 +仗著作 仗著作 +仗著名 仗著名 +仗著錄 仗著录 +仗著稱 仗著称 +仗著者 仗著者 +仗著述 仗著述 +代表著 代表着 +代表著書 代表著书 +代表著作 代表著作 +代表著名 代表著名 +代表著錄 代表著录 +代表著稱 代表著称 +代表著者 代表著者 +代表著述 代表著述 +伴著 伴着 +伴著書 伴著书 +伴著作 伴著作 +伴著名 伴著名 +伴著錄 伴著录 +伴著稱 伴著称 +伴著者 伴著者 +伴著述 伴著述 +低著 低着 +低著書 低著书 +低著作 低著作 +低著名 低著名 +低著錄 低著录 +低著稱 低著称 +低著称 低著称 +低著者 低著者 +低著述 低著述 +住著 住着 +住著書 住著书 +住著作 住著作 +住著名 住著名 +住著錄 住著录 +住著稱 住著称 +住著称 住著称 +住著者 住著者 +住著述 住著述 +側著 侧着 +側著書 侧著书 +側著作 侧著作 +側著名 侧著名 +側著錄 侧著录 +側著稱 侧著称 +側著者 侧著者 +側著述 侧著述 +保障著 保障着 +保障著書 保障著书 +保障著作 保障著作 +保障著名 保障著名 +保障著錄 保障著录 +保障著稱 保障著称 +保障著称 保障著称 +保障著者 保障著者 +保障著述 保障著述 +信著 信着 +信著書 信著书 +信著作 信著作 +信著名 信著名 +信著錄 信著录 +信著稱 信著称 +信著称 信著称 +信著者 信著者 +信著述 信著述 +候著 候着 +候著書 候著书 +候著作 候著作 +候著名 候著名 +候著錄 候著录 +候著稱 候著称 +候著者 候著者 +候著述 候著述 +借著 借着 +借著書 借著书 +借著作 借著作 +借著名 借著名 +借著錄 借著录 +借著稱 借著称 +借著者 借著者 +借著述 借著述 +做著 做着 +做著書 做著书 +做著作 做著作 +做著名 做著名 +做著錄 做著录 +做著稱 做著称 +做著者 做著者 +做著述 做著述 +偷著 偷着 +偷著書 偷著书 +偷著作 偷著作 +偷著名 偷著名 +偷著錄 偷著录 +偷著稱 偷著称 +偷著者 偷著者 +偷著述 偷著述 +光著 光着 +光著書 光著书 +光著作 光著作 +光著名 光著名 +光著錄 光著录 +光著稱 光著称 +光著称 光著称 +光著者 光著者 +光著述 光著述 +關著 关着 +關著書 关著书 +關著作 关著作 +關著名 关著名 +關著錄 关著录 +關著稱 关著称 +關著者 关著者 +關著述 关著述 +希冀著 希冀着 +冒著 冒着 +冒著書 冒著书 +冒著作 冒著作 +冒著名 冒著名 +冒著錄 冒著录 +冒著稱 冒著称 +冒著者 冒著者 +冒著述 冒著述 +寫著 写着 +寫著書 写著书 +寫著作 写著作 +寫著名 写著名 +寫著錄 写著录 +寫著稱 写著称 +寫著者 写著者 +寫著述 写著述 +涼著 凉着 +涼著書 凉著书 +涼著作 凉著作 +涼著名 凉著名 +涼著錄 凉著录 +涼著稱 凉著称 +涼著者 凉著者 +涼著述 凉著述 +制著 制着 +制著書 制著书 +制著作 制著作 +制著名 制著名 +制著錄 制著录 +制著稱 制著称 +制著者 制著者 +制著述 制著述 +刻著 刻着 +刻著書 刻著书 +刻著作 刻著作 +刻著名 刻著名 +刻著錄 刻著录 +刻著稱 刻著称 +刻著称 刻著称 +刻著者 刻著者 +刻著述 刻著述 +辦著 办着 +辦著書 办著书 +辦著作 办著作 +辦著名 办著名 +辦著錄 办著录 +辦著稱 办著称 +辦著者 办著者 +辦著述 办著述 +動著 动着 +動著書 动著书 +動著作 动著作 +動著名 动著名 +動著錄 动著录 +動著稱 动著称 +動著者 动著者 +動著述 动著述 +努力著 努力着 +努力著書 努力著书 +努力著作 努力著作 +努力著名 努力著名 +努力著錄 努力著录 +努力著稱 努力著称 +努力著称 努力著称 +努力著者 努力著者 +努力著述 努力著述 +印著 印着 +印著書 印著书 +印著作 印著作 +印著名 印著名 +印著錄 印著录 +印著稱 印著称 +印著者 印著者 +印著述 印著述 +壓著 压着 +壓著書 压著书 +壓著作 压著作 +壓著名 压著名 +壓著錄 压著录 +壓著稱 压著称 +壓著者 压著者 +壓著述 压著述 +受著 受着 +受著書 受著书 +受著作 受著作 +受著名 受著名 +受著錄 受著录 +受著稱 受著称 +受著者 受著者 +受著述 受著述 +變著 变着 +變著書 变著书 +變著作 变著作 +變著名 变著名 +變著錄 变著录 +變著稱 变著称 +變著者 变著者 +變著述 变著述 +叫著 叫着 +叫著書 叫著书 +叫著作 叫著作 +叫著名 叫著名 +叫著錄 叫著录 +叫著稱 叫著称 +叫著者 叫著者 +叫著述 叫著述 +向著 向着 +向著書 向著书 +向著作 向著作 +向著名 向著名 +向著錄 向著录 +向著稱 向著称 +向著者 向著者 +向著述 向著述 +含著 含着 +含著書 含著书 +含著作 含著作 +含著名 含著名 +含著錄 含著录 +含著稱 含著称 +含著者 含著者 +含著述 含著述 +聽著 听着 +聽著書 听著书 +聽著作 听著作 +聽著名 听著名 +聽著錄 听著录 +聽著稱 听著称 +聽著者 听著者 +聽著述 听著述 +吹著 吹着 +吹著書 吹著书 +吹著作 吹著作 +吹著名 吹著名 +吹著錄 吹著录 +吹著稱 吹著称 +吹著者 吹著者 +吹著述 吹著述 +味著 味着 +味著書 味著书 +味著作 味著作 +味著名 味著名 +味著錄 味著录 +味著稱 味著称 +味著称 味著称 +味著者 味著者 +味著述 味著述 +響著 响着 +響著書 响著书 +響著作 响著作 +響著名 响著名 +響著錄 响著录 +響著稱 响著称 +響著者 响著者 +響著述 响著述 +哭著 哭着 +哭著書 哭著书 +哭著作 哭著作 +哭著名 哭著名 +哭著錄 哭著录 +哭著稱 哭著称 +哭著者 哭著者 +哭著述 哭著述 +唱著 唱着 +唱著書 唱著书 +唱著作 唱著作 +唱著名 唱著名 +唱著錄 唱著录 +唱著稱 唱著称 +唱著者 唱著者 +唱著述 唱著述 +喝著 喝着 +喝著書 喝著书 +喝著作 喝著作 +喝著名 喝著名 +喝著錄 喝著录 +喝著稱 喝著称 +喝著者 喝著者 +喝著述 喝著述 +嚷著 嚷着 +嚷著書 嚷著书 +嚷著作 嚷著作 +嚷著名 嚷著名 +嚷著錄 嚷著录 +嚷著稱 嚷著称 +嚷著者 嚷著者 +嚷著述 嚷著述 +因著 因着 +因著書 因著书 +因著作 因著作 +因著名 因著名 +因著錄 因著录 +因著录 因著录 +因著稱 因著称 +因著者 因著者 +因著述 因著述 +因著《 因著《 +因著〈 因著〈 +困著 困着 +困著書 困著书 +困著作 困著作 +困著名 困著名 +困著錄 困著录 +困著稱 困著称 +困著者 困著者 +困著述 困著述 +圍著 围着 +圍著書 围著书 +圍著作 围著作 +圍著名 围著名 +圍著錄 围著录 +圍著稱 围著称 +圍著者 围著者 +圍著述 围著述 +存在著 存在着 +坐著 坐着 +坐著書 坐著书 +坐著作 坐著作 +坐著名 坐著名 +坐著錄 坐著录 +坐著稱 坐著称 +坐著者 坐著者 +坐著述 坐著述 +備著 备着 +備著書 备著书 +備著作 备著作 +備著名 备著名 +備著錄 备著录 +備著稱 备著称 +備著者 备著者 +備著述 备著述 +夾著 夹着 +夾著書 夹著书 +夾著作 夹著作 +夾著名 夹著名 +夾著錄 夹著录 +夾著稱 夹著称 +夾著者 夹著者 +夾著述 夹著述 +學著 学着 +學著書 学著书 +學著作 学著作 +學著名 学著名 +學著錄 学著录 +學著稱 学著称 +學著者 学著者 +學著述 学著述 +守著 守着 +守著書 守著书 +守著作 守著作 +守著名 守著名 +守著錄 守著录 +守著稱 守著称 +守著称 守著称 +守著者 守著者 +守著述 守著述 +定著 定着 +定著書 定著书 +定著作 定著作 +定著名 定著名 +定著錄 定著录 +定著稱 定著称 +定著称 定著称 +定著者 定著者 +定著述 定著述 +對著 对着 +對著書 对著书 +對著作 对著作 +對著名 对著名 +對著錄 对著录 +對著稱 对著称 +對著者 对著者 +對著述 对著述 +尋著 寻着 +尋著書 寻著书 +尋著作 寻著作 +尋著名 寻著名 +尋著錄 寻著录 +尋著稱 寻著称 +尋著者 寻著者 +尋著述 寻著述 +展著 展着 +展著書 展著书 +展著作 展著作 +展著名 展著名 +展著錄 展著录 +展著稱 展著称 +展著者 展著者 +展著述 展著述 +帶著 带着 +帶著書 带著书 +帶著作 带著作 +帶著名 带著名 +帶著錄 带著录 +帶著稱 带著称 +帶著者 带著者 +帶著述 带著述 +幫著 帮着 +幫著書 帮著书 +幫著作 帮著作 +幫著名 帮著名 +幫著錄 帮著录 +幫著稱 帮著称 +幫著者 帮著者 +幫著述 帮著述 +應著 应着 +應著書 应著书 +應著作 应著作 +應著名 应著名 +應著錄 应著录 +應著稱 应著称 +應著者 应著者 +應著述 应著述 +開著 开着 +開著書 开著书 +開著作 开著作 +開著名 开著名 +開著錄 开著录 +開著稱 开著称 +開著者 开著者 +開著述 开著述 +當著 当着 +當著書 当著书 +當著作 当著作 +當著名 当著名 +當著錄 当著录 +當著稱 当著称 +當著者 当著者 +當著述 当著述 +待著 待着 +待著書 待著书 +待著作 待著作 +待著名 待著名 +待著錄 待著录 +待著稱 待著称 +待著者 待著者 +待著述 待著述 +得著 得着 +得著書 得著书 +得著作 得著作 +得著名 得著名 +得著錄 得著录 +得著稱 得著称 +得著者 得著者 +得著述 得著述 +循著 循着 +循著書 循著书 +循著作 循著作 +循著名 循著名 +循著錄 循著录 +循著稱 循著称 +循著者 循著者 +循著述 循著述 +心著 心着 +心著書 心著书 +心著作 心著作 +心著名 心著名 +心著錄 心著录 +心著稱 心著称 +心著称 心著称 +心著者 心著者 +心著述 心著述 +忍著 忍着 +忍著書 忍著书 +忍著作 忍著作 +忍著名 忍著名 +忍著錄 忍著录 +忍著稱 忍著称 +忍著者 忍著者 +忍著述 忍著述 +標志著 标志着 +忙著 忙着 +忙著書 忙著书 +忙著作 忙著作 +忙著名 忙著名 +忙著錄 忙著录 +忙著稱 忙著称 +忙著者 忙著者 +忙著述 忙著述 +懷著 怀着 +懷著書 怀著书 +懷著作 怀著作 +懷著名 怀著名 +懷著錄 怀著录 +懷著稱 怀著称 +懷著者 怀著者 +懷著述 怀著述 +急著 急着 +急著書 急著书 +急著作 急著作 +急著名 急著名 +急著錄 急著录 +急著稱 急著称 +急著者 急著者 +急著述 急著述 +戀著 恋着 +戀著書 恋著书 +戀著作 恋著作 +戀著名 恋著名 +戀著錄 恋著录 +戀著稱 恋著称 +戀著者 恋著者 +戀著述 恋著述 +悠著 悠着 +悠著書 悠著书 +悠著作 悠著作 +悠著名 悠著名 +悠著錄 悠著录 +悠著稱 悠著称 +悠著者 悠著者 +悠著述 悠著述 +慣著 惯着 +慣著書 惯著书 +慣著作 惯著作 +慣著名 惯著名 +慣著錄 惯著录 +慣著稱 惯著称 +慣著者 惯著者 +慣著述 惯著述 +想著 想着 +想著書 想著书 +想著作 想著作 +想著名 想著名 +想著錄 想著录 +想著稱 想著称 +想著称 想著称 +想著者 想著者 +想著述 想著述 +戰著 战着 +戰著書 战著书 +戰著作 战著作 +戰著名 战著名 +戰著錄 战著录 +戰著稱 战著称 +戰著者 战著者 +戰著述 战著述 +戴著 戴着 +戴著書 戴著书 +戴著作 戴著作 +戴著名 戴著名 +戴著錄 戴著录 +戴著稱 戴著称 +戴著者 戴著者 +戴著述 戴著述 +紮著 扎着 +紮著書 扎著书 +紮著作 扎著作 +紮著名 扎著名 +紮著錄 扎著录 +紮著稱 扎著称 +紮著者 扎著者 +紮著述 扎著述 +打著 打着 +打著書 打著书 +打著作 打著作 +打著名 打著名 +打著錄 打著录 +打著稱 打著称 +打著者 打著者 +打著述 打著述 +扛著 扛着 +扛著書 扛著书 +扛著作 扛著作 +扛著名 扛著名 +扛著錄 扛著录 +扛著稱 扛著称 +扛著者 扛著者 +扛著述 扛著述 +抓著 抓着 +抓著作 抓著作 +抓著名 抓著名 +抓著錄 抓著录 +抓著稱 抓著称 +抓著者 抓著者 +抓著述 抓著述 +披著 披着 +披著書 披著书 +披著作 披著作 +披著名 披著名 +披著錄 披著录 +披著稱 披著称 +披著者 披著者 +披著述 披著述 +抬著 抬着 +抬著作 抬著作 +抬著名 抬著名 +抬著錄 抬著录 +抬著稱 抬著称 +抬著者 抬著者 +抬著述 抬著述 +抱著 抱着 +抱著作 抱著作 +抱著名 抱著名 +抱著錄 抱著录 +抱著稱 抱著称 +抱著者 抱著者 +抱著述 抱著述 +拉著 拉着 +拉著書 拉著书 +拉著作 拉著作 +拉著名 拉著名 +拉著錄 拉著录 +拉著稱 拉著称 +拉著者 拉著者 +拉著述 拉著述 +拎著 拎着 +拎著作 拎著作 +拎著名 拎著名 +拎著錄 拎著录 +拎著稱 拎著称 +拎著者 拎著者 +拎著述 拎著述 +拖著 拖着 +拖著作 拖著作 +拖著名 拖著名 +拖著錄 拖著录 +拖著稱 拖著称 +拖著者 拖著者 +拖著述 拖著述 +拼著 拼着 +拼著作 拼著作 +拼著名 拼著名 +拼著錄 拼著录 +拼著稱 拼著称 +拼著者 拼著者 +拼著述 拼著述 +拿著 拿着 +拿著作 拿著作 +拿著名 拿著名 +拿著錄 拿著录 +拿著稱 拿著称 +拿著者 拿著者 +拿著述 拿著述 +持著 持着 +持著作 持著作 +持著名 持著名 +持著錄 持著录 +持著稱 持著称 +持著者 持著者 +持著述 持著述 +挑著 挑着 +挑著作 挑著作 +挑著名 挑著名 +挑著錄 挑著录 +挑著稱 挑著称 +挑著者 挑著者 +挑著述 挑著述 +擋著 挡着 +擋著作 挡著作 +擋著名 挡著名 +擋著錄 挡著录 +擋著稱 挡著称 +擋著者 挡著者 +擋著述 挡著述 +掙著 挣着 +掙著書 挣著书 +掙著作 挣著作 +掙著名 挣著名 +掙著錄 挣著录 +掙著稱 挣著称 +掙著者 挣著者 +掙著述 挣著述 +揮著 挥着 +揮著作 挥著作 +揮著名 挥著名 +揮著錄 挥著录 +揮著稱 挥著称 +揮著者 挥著者 +揮著述 挥著述 +挨著 挨着 +挨著作 挨著作 +挨著名 挨著名 +挨著錄 挨著录 +挨著稱 挨著称 +挨著者 挨著者 +挨著述 挨著述 +捆著 捆着 +捆著作 捆著作 +捆著名 捆著名 +捆著錄 捆著录 +捆著稱 捆著称 +捆著者 捆著者 +捆著述 捆著述 +據著 据着 +據著書 据著书 +據著作 据著作 +據著名 据著名 +據著錄 据著录 +據著稱 据著称 +據著者 据著者 +據著述 据著述 +掖著 掖着 +掖著作 掖著作 +掖著名 掖著名 +掖著錄 掖著录 +掖著稱 掖著称 +掖著者 掖著者 +掖著述 掖著述 +接著 接着 +接著作 接著作 +接著名 接著名 +接著錄 接著录 +接著稱 接著称 +接著者 接著者 +接著述 接著述 +揉著 揉着 +揉著書 揉著书 +揉著作 揉著作 +揉著名 揉著名 +揉著錄 揉著录 +揉著稱 揉著称 +揉著者 揉著者 +揉著述 揉著述 +提著 提着 +提著作 提著作 +提著名 提著名 +提著錄 提著录 +提著稱 提著称 +提著者 提著者 +提著述 提著述 +摟著 搂着 +摟著作 搂著作 +摟著名 搂著名 +摟著錄 搂著录 +摟著稱 搂著称 +摟著者 搂著者 +摟著述 搂著述 +擺著 摆着 +擺著作 摆著作 +擺著名 摆著名 +擺著錄 摆著录 +擺著稱 摆著称 +擺著者 摆著者 +擺著述 摆著述 +撼著 撼着 +撼著書 撼著书 +撼著作 撼著作 +撼著名 撼著名 +撼著錄 撼著录 +撼著稱 撼著称 +撼著者 撼著者 +撼著述 撼著述 +敞著 敞着 +敞著作 敞著作 +敞著名 敞著名 +敞著錄 敞著录 +敞著稱 敞著称 +敞著者 敞著者 +敞著述 敞著述 +數著 数着 +數著作 数著作 +數著名 数著名 +數著錄 数著录 +數著稱 数著称 +數著者 数著者 +數著述 数著述 +鬥著 斗着 +鬥著書 斗著书 +鬥著作 斗著作 +鬥著名 斗著名 +鬥著錄 斗著录 +鬥著稱 斗著称 +鬥著者 斗著者 +鬥著述 斗著述 +斥著 斥着 +斥著書 斥著书 +斥著作 斥著作 +斥著名 斥著名 +斥著錄 斥著录 +斥著稱 斥著称 +斥著者 斥著者 +斥著述 斥著述 +昂著 昂着 +昂著書 昂著书 +昂著作 昂著作 +昂著名 昂著名 +昂著錄 昂著录 +昂著稱 昂著称 +昂著者 昂著者 +昂著述 昂著述 +映著 映着 +映著書 映著书 +映著作 映著作 +映著名 映著名 +映著錄 映著录 +映著稱 映著称 +映著者 映著者 +映著述 映著述 +晃著 晃着 +晃著作 晃著作 +晃著名 晃著名 +晃著錄 晃著录 +晃著稱 晃著称 +晃著者 晃著者 +晃著述 晃著述 +暗著 暗着 +暗著書 暗著书 +暗著作 暗著作 +暗著名 暗著名 +暗著錄 暗著录 +暗著稱 暗著称 +暗著者 暗著者 +暗著述 暗著述 +有著 有着 +有著書 有著书 +有著作 有著作 +有著名 有著名 +有著錄 有著录 +有著稱 有著称 +有著者 有著者 +有著述 有著述 +望著 望着 +望著作 望著作 +望著名 望著名 +望著錄 望著录 +望著稱 望著称 +望著者 望著者 +望著述 望著述 +朝著 朝着 +朝著作 朝著作 +朝著名 朝著名 +朝著錄 朝著录 +朝著稱 朝著称 +朝著者 朝著者 +朝著述 朝著述 +本著 本着 +本著書 本著书 +本著作 本著作 +本著名 本著名 +本著錄 本著录 +本著稱 本著称 +本著者 本著者 +本著述 本著述 +殺著 杀着 +殺著書 杀著书 +殺著作 杀著作 +殺著名 杀著名 +殺著錄 杀著录 +殺著稱 杀著称 +殺著者 杀著者 +殺著述 杀著述 +雜著 杂着 +雜著書 杂著书 +雜著作 杂著作 +雜著名 杂著名 +雜著錄 杂著录 +雜著稱 杂著称 +雜著者 杂著者 +雜著述 杂著述 +來著 来着 +來著書 来著书 +來著作 来著作 +來著名 来著名 +來著錄 来著录 +來著稱 来著称 +來著者 来著者 +來著述 来著述 +枕著 枕着 +枕著作 枕著作 +枕著名 枕著名 +枕著錄 枕著录 +枕著稱 枕著称 +枕著者 枕著者 +枕著述 枕著述 +夢著 梦着 +夢著書 梦著书 +夢著作 梦著作 +夢著名 梦著名 +夢著錄 梦著录 +夢著稱 梦著称 +夢著者 梦著者 +夢著述 梦著述 +梳著 梳着 +梳著作 梳著作 +梳著名 梳著名 +梳著錄 梳著录 +梳著稱 梳著称 +梳著者 梳著者 +梳著述 梳著述 +求著 求着 +求著書 求著书 +求著作 求著作 +求著名 求著名 +求著錄 求著录 +求著稱 求著称 +求著者 求著者 +求著述 求著述 +沉著 沉着 +沉著書 沉著书 +沉著作 沉著作 +沉著名 沉著名 +沉著錄 沉著录 +沉著稱 沉著称 +沉著者 沉著者 +沉著述 沉著述 +沿著 沿着 +沿著書 沿著书 +沿著作 沿著作 +沿著名 沿著名 +沿著錄 沿著录 +沿著稱 沿著称 +沿著者 沿著者 +沿著述 沿著述 +活著 活着 +活著書 活著书 +活著作 活著作 +活著名 活著名 +活著錄 活著录 +活著稱 活著称 +活著者 活著者 +活著述 活著述 +流著 流着 +流著書 流著书 +流著作 流著作 +流著名 流著名 +流著錄 流著录 +流著稱 流著称 +流著者 流著者 +流著述 流著述 +浮著 浮着 +浮著書 浮著书 +浮著作 浮著作 +浮著名 浮著名 +浮著錄 浮著录 +浮著稱 浮著称 +浮著者 浮著者 +浮著述 浮著述 +潤著 润着 +潤著書 润著书 +潤著作 润著作 +潤著名 润著名 +潤著錄 润著录 +潤著稱 润著称 +潤著者 润著者 +潤著述 润著述 +蘊涵著 蕴涵着 +渴著 渴着 +渴著書 渴著书 +渴著作 渴著作 +渴著名 渴著名 +渴著錄 渴著录 +渴著稱 渴著称 +渴著者 渴著者 +渴著述 渴著述 +溢著 溢着 +溢著書 溢著书 +溢著作 溢著作 +溢著名 溢著名 +溢著錄 溢著录 +溢著稱 溢著称 +溢著者 溢著者 +溢著述 溢著述 +演著 演着 +演著書 演著书 +演著作 演著作 +演著名 演著名 +演著錄 演著录 +演著稱 演著称 +演著者 演著者 +演著述 演著述 +漫著 漫着 +漫著書 漫著书 +漫著作 漫著作 +漫著名 漫著名 +漫著錄 漫著录 +漫著稱 漫著称 +漫著者 漫著者 +漫著述 漫著述 +點著 点着 +點著作 点著作 +點著名 点著名 +點著錄 点著录 +點著稱 点著称 +點著者 点著者 +點著述 点著述 +燒著 烧着 +燒著作 烧著作 +燒著名 烧著名 +燒著錄 烧著录 +燒著稱 烧著称 +燒著者 烧著者 +燒著述 烧著述 +照著 照着 +照著書 照著书 +照著作 照著作 +照著名 照著名 +照著錄 照著录 +照著稱 照著称 +照著者 照著者 +照著述 照著述 +愛著 爱着 +愛著書 爱著书 +愛著作 爱著作 +愛著名 爱著名 +愛著錄 爱著录 +愛著稱 爱著称 +愛著者 爱著者 +愛著述 爱著述 +牽著 牵着 +牽著書 牵著书 +牽著作 牵著作 +牽著名 牵著名 +牽著錄 牵著录 +牽著稱 牵著称 +牽著者 牵著者 +牽著述 牵著述 +猜著 猜着 +猜著書 猜着书 +猜著作 猜著作 +猜著名 猜著名 +猜著錄 猜著录 +猜著稱 猜著称 +猜著者 猜著者 +猜著述 猜著述 +甜著 甜着 +甜著書 甜著书 +甜著作 甜著作 +甜著名 甜著名 +甜著錄 甜著录 +甜著稱 甜著称 +甜著者 甜著者 +甜著述 甜著述 +用著 用着 +用著書 用著书 +用著作 用著作 +用著名 用著名 +用著錄 用著录 +用著稱 用著称 +用著者 用著者 +用著述 用著述 +留著 留着 +留著書 留着书 +留著作 留著作 +留著名 留著名 +留著錄 留著录 +留著稱 留著称 +留著者 留著者 +留著述 留著述 +疑著 疑着 +疑著書 疑著书 +疑著作 疑著作 +疑著名 疑著名 +疑著錄 疑著录 +疑著稱 疑著称 +疑著者 疑著者 +疑著述 疑著述 +皺著 皱着 +皺著書 皱著书 +皺著作 皱著作 +皺著名 皱著名 +皺著錄 皱著录 +皺著稱 皱著称 +皺著者 皱著者 +皺著述 皱著述 +盛著 盛着 +盛著書 盛著书 +盛著作 盛著作 +盛著名 盛著名 +盛著錄 盛著录 +盛著稱 盛著称 +盛著者 盛著者 +盛著述 盛著述 +盯著 盯着 +盯著書 盯着书 +盯著作 盯著作 +盯著名 盯著名 +盯著錄 盯著录 +盯著稱 盯著称 +盯著者 盯著者 +盯著述 盯著述 +矛盾著 矛盾着 +看著 看着 +看著書 看着书 +看著作 看著作 +看著名 看著名 +看著錄 看著录 +看著稱 看著称 +看著者 看著者 +看著述 看著述 +瞧著 瞧着 +瞧著書 瞧着书 +瞧著作 瞧著作 +瞧著名 瞧著名 +瞧著錄 瞧著录 +瞧著稱 瞧著称 +瞧著者 瞧著者 +瞧著述 瞧著述 +存著 存着 +存著名 存著名 +存著作 存著作 +劃著 划着 +別著 别着 +刮著 刮着 +掛著 挂着 +吊著 吊着 +回著 回着 +回著名 回著名 +塗著 涂着 +麼著 么着 +擔著 担着 +負著 负着 +板著臉 板着脸 +為著 为着 +為著作 为著作 +為著名 为著名 +為著錄 为著录 +為著稱 为著称 +為著者 为著者 +為著述 为著述 +為著《 为著《 +畫著 画着 +畫著作 画著作 +畫著名 画著名 +畫著稱 画著称 +畫著者 画著者 +發著 发着 +發著作 发著作 +發著名 发著名 +發著稱 发著称 +發著者 发著者 +發著《 发著《 +簽著 签着 +繃著 绷着 +覆著 覆着 +蓋著 蓋着 +說著 说着 +說著作 说著作 +說著稱 说著称 +說著者 说著者 +說著述 说著述 +湊合著 凑合着 +配合著 配合着 +配合著名 配合著名 +關係著 关系着 +鬧著 闹着 +蒙著 蒙着 +悶著 闷着 +占著 占着 +占著名 占著名 +占著作 占著作 +占著者 占著者 +呆著 呆着 +包著 包着 +駛著 驶着 +睡著 睡着 +睡著書 睡著书 +睡著作 睡著作 +睡著名 睡著名 +睡著錄 睡著录 +睡著稱 睡著称 +睡著者 睡著者 +睡著述 睡著述 +瞞著 瞒着 +瞞著書 瞒著书 +瞞著作 瞒著作 +瞞著名 瞒著名 +瞞著錄 瞒著录 +瞞著稱 瞒著称 +瞞著者 瞒著者 +瞞著述 瞒著述 +瞪著 瞪着 +瞪著書 瞪著书 +瞪著作 瞪著作 +瞪著名 瞪著名 +瞪著錄 瞪著录 +瞪著稱 瞪著称 +瞪著者 瞪著者 +瞪著述 瞪著述 +福著 福着 +福著書 福著书 +福著作 福著作 +福著名 福著名 +福著錄 福著录 +福著稱 福著称 +福著者 福著者 +福著述 福著述 +空著 空着 +空著書 空著书 +空著作 空著作 +空著名 空著名 +空著錄 空著录 +空著稱 空著称 +空著者 空著者 +空著述 空著述 +穿著 穿着 +穿著書 穿著书 +穿著作 穿著作 +穿著名 穿著名 +穿著錄 穿著录 +穿著稱 穿著称 +穿著者 穿著者 +穿著述 穿著述 +豎著 竖着 +豎著書 竖著书 +豎著作 竖著作 +豎著名 竖著名 +豎著錄 竖著录 +豎著稱 竖著称 +豎著者 竖著者 +豎著述 竖著述 +站著 站着 +站著書 站著书 +站著作 站著作 +站著名 站著名 +站著錄 站著录 +站著稱 站著称 +站著者 站著者 +站著述 站著述 +笑著 笑着 +笑著書 笑著书 +笑著作 笑著作 +笑著名 笑著名 +笑著錄 笑著录 +笑著稱 笑著称 +笑著者 笑著者 +笑著述 笑著述 +管著 管着 +管著書 管著书 +管著作 管著作 +管著名 管著名 +管著錄 管著录 +管著稱 管著称 +管著者 管著者 +管著述 管著述 +綁著 绑着 +綁著書 绑著书 +綁著作 绑著作 +綁著名 绑著名 +綁著錄 绑著录 +綁著稱 绑著称 +綁著者 绑著者 +綁著述 绑著述 +繞著 绕着 +繞著書 绕著书 +繞著作 绕著作 +繞著名 绕著名 +繞著錄 绕著录 +繞著稱 绕著称 +繞著者 绕著者 +繞著述 绕著述 +纏著 缠着 +纏著書 缠著书 +纏著作 缠著作 +纏著名 缠著名 +纏著錄 缠著录 +纏著稱 缠著称 +纏著者 缠著者 +纏著述 缠著述 +罩著 罩着 +罩著書 罩著书 +罩著作 罩著作 +罩著名 罩著名 +罩著錄 罩著录 +罩著稱 罩著称 +罩著者 罩著者 +罩著述 罩著述 +美著 美着 +美著書 美著书 +美著作 美著作 +美著名 美著名 +美著錄 美著录 +美著稱 美著称 +美著称 美著称 +美著者 美著者 +美著述 美著述 +耀著 耀着 +耀著書 耀著书 +耀著作 耀著作 +耀著名 耀著名 +耀著錄 耀著录 +耀著稱 耀著称 +耀著者 耀著者 +耀著述 耀著述 +考著 考着 +考著書 考著书 +考著作 考著作 +考著名 考著名 +考著錄 考著录 +考著稱 考著称 +考著者 考著者 +考著述 考著述 +背著 背着 +背著書 背著书 +背著作 背著作 +背著名 背著名 +背著錄 背著录 +背著稱 背著称 +背著者 背著者 +背著述 背著述 +膠著 胶着 +膠著書 胶著书 +膠著作 胶著作 +膠著名 胶著名 +膠著錄 胶著录 +膠著稱 胶著称 +膠著者 胶著者 +膠著述 胶著述 +苦著 苦着 +苦著書 苦著书 +苦著作 苦著作 +苦著名 苦著名 +苦著錄 苦著录 +苦著稱 苦著称 +苦著者 苦著者 +苦著述 苦著述 +獲著 获着 +獲著書 获著书 +獲著作 获著作 +獲著名 获著名 +獲著錄 获著录 +獲著稱 获著称 +獲著者 获著者 +獲著述 获著述 +落著 落着 +落著書 落著书 +落著作 落著作 +落著名 落著名 +落著錄 落著录 +落著稱 落著称 +落著者 落著者 +落著述 落著述 +蒙著書 蒙著书 +蒙著作 蒙著作 +蒙著名 蒙著名 +蒙著錄 蒙著录 +蒙著稱 蒙著称 +蒙著者 蒙著者 +蒙著述 蒙著述 +藏著 藏着 +藏著書 藏著书 +藏著作 藏著作 +藏著名 藏著名 +藏著錄 藏著录 +藏著稱 藏著称 +藏著者 藏著者 +藏著述 藏著述 +蘸著 蘸着 +蘸著書 蘸著书 +蘸著作 蘸著作 +蘸著名 蘸著名 +蘸著錄 蘸著录 +蘸著稱 蘸著称 +蘸著者 蘸著者 +蘸著述 蘸著述 +行著 行着 +行著書 行著书 +行著作 行著作 +行著名 行著名 +行著錄 行著录 +行著稱 行著称 +行著者 行著者 +行著述 行著述 +衣著 衣着 +衣著書 衣著书 +衣著作 衣著作 +衣著名 衣著名 +衣著錄 衣著录 +衣著稱 衣著称 +衣著称 衣著称 +衣著者 衣著者 +衣著述 衣著述 +裝著 装着 +裝著書 装著书 +裝著作 装著作 +裝著名 装著名 +裝著錄 装著录 +裝著稱 装著称 +裝著者 装著者 +裝著述 装著述 +裹著 裹着 +裹著書 裹著书 +裹著作 裹著作 +裹著名 裹著名 +裹著錄 裹著录 +裹著稱 裹著称 +裹著者 裹著者 +裹著述 裹著述 +見著 见着 +見著書 见著书 +見著作 见著作 +見著名 见著名 +見著錄 见著录 +見著稱 见著称 +見著者 见著者 +見著述 见著述 +記著 记着 +記著書 记著书 +記著作 记著作 +記著名 记著名 +記著錄 记著录 +記著稱 记著称 +記著者 记著者 +記著述 记著述 +試著 试着 +試著書 试著书 +試著作 试著作 +試著名 试著名 +試著錄 试著录 +試著稱 试著称 +試著者 试著者 +試著述 试著述 +語著 语着 +語著書 语著书 +語著作 语著作 +語著名 语著名 +語著錄 语著录 +語著稱 语著称 +語著者 语著者 +語著述 语著述 +猶豫著 犹豫着 +堅貞著 坚贞着 +忠貞著 忠贞着 +走著 走着 +走著書 走著书 +走著作 走著作 +走著名 走著名 +走著錄 走著录 +走著稱 走著称 +走著者 走著者 +走著述 走著述 +趕著 赶着 +趕著書 赶著书 +趕著作 赶著作 +趕著名 赶著名 +趕著錄 赶著录 +趕著稱 赶著称 +趕著者 赶著者 +趕著述 赶著述 +趴著 趴着 +趴著書 趴著书 +趴著作 趴著作 +趴著名 趴著名 +趴著錄 趴著录 +趴著稱 趴著称 +趴著者 趴著者 +趴著述 趴著述 +躍著 跃着 +躍著書 跃著书 +躍著作 跃著作 +躍著名 跃著名 +躍著錄 跃著录 +躍著稱 跃著称 +躍著者 跃著者 +躍著述 跃著述 +跑著 跑着 +跑著書 跑著书 +跑著作 跑著作 +跑著名 跑著名 +跑著錄 跑著录 +跑著稱 跑著称 +跑著者 跑著者 +跑著述 跑著述 +跟著 跟着 +跟著書 跟著书 +跟著作 跟著作 +跟著名 跟著名 +跟著錄 跟著录 +跟著稱 跟著称 +跟著者 跟著者 +跟著述 跟著述 +跪著 跪着 +跪著書 跪著书 +跪著作 跪著作 +跪著名 跪著名 +跪著錄 跪著录 +跪著稱 跪著称 +跪著者 跪著者 +跪著述 跪著述 +跳著 跳着 +跳著書 跳著书 +跳著作 跳著作 +跳著名 跳著名 +跳著錄 跳著录 +跳著稱 跳著称 +跳著者 跳著者 +跳著述 跳著述 +踏著 踏着 +踏著書 踏著书 +踏著作 踏著作 +踏著名 踏著名 +踏著錄 踏著录 +踏著稱 踏著称 +踏著者 踏著者 +踏著述 踏著述 +踩著 踩着 +踩著書 踩著书 +踩著作 踩著作 +踩著名 踩著名 +踩著錄 踩著录 +踩著稱 踩著称 +踩著者 踩著者 +踩著述 踩著述 +身著 身着 +身著書 身著书 +身著作 身著作 +身著名 身著名 +身著錄 身著录 +身著稱 身著称 +身著者 身著者 +身著述 身著述 +躺著 躺着 +躺著書 躺著书 +躺著作 躺著作 +躺著名 躺著名 +躺著錄 躺著录 +躺著稱 躺著称 +躺著者 躺著者 +躺著述 躺著述 +轉著 转着 +轉著書 转著书 +轉著作 转著作 +轉著名 转著名 +轉著錄 转著录 +轉著稱 转著称 +轉著者 转著者 +轉著述 转著述 +載著 载着 +載著書 载著书 +載著作 载著作 +載著名 载著名 +載著錄 载著录 +載著稱 载著称 +載著者 载著者 +載著述 载著述 +達著 达着 +達著書 达著书 +達著作 达著作 +達著名 达著名 +達著錄 达著录 +達著稱 达著称 +達著者 达著者 +達著述 达著述 +連著 连着 +連著書 连著书 +連著作 连著作 +連著名 连著名 +連著錄 连著录 +連著稱 连著称 +連著者 连著者 +連著述 连著述 +追著 追着 +追著書 追著书 +追著作 追著作 +追著名 追著名 +追著錄 追著录 +追著稱 追著称 +追著者 追著者 +追著述 追著述 +逆著 逆着 +逆著書 逆著书 +逆著作 逆著作 +逆著名 逆著名 +逆著錄 逆著录 +逆著稱 逆著称 +逆著者 逆著者 +逆著述 逆著述 +逼著 逼着 +逼著書 逼著书 +逼著作 逼著作 +逼著名 逼著名 +逼著錄 逼著录 +逼著稱 逼著称 +逼著者 逼著者 +逼著述 逼著述 +遇著 遇着 +遇著書 遇著书 +遇著作 遇著作 +遇著名 遇著名 +遇著錄 遇著录 +遇著稱 遇著称 +遇著称 遇著称 +遇著者 遇著者 +遇著述 遇著述 +配著 配着 +配著書 配著书 +配著作 配著作 +配著名 配著名 +配著錄 配著录 +配著稱 配著称 +配著者 配著者 +配著述 配著述 +釀著 酿着 +釀著書 酿著书 +釀著作 酿著作 +釀著名 酿著名 +釀著錄 酿著录 +釀著稱 酿著称 +釀著者 酿著者 +釀著述 酿著述 +鋪著 铺着 +鋪著書 铺著书 +鋪著作 铺著作 +鋪著名 铺著名 +鋪著錄 铺著录 +鋪著稱 铺著称 +鋪著者 铺著者 +鋪著述 铺著述 +閉著 闭着 +閉著書 闭著书 +閉著作 闭著作 +閉著名 闭著名 +閉著錄 闭著录 +閉著稱 闭著称 +閉著者 闭著者 +閉著述 闭著述 +閑著 闲着 +閑著書 闲著书 +閑著作 闲著作 +閑著名 闲著名 +閑著錄 闲著录 +閑著稱 闲著称 +閑著者 闲著者 +閑著述 闲著述 +附著 附着 +附著書 附著书 +附著作 附著作 +附著名 附著名 +附著錄 附著录 +附著稱 附著称 +附著者 附著者 +附著述 附著述 +陋著 陋着 +陋著書 陋著书 +陋著作 陋著作 +陋著名 陋著名 +陋著錄 陋著录 +陋著稱 陋著称 +陋著者 陋著者 +陋著述 陋著述 +陪著 陪着 +陪著書 陪著书 +陪著作 陪著作 +陪著名 陪著名 +陪著錄 陪著录 +陪著稱 陪著称 +陪著者 陪著者 +陪著述 陪著述 +隨著 随着 +隨著書 随著书 +隨著作 随著作 +隨著名 随著名 +隨著錄 随著录 +隨著稱 随著称 +隨著者 随著者 +隨著述 随著述 +隔著 隔着 +隔著書 隔著书 +隔著作 隔著作 +隔著名 隔著名 +隔著錄 隔著录 +隔著稱 隔著称 +隔著者 隔著者 +隔著述 隔著述 +雅著 雅着 +雅著書 雅著书 +雅著作 雅著作 +雅著名 雅著名 +雅著錄 雅著录 +雅著稱 雅著称 +雅著称 雅著称 +雅著者 雅著者 +雅著述 雅著述 +頂著 顶着 +頂著書 顶著书 +頂著作 顶著作 +頂著名 顶著名 +頂著錄 顶著录 +頂著稱 顶著称 +頂著者 顶著者 +頂著述 顶著述 +順著 顺着 +順著書 顺著书 +順著作 顺著作 +順著名 顺著名 +順著錄 顺著录 +順著稱 顺著称 +順著者 顺著者 +順著述 顺著述 +領著 领着 +領著書 领著书 +領著作 领著作 +領著名 领著名 +領著錄 领著录 +領著稱 领著称 +領著者 领著者 +領著述 领著述 +飄著 飘着 +飄著書 飘著书 +飄著作 飘著作 +飄著名 飘著名 +飄著錄 飘著录 +飄著稱 飘著称 +飄著者 飘著者 +飄著述 飘著述 +駕著 驾着 +駕著書 驾著书 +駕著作 驾著作 +駕著名 驾著名 +駕著錄 驾著录 +駕著稱 驾著称 +駕著者 驾著者 +駕著述 驾著述 +罵著 骂着 +罵著書 骂著书 +罵著作 骂著作 +罵著名 骂著名 +罵著錄 骂著录 +罵著稱 骂著称 +罵著者 骂著者 +罵著述 骂著述 +騎著 骑着 +騎著書 骑著书 +騎著作 骑著作 +騎著名 骑著名 +騎著錄 骑著录 +騎著稱 骑著称 +騎著者 骑著者 +騎著述 骑著述 +騙著 骗着 +騙著書 骗著书 +騙著作 骗著作 +騙著名 骗著名 +騙著錄 骗著录 +騙著稱 骗著称 +騙著者 骗著者 +騙著述 骗著述 +高著 高着 +高著書 高著书 +高著作 高著作 +高著名 高著名 +高著錄 高著录 +高著稱 高著称 +高著称 高著称 +高著者 高著者 +高著述 高著述 +黏著 黏着 +黏著書 黏著书 +黏著作 黏著作 +黏著名 黏著名 +黏著錄 黏著录 +黏著稱 黏著称 +黏著者 黏著者 +黏著述 黏著述 +護著 护着 +護著書 护著书 +護著作 护著作 +護著名 护著名 +護著錄 护著录 +護著稱 护著称 +護著者 护著者 +護著述 护著述 +保護著 保护着 +愛護著 爱护着 +庇護著 庇护着 +傳著 传着 +傳著書 传著书 +傳著作 传著作 +傳著名 传著名 +傳著錄 传著录 +傳著稱 传著称 +傳著者 传著者 +傳著述 传著述 +標誌著 标志着 +流露著 流露着 +靠著 靠着 +靠著作 靠著作 +靠著名 靠著名 +靠著錄 靠著录 +靠著稱 靠著称 +靠著者 靠著者 +靠著述 靠著述 +玩著 玩着 +迫著 迫着 +吃著 吃着 +聞著 闻着 +嗅著 嗅着 +警戒著 警戒着 +過著 过着 +過著作 过著作 +過著名 过著名 +過著錄 过著录 +過著稱 过著称 +過著者 过著者 +過著述 过著述 +下著 下着 +下著作 下著作 +下著名 下著名 +下著錄 下著录 +下著录 下著录 +下著稱 下著称 +下著称 下著称 +下著者 下著者 +下著述 下著述 +下著有 下著有 +放著 放着 +放著作 放著作 +放著名 放著名 +放著稱 放著称 +放著称 放著称 +藉著 借着 +显著 显著 +顯著 显著 +標誌著 标志着 +幹著 干着 +幹著名 幹著名 +幹著稱 幹著称 +穫著 获着 +閒著 闲着 +飃著 飘着 +沈著 沉着 +擡著 抬着 +著甚麼 着什么 +滿著 满着 +滿著名 满著名 +滿著作 满著作 +滿著者 满著者 +衝著 冲着 +沖著 冲着 +沖著《 冲著《 +沖著( 冲著( +沖著。 冲著。 +沖著, 冲著, +立著 立着 +立著名 立著名 +立著作 立著作 +立著者 立著者 +立著稱 立著称 +立著称 立著称 +立著有 立著有 +立著《 立著《 +立著( 立著( +繫著 系着 +颳著 刮着 +鬥著 斗着 +縱著 纵着 +伏著 伏着 +視著 视着 +視著名 视著名 +視著作 视著作 +視著者 视著者 +視著稱 视著称 +蓋著 盖着 +蓋著名 盖著名 +蓋著稱 盖著称 +蓋著作 盖著作 +覆蓋著 覆盖着 #分词用 +象徵著 象征着 +象徵著名 象征著名 +固著 固着 +班固著 班固著 +分布著 分布着 +分佈著 分布着 +散布著 散布着 +散佈著 散布着 +遍佈著 遍布着 +遍布著 遍布着 +記錄著 记录着 +紀錄著 纪录着 +收錄著 收录着 +促著 促着 +咬著 咬着 +三十六著 三十六着 +走為上著 走为上着 +記憶體 内存 +乙太網 以太网 +點陣圖 位图 +光碟機 光驱 +雜訊 噪声 +功能變數名稱 域名 +音效卡 声卡 +字型大小 字号 +欄位 字段 +非同步 异步 +匯流排 总线 +介面 界面 +控制項 控件 +矽片 硅片 +矽谷 硅谷 +硬碟 硬盘 +磁碟 磁盘 +磁軌 磁道 +程式控制 程控 +運算元 算子 +演算法 算法 +晶片 芯片 +晶元 芯片 +片語 词组 +隻字片語 只字片语 +隻言片語 只言片语 +軟碟機 软驱 +快閃記憶體 闪存 +滑鼠 鼠标 +滑鼠蛇 滑鼠蛇 +二進位 二进制 +滿二進位 满二进位 +六進位 六进制 +滿六進位 满六进位 +滿十六進位 满十六进位 +八進位 八进制 +滿八進位 满八进位 +十進位 十进制 +滿十進位 满十进位 +16進位 16进制 +滿16進位 满16进位 +二進位制 二进位制 +六進位制 六进位制 +八進位制 八进位制 +十進位制 十进位制 +16進位制 16进位制 +優先順序 优先级 +攜帶型 便携式 +資訊理論 信息论 +資訊時代 信息时代 +迴圈 循环 +解析度 分辨率 +伺服器 服务器 +區域網 局域网 +區域網路 局域网络 +巨集 宏 +掃瞄器 扫描仪 +資料庫 数据库 +母音 元音 +印表機 打印机 +位元組 字节 +列印 打印 +硬體 硬件 +二極體 二极管 +三極體 三极管 +軟體 软件 +軟體動物 软体动物 +軟體生物 软体生物 +軟體家具 软体家具 +網路 网络 +人工智慧 人工智能 +太空梭 航天飞机 +穿梭機 航天飞机 +網際網路 互联网 +機械人 机器人 +行動電話 移动电话 +流動電話 移动电话 +數據機 调制解调器 +網域名稱 域名 +葉門 也门 +貝里斯 伯利兹 +維德角 佛得角 +克羅埃西亞 克罗地亚 +甘比亞 冈比亚 +幾內亞比索 几内亚比绍 +列支敦斯登 列支敦士登 +賴比瑞亞 利比里亚 +迦納 加纳 +加彭 加蓬 +波札那 博茨瓦纳 +盧安達 卢旺达 +瓜地馬拉 危地马拉 +厄瓜多爾 厄瓜多尔 +厄瓜多尔 厄瓜多尔 +厄瓜多 厄瓜多尔 +厄利垂亞 厄立特里亚 +吉布地 吉布提 +哥斯大黎加 哥斯达黎加 +吐瓦魯 图瓦卢 +聖露西亞 圣卢西亚 +聖吉斯納域斯 圣基茨和尼维斯 +聖克里斯多福及尼維斯 圣基茨和尼维斯 +聖文森及格瑞那丁 圣文森特和格林纳丁斯 +聖馬利諾 圣马力诺 +蓋亞那 圭亚那 +坦尚尼亞 坦桑尼亚 +衣索匹亞 埃塞俄比亚 +衣索比亞 埃塞俄比亚 +吉里巴斯 基里巴斯 +塞拉利昂 塞拉利昂 +塞普勒斯 塞浦路斯 +塞席爾 塞舌尔 +安地卡及巴布達 安提瓜和巴布达 +奈及利亞 尼日利亚 +尼日爾 尼日尔 +巴貝多 巴巴多斯 +布吉納法索 布基纳法索 +蒲隆地 布隆迪 +帛琉 帕劳 +義大利 意大利 +索羅門群島 所罗门群岛 +汶萊 文莱 +史瓦濟蘭 斯威士兰 +斯洛維尼亞 斯洛文尼亚 +紐西蘭 新西兰 +格瑞那達 格林纳达 +茅利塔尼亞 毛里塔尼亚 +毛里裘斯 毛里求斯 +模里西斯 毛里求斯 +沙地阿拉伯 沙特阿拉伯 +沙烏地阿拉伯 沙特阿拉伯 +波士尼亞與赫塞哥維納 波斯尼亚和黑塞哥维那 +辛巴威 津巴布韦 +宏都拉斯 洪都拉斯 +千里達托貝哥 特立尼达和托巴哥 +萬那杜 瓦努阿图 +溫納圖 瓦努阿图 +葛摩 科摩罗 +象牙海岸 科特迪瓦 +突尼西亞 突尼斯 +寮國 老挝 +貢寮 贡寮 #分詞用 +蘇利南 苏里南 +奈洛比 内罗毕 +莫三比克 莫桑比克 +賴索托 莱索托 +尚比亞 赞比亚 +亞塞拜然 阿塞拜疆 +阿拉伯聯合大公國 阿拉伯联合酋长国 +南韓 韩国 +馬爾地夫 马尔代夫 +馬爾他 马耳他 +馬利共和國 马里共和国 +汕埠 圣佩德罗苏拉 +笨豬跳 蹦极跳 +绑紧跳 蹦极跳 +狗隻 犬只 +士多啤梨 草莓 +忌廉 奶油 +撞球 台球 +賓士 奔驰 +積架 捷豹 +布殊 布什 +柯林頓 克林顿 +梵谷 梵高 +碧咸 贝克汉姆 +米高·奧雲 迈克尔·欧文 +卡佩雅蒂 卡普里亚蒂 +舒麥加 舒马赫 +希特拉 希特勒 +黛安娜 戴安娜 +雷諾瓦 雷诺阿 +達文西 达芬奇 +達·文西 达·芬奇 +辛康納利 肖恩·康纳利 +維根斯坦 维特根斯坦 +索忍尼辛 索尔仁尼琴 +索贊尼辛 索尔仁尼琴 +蘇辛尼津 索尔仁尼琴 +皮雅斯·布士南 皮尔斯·布鲁斯南 +甘迺迪 肯尼迪 +梅赫西迪 梅赛德斯 +李奧納多 列奥那多 +普利茲 普利策 +戈巴契夫 戈尔巴乔夫 +德希達 德里达 +席哈克 希拉克 +蘿拉 劳拉 +史達林 斯大林 +史特勞斯 斯特劳斯 +卡斯楚 卡斯特罗 +占士邦 詹姆斯·邦德 +傅利葉 傅里叶 +伊莉莎白 伊丽莎白 +派屈克 帕特里克 +蒲美蓬 普密蓬 +畢卡索 毕加索 +蒲朗克 普朗克 +薛丁格 薛定谔 +克卜勒 开普勒 +都卜勒 多普勒 +邱吉爾 丘吉尔 +狄托 铁托 +查維茲 查韦斯 +班傑明 本杰明 +柯德莉·夏萍 奥黛丽·赫本 +華勒沙 瓦文萨 +華里沙 瓦文萨 +賓拉登 本拉登 +賓·拉登 本·拉登 +歐巴馬 奥巴马 +唐納·川普 唐纳德·特朗普 +當勞·特朗普 唐纳德·特朗普 +當奴·特朗普 唐纳德·特朗普 +北韓 北朝鲜 +台北韓 台北韩 +寮人民民主共和國 老挝人民民主共和国 +寮語 老挝语 +蘭卡威 浮罗交怡 +雷伊泰灣 莱特湾 +耶加達 雅加达 +伊斯蘭瑪巴德 伊斯兰堡 +喀拉蚩 卡拉奇 +葉里溫 埃里温 +提比里西 第比利斯 +巴斯拉 巴士拉 +杜拜 迪拜 +坚杜拜 坚杜拜 +堅杜拜 坚杜拜 +賽普勒斯 塞浦路斯 +荷姆茲 霍尔木兹 +加薩走廊 加沙地带 +西臺人 赫梯人 +西臺族 赫梯族 +西臺文 赫梯文 +西臺語 赫梯语 +西臺王 赫梯王 +西臺國 赫梯国 +西臺帝 赫梯帝 +坎培拉 堪培拉 +玻里尼西亞 波利尼西亚 +紐幾內亞 新几内亚 +強斯頓環礁 约翰斯顿岛 +帕邁拉環礁 巴尔米拉环礁 +萌島 马恩岛 +伯明罕 伯明翰 +威爾斯 威尔士 +諾曼第 诺曼底 +土魯斯 图卢兹 +坎城 戛纳 +羅亞爾 卢瓦尔 +艾菲爾 埃菲尔 +羅浮宮 卢浮宫 +安哈特 安哈尔特 +布蘭登堡 勃兰登堡 +什勒斯維希 石勒苏益格 +霍爾斯坦 荷尔斯泰因 +前波莫瑞 前波美拉尼亚 +威斯伐倫 威斯特法伦 +德勒斯登 德累斯顿 +杜塞道夫 杜塞尔多夫 +漢諾瓦 汉诺威 +柏林圍牆 柏林墙 +巴塞隆拿 巴塞罗那 +巴塞隆納 巴塞罗那 +西維爾 塞维利亚 +塞維亞 塞维利亚 +華倫西亞 巴伦西亚 +瓦倫西亞 巴伦西亚 +雅爾達 雅尔塔 +車諾比 切尔诺贝利 +馬斯垂克 马斯特里赫特 +波士尼亞 波斯尼亚 +塞拉耶佛 萨拉热窝 +貝爾格勒 贝尔格莱德 +蒙特內哥羅 黑山 +塞爾維亞與蒙特內哥羅 塞尔维亚和黑山 +伊斯坦堡 伊斯坦布尔 +庇里牛斯 比利牛斯 +亞斯文 阿斯旺 +厄立特里亞 厄立特里亚 +厄利垂亚 厄立特里亚 +亞歷山卓 亚历山大 +雅穆索戈 亚穆苏克罗 +索馬利蘭 索马里兰 +吉力馬札羅 乞力马扎罗 +索馬利亞 索马里 +金夏沙 金沙萨 +三蘭港 达累斯萨拉姆 +布隆泉 布隆方丹 +馬拉威 马拉维 +百慕達 百慕大 +三藩市 旧金山 +荷里活 好莱坞 +荷里活道 荷里活道 +荷里活廣場 荷里活广场 +麻薩諸塞 马萨诸塞 +伊利諾 伊利诺伊 +伊利諾伊 伊利诺伊 +密执安 密歇根 +密西根 密歇根 +紐澤西 新泽西 +蒙特婁 蒙特利尔 +千里達及托巴哥 特立尼达和多巴哥 +千里達 特立尼达 +托巴哥 多巴哥 +多明尼加 多米尼加 +斯堪地那維亞 斯堪的纳维亚 +頻寬 带宽 +數位相機 数码相机 +數位照相機 数码照相机 +單眼相機 单反相机 +單鏡反光機 单反相机 +桌上型電腦 台式电脑 +韌體 固件 +唯讀 只读 +作業系統 操作系统 +行動作業系統 移动操作系统 +流動作業系統 移动操作系统 +外掛程式 插件 +電晶體 晶体管 +顯示卡 显卡 +主機板 主板 +網際網絡 互联网 +原始碼 源代码 +螢幕 屏幕 +螢屏 荧屏 +解像度 分辨率 +IP位址 IP地址 +程式設計師 程序员 +公尺 米 +公升 升 +英吋 英寸 +英呎 英尺 +高畫質 高清 +飛彈 导弹 +電視影集 电视系列剧 +原子筆 圆珠笔 +智慧卡 智能卡 +鐵達尼號 泰坦尼克号 +轉殖 克隆 +空中巴士 空中客车 +電視劇集 电视剧 +狂牛症 疯牛病 +結他 吉他 +了結他 了结他 +連結他 连结他 +鏈結 链接 +已開發國家 发达国家 +太空飛行員 宇航员 +太空衣 宇航服 +外部連結 外部链接 +網站連結 网站链接 +網頁連結 网页链接 +超連結 超链接 +動畫影集 系列动画片 +全球資訊網 万维网 +伊波拉 埃博拉 +C肝 丙肝 +C型肝炎 丙型肝炎 +B肝 乙肝 +B型肝炎 乙型肝炎 +A肝 甲肝 +A型肝炎 甲型肝炎 +錄影帶 录像带 +音樂錄影帶 音乐录影带 +健力士世界紀錄 吉尼斯世界纪录 +金氏世界紀錄 吉尼斯世界纪录 +祖雲達斯 尤文图斯 +若且唯若 当且仅当 +複製人 克隆人 +白朗寧 勃朗宁 +形上學 形而上学 +藍芽 蓝牙 +槍枝 枪支 +掃瞄 扫描 +愛滋 艾滋 +正體中文 繁体中文 +智慧財產權 知识产权 +智財權 知识产权 +哥德式 哥特式 +芮氏0 里氏0 +芮氏1 里氏1 +芮氏2 里氏2 +芮氏3 里氏3 +芮氏4 里氏4 +芮氏5 里氏5 +芮氏6 里氏6 +芮氏7 里氏7 +芮氏8 里氏8 +芮氏9 里氏9 +芮氏規模 里氏震级 +芮氏地震規模 里氏地震规模 +黎克特制 里氏 +行政總裁 首席执行官 +執行長, 首席执行官, +執行長、 首席执行官、 +執行長。 首席执行官。 +財務長, 首席财务官, +財務長、 首席财务官、 +財務長。 首席财务官。 +營運長, 首席运营官, +營運長、 首席运营官、 +營運長。 首席运营官。 +智慧型 智能 +智慧手機 智能手机 +可攜式 便携式 +電腦程式 计算机程序 +應用程式 应用程序 +雷射 激光 +鱼雷 鱼雷 #分詞用 +魚雷 鱼雷 +尖峰時間 高峰时间 +尖峰時段 高峰时段 +咖哩 咖喱 +東協 东盟 +東協會 东协会 +東協助 东协助 +東協議 东协议 +亚细安 东盟 +大英國協 英联邦 +共和联邦 英联邦 +阿布達比 阿布扎比 +柴契爾 撒切尔 +戴卓爾 撒切尔 +凱薩琳 凯瑟琳 +嘉芙蓮 凯瑟琳 +孟德爾頌 门德尔松 +孟德爾遜 门德尔松 +蕭士塔高維奇 肖斯塔科维奇 +蕭士達高維契 肖斯塔科维奇 +工具機 机床 +空氣品質 空气质量 +空氣質素 空气质量 +伏地挺身 俯卧撑 +掌上壓 俯卧撑 +數位電視 数字电视 +數碼電視 数字电视 +數位技術 数字技术 +數位訊號 数字信号 +數碼訊號 数字信号 +數位音樂 数字音乐 +數位化 数字化 +行動網路 移动网络 +流動網絡 移动网络 +咪高峰 麦克风 +幫浦 泵 +電單車 摩托车 +演化論 进化论 +搜尋引擎 搜索引擎 +福馬林 福尔马林 +海洛英 海洛因 +赫魯雪夫 赫鲁晓夫 +公厘 毫米 +公釐 毫米 +海浬 海里 +森巴舞 桑巴舞 +喬治·歐威爾 乔治·奥威尔 +西元1 公元1 +西元2 公元2 +西元3 公元3 +西元4 公元4 +西元5 公元5 +西元6 公元6 +西元7 公元7 +西元8 公元8 +西元9 公元9 +西元前 公元前 +翁山蘇姬 昂山素季 +昂山素姬 昂山素季 +西洋棋 国际象棋 +私隱 隐私 +格林美獎 格莱美奖 +葛萊美獎 格莱美奖 +史丹福大學 斯坦福大学 +賈伯斯 乔布斯 +波里活 宝莱坞 +庫德族 库尔德族 +庫德人 库尔德人 +希拉蕊 希拉里 +希拉莉 希拉里 +麻薩諸塞 马萨诸塞 +東南亞國家協會 东南亚国家联盟 +獨立國協 独联体 +獨立國家國協 独立国家联合体 +行人路 人行道 +行人路權 行人路权 +行人路权 行人路权 +塑膠袋 塑料袋 +烏龍麵 乌冬面 +披索 比索 diff --git a/www/wiki/maintenance/language/zhtable/toHK.manual b/www/wiki/maintenance/language/zhtable/toHK.manual new file mode 100644 index 00000000..b71764ad --- /dev/null +++ b/www/wiki/maintenance/language/zhtable/toHK.manual @@ -0,0 +1,3055 @@ +裡 裏 +鉤 鈎 +檯 枱 +醯 酰 +菸 煙 +汙 污 +溼 濕 +硅 矽 +幺 么 +計畫 計劃 +吧台 吧枱 +坐台 坐枱 +坐台铁 坐台鐵 +妆台 妝枱 +弹珠台 彈珠枱 +折台 摺枱 +台历 枱曆 +台灯 枱燈 +写字台 寫字枱 +工作台 工作枱 +弹子台 彈子枱 +上台面 上枱面 +台面上 枱面上 +台面化 枱面化 +柜台 櫃枱 +球台 球枱 +赌台 賭枱 +办公台 辦公枱 +餐台 餐枱 +凶殺 兇殺 +凶殘 兇殘 +凶惡 兇惡 +緝凶 緝兇 +買凶 買兇 +颁布 頒佈 +頒布 頒佈 +发布 發佈 +發布 發佈 +秀发布 秀發佈 +并发布 並發佈 +分布 分佈 +分布于 分佈於 +宣布 宣佈 +承宣布政 承宣布政 +公布 公佈 +摆布 擺佈 +擺布 擺佈 +遍布 遍佈 +散布 散佈 +密布 密佈 +布于 佈於 +布於 佈於 +布道 佈道 +布置 佈置 +布景 佈景 +布光 佈光 +布局 佈局 +布防 佈防 +布满 佈滿 +布滿 佈滿 +布告 佈告 +布阵 佈陣 +布陣 佈陣 +布点 佈點 +布點 佈點 +布警 佈警 +布控 佈控 +布设 佈設 +布設 佈設 +布展 佈展 +布下了 佈下了 +布下的 佈下的 +星罗棋布 星羅棋佈 +星羅棋布 星羅棋佈 +开诚布公 開誠佈公 +開誠布公 開誠佈公 +空投布雷 空投佈雷 +火箭布雷 火箭佈雷 +海湾布雷 海灣佈雷 +海灣布雷 海灣佈雷 +空中布雷 空中佈雷 +海上布雷 海上佈雷 +布雷的 佈雷的 +布雷, 佈雷, +布雷、 佈雷、 +布雷。 佈雷。 +布雷; 佈雷; +布雷舰 佈雷艦 +布雷艦 佈雷艦 +布雷艇 佈雷艇 +布雷速度 佈雷速度 +布雷封锁 佈雷封鎖 +布雷封鎖 佈雷封鎖 +准将 準將 +准將 準將 +准尉 準尉 +迭代 疊代 +彩排 綵排 +彩带 綵帶 +彩帶 綵帶 +彩楼 綵樓 +彩樓 綵樓 +彩牌楼 綵牌樓 +彩牌樓 綵牌樓 +彩球 綵球 +彩绸 綵綢 +彩綢 綵綢 +彩线 綵綫 +彩線 綵線 +彩船 綵船 +彩衣 綵衣 +结彩 結綵 +結彩 結綵 +戏彩娱亲 戲綵娛親 +戲彩娛親 戲綵娛親 +剪彩 剪綵 +占上风 佔上風 +占上風 佔上風 +占下 佔下 +占位 佔位 +占住 佔住 +占占 佔佔 +占便宜 佔便宜 +占个 佔個 +占個 佔個 +占先 佔先 +占光 佔光 +占到 佔到 +占取 佔取 +占在 佔在 +占地 佔地 +占好 佔好 +占得 佔得 +占掉 佔掉 +占据 佔據 +占據 佔據 +占有 佔有 +占满 佔滿 +占滿 佔滿 +占为 佔為 +占為 佔為 +占用 佔用 +占毕 佔畢 +占畢 佔畢 +占尽 佔盡 +占盡 佔盡 +占线 佔線 +占線 佔線 +占起 佔起 +占过 佔過 +占過 佔過 +占领 佔領 +占領 佔領 +占头筹 佔頭籌 +占頭籌 佔頭籌 +占高枝 佔高枝 +侵占 侵佔 +先占 先佔 +分占 分佔 +只占 只佔 +强占 強佔 +強占 強佔 +抢占 搶佔 +搶占 搶佔 +攻占 攻佔 +照占 照佔 +约占 約佔 +約占 約佔 +连占 連佔 +連占 連佔 +进占 進佔 +進占 進佔 +还占 還佔 +還占 還佔 +隐占 隱佔 +隱占 隱佔 +霸占 霸佔 +鸠占 鳩佔 +鳩占 鳩佔 +割占 割佔 +非占不可 非佔不可 +占1 佔1 +占2 佔2 +占3 佔3 +占4 佔4 +占5 佔5 +占6 佔6 +占7 佔7 +占8 佔8 +占9 佔9 +占0 佔0 +占零 佔零 +占〇 佔〇 +占一 佔一 +占二 佔二 +占两 佔兩 +占兩 佔兩 +占三 佔三 +占四 佔四 +占五 佔五 +占六 佔六 +占七 佔七 +占八 佔八 +占九 佔九 +占十 佔十 +占百 佔百 +占千 佔千 +占万 佔萬 +占萬 佔萬 +占亿 佔億 +占億 佔億 +占超过 佔超過 +占超過 佔超過 +占不足 佔不足 +占至少 佔至少 +占少 佔少 +占至多 佔至多 +占半 佔半 +占多 佔多 +占大 佔大 +占小 佔小 +占中 佔中 +占东 佔東 +占東 佔東 +占西 佔西 +占南 佔南 +占北 佔北 +占平均 佔平均 +占总 佔總 +占總 佔總 +独占 獨佔 +獨占 獨佔 +所占 所佔 +市占 市佔 +占率 佔率 +占市 佔市 +占世界 佔世界 +占全 佔全 +占国 佔國 +占國 佔國 +占国桥 占國橋 +占國橋 占國橋 +占美国 佔美國 +占美國 佔美國 +占台 佔台 +占臺 佔臺 +占香 佔香 +占澳 佔澳 +占加 佔加 +占新 佔新 +占马 佔馬 +占馬 佔馬 +占印 佔印 +占英 佔英 +占法 佔法 +占德 佔德 +占葡 佔葡 +占俄 佔俄 +占苏 佔蘇 +占蘇 佔蘇 +占缺 佔缺 +占A 佔A +占B 佔B +占C 佔C +占D 佔D +占E 佔E +占F 佔F +占G 佔G +占H 佔H +占I 佔I +占J 佔J +占K 佔K +占L 佔L +占M 佔M +占N 佔N +占O 佔O +占P 佔P +占Q 佔Q +占R 佔R +占S 佔S +占T 佔T +占U 佔U +占V 佔V +占W 佔W +占X 佔X +占Y 佔Y +占Z 佔Z +占不占 佔不佔 +不占 不佔 +占了 佔了 +占资 佔資 +占資 佔資 +占人便宜 佔人便宜 +占主要 佔主要 +占所有 佔所有 +占头 佔頭 +占頭 佔頭 +占道 佔道 +占屋 佔屋 +占网 佔網 +占網 佔網 +占床 佔床 +占座 佔座 +占分 佔分 +占个位 佔個位 +占個位 佔個位 +占後 佔後 +占山为 佔山為 +占山為 佔山為 +占比 佔比 +占下風 佔下風 +占下风 佔下風 +少占 少佔 +多占 多佔 +费占 費佔 +費占 費佔 +占查 佔查 +占压 佔壓 +占壓 佔壓 +占优 佔優 +占優 佔優 +占劣 佔劣 +稳占 穩佔 +穩占 穩佔 +占整 佔整 +占局部 佔局部 +日占 日佔 +擇日占星 擇日占星 +择日占星 擇日占星 +美占 美佔 +英占 英佔 +德占 德佔 +法占 法佔 +俄占 俄佔 +葡占 葡佔 +西占 西佔 +奥占 奧佔 +奧占 奧佔 +意占 意佔 +義占 意佔 +地占 地佔 +占场 佔場 +占場 佔場 +占耕 佔耕 +狂占 狂佔 +征占 徵佔 +徵占 徵佔 +圈占 圈佔 +已占 已佔 +占囁 佔囁 +占主 佔主 +占次 佔次 +寡占 寡佔 +占去 佔去 +将占 將佔 +將占 將佔 +将占卜 將占卜 +將占卜 將占卜 +要占 要佔 +要占卜 要占卜 +会占 會佔 +會占 會佔 +会占卜 會占卜 +會占卜 會占卜 +占卜 占卜 +梦有五不占 夢有五不占 +夢有五不占 夢有五不占 +占有五不 占有五不 +吞占 吞佔 +一地里 一地裏 +中文里 中文裏 +英文里 英文裏 +古文里 古文裏 +经文里 經文裏 +论文里 論文裏 +譯文里 譯文裏 +原文里 原文裏 +正文里 正文裏 +下文里 下文裏 +条文里 條文裏 +画里 畫裏 +事里 事裏 +井里 井裏 +作品里 作品裏 +个里 個裏 +假里 假裏 +傻里傻气 傻裏傻氣 +丛林里 叢林裏 +口里 口裏 +吃里扒外 吃裏扒外 +吃里爬外 吃裏爬外 +呆里呆气 呆裏呆氣 +哪里 哪裏 +嘴里 嘴裏 +圈里 圈裏 +园里 園裏 +土里 土裏 +坑里 坑裏 +城里 城裏 +域里 域裏 +场里 場裏 +壶里 壺裏 +夜里 夜裏 +梦里 夢裏 +天里 天裏 +子里 子裏 +字里行间 字裏行間 +学里 學裏 +宫里 宮裏 +家里 家裏 +宝里宝气 寶裏寶氣 +封面里 封面裏 +专辑里 專輯裏 +就里 就裏 +局里 局裏 +屋里 屋裏 +屯里 屯裏 +巷里 巷裏 +城市里 城市裏 +都市里 都市裏 +市里的 市裏的 +年代里 年代裏 +年里 年裏 +年里约 年里約 #里约奧運 +店里 店裏 +庙里 廟裏 +往里 往裏 +从里到外 從裏到外 +从里向外 從裏向外 +心里面 心裏面 +心里 心裏 +忙里 忙裏 +怪里怪气 怪裏怪氣 +慌里慌张 慌裏慌張 +怀里 懷裏 +戏里 戲裏 +游戏里 遊戲裏 +房里 房裏 +手里 手裏 +手里剑 手裏劍 +族里 族裏 +日里 日裏 +暗地里 暗地裏 +暗沟里 暗溝裏 +暗里 暗裏 +会里 會裏 +村里的 村裏的 +村里有 村裏有 +区里的 區裏的 +区里有 區裏有 +森林里 森林裏 +棺材里 棺材裏 +树林里 樹林裏 +历史里 歷史裏 +死里求生 死裏求生 +死里逃生 死裏逃生 +壳里 殼裏 +水来汤里去 水來湯裏去 +水里 水裏 +池里 池裏 +沙里淘金 沙裏淘金 +河里 河裏 +洞里 洞裏 +渊里 淵裏 +湖里 湖裏 +漠里 漠裏 +潜意识里 潛意識裏 +潭里 潭裏 +墙里 牆裏 +狱里 獄裏 +班里 班裏 +田里 田裏 +由表及里 由表及裏 +界里 界裏 +白里透红 白裏透紅 +百科里 百科裏 +皮里春秋 皮裏春秋 +皮里阳秋 皮裏陽秋 +盒里 盒裏 +盘里 盤裏 +眼眶里 眼眶裏 +眼睛里 眼睛裏 +眼里 眼裏 +社里 社裏 +私下里 私下裏 +窝里 窩裏 +笑里藏刀 笑裏藏刀 +箱里 箱裏 +节目里 節目裏 +糊里糊涂 糊裏糊塗 +系列里 系列裏 +系里 系裏 +组里 組裏 +网里 網裏 +县里 縣裏 +缝里 縫裏 +肚里 肚裏 +胃里 胃裏 +背地里 背地裏 +胡里胡涂 胡裏胡塗 +腰里 腰裏 +花盆里 花盆裏 +苑里 苑裏 +苦里 苦裏 +草丛里 草叢裏 +庄里 莊裏 +葫芦里卖甚么药 葫蘆裏賣甚麼藥 +蜜里调油 蜜裏調油 +表里 表裏 +表里一致 表裏一致 +表里不一 表裏不一 +表里如一 表裏如一 +表里山河 表裏山河 +袋里 袋裏 +袖里 袖裏 +被里 被裏 +里勾外连 裏勾外連 +行家里手 行家裏手 +里海 裏海 +里屋 裏屋 +里层 裏層 +里带 裏帶 +里弦 裏弦 +里应外合 裏應外合 +里脊 裏脊 +里衣 裏衣 +里通外国 裏通外國 +里通外敌 裏通外敵 +里边 裏邊 +里间 裏間 +里面 裏面 +里头 裏頭 +衬里 襯裏 +角落里 角落裏 +话里有话 話裏有話 +车库里 車庫裏 +车站里 車站裏 +网站里 網站裏 +车里 車裏 +车里雅宾斯克 車里雅賓斯克 +这里 這裏 +邋里邋遢 邋裏邋遢 +那里 那裏 +金装玉里 金裝玉裏 +钟在寺里 鐘在寺裏 +门里 門裏 +间里 間裏 +院里 院裏 +阴沟里翻船 陰溝裏翻船 +集里 集裏 +鸡蛋里挑骨头 雞蛋裏挑骨頭 +雪里 雪裏 +雾里 霧裏 +云里雾里 雲裏霧裏 +鞋里 鞋裏 +鞭辟入里 鞭辟入裏 +头里 頭裏 +风里 風裏 +馆里 館裏 +点里 點裏 +点里程 點里程 +鼓里 鼓裏 +殿里 殿裏 +队里 隊裏 +世纪里 世紀裏 +夜晚里 夜晚裏 +参数里 參數裏 +集数里 集數裏 +人数里 人數裏 +总数里 總數裏 +函数里 函數裏 +地图里 地圖裏 +版图里 版圖裏 +配图里 配圖裏 +路图里 路圖裏 +线图里 線圖裏 +幅图里 幅圖裏 +镜图里 鏡圖裏 +从图里 從圖裏 +的图里 的圖裏 +图里的 圖裏的 +图里, 圖裏, +深山里 深山裏 +冰山里 冰山裏 +火山里 火山裏 +在山里 在山裏 +的山里 的山裏 +到山里 到山裏 +去山里 去山裏 +从山里 從山裏 +山里的 山裏的 +山里有 山裏有 +棉里 棉裏 +语里 語裏 +言里 言裏 +境里 境裏 +方法里 方法裏 +语法里 語法裏 +看法里 看法裏 +宪法里 憲法裏 +用法里 用法裏 +法里, 法裏, +框里 框裏 +碗里 碗裏 +电梯里 電梯裏 +个月里 個月裏 +月裡来 月裏來 +分钟里 分鐘裏 +小时里 小時裏 +体里 體裏 +柜里 櫃裏 +告里 告裏 +电影里 電影裏 +广播里 廣播裏 +电视里 電視裏 +公寓里 公寓裏 +窝里斗 窩裏鬥 +镇里 鎮裏 +苑裡 苑裡 +霄裡 霄裡 +岸裡 岸裡 +裡冷 裡冷 +挨著 挨着 +愛著 愛着 +暗著 暗着 +昂著 昂着 +擺著 擺着 +伴著 伴着 +辦著 辦着 +幫著 幫着 +綁著 綁着 +抱著 抱着 +背著 背着 +備著 備着 +本著 本着 +逼著 逼着 +閉著 閉着 +變著 變着 +猜著 猜着 +踩著 踩着 +藏著 藏着 +側著 側着 +纏著 纏着 +敞著 敞着 +唱著 唱着 +朝著 朝着 +沉著 沉着 +乘著 乘着 +持著 持着 +斥著 斥着 +醜著 醜着 +穿著 穿着 +吹著 吹着 +達著 達着 +打著 打着 +待著 待着 +帶著 帶着 +戴著 戴着 +當著 當着 +擋著 擋着 +得著 得着 +瞪著 瞪着 +低著 低着 +點著 點着 +盯著 盯着 +頂著 頂着 +定著 定着 +動著 動着 +鬥著 鬥着 +斗着 鬥着 +對著 對着 +矛盾著 矛盾着 +犯得著 犯得着 +犯不著 犯不着 +福著 福着 +趕著 趕着 +高著 高着 +隔著 隔着 +跟著 跟着 +關著 關着 +管著 管着 +慣著 慣着 +光著 光着 +跪著 跪着 +裹著 裹着 +撼著 撼着 +喝著 喝着 +候著 候着 +懷著 懷着 +晃著 晃着 +揮著 揮着 +活著 活着 +獲著 獲着 +急著 急着 +記著 記着 +希冀著 希冀着 +夾著 夾着 +駕著 駕着 +見著 見着 +閑著 閑着 +叫著 叫着 +接著 接着 +借著 借着 +據著 據着 +開著 開着 +看得著 看得着 +看不著 看不着 +看著 看着 +扛著 扛着 +考著 考着 +渴著 渴着 +刻著 刻着 +空著 空着 +哭著 哭着 +苦著 苦着 +捆著 捆着 +困著 困着 +拉著 拉着 +來著 來着 +樂著 樂着 +努力著 努力着 +麗著 麗着 +連著 連着 +戀著 戀着 +涼著 涼着 +亮著 亮着 +臨著 臨着 +拎著 拎着 +領著 領着 +流著 流着 +留著 留着 +摟著 摟着 +陋著 陋着 +落著 落着 +罵著 罵着 +瞞著 瞞着 +漫著 漫着 +忙著 忙着 +冒著 冒着 +美著 美着 +夢著 夢着 +蒙著 蒙着 +拿著 拿着 +逆著 逆着 +釀著 釀着 +趴著 趴着 +跑著 跑着 +陪著 陪着 +配著 配着 +披著 披着 +騙著 騙着 +飄著 飄着 +拼著 拼着 +鋪著 鋪着 +騎著 騎着 +牽著 牽着 +求著 求着 +嚷著 嚷着 +繞著 繞着 +忍著 忍着 +揉著 揉着 +潤著 潤着 +燒著 燒着 +身著 身着 +盛著 盛着 +試著 試着 +守著 守着 +受著 受着 +梳著 梳着 +豎著 豎着 +數著 數着 +睡得著 睡得着 +睡不著 睡不着 +睡著 睡着 +順著 順着 +隨著 隨着 +踏著 踏着 +抬著 抬着 +躺著 躺着 +提著 提着 +甜著 甜着 +挑著 挑着 +跳著 跳着 +聽得著 聽得着 +聽不著 聽不着 +聽著 聽着 +偷著 偷着 +拖著 拖着 +望著 望着 +圍著 圍着 +味著 味着 +想著 想着 +響著 響着 +向著 向着 +笑著 笑着 +心著 心着 +信著 信着 +行著 行着 +學著 學着 +尋著 尋着 +循著 循着 +壓著 壓着 +雅著 雅着 +沿著 沿着 +耀著 耀着 +掖著 掖着 +衣著 衣着 +疑著 疑着 +溢著 溢着 +因著 因着 +印著 印着 +應著 應着 +映著 映着 +用得著 用得着 +用不著 用不着 +用著 用着 +悠著 悠着 +有著 有着 +與著 與着 +語著 語着 +猶豫著 猶豫着 +躍著 躍着 +雜著 雜着 +載著 載着 +存在著 存在着 +紮著 紮着 +展著 展着 +占着 佔着 +占著 佔着 +占著作 占著作 +占著者 佔著者 +占著名 佔著名 +占著述 占著述 +占著稱 占著稱 +占著錄 占著錄 +站著 站着 +戰著 戰着 +蘸著 蘸着 +仗著 仗着 +找得著 找得着 +找不著 找不着 +照著 照着 +罩著 罩着 +堅貞著 堅貞着 +忠貞著 忠貞着 +枕著 枕着 +爭著 爭着 +掙著 掙着 +制著 制着 +標志著 標志着 +皺著 皺着 +住著 住着 +抓著 抓着 +轉著 轉着 +裝著 裝着 +追著 追着 +走著 走着 +坐著 坐着 +做著 做着 +含著 含着 +蘊涵著 蘊涵着 +演著 演着 +保障著 保障着 +黏著 黏着 +膠著 膠着 +附著 附着 +代表著 代表着 +浮著 浮着 +寫著 寫着 +遇著 遇着 +殺著 殺着 +驶著 驶着 +著筆 着筆 +著鞭 着鞭 +著法 着法 +著火 着火 +著急 着急 +著艦 着艦 +著腳 着腳 +著她 着她 +著緊 着緊 +著力 着力 +著涼 着涼 +著陸 着陸 +著錄 着錄 +著落 着落 +著忙 着忙 +著迷 着迷 +著墨 着墨 +著妳 着妳 +著你 着你 +著色 着色 +著什 着什 +著實 着實 +著手 着手 +著數 着數 +著絲 着絲 +著他 着他 +著它 着它 +著祂 着祂 +著我 着我 +著想 着想 +著眼 着眼 +著衣 着衣 +著意 着意 +著重 着重 +著裝 着裝 +著地 着地 +不著邊際 不着邊際 +不著痕跡 不着痕跡 +挨著作 挨著作 +挨著者 挨著者 +挨著名 挨著名 +挨著述 挨著述 +挨著稱 挨著稱 +挨著錄 挨著錄 +愛著作 愛著作 +愛著者 愛著者 +愛著名 愛著名 +愛著述 愛著述 +愛著稱 愛著稱 +愛著錄 愛著錄 +愛著書 愛著書 +暗著作 暗著作 +暗著者 暗著者 +暗著名 暗著名 +暗著述 暗著述 +暗著稱 暗著稱 +暗著錄 暗著錄 +暗著書 暗著書 +昂著作 昂著作 +昂著者 昂著者 +昂著名 昂著名 +昂著述 昂著述 +昂著稱 昂著稱 +昂著錄 昂著錄 +昂著書 昂著書 +擺著作 擺著作 +擺著者 擺著者 +擺著名 擺著名 +擺著述 擺著述 +擺著稱 擺著稱 +擺著錄 擺著錄 +伴著作 伴著作 +伴著者 伴著者 +伴著名 伴著名 +伴著述 伴著述 +伴著稱 伴著稱 +伴著錄 伴著錄 +伴著書 伴著書 +辦著作 辦著作 +辦著者 辦著者 +辦著名 辦著名 +辦著述 辦著述 +辦著稱 辦著稱 +辦著錄 辦著錄 +辦著書 辦著書 +幫著作 幫著作 +幫著者 幫著者 +幫著名 幫著名 +幫著述 幫著述 +幫著稱 幫著稱 +幫著錄 幫著錄 +幫著書 幫著書 +綁著作 綁著作 +綁著者 綁著者 +綁著名 綁著名 +綁著述 綁著述 +綁著稱 綁著稱 +綁著錄 綁著錄 +綁著書 綁著書 +抱著作 抱著作 +抱著者 抱著者 +抱著名 抱著名 +抱著述 抱著述 +抱著稱 抱著稱 +抱著錄 抱著錄 +背著作 背著作 +背著者 背著者 +背著名 背著名 +背著述 背著述 +背著稱 背著稱 +背著錄 背著錄 +背著書 背著書 +備著作 備著作 +備著者 備著者 +備著名 備著名 +備著述 備著述 +備著稱 備著稱 +備著錄 備著錄 +備著書 備著書 +本著作 本著作 +本著者 本著者 +本著名 本著名 +本著述 本著述 +本著稱 本著稱 +本著錄 本著錄 +本著書 本著書 +逼著作 逼著作 +逼著者 逼著者 +逼著名 逼著名 +逼著述 逼著述 +逼著稱 逼著稱 +逼著錄 逼著錄 +逼著書 逼著書 +閉著作 閉著作 +閉著者 閉著者 +閉著名 閉著名 +閉著述 閉著述 +閉著稱 閉著稱 +閉著錄 閉著錄 +閉著書 閉著書 +變著作 變著作 +變著者 變著者 +變著名 變著名 +變著述 變著述 +變著稱 變著稱 +變著錄 變著錄 +變著書 變著書 +猜著作 猜著作 +猜著者 猜著者 +猜著名 猜著名 +猜著述 猜著述 +猜著稱 猜著稱 +猜著錄 猜著錄 +猜著書 猜著書 +踩著作 踩著作 +踩著者 踩著者 +踩著名 踩著名 +踩著述 踩著述 +踩著稱 踩著稱 +踩著錄 踩著錄 +踩著書 踩著書 +藏著作 藏著作 +藏著者 藏著者 +藏著名 藏著名 +藏著述 藏著述 +藏著稱 藏著稱 +藏著錄 藏著錄 +藏著書 藏著書 +側著作 側著作 +側著者 側著者 +側著名 側著名 +側著述 側著述 +側著稱 側著稱 +側著錄 側著錄 +側著書 側著書 +纏著作 纏著作 +纏著者 纏著者 +纏著名 纏著名 +纏著述 纏著述 +纏著稱 纏著稱 +纏著錄 纏著錄 +纏著書 纏著書 +敞著作 敞著作 +敞著者 敞著者 +敞著名 敞著名 +敞著述 敞著述 +敞著稱 敞著稱 +敞著錄 敞著錄 +唱著作 唱著作 +唱著者 唱著者 +唱著名 唱著名 +唱著述 唱著述 +唱著稱 唱著稱 +唱著錄 唱著錄 +唱著書 唱著書 +朝著作 朝著作 +朝著者 朝著者 +朝著名 朝著名 +朝著述 朝著述 +朝著稱 朝著稱 +朝著錄 朝著錄 +沉著作 沉著作 +沉著者 沉著者 +沉著名 沉著名 +沉著述 沉著述 +沉著稱 沉著稱 +沉著錄 沉著錄 +沉著書 沉著書 +乘著作 乘著作 +乘著者 乘著者 +乘著名 乘著名 +乘著述 乘著述 +乘著稱 乘著稱 +乘著称 乘著稱 +乘著錄 乘著錄 +乘著書 乘著書 +持著作 持著作 +持著者 持著者 +持著名 持著名 +持著述 持著述 +持著稱 持著稱 +持著錄 持著錄 +斥著作 斥著作 +斥著者 斥著者 +斥著名 斥著名 +斥著述 斥著述 +斥著稱 斥著稱 +斥著錄 斥著錄 +斥著書 斥著書 +醜著作 醜著作 +醜著者 醜著者 +醜著名 醜著名 +醜著述 醜著述 +醜著稱 醜著稱 +醜著錄 醜著錄 +醜著書 醜著書 +穿著作 穿著作 +穿著者 穿著者 +穿著名 穿著名 +穿著述 穿著述 +穿著稱 穿著稱 +穿著錄 穿著錄 +穿著書 穿著書 +吹著作 吹著作 +吹著者 吹著者 +吹著名 吹著名 +吹著述 吹著述 +吹著稱 吹著稱 +吹著錄 吹著錄 +吹著書 吹著書 +達著作 達著作 +達著者 達著者 +達著名 達著名 +達著述 達著述 +達著稱 達著稱 +達著錄 達著錄 +達著書 達著書 +打著作 打著作 +打著者 打著者 +打著名 打著名 +打著述 打著述 +打著稱 打著稱 +打著錄 打著錄 +打著書 打著書 +待著作 待著作 +待著者 待著者 +待著名 待著名 +待著述 待著述 +待著稱 待著稱 +待著錄 待著錄 +待著書 待著書 +帶著作 帶著作 +帶著者 帶著者 +帶著名 帶著名 +帶著述 帶著述 +帶著稱 帶著稱 +帶著錄 帶著錄 +帶著書 帶著書 +戴著作 戴著作 +戴著者 戴著者 +戴著名 戴著名 +戴著述 戴著述 +戴著稱 戴著稱 +戴著錄 戴著錄 +戴著書 戴著書 +當著作 當著作 +當著者 當著者 +當著名 當著名 +當著述 當著述 +當著稱 當著稱 +當著錄 當著錄 +當著書 當著書 +擋著作 擋著作 +擋著者 擋著者 +擋著名 擋著名 +擋著述 擋著述 +擋著稱 擋著稱 +擋著錄 擋著錄 +得著作 得著作 +得著者 得著者 +得著名 得著名 +得著述 得著述 +得著稱 得著稱 +得著錄 得著錄 +得著書 得著書 +瞪著作 瞪著作 +瞪著者 瞪著者 +瞪著名 瞪著名 +瞪著述 瞪著述 +瞪著稱 瞪著稱 +瞪著錄 瞪著錄 +瞪著書 瞪著書 +低著作 低著作 +低著者 低著者 +低著名 低著名 +低著述 低著述 +低著稱 低著稱 +低著称 低著稱 +低著錄 低著錄 +低著書 低著書 +點著作 點著作 +點著者 點著者 +點著名 點著名 +點著述 點著述 +點著稱 點著稱 +點著錄 點著錄 +點著書 點著書 +盯著作 盯著作 +盯著者 盯著者 +盯著名 盯著名 +盯著述 盯著述 +盯著稱 盯著稱 +盯著錄 盯著錄 +盯著書 盯著書 +頂著作 頂著作 +頂著者 頂著者 +頂著名 頂著名 +頂著述 頂著述 +頂著稱 頂著稱 +頂著錄 頂著錄 +頂著書 頂著書 +定著作 定著作 +定著者 定著者 +定著名 定著名 +定著述 定著述 +定著稱 定著稱 +定著称 定著稱 +定著錄 定著錄 +定著書 定著書 +動著作 動著作 +動著者 動著者 +動著名 動著名 +動著述 動著述 +動著稱 動著稱 +動著錄 動著錄 +動著書 動著書 +鬥著作 鬥著作 +鬥著者 鬥著者 +鬥著名 鬥著名 +鬥著述 鬥著述 +鬥著稱 鬥著稱 +鬥著錄 鬥著錄 +鬥著書 鬥著書 +對著作 對著作 +對著者 對著者 +對著名 對著名 +對著述 對著述 +對著稱 對著稱 +對著錄 對著錄 +對著書 對著書 +犯不著作 犯不著作 +犯不著者 犯不著者 +犯不著名 犯不著名 +犯不著述 犯不著述 +犯不著稱 犯不著稱 +犯不著錄 犯不著錄 +犯不著書 犯不著書 +福著作 福著作 +福著者 福著者 +福著名 福著名 +福著述 福著述 +福著稱 福著稱 +福著錄 福著錄 +福著書 福著書 +趕著作 趕著作 +趕著者 趕著者 +趕著名 趕著名 +趕著述 趕著述 +趕著稱 趕著稱 +趕著錄 趕著錄 +趕著書 趕著書 +高著作 高著作 +高著者 高著者 +高著名 高著名 +高著述 高著述 +高著稱 高著稱 +高著称 高著稱 +高著錄 高著錄 +高著書 高著書 +隔著作 隔著作 +隔著者 隔著者 +隔著名 隔著名 +隔著述 隔著述 +隔著稱 隔著稱 +隔著錄 隔著錄 +隔著書 隔著書 +跟著作 跟著作 +跟著者 跟著者 +跟著名 跟著名 +跟著述 跟著述 +跟著稱 跟著稱 +跟著錄 跟著錄 +跟著書 跟著書 +關著作 關著作 +關著者 關著者 +關著名 關著名 +關著述 關著述 +關著稱 關著稱 +關著錄 關著錄 +關著書 關著書 +管著作 管著作 +管著者 管著者 +管著名 管著名 +管著述 管著述 +管著稱 管著稱 +管著錄 管著錄 +管著書 管著書 +慣著作 慣著作 +慣著者 慣著者 +慣著名 慣著名 +慣著述 慣著述 +慣著稱 慣著稱 +慣著錄 慣著錄 +慣著書 慣著書 +光著作 光著作 +光著者 光著者 +光著名 光著名 +光著述 光著述 +光著稱 光著稱 +光著称 光著稱 +光著錄 光著錄 +光著書 光著書 +跪著作 跪著作 +跪著者 跪著者 +跪著名 跪著名 +跪著述 跪著述 +跪著稱 跪著稱 +跪著錄 跪著錄 +跪著書 跪著書 +裹著作 裹著作 +裹著者 裹著者 +裹著名 裹著名 +裹著述 裹著述 +裹著稱 裹著稱 +裹著錄 裹著錄 +裹著書 裹著書 +撼著作 撼著作 +撼著者 撼著者 +撼著名 撼著名 +撼著述 撼著述 +撼著稱 撼著稱 +撼著錄 撼著錄 +撼著書 撼著書 +喝著作 喝著作 +喝著者 喝著者 +喝著名 喝著名 +喝著述 喝著述 +喝著稱 喝著稱 +喝著錄 喝著錄 +喝著書 喝著書 +候著作 候著作 +候著者 候著者 +候著名 候著名 +候著述 候著述 +候著稱 候著稱 +候著錄 候著錄 +候著書 候著書 +懷著作 懷著作 +懷著者 懷著者 +懷著名 懷著名 +懷著述 懷著述 +懷著稱 懷著稱 +懷著錄 懷著錄 +懷著書 懷著書 +晃著作 晃著作 +晃著者 晃著者 +晃著名 晃著名 +晃著述 晃著述 +晃著稱 晃著稱 +晃著錄 晃著錄 +揮著作 揮著作 +揮著者 揮著者 +揮著名 揮著名 +揮著述 揮著述 +揮著稱 揮著稱 +揮著錄 揮著錄 +活著作 活著作 +活著者 活著者 +活著名 活著名 +活著述 活著述 +活著稱 活著稱 +活著錄 活著錄 +活著書 活著書 +獲著作 獲著作 +獲著者 獲著者 +獲著名 獲著名 +獲著述 獲著述 +獲著稱 獲著稱 +獲著錄 獲著錄 +獲著書 獲著書 +急著作 急著作 +急著者 急著者 +急著名 急著名 +急著述 急著述 +急著稱 急著稱 +急著錄 急著錄 +急著書 急著書 +記著作 記著作 +記著者 記著者 +記著名 記著名 +記著述 記著述 +記著稱 記著稱 +記著錄 記著錄 +記著書 記著書 +夾著作 夾著作 +夾著者 夾著者 +夾著名 夾著名 +夾著述 夾著述 +夾著稱 夾著稱 +夾著錄 夾著錄 +夾著書 夾著書 +駕著作 駕著作 +駕著者 駕著者 +駕著名 駕著名 +駕著述 駕著述 +駕著稱 駕著稱 +駕著錄 駕著錄 +駕著書 駕著書 +見著作 見著作 +見著者 見著者 +見著名 見著名 +見著述 見著述 +見著稱 見著稱 +見著錄 見著錄 +見著書 見著書 +閑著作 閑著作 +閑著者 閑著者 +閑著名 閑著名 +閑著述 閑著述 +閑著稱 閑著稱 +閑著錄 閑著錄 +閑著書 閑著書 +叫著作 叫著作 +叫著者 叫著者 +叫著名 叫著名 +叫著述 叫著述 +叫著稱 叫著稱 +叫著錄 叫著錄 +叫著書 叫著書 +接著作 接著作 +接著者 接著者 +接著名 接著名 +接著述 接著述 +接著稱 接著稱 +接著錄 接著錄 +借著作 借著作 +借著者 借著者 +借著名 借著名 +借著述 借著述 +借著稱 借著稱 +借著錄 借著錄 +借著書 借著書 +據著作 據著作 +據著者 據著者 +據著名 據著名 +據著述 據著述 +據著稱 據著稱 +據著錄 據著錄 +據著書 據著書 +開著作 開著作 +開著者 開著者 +開著名 開著名 +開著述 開著述 +開著稱 開著稱 +開著錄 開著錄 +開著書 開著書 +看著作 看著作 +看著者 看著者 +看著名 看著名 +看著述 看著述 +看著稱 看著稱 +看著錄 看著錄 +看著書 看著書 +扛著作 扛著作 +扛著者 扛著者 +扛著名 扛著名 +扛著述 扛著述 +扛著稱 扛著稱 +扛著錄 扛著錄 +扛著書 扛著書 +考著作 考著作 +考著者 考著者 +考著名 考著名 +考著述 考著述 +考著稱 考著稱 +考著錄 考著錄 +考著書 考著書 +渴著作 渴著作 +渴著者 渴著者 +渴著名 渴著名 +渴著述 渴著述 +渴著稱 渴著稱 +渴著錄 渴著錄 +渴著書 渴著書 +刻著作 刻著作 +刻著者 刻著者 +刻著名 刻著名 +刻著述 刻著述 +刻著稱 刻著稱 +刻著称 刻著稱 +刻著錄 刻著錄 +刻著書 刻著書 +空著作 空著作 +空著者 空著者 +空著名 空著名 +空著述 空著述 +空著稱 空著稱 +空著錄 空著錄 +空著書 空著書 +哭著作 哭著作 +哭著者 哭著者 +哭著名 哭著名 +哭著述 哭著述 +哭著稱 哭著稱 +哭著錄 哭著錄 +哭著書 哭著書 +苦著作 苦著作 +苦著者 苦著者 +苦著名 苦著名 +苦著述 苦著述 +苦著稱 苦著稱 +苦著錄 苦著錄 +苦著書 苦著書 +捆著作 捆著作 +捆著者 捆著者 +捆著名 捆著名 +捆著述 捆著述 +捆著稱 捆著稱 +捆著錄 捆著錄 +困著作 困著作 +困著者 困著者 +困著名 困著名 +困著述 困著述 +困著稱 困著稱 +困著錄 困著錄 +困著書 困著書 +拉著作 拉著作 +拉著者 拉著者 +拉著名 拉著名 +拉著述 拉著述 +拉著稱 拉著稱 +拉著錄 拉著錄 +拉著書 拉著書 +來著作 來著作 +來著者 來著者 +來著名 來著名 +來著述 來著述 +來著稱 來著稱 +來著錄 來著錄 +來著書 來著書 +樂著作 樂著作 +樂著者 樂著者 +樂著名 樂著名 +樂著述 樂著述 +樂著稱 樂著稱 +樂著錄 樂著錄 +樂著書 樂著書 +努力著作 努力著作 +努力著者 努力著者 +努力著名 努力著名 +努力著述 努力著述 +努力著稱 努力著稱 +努力著称 努力著稱 +努力著錄 努力著錄 +努力著書 努力著書 +麗著作 麗著作 +麗著者 麗著者 +麗著名 麗著名 +麗著述 麗著述 +麗著稱 麗著稱 +麗著錄 麗著錄 +麗著書 麗著書 +連著作 連著作 +連著者 連著者 +連著名 連著名 +連著述 連著述 +連著稱 連著稱 +連著錄 連著錄 +連著書 連著書 +戀著作 戀著作 +戀著者 戀著者 +戀著名 戀著名 +戀著述 戀著述 +戀著稱 戀著稱 +戀著錄 戀著錄 +戀著書 戀著書 +涼著作 涼著作 +涼著者 涼著者 +涼著名 涼著名 +涼著述 涼著述 +涼著稱 涼著稱 +涼著錄 涼著錄 +涼著書 涼著書 +亮著作 亮著作 +亮著者 亮著者 +亮著名 亮著名 +亮著述 亮著述 +亮著稱 亮著稱 +亮著称 亮著稱 +亮著錄 亮著錄 +亮著書 亮著書 +臨著作 臨著作 +臨著者 臨著者 +臨著名 臨著名 +臨著述 臨著述 +臨著稱 臨著稱 +臨著錄 臨著錄 +臨著書 臨著書 +拎著作 拎著作 +拎著者 拎著者 +拎著名 拎著名 +拎著述 拎著述 +拎著稱 拎著稱 +拎著錄 拎著錄 +領著作 領著作 +領著者 領著者 +領著名 領著名 +領著述 領著述 +領著稱 領著稱 +領著錄 領著錄 +領著書 領著書 +流著作 流著作 +流著者 流著者 +流著名 流著名 +流著述 流著述 +流著稱 流著稱 +流著錄 流著錄 +流著書 流著書 +留著作 留著作 +留著者 留著者 +留著名 留著名 +留著述 留著述 +留著稱 留著稱 +留著錄 留著錄 +留著書 留著書 +摟著作 摟著作 +摟著者 摟著者 +摟著名 摟著名 +摟著述 摟著述 +摟著稱 摟著稱 +摟著錄 摟著錄 +陋著作 陋著作 +陋著者 陋著者 +陋著名 陋著名 +陋著述 陋著述 +陋著稱 陋著稱 +陋著錄 陋著錄 +陋著書 陋著書 +落著作 落著作 +落著者 落著者 +落著名 落著名 +落著述 落著述 +落著稱 落著稱 +落著錄 落著錄 +落著書 落著書 +罵著作 罵著作 +罵著者 罵著者 +罵著名 罵著名 +罵著述 罵著述 +罵著稱 罵著稱 +罵著錄 罵著錄 +罵著書 罵著書 +瞞著作 瞞著作 +瞞著者 瞞著者 +瞞著名 瞞著名 +瞞著述 瞞著述 +瞞著稱 瞞著稱 +瞞著錄 瞞著錄 +瞞著書 瞞著書 +漫著作 漫著作 +漫著者 漫著者 +漫著名 漫著名 +漫著述 漫著述 +漫著稱 漫著稱 +漫著錄 漫著錄 +漫著書 漫著書 +忙著作 忙著作 +忙著者 忙著者 +忙著名 忙著名 +忙著述 忙著述 +忙著稱 忙著稱 +忙著錄 忙著錄 +忙著書 忙著書 +冒著作 冒著作 +冒著者 冒著者 +冒著名 冒著名 +冒著述 冒著述 +冒著稱 冒著稱 +冒著錄 冒著錄 +冒著書 冒著書 +美著作 美著作 +美著者 美著者 +美著名 美著名 +美著述 美著述 +美著稱 美著稱 +美著称 美著稱 +美著錄 美著錄 +美著書 美著書 +夢著作 夢著作 +夢著者 夢著者 +夢著名 夢著名 +夢著述 夢著述 +夢著稱 夢著稱 +夢著錄 夢著錄 +夢著書 夢著書 +蒙著作 蒙著作 +蒙著者 蒙著者 +蒙著名 蒙著名 +蒙著述 蒙著述 +蒙著稱 蒙著稱 +蒙著錄 蒙著錄 +蒙著書 蒙著書 +拿著作 拿著作 +拿著者 拿著者 +拿著名 拿著名 +拿著述 拿著述 +拿著稱 拿著稱 +拿著錄 拿著錄 +逆著作 逆著作 +逆著者 逆著者 +逆著名 逆著名 +逆著述 逆著述 +逆著稱 逆著稱 +逆著錄 逆著錄 +逆著書 逆著書 +釀著作 釀著作 +釀著者 釀著者 +釀著名 釀著名 +釀著述 釀著述 +釀著稱 釀著稱 +釀著錄 釀著錄 +釀著書 釀著書 +趴著作 趴著作 +趴著者 趴著者 +趴著名 趴著名 +趴著述 趴著述 +趴著稱 趴著稱 +趴著錄 趴著錄 +趴著書 趴著書 +跑著作 跑著作 +跑著者 跑著者 +跑著名 跑著名 +跑著述 跑著述 +跑著稱 跑著稱 +跑著錄 跑著錄 +跑著書 跑著書 +陪著作 陪著作 +陪著者 陪著者 +陪著名 陪著名 +陪著述 陪著述 +陪著稱 陪著稱 +陪著錄 陪著錄 +陪著書 陪著書 +配著作 配著作 +配著者 配著者 +配著名 配著名 +配著述 配著述 +配著稱 配著稱 +配著錄 配著錄 +配著書 配著書 +披著作 披著作 +披著者 披著者 +披著名 披著名 +披著述 披著述 +披著稱 披著稱 +披著錄 披著錄 +披著書 披著書 +騙著作 騙著作 +騙著者 騙著者 +騙著名 騙著名 +騙著述 騙著述 +騙著稱 騙著稱 +騙著錄 騙著錄 +騙著書 騙著書 +飄著作 飄著作 +飄著者 飄著者 +飄著名 飄著名 +飄著述 飄著述 +飄著稱 飄著稱 +飄著錄 飄著錄 +飄著書 飄著書 +拼著作 拼著作 +拼著者 拼著者 +拼著名 拼著名 +拼著述 拼著述 +拼著稱 拼著稱 +拼著錄 拼著錄 +鋪著作 鋪著作 +鋪著者 鋪著者 +鋪著名 鋪著名 +鋪著述 鋪著述 +鋪著稱 鋪著稱 +鋪著錄 鋪著錄 +鋪著書 鋪著書 +騎著作 騎著作 +騎著者 騎著者 +騎著名 騎著名 +騎著述 騎著述 +騎著稱 騎著稱 +騎著錄 騎著錄 +騎著書 騎著書 +牽著作 牽著作 +牽著者 牽著者 +牽著名 牽著名 +牽著述 牽著述 +牽著稱 牽著稱 +牽著錄 牽著錄 +牽著書 牽著書 +求著作 求著作 +求著者 求著者 +求著名 求著名 +求著述 求著述 +求著稱 求著稱 +求著錄 求著錄 +求著書 求著書 +嚷著作 嚷著作 +嚷著者 嚷著者 +嚷著名 嚷著名 +嚷著述 嚷著述 +嚷著稱 嚷著稱 +嚷著錄 嚷著錄 +嚷著書 嚷著書 +繞著作 繞著作 +繞著者 繞著者 +繞著名 繞著名 +繞著述 繞著述 +繞著稱 繞著稱 +繞著錄 繞著錄 +繞著書 繞著書 +忍著作 忍著作 +忍著者 忍著者 +忍著名 忍著名 +忍著述 忍著述 +忍著稱 忍著稱 +忍著錄 忍著錄 +忍著書 忍著書 +揉著作 揉著作 +揉著者 揉著者 +揉著名 揉著名 +揉著述 揉著述 +揉著稱 揉著稱 +揉著錄 揉著錄 +揉著書 揉著書 +潤著作 潤著作 +潤著者 潤著者 +潤著名 潤著名 +潤著述 潤著述 +潤著稱 潤著稱 +潤著錄 潤著錄 +潤著書 潤著書 +燒著作 燒著作 +燒著者 燒著者 +燒著名 燒著名 +燒著述 燒著述 +燒著稱 燒著稱 +燒著錄 燒著錄 +燒著書 燒著書 +身著作 身著作 +身著者 身著者 +身著名 身著名 +身著述 身著述 +身著稱 身著稱 +身著錄 身著錄 +身著書 身著書 +盛著作 盛著作 +盛著者 盛著者 +盛著名 盛著名 +盛著述 盛著述 +盛著稱 盛著稱 +盛著錄 盛著錄 +盛著書 盛著書 +試著作 試著作 +試著者 試著者 +試著名 試著名 +試著述 試著述 +試著稱 試著稱 +試著錄 試著錄 +試著書 試著書 +守著作 守著作 +守著者 守著者 +守著名 守著名 +守著述 守著述 +守著稱 守著稱 +守著称 守著稱 +守著錄 守著錄 +守著書 守著書 +受著作 受著作 +受著者 受著者 +受著名 受著名 +受著述 受著述 +受著稱 受著稱 +受著錄 受著錄 +受著書 受著書 +梳著作 梳著作 +梳著者 梳著者 +梳著名 梳著名 +梳著述 梳著述 +梳著稱 梳著稱 +梳著錄 梳著錄 +豎著作 豎著作 +豎著者 豎著者 +豎著名 豎著名 +豎著述 豎著述 +豎著稱 豎著稱 +豎著錄 豎著錄 +豎著書 豎著書 +數著作 數著作 +數著者 數著者 +數著名 數著名 +數著述 數著述 +數著稱 數著稱 +數著錄 數著錄 +睡著作 睡著作 +睡著者 睡著者 +睡著名 睡著名 +睡著述 睡著述 +睡著稱 睡著稱 +睡著錄 睡著錄 +睡著書 睡著書 +順著作 順著作 +順著者 順著者 +順著名 順著名 +順著述 順著述 +順著稱 順著稱 +順著錄 順著錄 +順著書 順著書 +隨著作 隨著作 +隨著者 隨著者 +隨著名 隨著名 +隨著述 隨著述 +隨著稱 隨著稱 +隨著錄 隨著錄 +隨著書 隨著書 +踏著作 踏著作 +踏著者 踏著者 +踏著名 踏著名 +踏著述 踏著述 +踏著稱 踏著稱 +踏著錄 踏著錄 +抬著作 抬著作 +抬著者 抬著者 +抬著名 抬著名 +抬著述 抬著述 +抬著稱 抬著稱 +抬著錄 抬著錄 +躺著作 躺著作 +躺著者 躺著者 +躺著名 躺著名 +躺著述 躺著述 +躺著稱 躺著稱 +躺著錄 躺著錄 +躺著書 躺著書 +提著作 提著作 +提著者 提著者 +提著名 提著名 +提著述 提著述 +提著稱 提著稱 +提著錄 提著錄 +甜著作 甜著作 +甜著者 甜著者 +甜著名 甜著名 +甜著述 甜著述 +甜著稱 甜著稱 +甜著錄 甜著錄 +甜著書 甜著書 +挑著作 挑著作 +挑著者 挑著者 +挑著名 挑著名 +挑著述 挑著述 +挑著稱 挑著稱 +挑著錄 挑著錄 +跳著作 跳著作 +跳著者 跳著者 +跳著名 跳著名 +跳著述 跳著述 +跳著稱 跳著稱 +跳著錄 跳著錄 +跳著書 跳著書 +聽著作 聽著作 +聽著者 聽著者 +聽著名 聽著名 +聽著述 聽著述 +聽著稱 聽著稱 +聽著錄 聽著錄 +聽著書 聽著書 +偷著作 偷著作 +偷著者 偷著者 +偷著名 偷著名 +偷著述 偷著述 +偷著稱 偷著稱 +偷著錄 偷著錄 +偷著書 偷著書 +拖著作 拖著作 +拖著者 拖著者 +拖著名 拖著名 +拖著述 拖著述 +拖著稱 拖著稱 +拖著錄 拖著錄 +望著作 望著作 +望著者 望著者 +望著名 望著名 +望著述 望著述 +望著稱 望著稱 +望著錄 望著錄 +望著書 望著書 +圍著作 圍著作 +圍著者 圍著者 +圍著名 圍著名 +圍著述 圍著述 +圍著稱 圍著稱 +圍著錄 圍著錄 +圍著書 圍著書 +味著作 味著作 +味著者 味著者 +味著名 味著名 +味著述 味著述 +味著稱 味著稱 +味著称 味著稱 +味著錄 味著錄 +味著書 味著書 +想著作 想著作 +想著者 想著者 +想著名 想著名 +想著述 想著述 +想著稱 想著稱 +想著称 想著稱 +想著錄 想著錄 +想著書 想著書 +響著作 響著作 +響著者 響著者 +響著名 響著名 +響著述 響著述 +響著稱 響著稱 +響著錄 響著錄 +響著書 響著書 +向著作 向著作 +向著者 向著者 +向著名 向著名 +向著述 向著述 +向著稱 向著稱 +向著錄 向著錄 +向著書 向著書 +笑著作 笑著作 +笑著者 笑著者 +笑著名 笑著名 +笑著述 笑著述 +笑著稱 笑著稱 +笑著錄 笑著錄 +笑著書 笑著書 +心著作 心著作 +心著者 心著者 +心著名 心著名 +心著述 心著述 +心著稱 心著稱 +心著称 心著稱 +心著錄 心著錄 +心著書 心著書 +信著作 信著作 +信著者 信著者 +信著名 信著名 +信著述 信著述 +信著稱 信著稱 +信著称 信著稱 +信著錄 信著錄 +信著書 信著書 +行著作 行著作 +行著者 行著者 +行著名 行著名 +行著述 行著述 +行著稱 行著稱 +行著錄 行著錄 +行著書 行著書 +學著作 學著作 +學著者 學著者 +學著名 學著名 +學著述 學著述 +學著稱 學著稱 +學著錄 學著錄 +學著書 學著書 +尋著作 尋著作 +尋著者 尋著者 +尋著名 尋著名 +尋著述 尋著述 +尋著稱 尋著稱 +尋著錄 尋著錄 +尋著書 尋著書 +循著作 循著作 +循著者 循著者 +循著名 循著名 +循著述 循著述 +循著稱 循著稱 +循著錄 循著錄 +循著書 循著書 +壓著作 壓著作 +壓著者 壓著者 +壓著名 壓著名 +壓著述 壓著述 +壓著稱 壓著稱 +壓著錄 壓著錄 +壓著書 壓著書 +雅著作 雅著作 +雅著者 雅著者 +雅著名 雅著名 +雅著述 雅著述 +雅著稱 雅著稱 +雅著称 雅著稱 +雅著錄 雅著錄 +雅著書 雅著書 +沿著作 沿著作 +沿著者 沿著者 +沿著名 沿著名 +沿著述 沿著述 +沿著稱 沿著稱 +沿著錄 沿著錄 +沿著書 沿著書 +耀著作 耀著作 +耀著者 耀著者 +耀著名 耀著名 +耀著述 耀著述 +耀著稱 耀著稱 +耀著錄 耀著錄 +耀著書 耀著書 +掖著作 掖著作 +掖著者 掖著者 +掖著名 掖著名 +掖著述 掖著述 +掖著稱 掖著稱 +掖著錄 掖著錄 +衣著作 衣著作 +衣著者 衣著者 +衣著名 衣著名 +衣著述 衣著述 +衣著稱 衣著稱 +衣著稱 衣著稱 +衣著錄 衣著錄 +衣著書 衣著書 +疑著作 疑著作 +疑著者 疑著者 +疑著名 疑著名 +疑著述 疑著述 +疑著稱 疑著稱 +疑著錄 疑著錄 +疑著書 疑著書 +溢著作 溢著作 +溢著者 溢著者 +溢著名 溢著名 +溢著述 溢著述 +溢著稱 溢著稱 +溢著錄 溢著錄 +溢著書 溢著書 +因著作 因著作 +因著者 因著者 +因著名 因著名 +因著述 因著述 +因著稱 因著稱 +因著錄 因著錄 +因著書 因著書 +因著《 因著《 +因著〈 因著〈 +印著作 印著作 +印著者 印著者 +印著名 印著名 +印著述 印著述 +印著稱 印著稱 +印著錄 印著錄 +印著書 印著書 +應著作 應著作 +應著者 應著者 +應著名 應著名 +應著述 應著述 +應著稱 應著稱 +應著錄 應著錄 +應著書 應著書 +映著作 映著作 +映著者 映著者 +映著名 映著名 +映著述 映著述 +映著稱 映著稱 +映著錄 映著錄 +映著書 映著書 +用著作 用著作 +用著者 用著者 +用著名 用著名 +用著述 用著述 +用著稱 用著稱 +用著錄 用著錄 +用著書 用著書 +悠著作 悠著作 +悠著者 悠著者 +悠著名 悠著名 +悠著述 悠著述 +悠著稱 悠著稱 +悠著錄 悠著錄 +悠著書 悠著書 +有著作 有著作 +有著者 有著者 +有著名 有著名 +有著述 有著述 +有著稱 有著稱 +有著錄 有著錄 +有著書 有著書 +與著作 與著作 +與著者 與著者 +與著名 與著名 +與著述 與著述 +與著稱 與著稱 +與著錄 與著錄 +與著書 與著書 +語著作 語著作 +語著者 語著者 +語著名 語著名 +語著述 語著述 +語著稱 語著稱 +語著錄 語著錄 +語著書 語著書 +躍著作 躍著作 +躍著者 躍著者 +躍著名 躍著名 +躍著述 躍著述 +躍著稱 躍著稱 +躍著錄 躍著錄 +躍著書 躍著書 +雜著作 雜著作 +雜著者 雜著者 +雜著名 雜著名 +雜著述 雜著述 +雜著稱 雜著稱 +雜著錄 雜著錄 +雜著書 雜著書 +載著作 載著作 +載著者 載著者 +載著名 載著名 +載著述 載著述 +載著稱 載著稱 +載著錄 載著錄 +載著書 載著書 +紮著作 紮著作 +紮著者 紮著者 +紮著名 紮著名 +紮著述 紮著述 +紮著稱 紮著稱 +紮著錄 紮著錄 +紮著書 紮著書 +展著作 展著作 +展著者 展著者 +展著名 展著名 +展著述 展著述 +展著稱 展著稱 +展著錄 展著錄 +展著書 展著書 +站著作 站著作 +站著者 站著者 +站著名 站著名 +站著述 站著述 +站著稱 站著稱 +站著錄 站著錄 +站著書 站著書 +戰著作 戰著作 +戰著者 戰著者 +戰著名 戰著名 +戰著述 戰著述 +戰著稱 戰著稱 +戰著錄 戰著錄 +戰著書 戰著書 +蘸著作 蘸著作 +蘸著者 蘸著者 +蘸著名 蘸著名 +蘸著述 蘸著述 +蘸著稱 蘸著稱 +蘸著錄 蘸著錄 +蘸著書 蘸著書 +仗著作 仗著作 +仗著者 仗著者 +仗著名 仗著名 +仗著述 仗著述 +仗著稱 仗著稱 +仗著錄 仗著錄 +仗著書 仗著書 +照著作 照著作 +照著者 照著者 +照著名 照著名 +照著述 照著述 +照著稱 照著稱 +照著錄 照著錄 +照著書 照著書 +罩著作 罩著作 +罩著者 罩著者 +罩著名 罩著名 +罩著述 罩著述 +罩著稱 罩著稱 +罩著錄 罩著錄 +罩著書 罩著書 +枕著作 枕著作 +枕著者 枕著者 +枕著名 枕著名 +枕著述 枕著述 +枕著稱 枕著稱 +枕著錄 枕著錄 +爭著作 爭著作 +爭著者 爭著者 +爭著名 爭著名 +爭著述 爭著述 +爭著稱 爭著稱 +爭著錄 爭著錄 +爭著書 爭著書 +掙著作 掙著作 +掙著者 掙著者 +掙著名 掙著名 +掙著述 掙著述 +掙著稱 掙著稱 +掙著錄 掙著錄 +掙著書 掙著書 +制著作 制著作 +制著者 制著者 +制著名 制著名 +制著述 制著述 +制著稱 制著稱 +制著錄 制著錄 +制著書 制著書 +皺著作 皺著作 +皺著者 皺著者 +皺著名 皺著名 +皺著述 皺著述 +皺著稱 皺著稱 +皺著錄 皺著錄 +皺著書 皺著書 +住著作 住著作 +住著者 住著者 +住著名 住著名 +住著述 住著述 +住著稱 住著稱 +住著錄 住著錄 +住著書 住著書 +抓著作 抓著作 +抓著者 抓著者 +抓著名 抓著名 +抓著述 抓著述 +抓著稱 抓著稱 +抓著錄 抓著錄 +轉著作 轉著作 +轉著者 轉著者 +轉著名 轉著名 +轉著述 轉著述 +轉著稱 轉著稱 +轉著錄 轉著錄 +轉著書 轉著書 +裝著作 裝著作 +裝著者 裝著者 +裝著名 裝著名 +裝著述 裝著述 +裝著稱 裝著稱 +裝著錄 裝著錄 +裝著書 裝著書 +追著作 追著作 +追著者 追著者 +追著名 追著名 +追著述 追著述 +追著稱 追著稱 +追著錄 追著錄 +追著書 追著書 +走著作 走著作 +走著者 走著者 +走著名 走著名 +走著述 走著述 +走著稱 走著稱 +走著錄 走著錄 +走著書 走著書 +坐著作 坐著作 +坐著者 坐著者 +坐著名 坐著名 +坐著述 坐著述 +坐著稱 坐著稱 +坐著錄 坐著錄 +坐著書 坐著書 +做著作 做著作 +做著者 做著者 +做著名 做著名 +做著述 做著述 +做著稱 做著稱 +做著錄 做著錄 +做著書 做著書 +含著作 含著作 +含著者 含著者 +含著名 含著名 +含著述 含著述 +含著稱 含著稱 +含著錄 含著錄 +含著書 含著書 +演著作 演著作 +演著者 演著者 +演著名 演著名 +演著述 演著述 +演著稱 演著稱 +演著錄 演著錄 +演著書 演著書 +保障著作 保障著作 +保障著者 保障著者 +保障著名 保障著名 +保障著述 保障著述 +保障著稱 保障著稱 +保障著錄 保障著錄 +保障著書 保障著書 +黏著作 黏著作 +黏著者 黏著者 +黏著名 黏著名 +黏著述 黏著述 +黏著稱 黏著稱 +黏著錄 黏著錄 +黏著書 黏著書 +膠著作 膠著作 +膠著者 膠著者 +膠著名 膠著名 +膠著述 膠著述 +膠著稱 膠著稱 +膠著錄 膠著錄 +膠著書 膠著書 +附著作 附著作 +附著者 附著者 +附著名 附著名 +附著述 附著述 +附著稱 附著稱 +附著錄 附著錄 +附著書 附著書 +代表著作 代表著作 +代表著者 代表著者 +代表著名 代表著名 +代表著述 代表著述 +代表著稱 代表著稱 +代表著錄 代表著錄 +代表著書 代表著書 +浮著作 浮著作 +浮著者 浮著者 +浮著名 浮著名 +浮著述 浮著述 +浮著稱 浮著稱 +浮著錄 浮著錄 +浮著書 浮著書 +寫著作 寫著作 +寫著者 寫著者 +寫著名 寫著名 +寫著述 寫著述 +寫著稱 寫著稱 +寫著錄 寫著錄 +寫著書 寫著書 +遇著作 遇著作 +遇著者 遇著者 +遇著名 遇著名 +遇著述 遇著述 +遇著稱 遇著稱 +遇著称 遇著稱 +遇著錄 遇著錄 +遇著書 遇著書 +殺著作 殺著作 +殺著者 殺著者 +殺著名 殺著名 +殺著述 殺著述 +殺著稱 殺著稱 +殺著錄 殺著錄 +殺著書 殺著書 +標誌著 標誌着 +幹著 幹着 +幹著名 幹著名 +幹著稱 幹著稱 +干着 幹着 +干着急 干着急 +流露著 流露着 +靠著 靠着 +靠著作 靠著作 +靠著名 靠著名 +靠著錄 靠著錄 +靠著录 靠著錄 +靠著稱 靠著稱 +靠著称 靠著稱 +靠著者 靠著者 +靠著述 靠著述 +迫著 迫着 +繫著 繫着 +藉著 藉着 +吃得著 吃得着 +吃不著 吃不着 +吃著 吃着 +聞得著 闻得着 +聞不著 闻不着 +聞著 闻着 +嗅得著 嗅得着 +嗅不著 嗅不着 +嗅著 嗅着 +警戒著 警戒着 +過著 過着 +過著作 當著作 +過著者 當著者 +過著名 當著名 +過著述 當著述 +過著稱 當著稱 +過著錄 當著錄 +過著書 當著書 +穫著 穫着 +閒著 閒着 +飃著 飃着 +沈著 沈着 +竪著 竪着 +擡著 擡着 +沖著 沖着 +沖著《 沖著《 +沖著。 沖著。 +沖著, 沖著, +衝著 衝着 +著甚麼 着甚麼 +存著 存着 +存著名 存著名 +存著作 存著作 +劃著 劃着 +別著 別着 +刮著 刮着 +掛著 掛着 +吊著 吊着 +回著 回着 +回著名 回著名 +塗著 塗着 +麼著 麼着 +擔著 擔着 +負著 負着 +板著臉 板着臉 +為著 為着 +為著作 為著作 +為著名 為著名 +為著錄 為著錄 +為著稱 為著稱 +為著者 為著者 +為著述 為著述 +為著《 為著《 +畫著 畫着 +畫著作 畫著作 +畫著名 畫著名 +畫著稱 畫著稱 +畫著者 畫著者 +發著 發着 +發著作 發著作 +發著名 發著名 +發著稱 發著稱 +發著者 發著者 +發著《 發著《 +簽著 簽着 +繃著 繃着 +覆著 覆着 +蓋著 蓋着 +說著 說着 +說著作 說著作 +說著稱 說著稱 +說著者 說著者 +說著述 說著述 +象徵著 象著着 +象徵著名 象徵著名 +湊合著 湊合着 +配合著 配合着 +配合著名 配合著名 +關係著 關係着 +下著 下着 +下著作 下著作 +下著名 下著名 +下著录 下著錄 +下著錄 下著錄 +下著称 下著稱 +下著稱 下著稱 +下著者 下著者 +下著述 下著述 +下著有 下著有 +放著 放着 +放著作 放著作 +放著名 放著名 +放著稱 放著稱 +放著称 放著稱 +縱著 縱着 +伏著 伏着 +視著 視着 +視著名 視著名 +視著作 視著作 +視著者 視著者 +視著稱 視著稱 +蓋著 蓋着 +蓋著名 蓋著名 +蓋著稱 蓋著稱 +蓋著作 蓋著作 +覆蓋著 覆蓋着 +立著 立着 +立著名 立著名 +立著作 立著作 +立著者 立著者 +立著稱 立著稱 +立著称 立著稱 +立著有 立著有 +立著《 立著《 +立著( 立著( +固著 固着 +班固著 班固著 +面包著 面包着 +分布著 分佈着 +分佈著 分佈着 +散布著 散佈着 +散佈著 散佈着 +遍佈著 遍佈着 +遍布著 遍佈着 +記錄著 記錄着 +紀錄著 紀錄着 +收錄著 收錄着 +咬著 咬着 +三十六著 三十六着 +走為上著 走為上着 +鬧著 鬧着 +悶著 悶着 +呆著 呆着 +包著 包着 +系着 繫着 +颳著 颳着 +促著 促着 +榴莲 榴槤 +榴蓮 榴槤 +叱吒 叱咤 +嘯吒 嘯咤 +醯醬 醯醬 +醯雞 醯雞 +醯酱 醯醬 +醯鸡 醯雞 +醯醋 醯醋 +醯醢 醯醢 +醯壶 醯壺 +醯壺 醯壺 +想象 想像 +係數 系數 +澈底 徹底 +雇员 僱員 +雇用 僱用 +糊口 餬口 +倒楣 倒霉 +径庭 逕庭 +径到 逕到 +径取 逕取 +径入 逕入 +径行 逕行 +径自 逕自 +径往 逕往 +径寄 逕寄 +径启 逕啟 +径迎 逕迎 +印表機 打印機 +0字节 0位元組 +1字节 1位元組 +2字节 2位元組 +3字节 3位元組 +4字节 4位元組 +5字节 5位元組 +6字节 6位元組 +7字节 7位元組 +8字节 8位元組 +9字节 9位元組 +列印 打印 +硬件 硬件 +硬體 硬件 +二極體 二極管 +三極體 三極管 +軟體 軟件 +軟體動物 軟體動物 +軟體家具 軟體家具 +網路 網絡 +人工智慧 人工智能 +航天飞机 穿梭機 +太空梭 穿梭機 +因特网 互聯網 +網際網路 互聯網 +机器人 機械人 +機器人 機械人 +移动电话 流動電話 +行動電話 流動電話 +操作系统 作業系統 +移动操作系统 流動作業系統 +行動作業系統 流動作業系統 +數據機 調制解調器 +短信 短訊 +簡訊 短訊 +葉門 也門 +貝里斯 伯利茲 +維德角 佛得角 +克羅埃西亞 克羅地亞 +甘比亞 岡比亞 +幾內亞比索 幾內亞比紹 +列支敦斯登 列支敦士登 +賴比瑞亞 利比里亞 +迦納 加納 +加彭 加蓬 +波札那 博茨瓦納 +盧安達 盧旺達 +瓜地馬拉 危地馬拉 +厄瓜多尔 厄瓜多爾 +厄瓜多爾 厄瓜多爾 +厄瓜多 厄瓜多爾 +厄利垂亞 厄立特里亞 +吉布地 吉布堤 +哥斯大黎加 哥斯達黎加 +吐瓦魯 圖瓦盧 +聖露西亞 聖盧西亞 +圣基茨和尼维斯 聖吉斯納域斯 +聖克里斯多福及尼維斯 聖吉斯納域斯 +聖文森及格瑞那丁 聖文森特和格林納丁斯 +聖馬利諾 聖馬力諾 +蓋亞那 圭亞那 +坦尚尼亞 坦桑尼亞 +衣索匹亞 埃塞俄比亞 +衣索比亞 埃塞俄比亞 +吉里巴斯 基里巴斯 +塞普勒斯 塞浦路斯 +塞席爾 塞舌爾 +安地卡及巴布達 安提瓜和巴布達 +巴貝多 巴巴多斯 +紐幾內亞 新幾內亞 +布吉納法索 布基納法索 +蒲隆地 布隆迪 +帕劳 帛琉 +義大利 意大利 +索羅門群島 所羅門群島 +文莱 汶萊 +史瓦濟蘭 斯威士蘭 +斯洛維尼亞 斯洛文尼亞 +紐西蘭 新西蘭 +格瑞那達 格林納達 +茅利塔尼亞 毛里塔尼亞 +毛里求斯 毛里裘斯 +模里西斯 毛里裘斯 +沙地阿拉伯 沙特阿拉伯 +沙烏地阿拉伯 沙特阿拉伯 +辛巴威 津巴布韋 +宏都拉斯 洪都拉斯 +千里達托貝哥 特立尼達和多巴哥 +萬那杜 瓦努阿圖 +葛摩 科摩羅 +寮國 老撾 +貢寮 貢寮 #分詞用 +肯尼亚 肯雅 +奈洛比 內羅畢 +莫三比克 莫桑比克 +賴索托 萊索托 +尚比亞 贊比亞 +亞塞拜然 阿塞拜疆 +阿拉伯聯合大公國 阿拉伯聯合酋長國 +馬爾地夫 馬爾代夫 +馬利共和國 馬里共和國 +斯堪地那維亞 斯堪的納維亞 +台球 桌球 +撞球 桌球 +冰淇淋 雪糕 +賓士 平治 +捷豹 積架 +沃尓沃 富豪 +马自达 萬事得 +馬自達 萬事得 +寶獅 標致 +布什 布殊 +柯林頓 克林頓 +萨达姆 薩達姆 +贝克汉姆 碧咸 +貝克漢 碧咸 +迈克尔·欧文 米高·奧雲 +卡普里亚蒂 卡佩雅蒂 +马拉特·萨芬 馬拉特·沙芬 +舒马赫 舒麥加 +希特勒 希特拉 +狄安娜 戴安娜 +黛安娜 戴安娜 +南朝鲜 南韓 +北朝鲜 北韓 +寮語 老撾語 +寮人民民主共和國 老撾人民民主共和國 +莱特湾 雷伊泰灣 +萊特灣 雷伊泰灣 +蘭卡威 浮羅交怡 +撒马尔罕 撒馬爾罕 +伊斯蘭瑪巴德 伊斯蘭堡 +喀拉蚩 卡拉奇 +帕塔亚 芭達亞 +葉里溫 埃里溫 +巴士拉 巴斯拉 +賽普勒斯 塞浦路斯 +荷姆茲 霍爾木茲 +加薩走廊 加沙地帶 +西臺語 赫梯語 +西臺王 赫梯王 +西臺族 赫梯族 +西臺文 赫梯文 +西臺帝 赫梯帝 +西臺國 赫梯國 +西臺人 赫梯人 +阿联酋 阿聯酋 +迪拜 杜拜 +格鲁吉亚 格魯吉亞 +提比里西 第比利斯 +諾鲁 瑙魯 +玻里尼西亞 波利尼西亞 +帛琉 帕勞 +堪培拉 坎培拉 +约翰斯顿岛 強斯頓環礁 +巴尔米拉环礁 帕邁拉環礁 +马恩岛 萌島 +伯明罕 伯明翰 +布里斯托尔 布里斯托 +威尔士 威爾斯 +威爾士 威爾斯 +·威尔士 ·威爾士 +·威爾士 ·威爾士 +土魯斯 圖盧茲 +戛纳 康城 +坎城 康城 +羅亞爾 盧瓦爾 +诺曼底 諾曼第 +卢浮宫 羅浮宮 +埃菲尔 艾菲爾 +霍爾斯坦 荷爾斯泰因 +漢諾瓦 漢諾威 +哥廷根 格丁根 +杜塞道夫 杜塞爾多夫 +德勒斯登 德累斯頓 +安哈特 安哈爾特 +威斯伐倫 威斯特法倫 +布蘭登堡 勃蘭登堡 +前波莫瑞 前波美拉尼亞 +什勒斯維希 石勒蘇益格 +不萊梅 不來梅 +柏林墙 柏林圍牆 +巴塞罗那 巴塞隆拿 +巴塞隆納 巴塞隆拿 +塞维利亚 西維爾 +塞維亞 西維爾 +巴伦西亚 華倫西亞 +巴倫西亞 華倫西亞 +瓦倫西亞 華倫西亞 +雅爾達 雅爾塔 +切尔诺贝利 切爾諾貝爾 +蒙特內哥羅 黑山 +馬斯垂克 馬斯特里赫特 +貝爾格勒 貝爾格萊德 +塞拉耶佛 薩拉熱窩 +波士尼亞 波斯尼亞 +塞爾維亞與蒙特內哥羅 塞爾維亞和黑山 +波士尼亞與赫塞哥維納 波斯尼亞和黑塞哥維那 +卢塞恩 琉森 +亞斯文 阿斯旺 +奈及利亞 尼日利亞 +雅穆索戈 雅穆蘇克雷 +衣索匹亞 埃塞俄比亚 +吉力馬札羅 乞力馬札羅 +厄利垂亚 厄立特里亞 +索馬利亞 索馬里 +索馬利里 索馬里 +马里兰 馬利蘭 +馬里蘭 馬利蘭 +好萊塢 荷里活 +好莱坞 荷里活 +舊金山 三藩市 +旧金山 三藩市 +紐澳良 新奧爾良 +密西根 密歇根 +愛荷華 艾奧瓦 +爱荷华 艾奧瓦 +得克萨斯 德克薩斯 +蒙特婁 蒙特利爾 +紐賓士域 紐賓士域 +默多克 梅鐸 +梅鐸 梅鐸 +麦克尔 米高 +迈克尔 米高 +錢尼 切尼 +里瓦尔多 李華度 +罗纳德·里根 朗奴·列根 +达芬奇 達文西 +达·芬奇 達·文西 +克卜勒 開普勒 +谢丽·布莱尔 彭雪玲 +葉爾欽 葉利欽 +菲利普親王 菲臘親王 +菲利普亲王 菲臘親王 +華勒沙 華里沙 +艾里爾·夏隆 阿里埃勒·沙龍 +罗纳尔迪尼奥 朗拿甸奴 +罗纳尔多 朗拿度 +索忍尼辛 索贊尼辛 +索尔仁尼琴 索贊尼辛 +瓦文萨 華里沙 +班傑明 本傑明 +狄托 鐵托 +柴契爾 戴卓爾 +撒切尔 戴卓爾 +斯蒂芬·斯皮尔伯格 史提芬·史匹堡 +斯皮尔伯格 史匹堡 +史蒂芬·史匹柏 史提芬·史匹堡 +史匹柏 史匹堡 +戈巴契夫 戈爾巴喬夫 +席哈克 希拉克 +希拉蕊 希拉莉 +布莱尔 貝理雅 +尼克松 尼克遜 +奧黛麗·赫本 柯德莉·夏萍 +奧黛莉·朵杜 柯德莉·塔圖 +奥黛丽·赫本 柯德莉·夏萍 +卡斯楚 卡斯特羅 +肖邦 蕭邦 +恺撒 凱撒 +肯尼迪 甘迺迪 +賓拉登 本拉登 +賓·拉登 本·拉登 +歐巴馬 奧巴馬 +唐納·川普 當勞·特朗普 +唐纳德·特朗普 當勞·特朗普 +戈登·布朗 白高敦 +狂牛症 瘋牛症 +A肝 甲肝 +A型肝炎 甲型肝炎 +B肝 乙肝 +B型肝炎 乙型肝炎 +C肝 丙肝 +C型肝炎 丙型肝炎 +艾滋 愛滋 +链接 連結 +分辨率 解像度 +解析度 解像度 +智慧卡 智能卡 +晶元 晶片 +芯片 晶片 +晶體管 電晶體 +晶体管 電晶體 +源代码 原始碼 +IP地址 IP位址 +屏幕 螢幕 +荧屏 螢屏 +版权信息 版權資訊 +信息时代 資訊時代 +蹦床 彈床 +擊劍 劍擊 +击剑 劍擊 +金氏世界紀錄 健力士世界紀錄 +牛轧 鳥結 +牛軋 鳥結 +數位相機 數碼相機 +數位照相機 数碼照相機 +数字照相机 数碼照相機 +單眼相機 單鏡反光機 +单反相机 單鏡反光機 +台式电脑 桌上型電腦 +形上學 形而上學 +吉尼斯世界纪录 健力士世界紀錄 +吉他 結他 +古柯鹼 可卡因 +咖哩 咖喱 +泰坦尼克号 鐵達尼號 +自行火炮 自走炮 +冰激凌 雪糕 +里氏0 黎克特制0 +里氏1 黎克特制1 +里氏2 黎克特制2 +里氏3 黎克特制3 +里氏4 黎克特制4 +里氏5 黎克特制5 +里氏6 黎克特制6 +里氏7 黎克特制7 +里氏8 黎克特制8 +里氏9 黎克特制9 +芮氏0 黎克特制0 +芮氏1 黎克特制1 +芮氏2 黎克特制2 +芮氏3 黎克特制3 +芮氏4 黎克特制4 +芮氏5 黎克特制5 +芮氏6 黎克特制6 +芮氏7 黎克特制7 +芮氏8 黎克特制8 +芮氏9 黎克特制9 +芮氏規模 黎克特制震級 +芮氏地震規模 黎克特制地震震級 +里氏震级 黎克特制震級 +里氏规模 黎克特制震級 +里氏地震规模 黎克特制地震震級 +埃博拉 伊波拉 +哥特式 哥德式 +正體中文 繁體中文 +板球 木球 +籃板球 籃板球 +篮板球 籃板球 +智慧財產權 知識產權 +智財權 知識產權 +首席执行官 行政總裁 +智慧型 智能 +智慧手機 智能手機 +计算机程序 電腦程式 +电脑程序 電腦程式 +应用程序 應用程式 +尖峰時間 繁忙時間 +尖峰時段 繁忙時段 +東協 東盟 +東協會 東協會 +東協助 東協助 +東協議 東協議 +亚细安 東盟 +大英國協 英聯邦 +共和联邦 英聯邦 +阿布達比 阿布扎比 +宇航员 太空人 +薛丁格 薛定諤 +凯瑟琳 嘉芙蓮 +凱薩琳 嘉芙蓮 +门德尔松 孟德爾遜 +孟德爾頌 孟德爾遜 +肖斯塔科维奇 蕭士達高維契 +蕭士塔高維奇 蕭士達高維契 +工具機 機床 +空气质量 空氣質素 +空氣品質 空氣質素 +俯卧撑 掌上壓 +伏地挺身 掌上壓 +数字电视 數碼電視 +數位電視 數碼電視 +数字技术 數碼技術 +數位技術 數碼技術 +数字信号 數碼訊號 +數碼訊號 數碼訊號 +数字音乐 數碼音樂 +數位音樂 數碼音樂 +数字化 數碼化 +數位化 數碼化 +行動網路 流動網絡 +移动网络 流動網絡 +麥克風 咪高峰 +麦克风 咪高峰 +幫浦 泵 +朝鲜战争 韓戰 +万历朝鲜战争 萬曆朝鮮戰爭 +演化論 進化論 +搜索引擎 搜尋引擎 +福馬林 福爾馬林 +海洛因 海洛英 +高畫質 高清 +赫魯雪夫 赫魯曉夫 +公厘 毫米 +公釐 毫米 +桑巴舞 森巴舞 +乔治·奥威尔 喬治·歐威爾 +程序员 程式設計師 +昂山素季 昂山素姬 +翁山蘇姬 昂山素姬 +西洋棋 國際象棋 +隐私 私隱 +隱私 私隱 +硅藻 硅藻 +格莱美奖 格林美獎 +葛萊美獎 格林美獎 +斯坦福大学 史丹福大學 +賈伯斯 喬布斯 +宝莱坞 波里活 +寶萊塢 波里活 +庫德族 庫爾德族 +庫德人 庫爾德人 +東南亞國家協會 東南亞國家聯盟 +獨立國協 獨聯體 +獨立國家國協 獨立國家聯合體 +人行道 行人路 +塑料袋 膠袋 +烏龍麵 烏冬麵 diff --git a/www/wiki/maintenance/language/zhtable/toSimp.manual b/www/wiki/maintenance/language/zhtable/toSimp.manual new file mode 100644 index 00000000..56400c35 --- /dev/null +++ b/www/wiki/maintenance/language/zhtable/toSimp.manual @@ -0,0 +1,287 @@ +」 ” +「 “ +『 ‘ +』 ’ +「 “ +」 ” +乾县 乾县 +萧乾 萧乾 +乾断 乾断 +乾图 乾图 +乾纲 乾纲 +乾红 乾红 +乾清 乾清 +乾仪 乾仪 +乾兴 乾兴 +乾冈 乾冈 +乾刘 乾刘 +乾刚 乾刚 +乾启 乾启 +乾宁 乾宁 +乾岗 乾岗 +乾录 乾录 +乾晖 乾晖 +乾构 乾构 +乾枢 乾枢 +乾栋 乾栋 +乾灵 乾灵 +乾窦 乾窦 +乾笃 乾笃 +乾纽 乾纽 +乾络 乾络 +乾统 乾统 +乾维 乾维 +乾罗 乾罗 +乾荫 乾荫 +乾象历 乾象历 +乾贞 乾贞 +乾贶 乾贶 +乾车 乾车 +乾轴 乾轴 +乾鉴 乾鉴 +乾钧 乾钧 +乾闼 乾闼 +乾顾 乾顾 +乾风 乾风 +乾马 乾马 +乾鹄 乾鹄 +乾鹊 乾鹊 +乾龙 乾龙 +张法乾 张法乾 +旋乾转坤 旋乾转坤 +天道为乾 天道为乾 +易经·乾 易经·乾 +易经乾 易经乾 +乾务 乾务 +黄润乾 黄润乾 +男性为乾 男性为乾 +男为乾 男为乾 +阳为乾 阳为乾 +男性為乾 男性为乾 +男為乾 男为乾 +陽為乾 阳为乾 +乾一组 乾一组 +乾一坛 乾一坛 +陈乾生 陈乾生 +陈公乾生 陈公乾生 +李乾顺 李乾顺 +孙乾 孙乾 +陈遇乾 陈遇乾 +曾运乾 曾运乾 +乾贵士 乾贵士 +乾东 乾东 +柳诒徵 柳诒徵 +於夫罗 於夫罗 +於梨华 於梨华 +於潜 於潜 +於志贺 於志贺 +於戏 於戏 +憑藉 凭借 +藉端 借端 +藉故 借故 +藉口 借口 +藉助 借助 +藉手 借手 +藉詞 借词 +藉機 借机 +藉此 借此 +藉由 借由 +沈積 沉积 +沈船 沉船 +沈默 沉默 +沈沒 沉没 +沈澱 沉淀 +沈重 沉重 +彷彿 仿佛 +項鍊 项链 +肘手鍊足 肘手链足 +鍊子 链子 +鍊條 链条 +拉鍊 拉链 +鉸鍊 铰链 +鍊鎖 链锁 +鎖鍊 锁链 +鐵鍊 铁链 +金鍊 金链 +銀鍊 银链 +鍊錘 链锤 +洗鍊 洗练 +手鍊 手链 +鍊表 链表 +反覆 反复 +回覆 回复 +答覆 答复 +反反覆覆 反反复复 +重覆 重复 +覆核 复核 +覆查 复查 +覆检 复检 +鬱姓 鬱姓 +鬱氏 鬱氏 +夥計 伙计 +乾泉水 干泉水 +么半 幺半 +么元 幺元 +么爹 幺爹 +么叔 幺叔 +么舅 幺舅 +么爸 幺爸 +么媽 幺妈 +么姨 幺姨 +么娘 幺娘 +么孃 幺娘 +么弟 幺弟 +么妹 幺妹 +么小 幺小 +么姓 幺姓 +么氏 幺氏 +么蛾子 幺蛾子 +么鳳 幺凤 +么二三 幺二三 +么篇 幺篇 +六么 六幺 +老么 老幺 +么正 幺正 +么女 幺女 +么九 幺九 +么子 幺子 +姓么 姓幺 +么兒 幺儿 +么喝 幺喝 +么爺 幺爷 +么雞 幺鸡 +么麼 幺麽 +幺麽 幺麽 +麽氏 麽氏 +麼氏 麽氏 +乾乾淨淨 干干净净 +乾乾脆脆 干干脆脆 +肉乾乾 肉干干 +魚乾乾 鱼干干 +於于同 於于同 +於乙于同 於乙于同 +閻懷禮 闫怀礼 +醯酱 醯酱 +醯鸡 醯鸡 +醯壶 醯壶 +苧烯 苧烯 +氾濫 泛滥 +近角聪信 近角聪信 +米泽瑠美 米泽瑠美 +候覆 候复 +待覆 待复 +批覆 批复 +矇眬 矇眬 +荠苧 荠苧 +噁心 恶心 +碁圣 碁圣 +慇懃 殷勤 +慇勤 殷勤 +崑崙 昆仑 +崑山 昆山 +崑劇 昆剧 +崑曲 昆曲 +崑腔 昆腔 +崑蘇 昆苏 +崑調 昆调 +崑島 昆岛 +諠譁 喧哗 +慫慂 怂恿 +陈元扞 陈元扞 +甦醒 苏醒 +復甦 复苏 +蒐證 搜证 +蒐索 搜索 +蒐藏 搜藏 +蒐羅 搜罗 +蒐購 搜购 +蒐錄 搜录 +蒐集 搜集 +蒐輯 搜辑 +蒐采 搜采 +蒐採 搜采 +偵蒐 侦搜 +情蒐 情搜 +蘋果 苹果 +蘋婆 苹婆 +於之莹 於之莹 +陆徵祥 陆徵祥 +瞭臺 瞭台 +瞭台 瞭台 +慘澹 惨淡 +鍾情 钟情 +鍾愛 钟爱 +鍾意 钟意 +所鍾 所钟 +情鍾 情钟 +獨鍾 独钟 +鍾靈 钟灵 +龍鍾 龙钟 +薰心 熏心 +薰習 熏习 +薰陶 熏陶 +薰沐 熏沐 +薰香 熏香 +餬口 糊口 +跼限 局限 +跼促 局促 +釐清 厘清 +釐訂 厘订 +釐革 厘革 +釐改 厘改 +釐整 厘整 +釐正 厘正 +毫釐 毫厘 +釐毫 厘毫 +剖釐 剖厘 +一釐 一厘 +昇平 升平 +飛昇 飞升 +提昇 提升 +高昇 高升 +初昇 初升 +昇天 升天 +上昇 上升 +昇汞 升汞 +昇華 升华 +昇仙 升仙 +昇降 升降 +竹昇 竹升 +直昇 直升 +高陞 高升 +晉陞 晋升 +歷陞 历升 +官陞 官升 +榮陞 荣升 +又陞 又升 +年陞 年升 +月陞 月升 +陞官 升官 +陞任 升任 +陞為 升为 +陞遷 升迁 +陞用 升用 +陞補 升补 +陞了 升了 +,陞 ,升 +。陞 。升 +爾冬陞 尔冬升 +內聯陞 内联升 +同陞和 同升和 +拿破崙 拿破仑 +酒麴 酒曲 +麴黴 曲霉 +造麴 造曲 +大麴 大曲 +黃麴毒素 黄曲毒素 +硃砂 朱砂 +硃紅 朱红 +硃色 朱色 +銀硃 银朱 +遶境 绕境 +侷促 局促 +侷限 局限 +馬鞌 马鞍 +觔斗 斤斗 +穀阳 穀阳 +伊東豊雄 伊东丰雄 diff --git a/www/wiki/maintenance/language/zhtable/toTW.manual b/www/wiki/maintenance/language/zhtable/toTW.manual new file mode 100644 index 00000000..16e27516 --- /dev/null +++ b/www/wiki/maintenance/language/zhtable/toTW.manual @@ -0,0 +1,790 @@ +着 著 +佈 布 +鈎 鉤 +钩 鉤 +账 帳 +枱 檯 +睾 睪 +酰 醯 +钫 鍅 +锫 鉳 +镎 錼 +镅 鋂 +锿 鑀 +锝 鎝 +锎 鉲 +钚 鈽 +硅 矽 +幺 么 +煙草 菸草 +烟草 菸草 +煙蒂 菸蒂 +烟蒂 菸蒂 +煙斗 菸斗 +烟斗 菸斗 +煙鬼 菸鬼 +烟鬼 菸鬼 +煙灰 菸灰 +烟灰 菸灰 +煙具 菸具 +烟具 菸具 +煙民 菸民 +烟民 菸民 +煙農 菸農 +烟农 菸農 +煙絲 菸絲 +烟丝 菸絲 +煙頭 菸頭 +烟头 菸頭 +煙葉 菸葉 +烟叶 菸葉 +煙癮 菸癮 +烟瘾 菸癮 +煙嘴 菸嘴 +烟嘴 菸嘴 +煙酒 菸酒 +烟酒 菸酒 +煙袋 菸袋 +烟袋 菸袋 +煙品 菸品 +烟品 菸品 +煙鹼 菸鹼 +烟碱 菸鹼 +煙捲 菸捲 +烟卷 菸捲 +香煙 香菸 +香烟 香菸 +捲煙 捲菸 +卷烟 捲菸 +旱煙 旱菸 +旱烟 旱菸 +烤煙 烤菸 +烤烟 烤菸 +禁煙 禁菸 +禁烟 禁菸 +戒煙 戒菸 +戒烟 戒菸 +拒煙 拒菸 +拒烟 拒菸 +紙煙 紙菸 +纸烟 紙菸 +抽煙 抽菸 +抽烟 抽菸 +吸煙 吸菸 +吸烟 吸菸 +反煙 反菸 +反烟 反菸 +私煙 私菸 +私烟 私菸 +點煙 點菸 +点烟 點菸 +洋煙 洋菸 +洋烟 洋菸 +二手煙 二手菸 +二手烟 二手菸 +電子煙 電子菸 +电子烟 電子菸 +呂宋煙 呂宋菸 +吕宋烟 呂宋菸 +雪茄煙 雪茄菸 +雪茄烟 雪茄菸 +無煙日 無菸日 +无烟日 無菸日 +無煙環境 無菸環境 +无烟环境 無菸環境 +榴莲 榴槤 +榴蓮 榴槤 +铆足 卯足 +霉素 黴素 +想象 想像 +迭代 疊代 +叱咤 叱吒 +嘯咤 嘯吒 +叱咤9 叱咤9 +叱咤M 叱咤M +叱咤樂壇 叱咤樂壇 +叱咤咤 叱咤咤 +叱咤叱 叱咤叱 +正在叱咤 正在叱咤 +氨基酸 胺基酸 +枪支 槍枝 +球杆 球桿 +推杆 推桿 +挥杆 揮桿 +揮杆 揮桿 +一杆 一桿 +二杆 二桿 +三杆 三桿 +四杆 四桿 +五杆 五桿 +六杆 六桿 +七杆 七桿 +八杆 八桿 +九杆 九桿 +十杆 十桿 +1杆 1桿 +2杆 2桿 +3杆 3桿 +4杆 4桿 +5杆 5桿 +6杆 6桿 +7杆 7桿 +8杆 8桿 +9杆 9桿 +0杆 0桿 +标准杆 標準桿 +標準杆 標準桿 +电杆 電桿 +电线杆 電線桿 +木杆 木桿 +铁杆 鐵桿 +鐵杆 鐵桿 +杆头 桿頭 +杆頭 桿頭 +杆身 桿身 +杆弟 桿弟 +锻炼 鍛鍊 +炼金 鍊金 +熏烤 燻烤 +烟熏 煙燻 +熏肉 燻肉 +熏黑 燻黑 +糊口 餬口 +径庭 逕庭 +径到 逕到 +径取 逕取 +径入 逕入 +径行 逕行 +径自 逕自 +径往 逕往 +径寄 逕寄 +径启 逕啟 +径迎 逕迎 +系着 繫著 +关系着 關係著 +冲着 衝著 +干着 幹著 +干着急 干著急 +对着干 對著幹 +斗着 鬥著 +面包着 面包著 +徵狀 症狀 +系数 係數 +汇编 彙編 +报道 報導 +划着船 划著船 +划着竹筏 划著竹筏 +划着独木舟 划著獨木舟 +着眼于 著眼於 +桃金娘 桃金孃 +粘膜 黏膜 +缺省 預設 +以太网 乙太網 +光盘 光碟 +光驱 光碟機 +声卡 音效卡 +字段 欄位 +存盘 存檔 +控件 控制項 +盘片 碟片 +硬盘 硬碟 +磁盘 磁碟 +磁道 磁軌 +端口 埠 +芯片 晶片 +译码 解碼 +软驱 軟碟機 +快闪存储器 快閃記憶體 +闪存 快閃記憶體 +鼠标 滑鼠 +进制 進位 +信息论 資訊理論 +信息时代 資訊時代 +写保护 防寫 +分辨率 解析度 +服务器 伺服器 +局域网 區域網 +局域网络 區域網路 +数据库 資料庫 +打印机 印表機 +打印機 印表機 +打印 列印 +攻打 攻打 #分詞用 +打印度 打印度 +0字节 0位元組 +1字节 1位元組 +2字节 2位元組 +3字节 3位元組 +4字节 4位元組 +5字节 5位元組 +6字节 6位元組 +7字节 7位元組 +8字节 8位元組 +9字节 9位元組 +硬件 硬體 +二极管 二極體 +二極管 二極體 +三极管 三極體 +三極管 三極體 +软件 軟體 +軟件 軟體 +人工智能 人工智慧 +航天飞机 太空梭 +穿梭機 太空梭 +因特网 網際網路 +互聯網 網際網路 +互聯網絡 網際網路 +机器人 機器人 +機械人 機器人 +移动电话 行動電話 +流動電話 行動電話 +调制解调器 數據機 +調制解調器 數據機 +短信 簡訊 +乍得 查德 +也门 葉門 +也門 葉門 +伯利兹 貝里斯 +伯利茲 貝里斯 +佛得角 維德角 +克罗地亚 克羅埃西亞 +克羅地亞 克羅埃西亞 +冈比亚 甘比亞 +岡比亞 甘比亞 +几内亚比绍 幾內亞比索 +幾內亞比紹 幾內亞比索 +列支敦士登 列支敦斯登 +利比里亚 賴比瑞亞 +利比里亞 賴比瑞亞 +加蓬 加彭 +博茨瓦纳 波札那 +博茨瓦納 波札那 +卡塔尔 卡達 +卡塔爾 卡達 +卢旺达 盧安達 +盧旺達 盧安達 +危地马拉 瓜地馬拉 +危地馬拉 瓜地馬拉 +厄瓜多尔 厄瓜多 +厄瓜多爾 厄瓜多 +厄立特里亚 厄利垂亞 +厄立特里亞 厄利垂亞 +吉布提 吉布地 +吉布堤 吉布地 +哥斯达黎加 哥斯大黎加 +哥斯達黎加 哥斯大黎加 +图瓦卢 吐瓦魯 +圖瓦盧 吐瓦魯 +圣卢西亚 聖露西亞 +聖盧西亞 聖露西亞 +圣基茨和尼维斯 聖克里斯多福及尼維斯 +聖吉斯納域斯 聖克里斯多福及尼維斯 +圣文森特和格林纳丁斯 聖文森及格瑞那丁 +聖文森特和格林納丁斯 聖文森及格瑞那丁 +圣马力诺 聖馬利諾 +聖馬力諾 聖馬利諾 +圭亚那 蓋亞那 +法属圭亚那 法屬蓋亞那 +坦桑尼亚 坦尚尼亞 +坦桑尼亞 坦尚尼亞 +埃塞俄比亚 衣索比亞 +埃塞俄比亞 衣索比亞 +基里巴斯 吉里巴斯 +塞拉利昂 獅子山 +塞浦路斯 塞普勒斯 +塞舌尔 塞席爾 +塞舌爾 塞席爾 +安提瓜和巴布达 安地卡及巴布達 +安提瓜和巴布達 安地卡及巴布達 +尼日利亚 奈及利亞 +尼日利亞 奈及利亞 +尼日尔 尼日 +尼日爾 尼日 +巴巴多斯 巴貝多 +布基纳法索 布吉納法索 +布基納法索 布吉納法索 +布隆迪 蒲隆地 +帕劳 帛琉 +意大利 義大利 +所罗门群岛 索羅門群島 +所羅門群島 索羅門群島 +文莱 汶萊 +斯威士兰 史瓦濟蘭 +斯威士蘭 史瓦濟蘭 +斯洛文尼亚 斯洛維尼亞 +斯洛文尼亞 斯洛維尼亞 +新西兰 紐西蘭 +新西蘭 紐西蘭 +格林纳达 格瑞那達 +格林納達 格瑞那達 +格鲁吉亚 喬治亞 +格魯吉亞 喬治亞 +佐治亚 喬治亞 +佐治亞 喬治亞 +毛里塔尼亚 茅利塔尼亞 +毛里塔尼亞 茅利塔尼亞 +毛里求斯 模里西斯 +毛里裘斯 模里西斯 +沙特阿拉伯 沙烏地阿拉伯 +沙地阿拉伯 沙烏地阿拉伯 +波斯尼亚和黑塞哥维那 波士尼亞與赫塞哥維納 +波斯尼亞和黑塞哥維那 波士尼亞與赫塞哥維納 +津巴布韦 辛巴威 +津巴布韋 辛巴威 +洪都拉斯 宏都拉斯 +特立尼达和托巴哥 千里達托貝哥 +特立尼達和多巴哥 千里達托貝哥 +瑙鲁 諾魯 +瑙魯 諾魯 +瓦努阿图 萬那杜 +瓦努阿圖 萬那杜 +溫納圖萬 那杜 +科摩罗 葛摩 +科摩羅 葛摩 +科特迪瓦 象牙海岸 +突尼斯 突尼西亞 +老挝 寮國 +老撾 寮國 +肯尼亚 肯亞 +内罗毕 奈洛比 +內羅畢 奈洛比 +苏里南 蘇利南 +莫桑比克 莫三比克 +莱索托 賴索托 +萊索托 賴索托 +赞比亚 尚比亞 +贊比亞 尚比亞 +阿塞拜疆 亞塞拜然 +阿拉伯联合酋长国 阿拉伯聯合大公國 +阿拉伯聯合酋長國 阿拉伯聯合大公國 +马尔代夫 馬爾地夫 +馬爾代夫 馬爾地夫 +马耳他 馬爾他 +马里共和国 馬利共和國 +馬里共和國 馬利共和國 +蹦极跳 笨豬跳 +绑紧跳 笨豬跳 +出租车 計程車 +台球 撞球 +積架 捷豹 +布什 布希 +布殊 布希 +克林顿 柯林頓 +克林頓 柯林頓 +侯赛因 海珊 +侯賽因 海珊 +本拉登 賓拉登 +本·拉登 賓·拉登 +梵高 梵谷 +狄安娜 黛安娜 +戴安娜 黛安娜 +南朝鲜 南韓 +北朝鲜 北韓 +乔戈里峰 K2 +老挝人民民主共和国 寮人民民主共和國 +老撾人民民主共和國 寮人民民主共和國 +老挝语 寮語 +老撾語 寮語 +浮罗交怡 蘭卡威 +浮羅交怡 蘭卡威 +莱特湾 雷伊泰灣 +萊特灣 雷伊泰灣 +耶加達 雅加達 +伊斯兰堡 伊斯蘭瑪巴德 +伊斯蘭堡 伊斯蘭瑪巴德 +卡拉奇 喀拉蚩 +帕塔亚 芭達亞 +埃里温 葉里溫 +埃里溫 葉里溫 +第比利斯 提比里西 +巴士拉 巴斯拉 +塞浦路斯 賽普勒斯 +霍尔木兹 荷姆茲 +霍爾木茲 荷姆茲 +加沙地带 加薩走廊 +加沙地帶 加薩走廊 +赫梯 西臺 +阿联酋 阿聯 +阿聯酋 阿聯 +迪拜 杜拜 +堪培拉 坎培拉 +悉尼 雪梨 +波利尼西亚 玻里尼西亞 +波利尼西亞 玻里尼西亞 +新几内亚 紐幾內亞 +新幾內亞 紐幾內亞 +约翰斯顿岛 強斯頓環礁 +巴尔米拉环礁 帕邁拉環礁 +马恩岛 曼島 +萌島 曼島 +伯明翰 伯明罕 +布里斯托尔 布里斯托 +威尔士 威爾斯 +威爾士 威爾斯 +·威尔士 ·威爾士 +·威爾士 ·威爾士 +图卢兹 土魯斯 +圖盧茲 土魯斯 +戛纳 坎城 +卢瓦尔 羅亞爾 +盧瓦爾 羅亞爾 +诺曼底 諾曼第 +卢浮宫 羅浮宮 +埃菲尔 艾菲爾 +荷爾斯泰因 霍爾斯坦 +荷尔斯泰因 霍爾斯坦 +石勒蘇益格 什勒斯維希 +石勒苏益格 什勒斯維希 +漢诺威 漢諾瓦 +汉诺威 漢諾瓦 +格丁根 哥廷根 +杜塞爾多夫 杜塞道夫 +杜塞尔多夫 杜塞道夫 +德累斯顿 德勒斯登 +德累斯頓 德勒斯登 +安哈爾特 安哈特 +安哈尔特 安哈特 +威斯特法倫 威斯伐倫 +威斯特法伦 威斯伐倫 +勃蘭登堡 布蘭登堡 +勃兰登堡 布蘭登堡 +前波美拉尼亞 前波莫瑞 +前波美拉尼亚 前波莫瑞 +不来梅 不萊梅 +不來梅 不萊梅 +柏林墙 柏林圍牆 +柏林牆 柏林圍牆 +巴塞罗那 巴塞隆納 +巴塞隆拿 巴塞隆納 +塞维利亚 塞維亞 +西維爾 塞維亞 +巴伦西亚 瓦倫西亞 +華倫西亞 瓦倫西亞 +佛罗伦萨 佛羅倫斯 +雅尔塔 雅爾達 +雅爾塔 雅爾達 +切尔诺贝利 車諾比 +黑山共和國 蒙特內哥羅共和國 +黑山共和国 蒙特內哥羅共和國 +马斯特里赫特 馬斯垂克 +馬斯特里赫特 馬斯垂克 +贝尔格莱德 貝爾格勒 +貝爾格萊德 貝爾格勒 +薩拉熱窩 塞拉耶佛 +萨拉热窝 塞拉耶佛 +波黑 波赫 +波斯尼亞 波士尼亞 +波斯尼亚 波士尼亞 +比利牛斯 庇里牛斯 +塞黑 塞蒙 +塞爾維亞與蒙特內哥羅 塞爾維亞與蒙特內哥羅 +塞爾維亞和黑山 塞爾維亞與蒙特內哥羅 +塞尔维亚和黑山 塞爾維亞與蒙特內哥羅 +伊斯坦布尔 伊斯坦堡 +伊斯坦布爾 伊斯坦堡 +卢塞恩 琉森 +阿斯旺 亞斯文 +雅穆苏克雷 雅穆索戈 +雅穆蘇克雷 雅穆索戈 +索马里兰 索馬利蘭 +索馬里蘭 索馬利蘭 +乞力马扎罗 吉力馬札羅 +乞力馬札羅 吉力馬札羅 +厄利垂亚 厄利垂亞 +索马里 索馬利亞 +索馬里 索馬利亞 +扎伊尔 薩伊 +扎伊爾 薩伊 +金沙萨 金夏沙 +金沙薩 金夏沙 +达累斯萨拉姆 三蘭港 +马拉维 馬拉威 +留尼汪 留尼旺 +布隆方丹 布隆泉 +厄瓜多 厄瓜多 +百慕大 百慕達 +圣赫勒拿 聖赫倫那 +马萨诸塞 麻薩諸塞 +馬利蘭 馬里蘭 +里士满 里奇蒙 +荷里活 好萊塢 +荷里活道 荷里活道 +荷里活廣場 荷里活廣場 +维尔京群岛 維京群島 +維爾京群島 維京群島 +纽黑文 紐哈芬 +特拉華 德拉瓦 +特拉华 德拉瓦 +爱德华州 愛達荷州 +新罕布什尔 新罕布夏 +新奥尔良 紐奧良 +新奧爾良 紐奧良 +得克萨斯 德克薩斯 +弗吉尼亚 維吉尼亞 +康涅狄格 康乃狄克 +密歇根 密西根 +宾西法尼亚 賓夕法尼亞 +威士顿康星 威斯康辛 +伊利诺伊州 伊利諾州 +亚拉巴马 阿拉巴馬 +三藩市 舊金山 +艾奧瓦 愛荷華 +得克薩斯 德克薩斯 +蒙特利尔 蒙特婁 +蒙特利爾 蒙特婁 +斯堪的纳维亚 斯堪地那維亞 +斯堪的納維亞 斯堪地那維亞 +圣佩德罗苏拉 汕埠 +麦克尔 麥可 +迈克尔 麥可 +魯賓斯·巴里切羅 魯本·巴瑞切羅 +雷诺阿 雷諾瓦 +阿里埃勒·沙龙 艾里爾·夏隆 +阿里埃勒·沙龍 艾里爾·夏隆 +铁托 狄托 +鐵托 狄托 +邁凱輪 麥拿輪 +迈凯轮 麥拿輪 +达芬奇 達文西 +达·芬奇 達·文西 +赫鲁晓夫 赫魯雪夫 +赫丘勒·波洛 赫丘勒·白羅 +薛定谔 薛丁格 +葉利欽 葉爾欽 +華里沙 華勒沙 +瓦文萨 華勒沙 +艾森豪威尔 艾森豪 +罗纳德·里根 隆納·雷根 +维特根斯坦 維根斯坦 +约翰逊 詹森 +索尔仁尼琴 索忍尼辛 +索贊尼辛 索忍尼辛 +瓦格纳 華格納 +毕加索 畢卡索 +碧咸 貝克漢 +梅尔·吉布森 梅爾·吉勃遜 +查韦斯 查維茲 +本杰明 班傑明 +本傑明 班傑明 +普密蓬 蒲美蓬 +普利策 普利茲 +施罗德 施洛德 +斯蒂芬 史蒂芬 +斯皮尔伯格 史匹柏 +斯特劳斯 史特勞斯 +斯大林 史達林 +斯坦福大学 史丹福大學 +撒切尔 柴契爾 +戴卓爾 柴契爾 +摩根士丹利 摩根史坦利 +戴克里先 戴克里先 +戈爾巴喬夫 戈巴契夫 +戈尔巴乔夫 戈巴契夫 +愛德文 愛德溫 +德里达 德希達 +帕特里克 派屈克 +希拉里 希拉蕊 +希拉莉 希拉蕊 +希拉克 席哈克 +尼克松 尼克森 +威廉姆斯 威廉士 +多普勒 都卜勒 +开普勒 克卜勒 +開普勒 克卜勒 +叶利钦 葉爾欽 +卡斯特罗 卡斯楚 +包豪斯 包浩斯 +勃朗宁 白朗寧 +劳拉 蘿拉 +列奥纳多 李奧納多 +克里斯托弗 克里斯多福 +傅里叶 傅立葉 +伊丽莎白 伊莉莎白 +丘吉尔 邱吉爾 +肖邦 蕭邦 +理查德 理察 +肯尼迪 甘迺迪 +奥巴马 歐巴馬 +奧巴馬 歐巴馬 +特朗普 川普 +唐纳德·特朗普 唐納·川普 +當勞·特朗普 唐納·川普 +當奴·特朗普 唐納·川普 +概率 機率 +疯牛症 狂牛症 +甲肝 A肝 +甲型肝炎 A型肝炎 +乙肝 B肝 +乙型肝炎 B型肝炎 +丙肝 C肝 +丙型肝炎 C型肝炎 +艾滋 愛滋 +链接 連結 +程序员 程式設計師 +源代码 原始碼 +智能卡 智慧卡 +數據庫 資料庫 +操作系统 作業系統 +移动操作系统 行動作業系統 +流動作業系統 行動作業系統 +人机交互 人機互動 +交互设计 互動設計 +互联网络 網際網路 +互联网 網際網路 +万维网 全球資訊網 +编程语言 程式語言 +晶體管 電晶體 +晶体管 電晶體 +IP地址 IP位址 +解像度 解析度 +屏幕 螢幕 +荧屏 螢屏 +版权信息 版權資訊 +航天器 太空飛行器 +导弹 飛彈 +宇航服 太空衣 +宇航员 太空人 +太空飛行員 太空人 +独联体 獨立國協 +獨聯體 獨立國協 +独立国家联合体 獨立國家國協 +獨立國家聯合體 獨立國家國協 +东南亚国家联盟 東南亞國家協會 +東南亞國家聯盟 東南亞國家協會 +发达国家 已開發國家 +哥特式 哥德式 +落車 下車 +上落客 上下客 +集装箱 貨櫃 +雅马哈 山葉 +避孕套 保險套 +素檀 蘇丹 +珍寶客機 巨無霸客機 +泰坦尼克号 鐵達尼號 +樂行童軍 羅浮童軍 +朝鲜战争 韓戰 +万历朝鲜战争 萬曆朝鮮戰爭 +數碼相機 數位相機 +單鏡反光機 單眼相機 +数码相机 數位相機 +数字照相机 數位照相機 +数码照相机 數位照相機 +數碼照相機 數位照相機 +单反相机 單眼相機 +台式电脑 桌上型電腦 +形而上學 形上學 +形而上学 形上學 +当且仅当 若且唯若 +圆珠笔 原子筆 +可卡因 古柯鹼 +公共交通 公共運輸 +吉尼斯世界纪录 金氏世界紀錄 +健力士世界纪录 金氏世界紀錄 +健力士世界紀錄 金氏世界紀錄 +沙律 沙拉 +忌廉 奶油 +味美思 苦艾酒 +埃博拉 伊波拉 +克隆人 複製人 +荧光 螢光 +里氏0 芮氏0 +里氏1 芮氏1 +里氏2 芮氏2 +里氏3 芮氏3 +里氏4 芮氏4 +里氏5 芮氏5 +里氏6 芮氏6 +里氏7 芮氏7 +里氏8 芮氏8 +里氏9 芮氏9 +里氏震级 芮氏規模 +里氏规模 芮氏規模 +里氏地震规模 芮氏地震規模 +黎克特制 芮氏 +知识产权 智慧財產權 +知識產權 智慧財產權 +知识产权局 知識產權局 +知識產權局 知識產權署 +知识产权署 知識產權署 +知識產權署 知識產權署 +乒乓球 桌球 +乒乓 桌球 +首席执行官 執行長 +首席财务官 財務長 +首席运营官 營運長 +智能手机 智慧型手機 +智能手機 智慧型手機 +智能电话 智慧型電話 +智能電話 智慧型電話 +便携式 可攜式 +计算机程序 電腦程式 +电脑程序 電腦程式 +应用程序 應用程式 +激光 雷射 +高峰时间 尖峰時間 +高峰时段 尖峰時段 +东盟 東協 +東盟 東協 +亚细安 東協 +英联邦 大英國協 +英聯邦 大英國協 +共和联邦 大英國協 +阿布扎比 阿布達比 +凯瑟琳 凱薩琳 +嘉芙蓮 凱薩琳 +门德尔松 孟德爾頌 +孟德爾遜 孟德爾頌 +肖斯塔科维奇 蕭士塔高維奇 +蕭士達高維契 蕭士塔高維奇 +希特拉 希特勒 +自由泳 自由式 +机床 工具機 +機床 工具機 +空气质量 空氣品質 +空氣質素 空氣品質 +俯卧撑 伏地挺身 +掌上壓 伏地挺身 +数字电视 數位電視 +數碼電視 數位電視 +数字技术 數位技術 +數碼技術 數位技術 +数字信号 數位訊號 +數碼訊號 數位訊號 +数字化 數位化 +數碼化 數位化 +移动网络 行動網路 +流動網絡 行動網路 +网络游戏 網路遊戲 +網絡遊戲 網路遊戲 +电脑网络 電腦網路 +電腦網絡 電腦網路 +咪高峰 麥克風 +電單車 機車 +搜索引擎 搜尋引擎 +福尔马林 福馬林 +福爾馬林 福馬林 +海洛英 海洛因 +高清电视 高畫質電視 +桑巴舞 森巴舞 +乔治·奥威尔 喬治·歐威爾 +結他 吉他 +了結他 了結他 +連結他 連結他 +昂山素季 翁山蘇姬 +昂山素姬 翁山蘇姬 +国际象棋 西洋棋 +國際象棋 西洋棋 +私隱 隱私 +硅藻 硅藻 +格林美獎 葛萊美獎 +格莱美奖 葛萊美獎 +乔布斯 賈伯斯 +波里活 寶萊塢 +库尔德族 庫德族 +库尔德人 庫德人 +行人路 人行道 +行人路權 行人路權 +行人路权 行人路權 +塑料袋 塑膠袋 +触摸屏 觸控螢幕 +乌冬面 烏龍麵 diff --git a/www/wiki/maintenance/language/zhtable/toTrad.manual b/www/wiki/maintenance/language/zhtable/toTrad.manual new file mode 100644 index 00000000..1d536154 --- /dev/null +++ b/www/wiki/maintenance/language/zhtable/toTrad.manual @@ -0,0 +1,568 @@ +” 」 +“ 「 +‘ 『 +’ 』 +’s ’s +手塚治虫 手塚治虫 +寇仇 寇讎 +往日无仇 往日無讎 +近日无仇 近日無讎 +李連杰 李連杰 +杰倫 杰倫 +杰威爾 杰威爾 +黃詩杰 黃詩杰 +陳士杰 陳士杰 +林杰樑 林杰樑 +許聖杰 許聖杰 +張杰 張杰 +孫杰 孫杰 +陳杰 陳杰 +黃杰 黃杰 +謝杰 謝杰 +寶曆 寶曆 +涂謹申 涂謹申 +涂鴻欽 涂鴻欽 +涂壯勳 涂壯勳 +鄭凱云 鄭凱云 +筑陽 筑陽 +筑後 筑後 +采石磯 采石磯 +采石之戰 采石之戰 +張三丰 張三丰 +丰韻 丰韻 +丰儀 丰儀 +丰標不凡 丰標不凡 +二里頭 二里頭 +水里鄉 水里鄉 +蒙胧 朦朧 +酒曲 酒麴 +呆里呆气 呆裡呆氣 +拜托 拜託 +委托书 委託書 +委托 委託 +府干預 府干預 +府干擾 府干擾 +頁面 頁面 +面條目 面條目 +黃鈺筑 黃鈺筑 +答复 答覆 +反复 反覆 +反反复复 反反覆覆 +候复 候覆 +待复 待覆 +批复 批覆 +复信 覆信 +复核 覆核 +的回复 的回覆 +回复中 回覆中 +回复说 回覆說 +回复你 回覆你 +有回复 有回覆 +回复邮件 回覆郵件 +回复意见 回覆意見 +回复帖子 回覆帖子 +得到回复 得到回覆 +回复: 回覆: +索馬里 索馬里 +洗练 洗鍊 +朝乾夕惕 朝乾夕惕 +乾象曆 乾象曆 +乾象历 乾象曆 +不好干預 不好干預 +范文瀾 范文瀾 +機械系 機械系 +頂多 頂多 +馬占山 馬占山 +闫怀礼 閆懷禮 +薴烯 薴烯 +于謙 于謙 +詩云 詩云 +云為 云為 +古書云 古書云 +古語云 古語云 +經有云 經有云 +語有云 語有云 +采納 採納 +風采 風采 +于樂 于樂 +于軍 于軍 +于堅 于堅 +于帥 于帥 +于濤 于濤 +于贈 于贈 +于會泳 于會泳 +于偉國 于偉國 +于光遠 于光遠 +于鳳至 于鳳至 +于台煙 于台煙 +于國楨 于國楨 +于大寶 于大寶 +于學忠 于學忠 +于小偉 于小偉 +于山國 于山國 +于幼軍 于幼軍 +于廣洲 于廣洲 +于從濂 于從濂 +于志寧 于志寧 +于成龍 于成龍 +于明濤 于明濤 +于根偉 于根偉 +于樹潔 于樹潔 +于漢超 于漢超 +于洪區 于洪區 +于湘蘭 于湘蘭 +于蔭霖 于蔭霖 +于遠偉 于遠偉 +于都縣 于都縣 +于震寰 于震寰 +于震環 于震環 +于非闇 于非闇 +于風政 于風政 +于鳳桐 于鳳桐 +于默奧 于默奧 +于爾岑 于爾岑 +于貝爾 于貝爾 +于爾根 于爾根 +于雙戈 于雙戈 +于澤爾 于澤爾 +于斯達爾 于斯達爾 +于爾里克 于爾里克 +于奇庫杜克 于奇庫杜克 +于韋斯屈萊 于韋斯屈萊 +于克-蘭多縣 于克-蘭多縣 +于斯納爾斯貝里 于斯納爾斯貝里 +夏于喬 夏于喬 +于逸堯 于逸堯 +涂澤民 涂澤民 +涂長望 涂長望 +涂敏恆 涂敏恆 +后豐 后豐 +艷后 艷后 +廢后 廢后 +后髮座 后髮座 +后髮星系團 后髮星系團 +后髮FK型星 后髮FK型星 +后海灣 后海灣 +賈后 賈后 +賢后 賢后 +呂后 呂后 +蟻后 蟻后 +陳有后 陳有后 +天神之后 天神之后 +豔后 豔后 +后綜 后綜 +葉陽后 葉陽后 +后庄 后庄 +後庄 後庄 +后蒼 后蒼 +馬格里布 馬格里布 +佳里鎮 佳里鎮 +有只採 有只採 +會干擾 會干擾 +党項 党項 +余三勝 余三勝 +簡筑翎 簡筑翎 +楊雅筑 楊雅筑 +尸羅精舍 尸羅精舍 +騰格里 騰格里 +進制 進制 +強制 強制 +總裁制 總裁制 +獨裁制 獨裁制 +模范三軍 模范三軍 +陳冲 陳冲 +劉佳怜 劉佳怜 +范賢惠 范賢惠 +于國治 于國治 +于楓 于楓 +黎吉雲 黎吉雲 +于飛 于飛 +鄉愿 鄉愿 +愿樸 愿樸 +謹愿 謹愿 +奇迹 奇蹟 +划槳 划槳 +折子戲 折子戲 +佣錢 佣錢 +佣鈿 佣鈿 +阁府 閤府 +太阁 太閤 +昆仑 崑崙 +昆山 崑山 +昆剧 崑劇 +昆曲 崑曲 +昆腔 崑腔 +昆苏 崑蘇 +昆调 崑調 +昆冈 崑岡 +西昆 西崑 +苏昆 蘇崑 +苏醒 甦醒 +复苏 復甦 +苹果 蘋果 +苹果干 蘋果乾 +苹婆 蘋婆 +龜山庄 龜山庄 +寶山庄 寶山庄 +員山庄 員山庄 +昵称 暱稱 +單于 單于 +鮮于 鮮于 +賦范 賦范 +茅于軾 茅于軾 +壽天里 壽天里 +貴子里 貴子里 +東湖里 東湖里 +鹿場里 鹿場里 +水里高級商工 水里高級商工 +水里鳳林 水里鳳林 +水里濁水溪 水里濁水溪 +洞里薩 洞里薩 +愛河里花子 愛河里花子 +划不來 划不來 +划來划去 划來划去 +划動 划動 +划得來 划得來 +划著 划著 +划著 劃著名 +划進 划進 +划過 划過 +划龍舟 划龍舟 +划龍船 划龍船 +只影響 只影響 +義联 義联 +杠轂 杠轂 +局促 侷促 +開山辟谷 開山辟谷 +戲院里 戲院里 +么半 么半 +么元 么元 +么爹 么爹 +么叔 么叔 +么舅 么舅 +么爸 么爸 +么媽 么媽 +么姨 么姨 +么娘 么娘 +么孃 么孃 +么弟 么弟 +么妹 么妹 +么小 么小 +么姓 么姓 +么氏 么氏 +么蛾子 么蛾子 +么鳳 么鳳 +么二三 么二三 +么篇 么篇 +六么 六么 +老么 老么 +么正 么正 +么女 么女 +么九 么九 +么子 么子 +姓么 姓么 +么兒 么兒 +么喝 么喝 +么爺 么爺 +么雞 么雞 +么麼 么麼 +惨淡 慘澹 +恶心 噁心 +证谏 証諫 +项链 項鍊 +手链 手鍊 +金链 金鍊 +链表 鍊表 +熏心 薰心 +熏习 薰習 +熏陶 薰陶 +熏沐 薰沐 +熏染 薰染 +熏香 薰香 +熏风 薰風 +雨蒙蒙 雨濛濛 +夹衣 袷衣 +夹裙 袷裙 +局蹐 跼蹐 +拳局 拳跼 +踡局 踡跼 +局躅 跼躅 +蹒局 蹣跼 +厘清 釐清 +厘订 釐訂 +厘革 釐革 +厘改 釐改 +厘整 釐整 +厘正 釐正 +毫厘 毫釐 +厘毫 釐毫 +剖厘 剖釐 +一厘一毫 一釐一毫 +升州 昇州 +升平 昇平 +升阳 昇陽 +陈升 陳昇 +尔冬升 爾冬陞 +南宮适 南宮适 +舊庄 舊庄 +拿破仑 拿破崙 +冗余 冗餘 +课余 課餘 +节余 節餘 +盈余 盈餘 +病余 病餘 +余地 餘地 +余力 餘力 +余子 餘子 +余事 餘事 +扶余 扶餘 +腐余 腐餘 +富余 富餘 +之余 之餘 +余泽 餘澤 +流风余俗 流風餘俗 +流风余韵 流風餘韻 +淋余土 淋餘土 +余一 餘一 +余二 餘二 +余三 餘三 +余四 餘四 +余五 餘五 +余六 餘六 +余七 餘七 +余八 餘八 +余九 餘九 +余十 餘十 +零余 零餘 +〇余 〇餘 +余零 餘零 +余〇 餘〇 +余1 餘1 +余2 餘2 +余3 餘3 +余4 餘4 +余5 餘5 +余6 餘6 +余7 餘7 +余8 餘8 +余9 餘9 +余0 餘0 +余数 餘數 +其余 其餘 +尸居余气 尸居餘氣 +剩余 賸餘 +余孽 餘孽 +残余 殘餘 +业余 業餘 +余割 餘割 +余款 餘款 +余角 餘角 +余切 餘切 +余霞 餘霞 +余下 餘下 +余弦 餘弦 +余震 餘震 +余貾 餘貾 +余额 餘額 +余人 餘人 +余俗 餘俗 +余倍 餘倍 +同余 同餘 +空余 空餘 +余量 餘量 +余年 餘年 +余留 餘留 +余项 餘項 +余式 餘式 +余部 餘部 +编余 編餘 +余墨 餘墨 +唾余 唾餘 +余韵 餘韻 +归余 歸餘 +公余 公餘 +宽余 寬餘 +余粮 餘糧 +余庆 餘慶 +余殃 餘殃 +余烬 餘燼 +劫余 劫餘 +结余 結餘 +烬余 燼餘 +净余 淨餘 +馂余 餕餘 +余晖 餘暉 +余辉 餘輝 +羡余 羨餘 +余悸 餘悸 +心余 心餘 +刑余 刑餘 +绪余 緒餘 +血余 血餘 +朱庆余 朱慶餘 +诸余 諸餘 +余论 餘論 +茶余 茶餘 +厨余 廚餘 +余裕 餘裕 +余气 餘氣 +诗余 詩餘 +词余 詞餘 +余僇 餘僇 +余辜 餘辜 +余责 餘責 +余罪 餘罪 +无余 無餘 +耳余 耳餘 +余烈 餘烈 +余思 餘思 +盐余 鹽餘 +嬴余 嬴餘 +赢余 贏餘 +王余鱼 王餘魚 +纡余 紆餘 +余波 餘波 +余杯 餘杯 +余步 餘步 +余妙 餘妙 +余音 餘音 +余声 餘聲 +余明 餘明 +余风 餘風 +余党 餘黨 +余毒 餘毒 +余桃 餘桃 +余桶 餘桶 +余利 餘利 +余沥 餘瀝 +余膏 餘膏 +余光 餘光 +余杭 餘杭 +余窍 餘竅 +余缺 餘缺 +余暇 餘暇 +余闲 餘閒 +余羡 餘羨 +余响 餘響 +余兴 餘興 +余蓄 餘蓄 +余绪 餘緒 +余珍 餘珍 +余众 餘眾 +余酲 餘酲 +余喘 餘喘 +余食 餘食 +余热 餘熱 +余刃 餘刃 +余闰 餘閏 +余存 餘存 +余业 餘業 +余姚 餘姚 +余荫 餘蔭 +余映 餘映 +余外 餘外 +余威 餘威 +余味 餘味 +余温 餘溫 +余勇 餘勇 +多余 多餘 +剩余 剩餘 +余生 餘生 +余欢 餘歡 +有余 有餘 +一余 一餘 +二余 二餘 +两余 兩餘 +三余 三餘 +四余 四餘 +五余 五餘 +六余 六餘 +七余 七餘 +八余 八餘 +九余 九餘 +十余 十餘 +百余 百餘 +千余 千餘 +万余 萬餘 +亿余 億餘 +兆余 兆餘 +仅余 僅餘 +0余 0餘 +1余 1餘 +2余 2餘 +3余 3餘 +4余 4餘 +5余 5餘 +6余 6餘 +7余 7餘 +8余 8餘 +9余 9餘 +米余 米餘 +带余 帶餘 +余干 餘干 +余江 餘江 +于余曲折 于餘曲折 +尸居余气 尸居餘氣 +余光生 余光生 +余光中 余光中 +余思敏 余思敏 +余威德 余威德 +余子明 余子明 +余三胜 余三勝 +咨询 諮詢 +酒曲 酒麴 +曲霉 麴黴 +曲秀才 麴秀才 +曲尘 麴塵 +曲櫱 麴櫱 +黄曲毒素 黃麴毒素 +曲道士 麴道士 +曲钱 麴錢 +曲车 麴車 +鼠曲草 鼠麴草 +曲酒 麯酒 +泸州大曲 瀘州大麯 #商標名 +洋河大曲 洋河大麯 +沟大曲 溝大麯 +朱砂 硃砂 +银朱 銀硃 +喲喂 喲喂 +鳥栖 鳥栖 +澄江县 澂江縣 #以下為含異體字地名 +横峰县 橫峯縣 +鹤峰县 鶴峯縣 +五峰县 五峯縣 +兰溪市 蘭谿市 +金溪县 金谿縣 +竹溪县 竹谿縣 +辰溪县 辰谿縣 +松溪县 松谿縣 +慈溪 慈谿 +浚州 濬州 +浚县 濬縣 +穆棱 穆稜 +绥棱 綏稜 +丹棱 丹稜 +仙游 仙遊 +麟游 麟遊 +乐游原 樂遊原 +托克逊 託克遜 +托里县 託里縣 +沾化 霑化 +沾益 霑益 +岫岩 岫巖 +黄岩县 黃巖縣 +黄岩区 黃巖區 +北仑河 北崙河 +昆嵛 崑嵛 +昆承湖 崑承湖 +灵昆 靈崑 +龙岩 龍巖 +扑冬 撲鼕 +冬冬鼓 鼕鼕鼓 +苧麻 苧麻 +张柏芝 張栢芝 +杜琪峰 杜琪峯 +單向 單向 +轉向 轉向 #分詞用 +十出頭 十出頭 diff --git a/www/wiki/maintenance/language/zhtable/trad2simp.manual b/www/wiki/maintenance/language/zhtable/trad2simp.manual new file mode 100644 index 00000000..1912bcf2 --- /dev/null +++ b/www/wiki/maintenance/language/zhtable/trad2simp.manual @@ -0,0 +1,799 @@ +U+034BA㒺|U+07F54罔| +U+034C2㓂|U+05BC7寇| +U+03541㕁|U+05374却| +U+03551㕑|U+053A8厨| +U+03558㕘|U+053C2参| +U+03565㕥|U+04EE5以| +U+0362D㘭|U+05773坳| +U+0375B㝛|U+05BBF宿| +U+03760㝠|U+051A5冥| +U+03800㠀|U+05C9B岛| +U+0382F㠯|U+04EE5以| +U+03836㠶|U+05E06帆| +U+0384C㡌|U+05E3D帽| +U+03898㢘|U+05EC9廉| +U+03919㤙|U+06069恩| +U+03966㥦|U+060EC惬| +U+03A17㨗|U+06377捷| +U+03A2A㨪|U+06643晃| +U+03A3F㨿|U+0636E据| +U+03A57㩗|U+0643A携| +U+03A66㩦|U+0643A携| +U+03A9A㪚|U+06563散| +U+03A9F㪟|U+06566敦| +U+03B09㬉|U+06696暖| +U+03B2A㬪|U+053E0叠| +U+03BED㯭|U+06A79橹| +U+03C43㱃|U+0996E饮| +U+03CD2㳒|U+06CD5法| +U+03D31㴱|U+06DF1深| +U+03F1D㼝|U+07897碗| +U+03F5E㽞|U+07559留| +U+03FDC㿜|U+0762A瘪| +U+04039䀹|U+25174𥅴| +U+04230䈰|U+07B72筲| +U+04280䊀|U+07CCA糊| +U+045EC䗬|U+08702蜂| +U+0460F䘏|U+06064恤| +U+04611䘑|U+08109脉| +U+0461A䘚|U+05352卒| +U+046D0䛐|U+08BCD词| +U+046E1䛡|U+08BDD话| +U+04754䝔|U+0737E獾| +U+04800䠀|U+08E5A蹚| +U+04836䠶|U+05C04射| +U+04965䥥|U+09570镰| +U+04B03䬃|U+098D2飒| +U+04B7E䭾|U+09A6E驮| +U+04C1F䰟|U+09B42魂| +U+04CD8䳘|U+09E45鹅| +U+04D8A䶊|U+08844衄| +U+04E23丣|U+0536F卯| +U+04E57乗|U+04E58乘| +U+04E79乹|U+05E72干| +U+04E81亁|U+05E72干| +U+04E99亙|U+04E98亘| +U+04E9D亝|U+0658B斋| +U+04EB1亱|U+0591C夜| +U+04EB7亷|U+05EC9廉| +U+04EBE亾|U+04EA1亡| +U+04F48佈|U+05E03布| +U+04F54佔|U+05360占| +U+04FFB俻|U+05907备| +U+05010倐|U+0500F倏| +U+05016倖|U+05E78幸| +U+05023倣|U+04EFF仿| +U+05038倸|U+0776C睬| +U+0509A傚|U+06548效| +U+050A2傢|U+05BB6家| +U+050CA僊|U+04ED9仙| +U+050CD働|U+052A8动| +U+050F1僱|U+096C7雇| +U+0510C儌|U+04FA5侥| +U+05138儸|U+03469㑩|U+07F57罗| +U+05147兇|U+051F6凶| +U+0514E兎|U+05154兔| +U+05160兠|U+0515C兜| +U+05184冄|U+05189冉| +U+05190冐|U+05192冒| +U+05191冑|U+080C4胄| +U+051BA冺|U+06CEF泯| +U+051E2凢|U+051E1凡| +U+051F4凴|U+051ED凭| +U+05226刦|U+052AB劫| +U+05227刧|U+052AB劫| +U+0523C刼|U+052AB劫| +U+05249剉|U+09509锉| +U+0524F剏|U+0521B创| +U+05259剙|U+0521B创| +U+05273剳|U+0672D札| +U+05277剷|U+094F2铲| +U+05279剹|U+0622E戮| +U+05284劄|U+0672D札| +U+05292劒|U+05251剑| +U+052B9効|U+06548效| +U+052C5勅|U+06555敕| +U+052CC勌|U+05026倦| +U+052D1勑|U+06555敕| +U+052E6勦|U+0527F剿| +U+052F3勳|U+052CB勋| +U+0531F匟|U+07095炕| +U+05332匲|U+05941奁| +U+05333匳|U+05941奁| +U+05379卹|U+06064恤| +U+0537D卽|U+05373即| +U+05380厀|U+0819D膝| +U+053A0厠|U+05395厕| +U+053A4厤|U+05386历| +U+053B0厰|U+05382厂| +U+0541A吚|U+054BF咿| +U+0544C呌|U+053EB叫| +U+0546A呪|U+05492咒| +U+0548A咊|U+0548C和| +U+054F6哶|U+054A9咩| +U+05515唕|U+05523唣| +U+05518唘|U+0542F启| +U+05538唸|U+05FF5念| +U+0554E啎|U+05FE4忤| +U+05551啑|U+0558B喋| +U+05553啓|U+0542F启| +U+05557啗|U+05556啖| +U+05563啣|U+08854衔| +U+055AB喫|U+05403吃| +U+055C1嗁|U+0557C啼| +U+05605嘅|U+06168慨| +U+05611嘑|U+0547C呼| +U+05620嘠|U+0560E嘎| +U+05637嘷|U+055E5嗥| +U+05649噉|U+05556啖| +U+05690嚐|U+05C1D尝| +U+056A5嚥|U+054BD咽| +U+056AE嚮|U+05411向| +U+056CC囌|U+082CF苏| +U+056D3囓|U+0556E啮| +U+056D9囙|U+056E0因| +U+05705圅|U+051FD函| +U+0577F坿|U+09644附| +U+0579C垜|U+0579B垛| +U+057BB垻|U+0575D坝| +U+0585A塚|U+051A2冢| +U+0585F塟|U+0846C葬| +U+05872塲|U+0573A场| +U+05896墖|U+05854塔| +U+058B0墰|U+0575B坛| +U+058BB墻|U+05899墙| +U+058CE壎|U+057D9埙| +U+058DC壜|U+0575B坛| +U+058FB壻|U+05A7F婿| +U+05918夘|U+0536F卯| +U+05925夥|U+04F19伙|U+05925夥| +U+0596C奬|U+05956奖| +U+059AC妬|U+05992妒| +U+059B3妳|U+04F60你| +U+059B7妷|U+04F84侄| +U+059C9姉|U+059CA姊| +U+059D9姙|U+0598A妊| +U+059EA姪|U+04F84侄| +U+059F8姸|U+0598D妍| +U+05A63婣|U+059FB姻| +U+05A6C婬|U+06DEB淫| +U+05A8D媍|U+05987妇| +U+05ABF媿|U+06127愧| +U+05ACB嫋|U+08885袅| +U+05AF0嫰|U+05AE9嫩| +U+05AFA嫺|U+05A34娴| +U+05B00嬀|U+059AB妫| +U+05B1D嬝|U+08885袅| +U+05B2D嬭|U+05976奶| +U+05B3E嬾|U+061D2懒| +U+05B43孃|U+05A18娘| +U+05B7C孼|U+05B7D孽| +U+05B82宂|U+05197冗| +U+05BC0寀|U+091C7采| +U+05BC3寃|U+051A4冤| +U+05BD1寑|U+05BDD寝| +U+05BF3寳|U+05B9D宝| +U+05C05尅|U+0514B克| +U+05C12尒|U+05C14尔| +U+05C19尙|U+05C1A尚| +U+05C1F尟|U+09C9C鲜| +U+05C20尠|U+09C9C鲜| +U+05C5B屛|U+05C4F屏| +U+05C6D屭|U+05C43屃| +U+05C85岅|U+05742坂| +U+05CDD峝|U+05CD2峒| +U+05D57嵗|U+05C81岁| +U+05D83嶃|U+05D2D崭| +U+05DBD嶽|U+05CB3岳| +U+05DD6巖|U+05CA9岩| +U+05DD7巗|U+05CA9岩| +U+05DF5巵|U+0536E卮| +U+05E00帀|U+0531D匝| +U+05E0B帋|U+07EB8纸| +U+05E2C帬|U+088D9裙| +U+05E47幇|U+05E2E帮| +U+05E51幑|U+05FBD徽| +U+05E59幙|U+05E55幕| +U+05E5A幚|U+05E2E帮| +U+05EBB庻|U+05EB6庶| +U+05EBD庽|U+05BD3寓| +U+05ED0廐|U+053A9厩| +U+05ED5廕|U+0836B荫| +U+05EF5廵|U+05DE1巡| +U+05EF9廹|U+08FEB迫| +U+05EFB廻|U+056DE回| +U+05F14弔|U+0540A吊| +U+05F46彆|U+0522B别| +U+05F6B彫|U+096D5雕| +U+05F83徃|U+05F80往| +U+05FA7徧|U+0904D遍| +U+06031怱|U+05306匆| +U+06033怳|U+0604D恍| +U+06060恠|U+0602A怪| +U+06061恡|U+0541D吝| +U+060A4悤|U+05306匆| +U+060BD悽|U+051C4凄| +U+060CF惏|U+05A6A婪| +U+060E5惥|U+0607F恿| +U+060F7惷|U+08822蠢| +U+0613D愽|U+0535A博| +U+06159慙|U+060ED惭| +U+06164慤|U+060AB悫| +U+06174慴|U+06151慑| +U+0617C慼|U+0621A戚| +U+0617D慽|U+0621A戚| +U+0617E慾|U+06B32欲| +U+06187憇|U+061A9憩| +U+061DE懞|U+061DE懞|U+08499蒙| +U+0621E戞|U+0621B戛| +U+0622F戯|U+0620F戏| +U+06239戹|U+05384厄| +U+0625E扞|U+0634D捍| +U+0629D抝|U+062D7拗| +U+062DA拚|U+062FC拼| +U+06331挱|U+06332挲| +U+06335挵|U+05F04弄| +U+06344捄|U+06551救| +U+06372捲|U+05377卷| +U+063BD掽|U+078B0碰| +U+063D1揑|U+0634F捏| +U+063EB揫|U+063EA揪| +U+063F7揷|U+063D2插| +U+063F9揹|U+080CC背| +U+06406搆|U+06784构| +U+06407搇|U+063FF揿| +U+06409搉|U+069B7榷| +U+06424搤|U+0627C扼| +U+06425搥|U+06376捶| +U+06428搨|U+062D3拓| +U+0642F搯|U+0638F掏| +U+0643E搾|U+069A8榨| +U+06443摃|U+0625B扛| +U+0647A摺|U+06298折| +U+064A1撡|U+064CD操| +U+064A6撦|U+0626F扯| +U+064D5擕|U+0643A携| +U+064E7擧|U+04E3E举| +U+06529攩|U+06321挡| +U+06537攷|U+08003考| +U+06542敂|U+053E9叩| +U+0654D敍|U+053D9叙| +U+0657A敺|U+09A71驱| +U+065C2旂|U+065D7旗| +U+065E3旣|U+065E2既| +U+065E4旤|U+07978祸| +U+065F9旹|U+065F6时| +U+065FE旾|U+06625春| +U+06607昇|U+06607昇|U+05347升| +U+0662C昬|U+0660F昏| +U+066B1暱|U+06635昵| +U+066E1曡|U+053E0叠| +U+0671E朞|U+0671F期| +U+06722朢|U+0671B望| +U+0672E朮|U+0672F术| +U+06736朶|U+06735朵| +U+067B1枱|U+053F0台| +U+067FA柺|U+062D0拐| +U+067FB査|U+067E5查| +U+06801栁|U+067F3柳| +U+0681E栞|U+0520A刊| +U+06822栢|U+067CF柏| +U+06830栰|U+07B4F筏| +U+06852桒|U+06851桑| +U+0686E桮|U+0676F杯| +U+0687A桺|U+067F3柳| +U+068CA棊|U+068CB棋| +U+06917椗|U+07887碇| +U+06936椶|U+068D5棕| +U+06937椷|U+07F04缄| +U+0693E椾|U+07B3A笺| +U+06965楥|U+06966楦| +U+069A6榦|U+05E72干| +U+069D3槓|U+06760杠| +U+069D5槕|U+0684C桌| +U+06A11樑|U+06881梁| +U+06A5C橜|U+06A5B橛| +U+06AC8櫈|U+051F3凳| +U+06B05欅|U+06989榉| +U+06B1D欝|U+090C1郁| +U+06B35欵|U+06B3E款| +U+06B4E歎|U+053F9叹| +U+06B5B歛|U+0655B敛| +U+06B74歴|U+05386历| +U+06B80殀|U+0592D夭| +U+06BAD殭|U+050F5僵| +U+06BBB殻|U+058F3壳| +U+06BE7毧|U+07ED2绒| +U+06BEC毬|U+07403球| +U+06C0A氊|U+06BE1毡| +U+06C37氷|U+051B0冰| +U+06C59汙|U+06C61污| +U+06C5A汚|U+06C61污| +U+06C88瀋|U+06C88沈|U+0700B渖| +U+06CDD泝|U+06EAF溯| +U+06D29洩|U+06CC4泄| +U+06D96涖|U+08385莅| +U+06DD2淒|U+051C4凄| +U+06DDB淛|U+06D59浙| +U+06DE8淨|U+051C0净| +U+06DE9淩|U+051CC凌| +U+06E67湧|U+06D8C涌| +U+06E7C湼|U+06D85涅| +U+06EBC溼|U+06E7F湿| +U+06ED9滙|U+06C47汇| +U+06EDB滛|U+06DEB淫| +U+06EF7滷|U+05364卤| +U+06F44潄|U+06F31漱| +U+06F55潕|U+23C98𣲘| +U+06F59潙|U+06CA9沩| +U+06F81澁|U+06DA9涩| +U+06F90澐|U+06C84沄| +U+06FBE澾|U+03CE0㳠| +U+06FC7濇|U+06DA9涩| +U+06FDB濛|U+06FDB濛|U+08499蒙| +U+06FF6濶|U+09614阔| +U+07030瀰|U+05F25弥| +U+0704B灋|U+06CD5法| +U+070D6烖|U+0707E灾| +U+07151煑|U+0716E煮| +U+07157煗|U+06696暖| +U+07188熈|U+07199熙| +U+071C4燄|U+07130焰| +U+071C9燉|U+07096炖|U+071C9燉| +U+071EC燬|U+06BC1毁| +U+071FB燻|U+0718F熏| +U+07217爗|U+070E8烨| +U+07232爲|U+04E3A为| +U+07240牀|U+05E8A床| +U+0724B牋|U+07B3A笺| +U+0724E牎|U+07A97窗| +U+07250牐|U+095F8闸| +U+07253牓|U+0699C榜| +U+07255牕|U+07A97窗| +U+07260牠|U+05B83它| +U+07274牴|U+062B5抵| +U+072E5狥|U+05F87徇| +U+07302猂|U+0608D悍| +U+07328猨|U+0733F猿| +U+07343獃|U+05446呆| +U+07358獘|U+06BD9毙| +U+07367獧|U+072F7狷| +U+07385玅|U+05999妙| +U+07416琖|U+076CF盏| +U+07431琱|U+096D5雕| +U+07447瑇|U+073B3玳| +U+0746F瑯|U+07405琅| +U+074A2璢|U+07460瑠| +U+0750E甎|U+07816砖| +U+07515甕|U+074EE瓮| +U+07516甖|U+07F42罂| +U+0751E甞|U+05C1D尝| +U+07523産|U+04EA7产| +U+07526甦|U+07526甦|U+082CF苏| +U+0752F甯|U+0752F甯|U+05B81宁| +U+07542畂|U+04EA9亩| +U+07546畆|U+04EA9亩| +U+07567畧|U+07565略| +U+0756B畫|U+0753B画|U+05212划| +U+0756E畮|U+04EA9亩| +U+07571畱|U+07559留| +U+07575畵|U+0753B画|U+05212划| +U+0758E疎|U+0758F疏| +U+07598疘|U+0809B肛| +U+075BF疿|U+075F1痱| +U+075D0痐|U+086D4蛔| +U+075E0痠|U+09178酸| +U+075FA痺|U+075F9痹| +U+07609瘉|U+06108愈| +U+07616瘖|U+05591喑| +U+0763B瘻|U+07618瘘| +U+07644癄|U+06194憔| +U+07645癅|U+07624瘤| +U+07648癈|U+05E9F废| +U+07652癒|U+06108愈| +U+07661癡|U+075F4痴| +U+07681皁|U+07682皂| +U+07690皐|U+0768B皋| +U+0769C皜|U+07693皓| +U+076B7皷|U+09F13鼓| +U+076C3盃|U+0676F杯| +U+076C7盇|U+076CD盍| +U+076CC盌|U+07897碗| +U+0770E眎|U+089C6视| +U+0771E眞|U+0771F真| +U+07721眡|U+089C6视| +U+07760睠|U+07737眷| +U+0776A睪|U+0777E睾| +U+07787瞇|U+0772F眯| +U+07796瞖|U+07FF3翳| +U+077AD瞭|U+04E86了| +U+077C1矁|U+07785瞅| +U+077C7矇|U+08499蒙|U+077C7矇| +U+077D9矙|U+077B0瞰| +U+07832砲|U+070AE炮| +U+07881碁|U+068CB棋| +U+078AA碪|U+07827砧| +U+078DF磟|U+0788C碌| +U+07906礆|U+078B1碱| +U+0792E礮|U+070AE炮| +U+07955祕|U+079D8秘| +U+07958祘|U+07B97算| +U+079CA秊|U+05E74年| +U+079CC秌|U+079CB秋| +U+079D6秖|U+053EA只| +U+07A09稉|U+07CB3粳| +U+07A1C稜|U+068F1棱| +U+07A2C稬|U+07CEF糯| +U+07A2D稭|U+079F8秸| +U+07A3E稾|U+07A3F稿| +U+07A64穤|U+07CEF糯| +U+07A68穨|U+09893颓| +U+07A7D穽|U+09631阱| +U+07A93窓|U+07A97窗| +U+07AB0窰|U+07A91窑| +U+07ABB窻|U+07A97窗| +U+07AC8竈|U+07076灶| +U+07ADA竚|U+04F2B伫| +U+07ADD竝|U+05E76并| +U+07AE2竢|U+04FDF俟| +U+07AEA竪|U+07AD6竖| +U+07B5E筞|U+07B56策| +U+07B69筩|U+07B52筒| +U+07B6F筯|U+07BB8箸| +U+07B87箇|U+04E2A个| +U+07B92箒|U+05E1A帚| +U+07BA0箠|U+068F0棰| +U+07BDB篛|U+07BAC箬| +U+07C11簑|U+084D1蓑| +U+07C12簒|U+07BE1篡| +U+07C2E簮|U+07C2A簪| +U+07C37簷|U+06A90檐| +U+07C50籐|U+085E4藤| +U+07C64籤|U+07B7E签| +U+07C72籲|U+05401吁| +U+07C83粃|U+079D5秕| +U+07CA7粧|U+05986妆| +U+07CC9糉|U+07CBD粽| +U+07CF0糰|U+056E2团| +U+07D25紥|U+0624E扎| +U+07D2E紮|U+0624E扎| +U+07D43絃|U+05F26弦| +U+07D4F絏|U+07EC1绁| +U+07D76絶|U+07EDD绝| +U+07D89綉|U+07EE3绣| +U+07D91綑|U+06346捆| +U+07DAB綫|U+07EBF线| +U+07DB5綵|U+05F69彩|U+0433D䌽| +U+07DD0緐|U+07E41繁| +U+07DD1緑|U+07EFF绿| +U+07DD4緔|U+07EF1绱| +U+07DDA線|U+07EBF线|U+07F10缐| +U+07DDC緜|U+07EF5绵| +U+07DE5緥|U+08913褓| +U+07DFC緼|U+07F0A缊| +U+07E27縧|U+07EE6绦| +U+07E34縴|U+07EA4纤| +U+07E50繐|U+07A57穗| +U+07E56繖|U+04F1E伞| +U+07E59繙|U+07FFB翻| +U+07E66繦|U+08941襁| +U+07E6E繮|U+07F30缰| +U+07E94纔|U+0624D才| +U+07F47罇|U+06A3D樽| +U+07F4B罋|U+074EE瓮| +U+07F4E罎|U+0575B坛| +U+07F78罸|U+07F5A罚| +U+07F97羗|U+07F8C羌| +U+07FA2羢|U+07ED2绒| +U+07FA3羣|U+07FA4群| +U+07FA8羨|U+07FA1羡| +U+07FB6羶|U+081BB膻| +U+07FC4翄|U+07FC5翅| +U+07FEB翫|U+073A9玩| +U+07FF6翶|U+07FF1翱| +U+08021耡|U+09504锄| +U+0808E肎|U+080AF肯| +U+08090肐|U+080F3胳| +U+080A7肧|U+080DA胚| +U+080F7胷|U+080F8胸| +U+08103脃|U+08106脆| +U+08107脇|U+080C1胁| +U+08117脗|U+0543B吻| +U+08123脣|U+05507唇| +U+08141腁|U+080FC胼| +U+08193膓|U+080A0肠| +U+081C8臈|U+0814A腊| +U+081CB臋|U+081C0臀| +U+081D5臕|U+08198膘| +U+081D9臙|U+080ED胭| +U+081DD臝|U+088F8裸| +U+081E5臥|U+05367卧| +U+081EF臯|U+0768B皋| +U+08216舖|U+094FA铺| +U+08218舘|U+09986馆| +U+08229舩|U+08239船| +U+08262艢|U+06A2F樯| +U+08263艣|U+06A79橹| +U+0826A艪|U+06A79橹| +U+082B2芲|U+082B1花| +U+08318茘|U+08354荔| +U+08373荳|U+08C46豆| +U+083F8菸|U+070DF烟| +U+08432萲|U+08431萱| +U+08457著|U+08457著|U+07740着| +U+08460葠|U+053C2参| +U+0846F葯|U+0836F药| +U+08493蒓|U+083BC莼| +U+084C6蓆|U+05E2D席| +U+084E1蓡|U+053C2参| +U+084F4蓴|U+083BC莼| +U+08514蔔|U+0535C卜| +U+08515蔕|U+08482蒂| +U+08518蔘|U+053C2参| +U+0855A蕚|U+0843C萼| +U+0857F蕿|U+08431萱| +U+08591薑|U+059DC姜| +U+085C9藉|U+085C9藉|U+0501F借| +U+085F4藴|U+08574蕴| +U+085F7藷|U+085AF薯| +U+085FC藼|U+08431萱| +U+08610蘐|U+08431萱| +U+08613蘓|U+082CF苏| +U+08624蘤|U+082B1花| +U+08698蚘|U+086D4蛔| +U+086D5蛕|U+086D4蛔| +U+0870B蜋|U+08782螂| +U+08716蜖|U+086D4蛔| +U+08728蜨|U+08776蝶| +U+08768蝨|U+08671虱| +U+0876F蝯|U+0733F猿| +U+08771蝱|U+0867B虻| +U+0878E螎|U+0878D融| +U+087A1螡|U+0868A蚊| +U+087C1蟁|U+0868A蚊| +U+087C7蟇|U+087C6蟆| +U+0880D蠍|U+0874E蝎| +U+0880F蠏|U+087F9蟹| +U+08812蠒|U+08327茧| +U+08814蠔|U+0869D蚝| +U+0882D蠭|U+08702蜂| +U+08842衂|U+08844衄| +U+08846衆|U+04F17众| +U+08847衇|U+08109脉| +U+0884A衊|U+08511蔑| +U+0885E衞|U+0536B卫| +U+0887A衺|U+090AA邪| +U+0889F袟|U+05E19帙| +U+088B5袵|U+0887D衽| +U+088CC裌|U+088B7袷| +U+088CF裏|U+091CC里| +U+088E0裠|U+088D9裙| +U+0892D褭|U+08885袅| +U+08943襃|U+08912褒| +U+0894D襍|U+06742杂| +U+08986覆|U+08986覆|U+0590D复| +U+08987覇|U+09738霸| +U+08988覈|U+06838核| +U+0898A覊|U+07F81羁| +U+08994覔|U+089C5觅| +U+089A9覩|U+07779睹| +U+089DD觝|U+062B5抵| +U+08A17託|U+06258托|U+08BAC讬| +U+08A3C証|U+08BC1证| +U+08A76詶|U+0916C酬| +U+08A96誖|U+06096悖| +U+08AAC説|U+08BF4说| +U+08AEE諮|U+08C18谘|U+054A8咨| +U+08B0C謌|U+06B4C歌| +U+08B21謡|U+08C23谣| +U+08B2D謭|U+08C2B谫| +U+08B41譁|U+054D7哗| +U+08B46譆|U+0563B嘻| +U+08B4C譌|U+08BB9讹| +U+08B54譔|U+064B0撰| +U+08B5F譟|U+0566A噪| +U+08B6D譭|U+06BC1毁| +U+08B81讁|U+08C2A谪| +U+08B8E讎|U+04EC7仇|U+096E0雠| +U+08B90讐|U+096E0雠| +U+08B9A讚|U+08D5E赞| +U+08C53豓|U+08273艳| +U+08C54豔|U+08273艳| +U+08C8D貍|U+072F8狸| +U+08C9B貛|U+0737E獾| +U+08CC9賉|U+06064恤| +U+08CDB賛|U+08D5E赞| +U+08CEB賫|U+08D4D赍| +U+08CF7賷|U+08D4D赍| +U+08D0B贋|U+08D5D赝| +U+08D11贑|U+08D63赣| +U+08D1C贜|U+08D43赃| +U+08D82趂|U+08D81趁| +U+08DE5跥|U+08DFA跺| +U+08DF4跴|U+08E29踩| +U+08E01踁|U+080EB胫| +U+08E2B踫|U+078B0碰| +U+08E30踰|U+0903E逾| +U+08E4F蹏|U+08E44蹄| +U+08E54蹔|U+06682暂| +U+08E5F蹟|U+08FF9迹| +U+08E60蹠|U+08DD6跖| +U+08E67蹧|U+07CDF糟| +U+08E75蹵|U+08E74蹴| +U+08E98躘|U+28001𨀁| +U+08EAD躭|U+0803D耽| +U+08EB3躳|U+08EAC躬| +U+08EB6躶|U+088F8裸| +U+08F19輙|U+08F84辄| +U+08F2D輭|U+08F6F软| +U+08F3C輼|U+08F92辒| +U+08FA0辠|U+07F6A罪| +U+08FA2辢|U+08FA3辣| +U+08FA4辤|U+08F9E辞| +U+08FB3辳|U+0519C农| +U+08FF4迴|U+056DE回| +U+08FFB迻|U+079FB移| +U+09008逈|U+08FE5迥| +U+09025逥|U+056DE回| +U+09029逩|U+05954奔| +U+0902C逬|U+08FF8迸| +U+09031週|U+05468周| +U+09049遉|U+04FA6侦| +U+0904A遊|U+06E38游| +U+09061遡|U+06EAF溯| +U+0906F遯|U+09041遁| +U+09156酖|U+09E29鸩| +U+09167酧|U+0916C酬| +U+09183醃|U+0814C腌| +U+09186醆|U+076CF盏| +U+09195醕|U+09187醇| +U+091A3醣|U+07CD6糖| +U+091AF醯|U+09170酰| +U+091BB醻|U+0916C酬| +U+091BC醼|U+05BB4宴| +U+091E6釦|U+06263扣| +U+091EC釬|U+0710A焊| +U+09205鈅|U+094A5钥| +U+0920E鈎|U+094A9钩| +U+09244鉄|U+094C1铁| +U+09246鉆|U+094BB钻| +U+09262鉢|U+094B5钵| +U+092B2銲|U+0710A焊| +U+092ED鋭|U+09510锐| +U+09332録|U+05F55录| +U+09341鍁|U+09528锨| +U+0934A鍊|U+070BC炼|U+094FE链| +U+0936B鍫|U+09539锹| +U+09373鍳|U+09274鉴| +U+0937E鍾|U+0953A锺|U+0949F钟| +U+0938C鎌|U+09570镰| +U+09397鎗|U+067AA枪| +U+0939A鎚|U+09524锤| +U+093AD鎭|U+093AE镇| +U+093AD鎭|U+09547镇| +U+093B8鎸|U+0954C镌| +U+093BB鎻|U+09501锁| +U+093DA鏚|U+0621A戚| +U+0941D鐝|U+09562镢| +U+09451鑑|U+09274鉴| +U+0945A鑚|U+094BB钻| +U+0945B鑛|U+077FF矿| +U+09464鑤|U+05228刨| +U+09475鑵|U+07F50罐| +U+09482钂|U+0954B镋| +U+09592閒|U+095F2闲| +U+09599閙|U+095F9闹| +U+095A4閤|U+09601阁|U+05408合| +U+095A7閧|U+054C4哄| +U+095B2閲|U+09605阅| +U+095C7闇|U+06697暗| +U+095DA闚|U+07AA5窥| +U+095E2闢|U+08F9F辟| +U+09628阨|U+05384厄| +U+0962A阪|U+0962A阪|U+05742坂| +U+0962C阬|U+05751坑| +U+09657陗|U+05CED峭| +U+0965C陜|U+09655陕| +U+0965E陞|U+0965E陞|U+05347升| +U+0967B陻|U+05819堙| +U+0967F陿|U+072ED狭| +U+09682隂|U+09634阴| +U+09684隄|U+05824堤| +U+09696隖|U+0575E坞| +U+096A3隣|U+090BB邻| +U+096B7隷|U+096B6隶| +U+0976D靭|U+097E7韧| +U+09771靱|U+097E7韧| +U+097A6鞦|U+079CB秋|U+097A7鞧| +U+097B5鞵|U+0978B鞋| +U+097BE鞾|U+09774靴| +U+097C6韆|U+05343千| +U+097C8韈|U+0889C袜| +U+097E4韤|U+0889C袜| +U+097EE韮|U+097ED韭| +U+0981F頟|U+0989D额| +U+0983C頼|U+08D56赖| +U+0983D頽|U+09893颓| +U+09847顇|U+060B4悴| +U+0984B顋|U+0816E腮| +U+09854顔|U+0989C颜| +U+09858願|U+0613F愿| +U+09866顦|U+06194憔| +U+098C3飃|U+098D8飘| +U+098DC飜|U+07FFB翻| +U+098E4飤|U+09972饲| +U+098F1飱|U+098E7飧| +U+09901餁|U+0996A饪| +U+09908餈|U+07CCD糍| +U+09918餘|U+09980馀|U+04F59余| +U+09935餵|U+05582喂| +U+09939餹|U+07CD6糖| +U+0993B餻|U+07CD5糕| +U+0993D餽|U+09988馈| +U+0994D饍|U+081B3膳| +U+09951饑|U+09965饥| +U+0995D饝|U+0998D馍| +U+099C8駈|U+09A71驱| +U+099E1駡|U+09A82骂| +U+09A10騐|U+09A8C验| +U+09A23騣|U+09B03鬃| +U+09A58驘|U+09AA1骡| +U+09ABD骽|U+0817F腿| +U+09ABE骾|U+09CA0鲠| +U+09AC8髈|U+08180膀| +U+09AE5髥|U+09AEF髯| +U+09B00鬀|U+05243剃| +U+09B09鬉|U+09B03鬃| +U+09B26鬦|U+06597斗| +U+09B28鬨|U+054C4哄| +U+09B2A鬪|U+06597斗| +U+09B30鬰|U+090C1郁| +U+09B8E鮎|U+09C87鲇| +U+09B9D鮝|U+09C9E鲞| +U+09BF0鯰|U+09CB6鲶|U+09C87鲇| +U+09C10鰐|U+09CC4鳄| +U+09C1B鰛|U+09CC1鳁| +U+09C2E鰮|U+09CC1鳁| +U+09CEC鳬|U+051EB凫| +U+09D08鴈|U+096C1雁| +U+09D5E鵞|U+09E45鹅| +U+09D70鵰|U+096D5雕|U+05F6B彫| +U+09D76鵶|U+09E26鸦| +U+09DC0鷀|U+09E5A鹚| +U+09DC4鷄|U+09E21鸡| +U+09DF0鷰|U+071D5燕| +U+09DF4鷴|U+09E47鹇| +U+09E0E鸎|U+083BA莺| +U+09E7B鹻|U+078B1碱| +U+09E7C鹼|U+078B1碱|U+07877硷| +U+09EAA麪|U+09762面| +U+09EAB麫|U+09762面| +U+09EAF麯|U+066F2曲| +U+09EB4麴|U+09EB9麹|U+066F2曲| +U+09EB5麵|U+09762面|U+09EBA麺| +U+09EF4黴|U+09709霉| +U+09F03鼃|U+086D9蛙| +U+09F07鼇|U+09CCC鳌| +U+09F08鼈|U+09CD6鳖| +U+09F15鼕|U+0549A咚| +U+09F63齣|U+051FA出| +U+09F67齧|U+0556E啮| +U+09F69齩|U+054AC咬| +U+20542𠕂|U+0518D再| +U+20545𠕅|U+0518D再| +U+207B0𠞰|U+0527F剿| +U+21681𡚁|U+05F0A弊| +U+21A25𡨥|U+05BC7寇| +U+21ED5𡻕|U+05C81岁| +U+2365C𣙜|U+069B7榷| +U+242EE𤋮|U+07199熙| +U+24A0F𤨏|U+07410琐| +U+24C48𤱈|U+04EA9亩| +U+24EA5𤺥|U+07629瘩| +U+262B1𦊱|U+06302挂| +U+26351𦍑|U+07F8C羌| +U+26548𦕈|U+07707眇| +U+26D4F𦵏|U+0846C葬| +U+28F7B𨽻|U+096B6隶| +U+294D0𩓐|U+08116脖| +U+295D7𩗗|U+098D3飓| diff --git a/www/wiki/maintenance/language/zhtable/trad2simp_noconvert.manual b/www/wiki/maintenance/language/zhtable/trad2simp_noconvert.manual new file mode 100644 index 00000000..8b6dd7a8 --- /dev/null +++ b/www/wiki/maintenance/language/zhtable/trad2simp_noconvert.manual @@ -0,0 +1,19 @@ +余 +碁 +藉 +𫚭 +咤 +吒 +曏 +痾 +枒 +幺 +苹 +厘 +𫍟 +垴 +岙 +㳕 +䓕 +埯 +埰 diff --git a/www/wiki/maintenance/language/zhtable/trad2simp_supp_set.manual b/www/wiki/maintenance/language/zhtable/trad2simp_supp_set.manual new file mode 100644 index 00000000..d1728f0a --- /dev/null +++ b/www/wiki/maintenance/language/zhtable/trad2simp_supp_set.manual @@ -0,0 +1,3 @@ +著 着 +藉 借 +濛 蒙
\ No newline at end of file diff --git a/www/wiki/maintenance/language/zhtable/tradphrases.manual b/www/wiki/maintenance/language/zhtable/tradphrases.manual new file mode 100644 index 00000000..c5d5fd73 --- /dev/null +++ b/www/wiki/maintenance/language/zhtable/tradphrases.manual @@ -0,0 +1,3722 @@ +零隻 +〇隻 +一隻 +二隻 +兩隻 +三隻 +四隻 +五隻 +六隻 +七隻 +八隻 +九隻 +0隻 +1隻 +2隻 +3隻 +4隻 +5隻 +6隻 +7隻 +8隻 +9隻 +0只支援 +1只支援 +2只支援 +3只支援 +4只支援 +5只支援 +6只支援 +7只支援 +8只支援 +9只支援 +0只支持 +1只支持 +2只支持 +3只支持 +4只支持 +5只支持 +6只支持 +7只支持 +8只支持 +9只支持 +百隻 +千隻 +萬隻 +億隻 +最多 +至多 +頂多 +多隻 +這只能 +這只可 +這只在 +這只是 +這只需 +這只須 +這只會 +這只用 +這只比 +這只限 +這只應 +這只不過 +這只包括 +那只能 +那只可 +那只在 +那只是 +那只需 +那只須 +那只會 +那只用 +那只怕 +那只比 +那只限 +那只應 +那只不過 +那只包括 +多只能 +多只可 +多只在 +多只有 +多只是 +多只需 +多只須 +多只會 +多只用 +多只含 +多只比 +多只限 +多只包括 +大只能 +大只可 +大只在 +大只有 +大只是 +大只需 +大只會 +小只能 +小只可 +小只在 +小只有 +小只是 +小只需 +小只會 +數隻 +數只能 +數只可 +數只在 +數只有 +數只是 +數只需 +數只須 +數只會 +數只含 +數只比 +數只限 +數只應 +數只包括 +人數只 +參數只 +總數只 +隻身 +形單影隻 +首隻 +數天後 +幾天後 +多天後 +零天後 +一天後 +二天後 +兩天後 +三天後 +四天後 +五天後 +六天後 +七天後 +八天後 +九天後 +十天後 +百天後 +千天後 +萬天後 +億天後 +0天後 +1天後 +2天後 +3天後 +4天後 +5天後 +6天後 +7天後 +8天後 +9天後 +天後來 +天後天 +天後半 +後印 +萬象 +乾絲 +乾魚 +魚乾 +乾梅 +糕乾 +黃乾黑瘦 +馬乾 +香乾 +趲幹 +謀幹 +詞幹 +蟶乾 +薄幹 +腦幹 +營幹 +老乾 +老幹部 +管幹 +盲幹 +煨乾 +海乾 +乾漆 +淚乾 +沒幹 +沒乾沒淨 +杯乾 +打幹 +打乾噦 +徐幹 +府幹 +乾館 +乾顙 +幹革命 +乾霍亂 +乾雷 +乾阿奶 +乾量 +乾醋 +乾逼 +乾貨 +乾衣 +幹蠱 +乾虔 +乾落 +幹營生 +乾茶錢 +乾茨臘 +乾苔 +乾花 +乾肥 +乾耗 +幹缺 +乾繃 +乾結 +乾餱 +乾篾片 +乾稿 +乾禮 +乾瞪眼 +乾白兒 +乾疥 +乾生子 +乾生受 +幹父之蠱 +乾熬 +乾燈盞 +乾濕 +乾澀 +幹濟 +乾沒 +乾死 +乾村沙 +乾暖 +乾料 +乾支支 +乾支剌 +乾擦 +乾撇下 +乾撂台 +乾折 +乾急 +幹當 +乾式 +乾屎橛 +幹家 +乾奴才 +幹頭 +乾塢 +乾圓潔淨 +乾回付 +乾啼 +乾哭 +乾噦 +乾咽 +乾和 +幹吏 +乾號 +乾颱 +乾卦 +乾剝剝 +乾刻版 +乾芻 +乾產 +乾喬 +大目乾連 +國之楨榦 +唇乾 +單幹 +勾幹 +豆乾 +果乾 +如果幹 +乾麵 +乾柴 +枯乾 +曬乾 +顛乾倒坤 +強幹 +乾眼 +井幹 +乾巴 +偎乾 +眼乾 +瀝乾 +白乾兒 +肉絲麵 +薑絲 +反覆 +豐濱 +豐濱鄉 +豐度 +雞絲 +雞絲麵 +髮絲 +斷髮 +不斷發 +判斷發 +評斷發 +買斷發 +賣斷發 +打斷發 +披頭散髮 +髮禁 +世界盃 +其次辟地 +開闢 +闢地 +精闢 +別闢 +另闢 +闢佛 +闢田 +闢築 +闢謠 +闢辟 +透闢 +墾闢 +翕闢 +軒闢 +闢建 +闢室 +各闢 +增闢 +闢邪以律 +錶盤 +錶板 +錶帶 +錶針 +錶蒙子 +袋錶 +腕錶 +碼錶 +錶冠 +魔錶 +并州 +幽并 +併力 +,並力 +,并力討 +兼併 +併兼 +併骨 +併網 +併線 +江併流 +水併流 +逼併 +併名 +併肩子 +併疊 +簡併 +並發表 +並發現 +並發展 +並發動 +並發布 +火並非 +舉手表 +揮手表 +併一不二 +連三併四 +相併 +撤併 +數罪併罰 +催併 +狂併潮 +合併 +併為一體 +併為一家 +併吞 +並吞下 +提摩太後書 +裏海 +不採 +披榛採蘭 +謬採虛聲 +採樵人 +回採 +觀採 +開採 +揪採 +樵採 +改採 +採訪 +採辦 +採補 +採買 +採風問俗 +採納 +採獵 +採蓮 +採錄 +採購 +採光 +採礦 +採花 +採集 +採擷 +採掘 +採芹人 +採取 +採選 +採摭 +採摘 +採珠 +採種 +採茶 +採石 +採拾 +採收 +採生折割 +採樹種 +採擇 +採藥 +採薇 +採用 +盜採 +採信 +採行 +採證 +採菊 +博採 +採空採穗 +採挖 +採鐵 +採金 +採氣 +採油 +採煤 +採鹽 +採區 +採運 +採風 +採血 +花不要採 +官地為寀 +寮寀 +蔘綏 +蕭蔘 +東衝西突 +天克地衝 +六衝 +撞陣衝軍 +衝波 +衝風 +衝頭陣 +衝堅陷陣 +衝陷 +衝心 +衝州撞府 +衝殺 +衝然 +衝盹 +左衝右突 +虫部 +手塚治虫 +群醜 +百拙千醜 +大醜 +地醜德齊 +丟醜 +亮醜 +揭醜 +倛醜 +嫌好道醜 +醜巴怪 +醜末 +醜婦 +醜地 +醜頭怪臉 +醜女效顰 +醜剌剌 +醜話 +醜媳 +醜吒 +醜聲遠播 +醜夷 +弄醜 +露醜 +摧堅獲醜 +謷醜 +不嫌母醜 +一爭兩醜 +惡直醜正 +很醜 +醜男 +醜斃了 +醜奴兒 +醜言 +醜徒 +醜雜 +醜儕 +醜沮 +醜辭 +醜比 +醜辱 +醜逆 +醜史 +醜賊生 +真醜 +出乖弄醜 +出乖露醜 +獲匪其醜 +乙丑 +丁丑 +己丑 +辛丑 +癸丑 +丑時 +丑日 +丑月 +丑年 +文丑 +武丑 +女丑 +小丑 +大丑 +丑旦 +丑角 +丑三 +丑表功 +公孫丑 +平平當當 +滿滿當當 +當當丁丁 +丁丁當當 +停停當當 +快快當當 +咯噹 +啷噹 +党進 +党太尉 +党項 +撲鼕 +洗髮 +牽一髮 +白發其事 +后髮座 +后髮星系團 +后髮FK型星 +波髮藻 +辮髮 +逋髮 +抿髮 +髮漂 +髮匪 +髮腳 +髮癬 +髮釵 +髮飾 +髮紗 +髮簪 +髮上指冠 +髮上沖冠 +髮乳 +髮引千鈞 +髮踴沖冠 +董氏封髮 +胎髮 +禿妃之髮 +捉髮 +綠髮 +括髮 +髡髮 +鵠髮 +截髮 +解髮佯狂 +淨髮 +噙齒戴髮 +青山一髮 +晞髮 +細不容髮 +心細如髮 +祝髮 +擢髮 +齒髮 +齒危髮秀 +沖冠髮怒 +甩髮 +絲髮 +絲恩髮怨 +蒜髮 +有髮頭陀寺 +髮箋 +髮屋 +櫛髮工 +鬒髮 +人髮指 +爆發 #分詞 +引發 +開發 +剪其髮 +吐哺捉髮 +吐哺握髮 +含齒戴髮 +大金髮苔 +寸髮千金 +心長髮短 +戴髮含齒 +拔髮 +拔鬚 +揪髮 +揪鬚 +整髮用品 +斷髮文身 +滿頭洋髮 +燙一個髮 +燙一次髮 +燙個髮 +燙完髮 +燙次髮 +理一個髮 +理一次髮 +理個髮 +理完髮 +理次髮 +細如髮 +繫於一髮 +皮膚 +生華髮 +蒼髮 +被髮佯狂 +被髮入山 +被髮左衽 +被髮纓冠 +被髮陽狂 +身體髮膚 +髮光可鑑 +髮已霜白 +髮油 +髮為血之本 +髮網菌 +髮踊沖冠 +髮際 +黃髮 +齒落髮白 +長髮姑娘 +長髮公主 +長髮妹 +的髮小 +是髮小 +代理發行 +美髮店 +美髮館 +美髮師 +美髮學 +美髮業 +美髮沙龍 +美容美髮 +程十髮 +模范棒棒堂 +模范三軍 +模范七棒 +顏範 +儀範 +典範 +坤範 +壼範 +容範 +懿範 +明範 +格範 +模範 +樣範 +母範 +洪範 +淑範 +遺範 +科範 +立範 +貽範 +道範 +閨範 +閫範 +雅範 +霽範 +鴻範 +沒樣範 +錢範 +銅範 +金範 +範金 +垂範 +範性形變 +範字 +有事之無範 +置言成範 +吾爲之範我馳驅 +天地為範 +範數 +範亭 +丰采 +丰標不凡 +丰神 +丰茸 +丰儀 +丰度 +丰情 +丰韵 +子之丰兮 +艸木丰丰 +張三丰 +復始 +往復式 +複分析 +複輔音 +複元音 +複平面 +複函數 +複流 +反複製 +複對數 +複分解 +複合 #因複合詞頻遠高於復合 +複方 +複穗 +撥穀 +扁擬穀盜蟲 +不穀 +辟穀 +脫穀機 +年穀 +礱穀 +穀米 +穀旦 +穀圭 +穀貴餓農 +穀食 +穀日 +館穀 +禾穀 +積穀 +嘉穀 +嚼穀 +九穀 +戩穀 +錢穀 +息穀 +殖穀 +曬穀 +臧穀亡羊 +種穀 +陽穀 +布穀鳥 +穀祿 +穀城縣 +穀氨 +穀胱 +颳雪 +广部 +亂鬨鬨 +斗鬨 +開鬨 +花鬨 +鬨動 +交鬨 +喧鬨 +起鬨 +內鬨 +猜三划五 +划龍舟 +划龍船 +南迴線 +南迴鐵路 +北迴線 +北迴鐵路 +迴文詩 +迴文數 +迴文錦 +迴文聯 +迴文序列 +迴文結構 +迴文構詞 +滙豐 +伙頭 +伏几 +高几 +雪窗螢几 +燕几 +隱几 +几筵 +饑饉 +乾薑 +毛薑 +薑母 +薑湯 +薑桂 +薑還是老的辣 +吃薑 +薑老辣 +野薑 +咬薑呷醋 +薑蓉 +薑黃 +狐藉虎威 +滑藉 +藉寇兵 +藉箸代籌 +藉手 +藉此 +龍捲 +捲舌 +不捲 +漫捲 +捲地 +捲瓣 +捲葉蛾 +捲尾猴 +捲積雲 +夸父 +夸克 +夸特 +夸毗 +夸麗 +夸姣 +夸人 +夸容 +大言非夸 +言大而夸 +睏覺 +愛睏 +纍堆 +纍紲 +纍臣 +纍瓦結繩 +湘纍 +印纍綬若 +灕湘 +灕然 +滲灕 +裏勾外連 +水里溪 +二里頭 +年歷史 +年歷次 +西歷史 +西歷次 +西歷代 +西歷任 +國歷史 +國歷代 +國歷任 +國歷屆 +國歷經 +國歷來 +新歷史 +夏歷史 +百花曆 +寶曆 +穆罕默德曆 +大明曆 +大曆 +檯曆 +太初曆 +通曆 +曆本 +曆命 +曆紀 +曆始 +曆室 +曆日 +曆尾 +曆元 +律曆志 +官曆 +回曆 +巧曆 +慶曆 +朱理安曆 +長曆 +藏曆 +四分曆 +三統曆 +額我略曆 +埃及曆 +伊斯蘭教曆 +合曆 +玉曆 +農民曆 +桌曆 +商曆 +周曆 +大衍曆 +皇極曆 +儒略改革曆 +希伯來曆 +格里曆 +格里高利曆 +共和曆 +掛曆 +曆獄 +天文曆表 +日心曆表 +地心曆表 +復活節曆表 +月球曆表 +伊爾汗曆表 +延曆 +萬曆 +永曆 +聖人曆 +羅馬曆 +羅馬歷史 +羅馬歷代 +曆數書 +曆局 +授時曆 +顓頊曆 +共和歷史 +厤物之意 +爰定祥厤 +白黴 +黴黧 +黴黑 +麴黴 +蒙霧露 +懞懞懂懂 +懞直 +老懞 +放懞掙 +矇聵 +矇瞍 +矇事 +矇頭轉 +矇松雨 +藏矇歌兒 +朦朧 +濛濛細雨 +濛汜 +冥濛 +溟濛 +淡濛濛 +凌濛初 +涳濛 +灰濛濛 +澒濛 +瀰山遍野 +瀰瀰 +冷麵 +撈麵 +煮麵 +炆麵 +煎麵 +泡麵 +食麵 +公仔麵 +方便麵 +白粉麵 +棒子麵 +麵缸 +麵坯兒 +麵碼兒 +麵坊 +麵湯 +麵疙瘩 +麵館 +麵漿 +甜水麵 +麵人兒 +麵塑 +捏麵人 +趕麵棍 +擀麵 +過水麵 +蕎麥麵 +削麵 +小米麵 +壯麵 +吃板刀麵 +扯麵 +搋麵 +重羅麵 +雜麵 +雜合麵兒 +溲麵 +索麵 +一鍋麵 +伊府麵 +藥麵兒 +意大利麵 +湯下麵 +茶麵 +麵團 +北山索麵 +土索麵 +米麵 +椒麵 +掛麵 +臊子麵 +龍鬚麵 +油潑麵 +辣麵 +肉麵 +燴麵 +蝦麵 +雲吞 +一碗麵 +吃碗麵 +吃麵 +麵點師 +麵點、 +、麵點 +麵製品 +乾脆麵 +磨麵 +莜麵 +雲吞麵 +拌麵 +乾拌麵 +冷面相 +糞穢衊面 +僕僕 +有僕 +冉有僕 +屢顧爾僕 +僕少 +僕雖罷駑 +僕夫 +僕僮 +僕吏 +僕姑 +僕固懷恩 +僕程 +僕使 +僕憎 +僕歐 +僕射 +太僕 +僮僕 +金僕姑 +僕婢 +惡僕 +從僕 +樸實 +樸訥 +樸念仁 +白樸 +抱素懷樸 +抱朴而長吟兮 +樸鄙 +樸馬 +樸父 +樸陋 +樸魯 +樸厚 +樸學 +樸質 +樸拙 +樸重 +樸素 +樸樕 +樸野 +反樸 +古樸 +胡樸安 +返樸 +渾樸 +儉樸 +簡樸 +拙樸 +斫雕為樸 +斲雕為樸 +質樸 +誠樸 +純樸 +曾樸 +郁樸 +棫樸 +敦樸 +樸鈍 +樸直 +見素抱樸 +掣籤 +標籤 +書籤 +發籤 +粉籤子 +路籤 +更籤 +好籤 +火籤 +籤幐 +籤押 +照入籤 +制籤 +抽公籤 +瑤籤 +藥籤 +萬籤插架 +雲笈七籤 +上簽名 +上簽字 +上簽收 +上簽寫 +上簽訂 +上簽定 +上簽署 +上簽發 +上簽約 +上簽了 +上簽證 +中簽名 +中簽字 +中簽收 +中簽寫 +中簽訂 +中簽定 +中簽署 +中簽發 +中簽約 +中簽了 +中簽證 +下簽名 +下簽字 +下簽收 +下簽寫 +下簽訂 +下簽定 +下簽署 +下簽發 +下簽約 +下簽了 +下簽證 +犖确 +磽确 +确瘠 +拚捨 +廣捨 +齊王捨牛 +捨墮 +捨實 +棄捨 +捨安就危 +施舍之道 +瀋河 +瀋水 +瀋州 +瀋北 +瀋吉 +瀋山線 +瀋山鐵路 +瀋海鐵路 +瀋海高速 +瀋丹線 +瀋丹鐵路 +瀋丹客運 +瀋丹高 +瀋大線 +瀋大鐵路 +瀋大高速 +秦瀋客運 +遼瀋 +京瀋 +胜肽 +胜鍵 +雙胜類 +兀朮 +白朮 +蒼朮 +赤朮 +朮赤 +莪朮 +博爾朮 +巴而朮 +朮虎高 +耶律朮烈 +髼鬆 +皮鬆 +濛鬆雨 +發鬆 +翻鬆 +浮鬆 +弄鬆 +旋鬆 +精鬆 +懈鬆 +鬆蛋 +鬆寬 +鬆氣 +鬆一口氣 +鬆元音 +鬆喉 +鬆化 +很鬆 +寬鬆鬆 +蓬鬆鬆 +輕鬆鬆 +鬆鬆地 +囉囉囌囌 +囉囌 +骨罈 +菜罈 +罈騞 +鹹粥 +鹹食 +鹹潟 +鹹嘴淡舌 +鹽打怎麼鹹 +鹹派 +鹹批 +鹹濕 +鹹豬 +甜鹹 +鹹甜 +甜、鹹 +鹹、甜 +錦綉花園 +籲天 +勃鬱 +怫鬱 +氣鬱 +沉鬱 +神荼鬱壘 +躁鬱 +蒼鬱 +漚鬱 +伊鬱 +壹鬱 +悒鬱 +氤鬱 +湮鬱 +陰鬱 +泱鬱 +坱鬱 +滃鬱 +蓊鬱 +紆鬱 +鬱勃 +鬱陶 +鬱律 +鬱壘 +鬱火 +鬱積 +鬱金 +鬱江 +鬱血 +鬱蒸 +鬱症 +鬱沉沉 +鬱熱 +鬱塞 +鬱伊 +鬱邑 +鬱挹 +鬱堙不偶 +鬱泱 +鬱蓊 +鬱紆 +鬱燠 +肝鬱 +鬱卒 +鬱鬱不平 +鬱鬱不樂 +鬱鬱寡歡 +鬱鬱蔥蔥 +鬱鬱蒼蒼 +鬱鬱而終 +愿樸 +愿而恭 +許愿起經 +北嶽 +嶽麓 +但云 +胡云 +詩云 +注云 +鄭凱云 +云乎 +云然 +云為 +對摺 +網誌 +標標致致 +澄澹精致 +呆緻緻 +光緻緻 +縝緻 +堅緻 +种放 +种師道 +种師中 +正官庄 +冬山庄 +松山庄 +香山庄 +中庄子 +新庄子 +田庄英雄 +本庄 +庄司 +厂部 +衝量 +衝車 +相干 +府干預 +府干涉 +府干政 +府干擾 +府干犯 +府干卿 +一干人 +未乾 +未干涉 +未干預 +抹乾 +餅乾 +拭乾 +擦乾 +晾乾 +烘乾 +肉乾 +菜乾 +腐乾 +乾脆 +乾淨 +乾燥 +乾旱 +乾涸 +乾洗 +乾女 +乾等 +乾糧 +乾枯 +乾薪 +乾爹 +乾粉 +乾爽 +乾兒 +乾子 +乾渴 +乾股 +乾果 +乾草 +乾菜 +乾笑 +乾餾 +乾電 +乾飯 +乾冰 +乾嘔 +乾材 +乾媽 +乾季 +葡萄乾 +提子乾 +芒果乾 +菠蘿乾 +鳳梨乾 +豆腐乾 +果子乾 +龍眼乾 +乾乾淨淨 +乾柴烈火 +桑乾 +撈乾 +搭乾鋪 +揩乾 +敢幹 +幹探 +幹事 +幹什麼 +幹細胞 +樹幹 +口燥唇乾 +舌乾唇焦 +不食乾腊 +不乾不淨 +乾重 +蒸乾 +乾物 +乾食 +乾鍋 +自乾五 +不乾膠 +老白乾 +乾姐 +乾紅葡萄酒 +乾白葡萄酒 +抽乾 +排乾 +排幹部 +吸乾 +楨幹 +新幹縣 +誰幹的 +他幹的 +們幹的 +人幹的 +幹的事 +幹的好事 +得力幹將 +黑幹將 +的幹將 +幹大事 +對着幹 +怎麼幹 +這麼幹 +幹這 +幹仗 +李連杰 +周杰 +杰倫 +文杰 +杰威爾 +黃詩杰 +何杰 +狄志杰 +伊適杰 +張杰 +孫杰 +胡杰 +陳杰 +黃杰 +謝杰 +正杰 +柳斌杰 +稜鏡 +稜角 +稜台 +稜錐 +觚稜 +稜子 +稜層 +稜柱 +盧稜伽 +波稜菜 +菠稜菜 +稜縫 +稜等登 +稜稜 +嶒稜 +蹭稜子 +稜體 +二不稜登 +有稜有角 +威稜 +債纍纍 +果纍纍 +實纍纍 +儒略曆 +伊斯蘭曆 +酒麴 +澹臺 +拜託 +委託 +輓曲 +敬輓 +輓車 +輓輸 +輓辭 +万俟 +万旗 +鬚鯨 +鬚鯊 +兇手 +兇徒 +兇案 +兇器 +兇殺 +兇殘 +行兇 +緝兇 +追兇 +真兇 +疑兇 +買兇 +元兇 +叶韻 +叶音 +叶恭弘 +新紮 +紙紮 +紮鐵 +紮寨 +一紮 +兩紮 +三紮 +四紮 +五紮 +六紮 +七紮 +八紮 +九紮 +十紮 +百紮 +千紮 +萬紮 +誌異 +筑前 +修築前 +建築前 +筑後 +修築後 +建築後 +筑紫 +筑波 +筑州 +筑肥 +筑西 +筑北 +肥筑方言 +筑邦 +筑陽 +南筑 +悲筑 +批准 +核准 +為準 +準直 +擺鐘 +編鐘 +碰鐘 +鳴鐘 +晨鐘 +鐘體 +飯後鐘 +盜鐘 +一天鐘 +撞鐘 +殿鐘自鳴 +天文鐘 +天文學鐘 +洛鐘東應 +亮鐘 +郘鐘 +歌鐘 +鐘不撞不鳴 +毀鐘為鐸 +洪鐘 +擊鐘 +警世鐘 +竊鐘掩耳 +琴鐘 +見鐘不打 +釁鐘 +朝鐘 +木鐘 +鐘不扣不鳴 +鐘鳴 +鐘塔 +鐘漏 +鐘琴 +鐘磬 +鐘形蟲 +鐘乳洞 +鐘乳石 +鐘在寺裡 +詩鐘 +懸鐘 +山崩鐘應 +坐鐘 +宗周鐘 +塞耳盜鐘 +二缶鐘惑 +口鐘 +鐘的 +的鐘 +這鐘 +叩鐘 +音聲如鐘 +應鐘 +原子鐘 +泳氣鐘 +電子鐘 +電子鐘錶 +石英鐘錶 +石英鐘 +鐘錶王 +鐘律 +看鐘 +看錶 +看表面 +鐵鐘 +鐘不敲不響 +對準鐘 +對準鐘錶 +對準錶 +鐘錶快 +鐘快 +錶快 +鐘錶慢 +鐘慢 +錶慢 +響鐘 +鐘敲 +世紀鐘錶 +世紀鐘 +錶王 +鐘王 +鐘錶 +古鐘 +古鐘錶 +鐘面 +鐘表面 +南京鐘 +南京鐘錶 +造鐘 +鐘行 +小型鐘表面 +小型鐘面 +小型鐘錶 +小型鐘 +中型鐘表面 +中型鐘面 +中型鐘錶 +中型鐘 +大型鐘表面 +大型鐘面 +大型鐘錶 +大型鐘 +鐘匠 +深山何處鐘 +下課鐘 +上課鐘 +老爺鐘 +萬年曆錶 +個鐘 +個鐘錶 +喜歡鐘 +喜歡鐘錶 +喜歡錶 +大鐘 +佛鐘 +鐘壁 +鐘腰 +鐘口 +鐘身 +鐘模 +鐘頂 +鐘紐 +鐘座 +寺鐘 +座鐘 +大笨鐘 +大本鐘 +點多鐘 +點半鐘 +分多鐘 +刻多鐘 +分半鐘 +刻半鐘 +教學鐘 +操作鐘 +南屏晚鐘 +敲鐘 +警報鐘 +猶如鐘 +猶如鐘錶 +猶如錶 +舊鐘錶 +繁鐘 +四面鐘 +更鐘 +警示鐘 +鐘差 +任何鐘錶 +任何鐘 +手錶 +選手表現 +選手表達 +選手表示 +選手表明 +選手表決 +分子鐘 +飛行鐘 +鐘罩 +主鐘差 +花鐘 +磬鐘 +主鐘曲線 +鐘速 +紅鐘 +各類鐘 +衛星鐘 +該鐘 +錶轉 +鐘調 +調鐘錶 +調錶 +原鐘 +鐘錶速 +件鐘 +鐘發音 +逆鐘 +拂鐘無聲 +鐘不空則啞 +晚鐘 +潛水鐘錶 +潛水鐘 +潛水錶 +樂器鐘 +鐘左右 +鐘陳列 +驚鐘 +鐘錶停 +鐘停 +銫鐘 +數字鐘錶 +數字鐘 +顯示鐘錶 +顯示鐘 +顯示錶 +坐如鐘 +錶停 +西周鐘 +東周鐘 +錶速 +機械鐘錶 +機械鐘 +機械錶 +之鐘 +鐘形 +架鐘 +順鐘向 +逆鐘向 +遺傳鐘 +鬧錶 +華嚴鐘 +懷鐘 +生物鐘 +鐘好 +鐘太 +鐘不 +鐘有 +鐘盤 +鐘錶盤 +鐘沒 +鐘被 +制鐘 +布穀鳥鐘 +咕咕鐘 +拉克施爾德鐘 +鐘上 +鐘下 +摸鐘 +舊鐘 +舊錶 +台鐘 +鐘響 +船鐘 +電波鐘 +石鐘 +自由鐘 +鐘螺 +鐘花 +馬德鐘 +計時錶 +防水錶 +顯示表格 +顯示表頭 +顯示表面 +顯示表達 +顯示表明 +顯示表現 +顯示表示 +電錶 +水錶 +水表示 +咪錶 +射鵰 +神鵰 +神雕像 +采石磯 +采石之戰 +采石之役 +聊齋志異 +部落發 +角落發 +村落發 +蛇髮女妖 +畢生發展 +對華發 +尸魂界 +樹樑 +屋樑 +樑柱 +柱樑 +下樑 +上梁山 +僥倖 +夏遊 +秋遊 +冬遊 +傲遊 # 浏览器名 +網遊 +桌遊 +手遊 +遊輪 +遊牧 +遊蕩 +遊刃 +遊廊 +遊春 +遊美學務 +黑奴籲天錄 +林郁方 +讚歌 +崑山 +崑曲 +崑腔 +崑調 +崑劇 +崑蘇 +蘇崑 +一干家中 +星期後 +依依不捨 +戀戀不捨 +窮追不捨 +緊追不捨 +鍥而不捨 +稜登 +繃扒弔拷 +不弔, +不通弔慶 +陪弔 +盆弔 +撇弔 +憑弔 +門弔兒 +伐罪弔民 +打出弔入 +搗鬼弔白 +弔膀子 +弔民 +弔奠 +弔頭 +弔古 +弔詭 +弔客 +弔拷 +弔扣 +弔賀迎送 +弔鶴 +弔喉 +弔謊 +弔祭 +弔恤 +弔腳兒事 +弔取 +弔孝 +弔紙 +弔者大悅 +弔詞 +弔撒 +弔喪 +弔腰撒跨 +弔唁 +弔宴 +弔喭 +弔影 +弔慰 +弔文 +弔問 +弄鬼弔猴 +開弔 +鶴弔 +昊天不弔 +花馬弔嘴 +會弔 +吉凶慶弔 +蟣蝨相弔 +祭弔 +慶弔 +影相弔 +哀弔 +唁弔 +鬼谷子 +谷子敬 +洪谷子 +聖馬爾谷日 +澀谷區 +開山闢谷 +山谷 #分詞用 +溝谷 +曼谷 +星露谷物語 +于美人 +緊緻 +曰云 +若干 +徵婚 +鬥鬨 +事有鬥巧 +歹鬥 +鬥茶 +鬥鴨 +爭奇鬥妍 +誇能鬥智 +春香鬥學 +鬥引 +鬥彩 +鬥武 +鬥悶 +鬥牙拌齒 +鬥幌子 +鬥腳 +雞吵鵝鬥 +辯鬥 +廝鬥 +誇多鬥靡 +臨潼鬥寶 +鬥趣 +撩鬥 +傲霜鬥雪 +賭鬥 +搬鬥 +鬥爭鬥合 +鬥疊 +鬥文 +耍鬥 +鬥巧 +油鬥 +蚊動牛鬥 +卵與石鬥 +挑鬥 +爭奇鬥異 +鬥葉子 +鬥分子 +爭妍鬥奇 +不鬥 +鬥心眼 +鬥頭 +挌鬥 +好鬥 +鬥合 +拚鬥 +兩虎共鬥 +兩鼠鬥穴 +鬥犀臺 +鬥牙鬥齒 +惡鬥 +鬥勝 +鬥富 +鬥艦 +鬥葉兒 +鬥彆氣 +鬥話 +鬥牌 +鬥百草 +鬥打 +鬥犬 +鬥風 +鬥雪紅 +鬥暴 +鬥閒氣 +龍鬥虎傷 +殷師牛鬥 +二虎相鬥 +鬥力 +爭紅鬥紫 +鬥麗 +鬥狠 +鬥飣 +虎鬥 +引鬥 +爭妍鬥豔 +轉鬥千里 +鬥而鑄兵 +困鬥 +好勇鬥狠 +爭奇鬥豔 +使其鬥 +鬥地主 +鈎心鬥角 +鬥劍 +激鬥 +政鬥 +鬥獸 +鬥龍 +鬥勇 +鬥狗 +鬥蛐 +鬥垮 +鬥敗 +鬥戰 +窩裡鬥 +亂鬥 +石樑 +木樑 +藏歷史 +頁面 +面條目 +大讚 +唄讚 +褒讚 +謬讚 +誄讚 +祝讚 +詩讚 +賞讚 +讚嘆 +讚唄 +點讚 +點個讚 +讚一個 +超讚 +飛紮 +紮裹 +紮腳 +紮詐 +紮囮 +住紮 +佔畢 +佔頭籌 +佔高枝兒 +隱佔 +憑摺 +沒摺至 +大摺兒 +大週摺 +火摺子 +裝摺 +變徵 +談徵 +納徵 +流徵 +柳詒徵 +固徵 +貴徵 +考徵 +咎徵 +杞宋無徵 +休徵 +徵辟 +徵名責實 +徵發 +徵風召雨 +徵答 +徵啟 +徵選 +徵招 +徵士 +徵庸 +之徵 +瑞徵 +三徵七辟 +額徵 +有徵 +有征服 +有征戰 +有征伐 +有征討 +無徵不信 +文徵明 +徵跡 +徵車 +徵效 +徵怪 +徵聖 +徵咎 +徵吏 +徵令 +本徵 +黃鈺筑 +當準 +憑準 +沒準 +蜂準 +推情準理 +寇準 +合準 +準保 +準譜 +準分子 +準點 +一個準 +準擬 +準貨幣 +準軍事 +準式 +認準 +三準 +鵝準 +有準 +鎌倉 +請君入甕 +甕安 +痊癒 +治癒 +病癒 +大病初癒 +癒合 +槓桿 +宣洩 +鑑別 +鑑察 +鑑定 +鑑戒 +鑑諒 +鑑賞 +鑑於 +鑑證 +鑑湖 +鑑識 +鑑藏 +鑑往知來 +品鑑 +評鑑 +可鑑 +為鑑 +之鑑 +鑑古 +明鑑 +寶鑑 +破鑑 +年鑑 +圖鑑 +通鑑 +綱鑑 +鸞鑑 +借鑑 +龜鑑 +衡鑑 +史鑑 +殷鑑 +印鑑 +王鑑 +勳章 +張勳 +趙治勳 +殭屍 +有栖川 +栗栖溪 +鳥栖市 +兇惡 +兇狠 +兇猛 +兇橫 +兇悍 +兇險 +兇相 +兇犯 +嫌兇 +兇嫌 +兇疑 +兇刀 +兇槍 +很兇 +兇巴巴 +頂兇 +太兇 +好兇 +凝鍊 +鍊貧 +鍊度 +鍊形 +鍊師 +鍊石 +鍊字 +鍊冶 +細鍊 +陳鍊 +闖鍊 +鍊汞 +淬鍊 +鋼之鍊金術師 +索馬里 +范登堡 +製漿 +三統歷史 +伊斯蘭教歷史 +伊斯蘭歷史 +儒略改革歷史 +儒略歷史 +公歷史 +台歷史 +合歷史 +周歷史 +商歷史 +四分歷史 +回歷史 +埃及歷史 +大明歷史 +大歷史 +大衍歷史 +太初歷史 +官歷史 +寶歷史 +巧歷史 +希伯來歷史 +弘歷史 +慶歷史 +日歷史 +星歷史 +月歷史 +朱理安歷史 +桌歷史 +永歷史 +玉歷史 +百花歷史 +皇歷史 +皇極歷史 +穆罕默德歷史 +算歷史 +紀歷史 +舊歷史 +航海歷史 +萬歷史 +行事歷史 +農歷史 +農民歷史 +通歷史 +長歷史 +陰歷史 +陽歷史 +額我略歷史 +黃歷史 +天曆 +天歷史 +美醜 +獻醜 +出醜 +家醜 +遮醜 +醜八怪 +醜名 +醜詆 +醜態 +醜女 +醜類 +醜陋 +醜虜 +醜化 +醜劇 +醜媳婦 +醜小鴨 +醜行 +醜事 +醜聲 +醜人 +醜惡 +醜丫頭 +醜聞 +醜語 +母醜 +一齣子 +丰標 +丰姿 +丰韻 +鵰翎 +鵰心雁爪 +鵰鶚 +雙鵰 +撲鼕鼕 +普鼕鼕 +鼕鼕鼓 +剷頭 +剷刈 +花菴詞選 +渾箇 +箇中原因 +箇中理由 +箇中高手 +箇中好手 +箇中強手 +箇中滋味 +箇中奧 +箇中道理 +箇中玄機 +箇中翹楚 +,箇中 +。箇中 +的箇中 +對表達 +對表現 +對表演 +對表揚 +對表中 +對表明 +一伙頭 +一伙食 +一半只 +一干弟兄 +一干弟子 +一干部下 +一斗斗 +一面食 +萬一只 +上面糊 +不克自制 +不加自制 +不占凶吉 +不占卜 +不占吉凶 +不占算 +不好干涉 +不好干預 +不斗膽 +不每只 +不采聲 +向往常 +向往日 +向往時 +向往來 +方向 +轉向 +單向 #分詞用 +丰容 +之一只 +之二只 +之八九只 +二只得 +亦云 +人云 +以自制 +其一只 +其二只 +其八九只 +內面包 +內面包的 +准保護 +准保釋 +几上 +几淨窗明 +几凳 +几子 +几旁 +几椅 +几榻 +几面上 +出征收 +擊扑 +划一槳 +划了一會 +划到岸 +划到江心 +前面店 +千只可 +千只夠 +千只怕 +千只能 +千只足夠 +半只可 +半只夠 +占了卜 +口干冒 +口干政 +口干涉 +口干犯 +口干預 +古書云 +古語云 +只占卜 +只占吉 +只占神問卜 +只占算 +只身上已 +只身上無 +只身上有 +只身上沒 +只身上的 +只身世 +只身為 +只身份 +只身體 +只身前 +只身受 +只身後 +只身子 +只身形 +只身影 +只身心 +只身旁 +只身材 +只身段 +只身邊 +只身首 +只身高 +只采聲 +可自制 +台子女 +台子孫 +台州 +台風穩健 +穩健的台風 +台風獎 +電視台風 +足球台 +網球台 +合府上 +後面店 +唯一只 +喂了一聲 +四出徵收 +四面包 +多半只 +好斗大 +好斗室 +好斗笠 +好斗篷 +好斗膽 +好斗蓬 +墨斗 +小几 +尸利 +尸祿 +尸臣 +尸鳩 +尸佼 +尸子 +尸羅 +尸羅精舍 +毗婆尸佛 +尸棄佛 +已占卜 +已占算 +并迭 +所云 +所云云 +所占卜 +所占星 +所占算 +手表決 +手表態 +手表明 +手表演 +手表現 +手表示 +手表達 +手表露 +手表面 +才干休 +才干戈 +才干擾 +才干政 +才干涉 +才干預 +扎好底子 +扎好根 +扑撻 +打吨 +拉面上 +拉面具 +拉面前 +拉面巾 +拉面無 +拉面皮 +拉面罩 +拉面色 +拉面部 +捉奸黨 +捉奸徒 +捉奸細 +捉奸賊 +敢情欲 +敢斗了膽 +敲扑 +望了望 +桌几 +每每只 +法自制 +洒淅 +洒濯 +洒然 +灘涂 +特制住 +特制定 +特制止 +特制訂 +百只可 +百只夠 +百只怕 +百只足夠 +皮制服 +相克制 +相克服 +短几 +石几 +秒表明 +秒表示 +窗明几亮 +竹几 +精制伏 +精制住 +精制服 +經有云 +編制法 +防制法 +能干休 +能干戈 +能干擾 +能干政 +能干涉 +能干預 +能自制 +自制一下 +自制下來 +自制不 +自制之力 +自制之能 +自制他 +自制伏 +自制你 +自制地 +自制她 +自制情 +自制我 +自制服 +自制的能 +自制能力 +船只得 +船只有 +船只能 +草荐 +荐居 +荐臻 +荐饑 +要自制 +語有云 +跌扑 +酒帘 +金表態 +金表情 +金表揚 +金表明 +金表演 +金表現 +金表示 +金表達 +金表露 +金表面 +長几 +隆准許 +雄斗斗 +裡面包 +表面包 +外面包 +面包住 +面包辦 +面包廂 +面包含 +面包圍 +面包容 +面包庇 +面包紮 +面包抄 +面包括 +面包攬 +面包涵 +面包管 +面包羅 +面包藏 +面包裝 +面包裹 +面包起 +面包着 +面包著 +面店鋪 +面粉碎 +面粉紅 +面食飯 +水表面 +費米面 +顛顛仆仆 +高干擾 +高干預 +高度自制 +黃金表 +天后宮 +一吊錢 +傳位于四太子 +儉确之教 +党懷英 +八蜡 +憑几 +南宮适 +洪适 +李适 +大蜡 +子云 +分子雲 +小价 +歲聿云暮 +崖广 +恕乏价催 +悲筑 +折子戲 +搤肮拊背 +文采郁郁 +腊毒 +蜡月 +蜡祭 +言云 +宜云 +貴价 +郁郁菲菲 +生發生 +必須 +須根據 +·范 +剋剝 +休克期 +克期間 +溫洛克期 +科尼亞克期 +馬斯垂克期 +滿拚自盡 +拚生盡死 +拚卻 +拚老命 +拚絕 +成於思 +單單於 +名單於 +積澱 +澱積 +澱北片 +澱解物 +澱謂之滓 +淺澱 +堙澱 +茂都澱 +並曰入澱 +澱乃不耕之地 +藍澱 +皆可作澱 +澱山 +海淀山後 +澱澱 +掛鈎 +薴悴 +絡腮鬍 +落腮鬍 +山羊鬍 +幸運鬍 +刮鬍 +剃鬍 +蓄鬍 +鬍髯 +髯鬍 +髭鬍 +鬚鬍 +范文瀾 +范文同 +范文正公 +范文程 +范文芳 +范文藤 +范文虎 +范文照 +機械系 +體系 +維系統 +心理 +鹰鵰 +天地志狼 +薴烯 +雙折射 +心繫家 +心繫國 +心繫祖 +心繫北 +心繫京 +心繫南 +心繫西 +心繫東 +心繫四 +心繫川 +心繫浙 +心繫汶 +心繫廣 +心繫湖 +心繫山 +心繫台 +心繫江 +心繫昌 +心繫香 +心繫澳 +心繫港 +心繫泰 +心繫健 +心繫天 +心繫地 +心繫大 +心繫小 +心繫全 +心繫眾 +心繫奧 +心繫世 +心繫中 +心繫高 +心繫災 +心繫非 +心繫群 +心繫新 +心繫沈 +心繫唐 +心繫黃 +心繫喬 +心繫阮 +心繫父 +心繫母 +心繫病 +心繫故 +心繫哪 +心繫英 +心繫美 +心繫日 +心繫德 +心繫功 +心繫曉 +心繫神 +心繫萬 +心繫的 +心繫在 +心繫兩 +心繫社 +心繫曼 +心繫彼 +心繫風 +心繫募 +心繫一 +心繫何 +心繫困 +心繫輸 +心繫人 +心繫民 +心繫十 +心繫百 +心繫千 +心繫和 +心繫選 +心繫囑 +心繫我 +心繫你 +心繫您 +心繫他 +心繫她 +心繫它 +心繫伊 +心繫長 +心繫舞 +心繫蘭 +心繫五 +心繫生 +心繫婦 +心繫幼 +心繫茶 +心繫動 +心繫沙 +心繫林 +心繫摩 +心繫农 +心繫慈 +心繫麥 +心繫貧 +心繫富 +心繫遠 +心繫近 +心繫宣 +心繫傳 +心繫紅 +心繫老 +心繫重 +心繫震 +心繫妻 +心繫夫 +心繫女 +心繫子 +繫鞋帶 +繫船 +繫着 +重回 +挑大樑 +扛大樑 +后豐 +心臟 +肝臟 +脾臟 +肺臟 +腎臟 +浮誇 +誇人 +誇姣 +誇容 +誇毗 +誇麗 +于謙 +于寘 +淳于 +于禁 +于敏中 +註:# 不作“注:” +劃為# 不作“划為” +一個# 避免“個裡”的錯誤 +兩個 +二個 +三個 +四個 +五個 +六個 +七個 +八個 +九個 +十個 +百個 +千個 +萬個 +億個 +兆個 +零個 +云:# 不作“雲:” +電子表格 +雪裡紅 +雪裡蕻 +樹林裡 +叢林裡 +森林裡 +水裡 +子裡 +事裡 +域裡 +間裡 +淵裡 +院裡 +假裡 +天裡 +日裡 +嘴裡 +心裡 +皮裡陽秋 +肚裡 +苦裡 +裡勾外連 +裡面 +這裡 +中文裡 +英文裡 +古文裡 +經文裡 +論文裡 +譯文裡 +原文裡 +正文裡 +下文裡 +條文裡 +畫裡 +洞裡 +洞里薩 +界裡 +眼睛裡 +百科裡 +歷史裡 +戲裡 +遊戲裡 +作品裡 +專輯裡 +年代裡 +棺材裡 +天里村 +上天里 +天里昂 +人生天里 +百子里 +朴子里 +翁子里 +田子里 +部子里 +曹子里 +埔子里 +廍子里 +廓子里 +堡子里 +墨子里 +瑞城里 +金城里 +西湖里 +坑口里 +子里甲 +水里商工 +車里雅賓斯克 +漠裡 +集裡 +節目裡 +場裡 +世紀裡 +注釋 +月面 +路面 +修杰楷 +修杰麟 +學裡 +獄裡 +館裡 +箱裡 +系列裡 +點裡 +點里程 +家裡 +忙裡偷閒 +夜晚裡 +參數裡 +集數裡 +人數裡 +總數裡 +代數裡 +函數裡 +素數裡 +質數裡 +自然數裡 +索馬里 # (及以下)避免里海=>裏海的轉換 +西西里 +騰格里 +阿里 +峇里海 +里海崖 +里海茨 +里海大學 +孛里海 +布里海 +公里海 +地圖裡 +版圖裡 +配圖裡 +路圖裡 +線圖裡 +幅圖裡 +鏡圖裡 +從圖裡 +的圖裡 +圖裡的 +圖裡, +深山裡 +冰山裡 +火山裡 +在山裡 +的山裡 +到山裡 +去山裡 +從山裡 +山裡的 +山裡有 +棉裡 +語裡 +言裡 +境裡 +境里程 +中境里 +方法裡 +語法裡 +看法裡 +憲法裡 +用法裡 +法裡, +框裡 +碗裡 +電梯裡 +網站裡 +行家裡手 +雲裡霧裡 +城市裡 +都市裡 +市裡的 +個月裡 +月裡來 +分鐘裡 +小時裡 +體裡 +櫃裡 +片裡 +告裡 +電影裡 +廣播裡 +電視裡 +公寓裡 +公寓里弄 +村裡的 +村裡有 +鎮裡 +區裡的 +區裡有 +實驗裡 +註裡 +殿裡 +隊裡 +裏白 #植物常用名 +烏蘇里 #分詞用 +首發 +夸脫 +風采 +代碼表 +編碼表 +字碼表 +電碼表 +科斗 +灕水 +這只不 +這只容 +這只允 +這只採 +有只是 +有只不 +有只容 +有只允 +有只採 +有只用 +所有只 +葉叶琹 +胡子昂 +胡子嬰 +包括 +特别致 +分别致 +韶山沖 +于丹 +于冕 +于吉 +于堅 +于姓 +于氏 +于娜 +于娟 +于山 +于帥 +于慧 +于振 +于敏 +于斌 +于晴 +于波 +于濤 +于衡 +于贈 +于越 +于靖 +于勒 +于格 +于飛 +于仁泰 +于會泳 +于偉國 +于佳卉 +于光遠 +于克勒 +于凌奎 +于鳳至 +于化虎 +于占元 +于台煙 +于品海 +于國楨 +于大寶 +于天仁 +于子千 +于孔兼 +于學忠 +于家堡 +于小偉 +于小彤 +于山國 +于幼軍 +于廣洲 +于康震 +于式枚 +于從濂 +于德海 +于志寧 +于慎行 +于成龍 +于振武 +于明濤 +于是之 +于晨楠 +于根偉 +于樹潔 +于欣源 +于正昇 +于正昌 +于永波 +于漢超 +于江震 +于洪區 +于浩威 +于海洋 +于湘蘭 +于特森 +于玉立 +于秀敏 +于素秋 +于若木 +于蔭霖 +于西翰 +于遠偉 +于道泉 +于都縣 +于震寰 +于震環 +于非闇 +于風政 +于鳳桐 +于默奧 +于爾岑 +于貝爾 +于爾根 +于雙戈 +于里察 +于澤爾 +于斯塔德 +于斯達爾 +于爾里克 +于奇庫杜克 +于韋斯屈萊 +于克-蘭多縣 +于斯納爾斯貝里 +夏于喬 +涂姓 +涂坤 +涂天相 +涂序瑄 +涂澤民 +涂紹煃 +涂羽卿 +涂逢年 +涂長望 +涂謹申 +涂鴻欽 +涂壯勳 +涂醒哲 +涂善妮 +涂敏恆 +涂爾幹 +故云 +強制作用 +鬱南 +鬱林 +饑荒 +免徵 +艷后 +廢后 +妖后 +后海灣 +仙后 +賈后 +賢后 +蜂后 +皇后 +王后 +王侯后 +母后 +字母後 +聲母後 +武后 +歌后 +影后 +封后 +查封後 +解封後 +太后 +天后 +呂后 +后里 +后街 +后羿 +后稷 +仙后座 +六樓后座 +后平路 +后安路 +后土 +后北街 +后冠 +望后石 +后角 +蟻后 +后妃 +大周后 +小周后 +染殿后 +准三后 +風后 +后母戊 +風後, +人如風後入江雲 +中風後 +屏風後 +颱風後 +颳風後 +整風後 +打風後 +遇風後 +聞風後 +逆風後 +順風後 +大風後 +賭后 +山仔后 +甲后路 +劉芸后 +謝華后 +趙惠后 +昭惠后 +周惠后 +孝惠后 +趙威后 +聖后 +陳有后 +惠文后 +葉陽后 +后蒼 +馬格里布 +伊里布 +劃入 +埔裏社 +手裏劍 +裏水鎮 +裏運河 +懸掛 +僱傭 +四捨六入 +宿舍 +校舍 +會干擾 +高清愿 +瓷製 +陶製 +竹製 +絲製 +簡筑翎 +楊雅筑 +彭于晏 +進制 +劉佳怜 +于小惠 +于耘婕 +于洋 +于澄 +于光新 +范賢惠 +于國治 +于楓 +于熙珍 +邱于庭 +熊杰 +卜云吉 +黎吉雲 +代表 +怜奈 +于冠華 +于雲鶴 +于忠肅集 +于友澤 +于和偉 +于來山 +于天龍 +于謹 +于榮光 +掛名 +舞后 +甄后 +郭后 +高后 +升高後 +提高後 +周后 +0周後 +1周後 +2周後 +3周後 +4周後 +5周後 +6周後 +7周後 +8周後 +9周後 +零周後 +〇周後 +一周後 +二周後 +兩周後 +三周後 +四周後 +五周後 +六周後 +七周後 +八周後 +九周後 +十周後 +百周後 +千周後 +萬周後 +億周後 +幾周後 +多周後 +后瑞站 +帝后臺 +紅后假說 +尊后 +前往 +新井里美 +樗里子 +伊達里子 +濱田里佳子 +王田里 +田里穗 +小井里 +西井里 +碧河里 +愛河里花子 +叶志穗 +叶不二子 +于立成 +李志喜 +于欣 +于少保 +于海 +於海邊 +於海上 +於海拔 +於山東 +於山西 +于凌辰 +于魁智 +于鬯 +于仲文 +于再清 +茅于軾 +張樂于張徐 +鮮于 +朝鮮於 +于寶軒 +于承惠 +于震 +于建嶸 +於震前 +於震後 +於震中 +固定制 +划船 +划不來 +划拳 +划槳 +划動 +划艇 +划行 +划算 +划着船 +划着竹筏 +划着獨木舟 +總裁制 +仲裁制 +獨裁制 +恒生 +恒基 +恒隆 +嚴云農 +伊東怜 +衛後莊公 +並行 +郁郁青青 +協防 +了然後 +戴表元 +余力為 +葉叶琴 +幾個 +併發症 +併發重症 +併發模式 +併發型模式 +啟發式 +連發式 +色長髮 +頭長髮 +的長髮 +黑長髮 +留長髮 +髮披肩 +髮及腰 +飄髮自由女神 +後天 +學家 +游離 +書面 +不只 +湧水 +高涌泉 +涌水塘 +后姓 +計劃 +党姓 +党家 +种丹妮 +當當網 +縴繩 +佣金 +佣錢 +佣鈿 +回佣 +蕓薹 +宋王臺 +臺佟 +臺靜農 +林鵞峰 +沙羡 +最多只 +大多只 +至多只 +只影響 +測不準 +說不準 +對不準 +量不準 +準不準 +音不準 +預報不準 +時間不準 +不太準 +非常準 +很準 +囓蟲 +勳勞 +勳績 +勳爵 +勳業 +授勳 +奇勳 +功勳 +蝎虎 +磨蝎 +古蹟 +瀋撫 +賦范 +騰衝 +沖天 +豐臺 +煙臺 +太醜 +御製 +電影後 +封為后 +皮托管 +白面包青天 +天神之后 +你誇 +誇你 +誇我 +誇他 +誇她 +誇了 +被誇 +誇辯 +誇大 +誇誕 +誇官 +誇口 +誇誇其談 +誇海口 +誇獎 +誇強說會 +誇下海口 +誇詡 +誇張 +誇示 +誇飾 +誇勝道強 +誇說 +誇才 +誇耀 +矜誇 +誇能 +自誇 +誇稱 +誇讚 +黎克特制 +筆桿 +袋桿 +槍桿 +秤桿 +兩桿 +桿菌 +桿秤 +桿槍 +四桿鐵筆 +徒杠 +杠梁 +杠轂 +杠人 +石杠 +墨瀋 +米瀋 +拾瀋 +姦污 +託兒 +同人誌 +文學誌 +衝着 +確係 +乃係 +製衣 +巨製 +窗簾 +吉徵 +凶徵 +臟腑 +臟胸 +弄髒胸 +腸臟 +養臟 +膵臟 +驚慄 +支配慾 +利慾 +悽美 +滷煮 +滷蝦 +滷鴨 +滷鵝 +滷牛 +滷五花 +滷子 +滷料 +滷豆 +滷了 +滷的 +滷好 +打滷 +滷麵 +烤滷 +錦滷 +汤滷 +浸滷 +花葯 +聚葯雄蕊 +遺蹟 +受僱 +僱請 +僱車 +僱船 +米糰 +集團 +謹愿 +瞎矇 +里舖 +喧譁 +譁眾 +譁囂 +譁然 +譁噪 +譁笑 +譁變 +鼓譟 +譟詐 +蕩氣 +木籤 +薝蔔 +斗牛星 +告劄 +點劄 +嚮慕 +儘自 +憑閑 +倚閑 +踰閑 +閑邪 +摺檯 +球檯 +櫃檯 +吧檯 +賭檯 +坐檯 +坐台鐵 +妝檯 +餐檯 +工作檯 +辦公檯 +檯面上 +上檯面 +檯面化 +牴觸 +牴牾 +角牴 +扼肮 +搤肮 +嫩薑 +酸薑 +薑啤 +騰湧 +草蓆 +竹蓆 +藤蓆 +涼蓆 +灘蓆 +麻將蓆 +被廢後 +蒸製 +烹製 +醃製 +鐵製 +鋼製 +銅製 +鋅製 +和製漢 +和製英語 +壓製機 +壓製出 +應制得 +反應製得 +製表鍵 +電子製表 +製毒 +製販 +製得 +製取 +譯製 +燉製 +煮製 +熬製 +遏制 #以下分詞用 +管制 +抑制 +控制 +限制 +強制 +改制成 +考試制度 +體徵 +綜合徵 +价川 +商標准許 +批准確定 +御嶽山 +兩齣 +進兩出 +幾進幾出 +十出生 +十出頭 +十出擊 +十出家 +十出祁山 +0齣 +0出現 +0出線 +這齣 +這出現 +這出乎 +這出人 +這出生 +這出世 +這出身 +這出色 +這出版 +這出道 +本齣戲 +整齣戲 +一齣戲 +三齣戲 +一齣好戲 +一齣電影 +首齣電影 +多齣電影 +整齣電影 +一齣劇 +整齣劇 +一齣悲劇 +一齣喜劇 +捨入 +舍入口 +繫上了 +繫上頭 +繫上紅 +繫上黑 +繫上絲 +繫上繩 +繫上安全 +上繫上 +被繫上 +繫上, +繫上。 +繫舟 +繫膜 +亂發生 +亂發脾氣 +秀發村 +秀發動 +秀發表 +秀發布 +秀發現 +秀發生 +秀發起 +秀發展 +留發生 +留發行 +留發展 +縮短發 +簡短發 +短發生 +頭發現 +蛋白發 +發狀態 +發狀況 +染發生 +古人有云 +昔人有云 +云敞 +喂, +喂! +喂喲 +喲喂 +啊喂 +呵喂 +呦喂 +哈囉喂 +松口鎮 +岩松了 +沙瑯 +琺瑯 +菜餚 +梁啓超 +王添灯 +腌臢 +風颳 +颳大風 +黃白術 +仁貴 #分詞用 +金聖歎 +天台 #分詞用 +性別扭曲 +箇舊市 +雲南箇舊 +關系列 +關系統 +關系所 +關系科 diff --git a/www/wiki/maintenance/language/zhtable/tradphrases_exclude.manual b/www/wiki/maintenance/language/zhtable/tradphrases_exclude.manual new file mode 100644 index 00000000..3ab14eb1 --- /dev/null +++ b/www/wiki/maintenance/language/zhtable/tradphrases_exclude.manual @@ -0,0 +1,780 @@ +三國誌 +聊齋誌異 +北迴 +南迴 +併排 +併進 +併在 +併成 +衝衝 +臺 +著 +佈 +纔 +采 +着 +借 +甦 +荐 +担 +可憐虫 +一齣 +上弔 +弔車 +弔橋 +弔嗓子 +弔床 +弔架 +弔桶 +弔桿 +弔燈 +弔環 +弔籃 +弔胃口 +弔臂 +弔銷 +形影相弔 +被髮 +散髮 +長髮 +髮毛 +髮端 +周而複始 +答複 +複興 +複舊 +顛複 +修複 +報複 +複活 +反複 +迴首 +彙總 +饑餓 +饑不擇食 +饑荒 +藉端 +藉酒 +蛋捲 +行李捲 +克裡 +纍纍 +華裡 +裡海 +瞭解 +明瞭 +發黴 +矇蔽 +矇住 +濛濛 +矇矇 +下麵 +白麵 +切麵 +和麵 +過水麵 +復甦 +複蘇 +甦醒 +体 +繫數 +遊擊 +馥鬱 +鬱鬱 +改製 +獃住 +獃氣 +獃子 +獃頭獃腦 +發獃 +希腊 +腊肉 +瞭如 +昇 +武鬆 +赤鬆 +黑鬆 +鬆林 +鬆科 +鬆濤 +鬆毛蟲 +鬆節油 +濕地鬆 +尼克鬆 +紮伊爾 +阿布紮比 +阿紮尼亞 +利比裡亞 +斯裡蘭卡 +烏蘇裡江 +加裡寧 +歐幾裡得 +格裡 +巴裡 +居裡 +卡裡 +墨索裡尼 +底裡 +裡人 +裡加 +裡裡 +馬裡 +裡拉 +阿裡 +裡斯 +鄰裡 +鄉裡 +百裡 +特裡 +海裡 +三元裡 +漏鬥 +春捲 +採邑 +嚮日 +佔城 +水錶 +名錶 +錶面 +彆腳 +併力 +併列 +併為 +豐富多採 +採採 +尼採 +小醜 +辛醜 +整齣 +嚴複 +枯幹 +干著急 +單於 +攻剋 +剋服 +闢邪 +釐米 +後樑 +石樑 +木樑 +舊莊 +介係詞 +介繫詞 +餘年 +大阪 +阪田 +豪杰 +七拚八湊 +一捲 +十捲 +上捲 +下捲 +加捲 +不捨 +不識檯舉 +稜登 +半弔子 +分布圖 +星鬥 +筋鬥 +斗鬨 +料鬥 +煙鬥 +熨鬥 +笆鬥 +箕鬥 +金鬥 +門鬥 +風鬥 +鬥子 +鬥笠 +老板娘 +剋制 +洋麵 +病癥 +製裁 +台製 +石家庄 +酒盃 +積极 +殭尸 +上梁不正 +項鍊 +鍊子 +鍊條 +拉鍊 +鉸鍊 +鍊鎖 +鐵鍊 +鍛鍊 +鍊乳 +鍊丹 +至于 +浮于 +附于 +次于 +于人 +助于 +行于 +于衷 +于事 +低于 +大于 +高于 +等于 +位于 +用于 +答覆 +複蓋 +反覆 +藉藉 +蘊藉 +蹈藉 +醞藉 +氆氌 +慰藉 +文藉 +枕藉 +狼藉 +別隻 +鼕鼕 +矇松雨 +佈雷 +丰度 +剪彩 +脣 +菴 +公裡 +箇中 +樑子 +樑書 +讚成 +讚同 +鐘表店 +精採 +鞭尸 +尸身 +尸首 +行尸走肉 +裹尸 +慼慼 +痠 +簑 +捱 +朝乾夕惕 +大曲酒 +神麴 +便于 +偏于 +勇于 +居于 +常見于 +強加于 +從事于 +忙于 +敢于 +服務于 +服從于 +樂于 +歸罪于 +歸諸于 +活動于 +瀕于 +苦于 +莫過于 +處于 +適于 +乾和 +鉤 +高陞 +大胆 +託福 +繫系 +酰 +醯 +大樑 +光採 +鍾錶 +複原 +浮夸 +剋日 +羡 +旅游 +穀風 +復讎 +避暑山庄 +遊牧 +烟草 +征 +占領 +入夥 +懸挂 +註釋 +浮遊 +冶鍊 +裡子 +裡外 +單隻 +聯係 +那裏 +殺虫藥 +好家伙 +姦污 +併發 +衚衕 +轉檯 +檯子 +佣人 +佣工 +佣仆 +男佣 +女佣 +傢俱 +傢具 +華冑 +裔冑 +貴冑 +美髮 +癥狀 +癥候 +不準 +囓合 +囓齒類 +編製 +索麵 +專註 +鬥上 +古迹 +划了 +合并 +划出 +划到 +題籤 +克複 +意麵 +明裡 +華髮 +迴流 +採的 +複名 +看錶 +嚮應 +複電 +綵排 +綵帶 +綵樓 +綵牌樓 +綵球 +綵綢 +綵線 +綵船 +綵衣 +結綵 +戲綵娛親 +剪綵 +複檢 +黃曲霉 +佔有慾 +不佔 +佔上風 +佔下 +佔了 +佔位 +佔住 +佔佔 +佔便宜 +佔個 +佔優勢 +佔先 +佔光 +佔到 +佔去 +佔取 +佔在 +佔地 +佔多數 +佔好 +佔得 +佔掉 +佔據 +佔有 +佔滿 +佔為 +佔用 +佔畢 +佔盡 +佔線 +佔起 +佔過 +佔領 +佔頭籌 +佔高枝兒 +侵佔 +先佔 +分佔 +只佔 +強佔 +搶佔 +攻佔 +會佔 +照佔 +約佔 +連佔 +進佔 +還佔 +隱佔 +霸佔 +非佔不可 +鳩佔鵲巢 +占 +佔0 +佔1 +佔2 +佔3 +佔4 +佔5 +佔6 +佔7 +佔8 +佔9 +佔A +佔B +佔C +佔D +佔E +佔F +佔G +佔H +佔I +佔J +佔K +佔L +佔M +佔N +佔O +佔P +佔Q +佔R +佔S +佔T +佔U +佔V +佔W +佔X +佔Y +佔Z +佔〇 +佔一 +佔七 +佔三 +佔下風 +佔不佔 +佔不足 +佔世界 +佔中 +佔主 +佔九 +佔二 +佔五 +佔人便宜 +佔俄 +佔個位 +佔億 +佔優 +佔全 +佔兩 +佔八 +佔六 +佔分 +佔加 +佔劣 +佔北 +佔十 +佔千 +佔半 +佔南 +佔印 +佔台 +佔囁 +佔四 +佔國 +佔場 +佔壓 +佔多 +佔大 +佔小 +佔少 +佔局部 +佔屋 +佔山為 +佔市 +佔平均 +佔床 +佔座 +佔後 +佔德 +佔整 +佔新 +佔東 +佔查 +佔次 +佔比 +佔法 +佔澳 +佔率 +佔百 +佔網 +佔總 +佔缺 +佔美 +佔耕 +佔至多 +佔至少 +佔臺 +佔英 +佔萬 +佔葡 +佔蘇 +佔西 +佔資 +佔超過 +佔道 +佔零 +佔頭 +佔香 +佔馬 +俄佔 +圈佔 +地佔 +多佔 +奧佔 +寡佔 +將佔 +少佔 +已佔 +市佔 +徵佔 +德佔 +意佔 +所佔 +日佔 +法佔 +狂佔 +獨佔 +穩佔 +美佔 +義佔 +英佔 +葡佔 +西佔 +要佔 +費佔 +標準桿 +單杠 +杠子 +杠鈴 +經濟杠桿 +高低杠 +陞官 +姦汙 +興緻 +景緻 +別緻 +雅緻 +崑 +表 +錶 +小夥子 +夸父 +夸特 +夸脫 +心臟痲痹 +心臟麻痺 +悽涼 +悽悽 +悽豔 +悽切 +悽楚 +家裏 +利欲熏心 +遊離票 +遊離份子 +閑 +鍊鋼 +事迹 +痕迹 +遺迹 +僱員 +僱用 +霉素 +遊盪 +搖蕩 +激蕩 +動蕩 +跌蕩 +震蕩 +充饑 +儘力 +彈葯 +炸葯 +醫葯 +骯臟 +釐升 +蓆地 +晒 +窗檯 +和尚撞一天鍾 +製為 +裡布 +里布 +圖裡 +山裡 +複發 +照準 +四齣 +五齣 +六齣 +弔兒郎當 +髮小 +修鍊 +麵線 +繫上 +清湯掛麵 +牛肉麵 +檯面 +庄 +複信 +複核 +三複 +來複 +匡複 +傾複 +墾複 +往複 +被複 +複仞年如 +複以百萬 +複位 +複合 +複員 +複壯 +複復 +複流 +複畝珍 +起複 +餘 +旋乾轉坤 +乾坤 +乾卦 +乾隆 +乾掉 +讚嘆不已 +讚歎 +好乾 +加註 +幹將 +鼕 +彙報 +彙整 +彙編 +彙集 +快幹 +快乾 +瀋海 +迴文 +迴向 +迴音 +美製 +麵灰 +麵價 +承製 +樹榦 +白乾 +白干兒 +市裡 +于飛 +髮指 +鬆鬆 +于是 +于七 +于今 +曆數 +發矇 +不幹 +作姦犯科 +游牧民族 +穀道 +託大 +藉詞 +摺合 +仇讎 +讎正 +校讎學 +姦細 +姦邪 +姦宄 +姦猾 +防颱 +慾望 +剋星 +挂 +掛 +陸遊 +徵人 +髮針 +供製 +并吞 +併吞下 +髮網 +精鍊 +腳鍊 +託人 +鬥口 +噴洒 +洒掃 +洒水 +洒洒 +洒脫 +瀟洒 +村裡 +振蕩 +重摺 +兼并 +并力 +弔死 +弔帶 +繫世 +划上 +划下 +洄遊 +洄游 +花捲 +乾地 +方誌 +編髮 +颳去 +刮去 +么 +換髮 +谷氨酸 +幸免於難 +勝于 +善于 +安于 +寓于 +對于 +屬于 +忠于 +急于 +歸于 +生于 +由于 +終于 +見于 +過于 +長于 +關于 +難于 +箇舊 +條幹 +檯布 diff --git a/www/wiki/maintenance/locking/file_locks.sql b/www/wiki/maintenance/locking/file_locks.sql new file mode 100644 index 00000000..f51d06b3 --- /dev/null +++ b/www/wiki/maintenance/locking/file_locks.sql @@ -0,0 +1,11 @@ +-- Table to handle resource locking (EX) with row-level locking +CREATE TABLE /*_*/filelocks_exclusive ( + fle_key binary(31) NOT NULL PRIMARY KEY +) ENGINE=InnoDB, CHECKSUM=0; + +-- Table to handle resource locking (SH) with row-level locking +CREATE TABLE /*_*/filelocks_shared ( + fls_key binary(31) NOT NULL, + fls_session binary(31) NOT NULL, + PRIMARY KEY (fls_key,fls_session) +) ENGINE=InnoDB, CHECKSUM=0; diff --git a/www/wiki/maintenance/makeTestEdits.php b/www/wiki/maintenance/makeTestEdits.php new file mode 100644 index 00000000..ca2f7c51 --- /dev/null +++ b/www/wiki/maintenance/makeTestEdits.php @@ -0,0 +1,68 @@ +<?php +/** + * Make test edits for a user to populate a test wiki + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ +require_once __DIR__ . '/Maintenance.php'; + +/** + * Make test edits for a user to populate a test wiki + * + * @ingroup Maintenance + */ +class MakeTestEdits extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Make test edits for a user' ); + $this->addOption( 'user', 'User name', true, true ); + $this->addOption( 'count', 'Number of edits', true, true ); + $this->addOption( 'namespace', 'Namespace number', false, true ); + $this->setBatchSize( 100 ); + } + + public function execute() { + $user = User::newFromName( $this->getOption( 'user' ) ); + if ( !$user->getId() ) { + $this->error( "No such user exists.", 1 ); + } + + $count = $this->getOption( 'count' ); + $namespace = (int)$this->getOption( 'namespace', 0 ); + + for ( $i = 0; $i < $count; ++$i ) { + $title = Title::makeTitleSafe( $namespace, "Page " . wfRandomString( 2 ) ); + $page = WikiPage::factory( $title ); + $content = ContentHandler::makeContent( wfRandomString(), $title ); + $summary = "Change " . wfRandomString( 6 ); + + $page->doEditContent( $content, $summary, 0, false, $user ); + + $this->output( "Edited $title\n" ); + if ( $i && ( $i % $this->mBatchSize ) == 0 ) { + wfWaitForSlaves(); + } + } + + $this->output( "Done\n" ); + } +} + +$maintClass = "MakeTestEdits"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/manageJobs.php b/www/wiki/maintenance/manageJobs.php new file mode 100644 index 00000000..32333b76 --- /dev/null +++ b/www/wiki/maintenance/manageJobs.php @@ -0,0 +1,97 @@ +<?php +/** + * Maintenance script that handles managing job queue admin tasks + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that handles managing job queue admin tasks (re-push, delete, ...) + * + * @ingroup Maintenance + */ +class ManageJobs extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Perform administrative tasks on a job queue' ); + $this->addOption( 'type', 'Job type', true, true ); + $this->addOption( 'action', 'Queue operation ("delete", "repush-abandoned")', true, true ); + } + + public function execute() { + $type = $this->getOption( 'type' ); + $action = $this->getOption( 'action' ); + + $group = JobQueueGroup::singleton(); + $queue = $group->get( $type ); + + if ( $action === 'delete' ) { + $this->delete( $queue ); + } elseif ( $action === 'repush-abandoned' ) { + $this->repushAbandoned( $queue ); + } else { + $this->error( "Invalid action '$action'.", 1 ); + } + } + + private function delete( JobQueue $queue ) { + $this->output( "Queue has {$queue->getSize()} job(s); deleting...\n" ); + $queue->delete(); + $this->output( "Done; current size is {$queue->getSize()} job(s).\n" ); + } + + private function repushAbandoned( JobQueue $queue ) { + $cache = ObjectCache::getInstance( CACHE_DB ); + $key = $cache->makeGlobalKey( 'last-job-repush', $queue->getWiki(), $queue->getType() ); + + $now = wfTimestampNow(); + $lastRepushTime = $cache->get( $key ); + if ( $lastRepushTime === false ) { + $lastRepushTime = wfTimestamp( TS_MW, 1 ); // include all jobs + } + + $this->output( "Last re-push time: $lastRepushTime; current time: $now\n" ); + + $count = 0; + $skipped = 0; + foreach ( $queue->getAllAbandonedJobs() as $job ) { + /** @var Job $job */ + if ( $job->getQueuedTimestamp() < wfTimestamp( TS_UNIX, $lastRepushTime ) ) { + ++$skipped; + continue; // already re-pushed in prior round + } + + $queue->push( $job ); + ++$count; + + if ( ( $count % $this->mBatchSize ) == 0 ) { + $queue->waitForBackups(); + } + } + + $cache->set( $key, $now ); // next run will ignore these jobs + + $this->output( "Re-pushed $count job(s) [$skipped skipped].\n" ); + } +} + +$maintClass = "ManageJobs"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/mcc.php b/www/wiki/maintenance/mcc.php new file mode 100644 index 00000000..784ba0ea --- /dev/null +++ b/www/wiki/maintenance/mcc.php @@ -0,0 +1,226 @@ +<?php +/** + * memcached diagnostic tool + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @todo document + * @ingroup Maintenance + */ + +$optionsWithArgs = [ 'cache' ]; +$optionsWithoutArgs = [ + 'debug', 'help' +]; +require_once __DIR__ . '/commandLine.inc'; + +$debug = isset( $options['debug'] ); +$help = isset( $options['help'] ); +$cache = isset( $options['cache'] ) ? $options['cache'] : null; + +if ( $help ) { + mccShowUsage(); + exit( 0 ); +} +$mcc = new MemcachedClient( [ + 'persistent' => true, + 'debug' => $debug, +] ); + +if ( $cache ) { + if ( !isset( $wgObjectCaches[$cache] ) ) { + print "MediaWiki isn't configured with a cache named '$cache'"; + exit( 1 ); + } + $servers = $wgObjectCaches[$cache]['servers']; +} elseif ( $wgMainCacheType === CACHE_MEMCACHED ) { + $mcc->set_servers( $wgMemCachedServers ); +} elseif ( isset( $wgObjectCaches[$wgMainCacheType]['servers'] ) ) { + $mcc->set_servers( $wgObjectCaches[$wgMainCacheType]['servers'] ); +} else { + print "MediaWiki isn't configured for Memcached usage\n"; + exit( 1 ); +} + +/** + * Show this command line tool usage. + */ +function mccShowUsage() { + echo <<<EOF +Usage: + mcc.php [--debug] + mcc.php --help + +MemCached Command (mcc) is an interactive command tool that let you interact +with the MediaWiki memcached cache. + +Options: + --debug Set debug mode on the memcached connection. + --help This help screen. + +Interactive commands: + +EOF; + print "\t"; + print str_replace( "\n", "\n\t", mccGetHelp( false ) ); + print "\n"; +} + +function mccGetHelp( $command ) { + $output = ''; + $commandList = [ + 'get' => 'grabs something', + 'getsock' => 'lists sockets', + 'set' => 'changes something', + 'delete' => 'deletes something', + 'history' => 'show command line history', + 'server' => 'show current memcached server', + 'dumpmcc' => 'shows the whole thing', + 'exit' => 'exit mcc', + 'quit' => 'exit mcc', + 'help' => 'help about a command', + ]; + if ( !$command ) { + $command = 'fullhelp'; + } + if ( $command === 'fullhelp' ) { + $max_cmd_len = max( array_map( 'strlen', array_keys( $commandList ) ) ); + foreach ( $commandList as $cmd => $desc ) { + $output .= sprintf( "%-{$max_cmd_len}s: %s\n", $cmd, $desc ); + } + } elseif ( isset( $commandList[$command] ) ) { + $output .= "$command: $commandList[$command]\n"; + } else { + $output .= "$command: command does not exist or no help for it\n"; + } + + return $output; +} + +do { + $bad = false; + $showhelp = false; + $quit = false; + + $line = Maintenance::readconsole(); + if ( $line === false ) { + exit; + } + + $args = explode( ' ', $line ); + $command = array_shift( $args ); + + // process command + switch ( $command ) { + case 'help': + // show an help message + print mccGetHelp( array_shift( $args ) ); + break; + + case 'get': + $sub = ''; + if ( array_key_exists( 1, $args ) ) { + $sub = $args[1]; + } + print "Getting {$args[0]}[$sub]\n"; + $res = $mcc->get( $args[0] ); + if ( array_key_exists( 1, $args ) ) { + $res = $res[$args[1]]; + } + if ( $res === false ) { + # print 'Error: ' . $mcc->error_string() . "\n"; + print "MemCached error\n"; + } elseif ( is_string( $res ) ) { + print "$res\n"; + } else { + var_dump( $res ); + } + break; + + case 'getsock': + $res = $mcc->get( $args[0] ); + $sock = $mcc->get_sock( $args[0] ); + var_dump( $sock ); + break; + + case 'server': + if ( $mcc->_single_sock !== null ) { + print $mcc->_single_sock . "\n"; + break; + } + $res = $mcc->get( $args[0] ); + $hv = $mcc->_hashfunc( $args[0] ); + for ( $i = 0; $i < 3; $i++ ) { + print $mcc->_buckets[$hv % $mcc->_bucketcount] . "\n"; + $hv += $mcc->_hashfunc( $i . $args[0] ); + } + break; + + case 'set': + $key = array_shift( $args ); + if ( $args[0] == "#" && is_numeric( $args[1] ) ) { + $value = str_repeat( '*', $args[1] ); + } else { + $value = implode( ' ', $args ); + } + if ( !$mcc->set( $key, $value, 0 ) ) { + # print 'Error: ' . $mcc->error_string() . "\n"; + print "MemCached error\n"; + } + break; + + case 'delete': + $key = implode( ' ', $args ); + if ( !$mcc->delete( $key ) ) { + # print 'Error: ' . $mcc->error_string() . "\n"; + print "MemCached error\n"; + } + break; + + case 'history': + if ( function_exists( 'readline_list_history' ) ) { + foreach ( readline_list_history() as $num => $line ) { + print "$num: $line\n"; + } + } else { + print "readline_list_history() not available\n"; + } + break; + + case 'dumpmcc': + var_dump( $mcc ); + break; + + case 'quit': + case 'exit': + $quit = true; + break; + + default: + $bad = true; + } // switch() end + + if ( $bad ) { + if ( $command ) { + print "Bad command\n"; + } + } else { + if ( function_exists( 'readline_add_history' ) ) { + readline_add_history( $line ); + } + } +} while ( !$quit ); diff --git a/www/wiki/maintenance/mctest.php b/www/wiki/maintenance/mctest.php new file mode 100644 index 00000000..60f94a5f --- /dev/null +++ b/www/wiki/maintenance/mctest.php @@ -0,0 +1,106 @@ +<?php +/** + * Makes several 'set', 'incr' and 'get' requests on every memcached + * server and shows a report. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that makes several 'set', 'incr' and 'get' requests + * on every memcached server and shows a report. + * + * @ingroup Maintenance + */ +class McTest extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( "Makes several 'set', 'incr' and 'get' requests on every" + . " memcached server and shows a report" ); + $this->addOption( 'i', 'Number of iterations', false, true ); + $this->addOption( 'cache', 'Use servers from this $wgObjectCaches store', false, true ); + $this->addArg( 'server[:port]', 'Memcached server to test, with optional port', false ); + } + + public function execute() { + global $wgMainCacheType, $wgMemCachedTimeout, $wgObjectCaches; + + $cache = $this->getOption( 'cache' ); + $iterations = $this->getOption( 'i', 100 ); + if ( $cache ) { + if ( !isset( $wgObjectCaches[$cache] ) ) { + $this->error( "MediaWiki isn't configured with a cache named '$cache'", 1 ); + } + $servers = $wgObjectCaches[$cache]['servers']; + } elseif ( $this->hasArg() ) { + $servers = [ $this->getArg() ]; + } elseif ( $wgMainCacheType === CACHE_MEMCACHED ) { + global $wgMemCachedServers; + $servers = $wgMemCachedServers; + } elseif ( isset( $wgObjectCaches[$wgMainCacheType]['servers'] ) ) { + $servers = $wgObjectCaches[$wgMainCacheType]['servers']; + } else { + $this->error( "MediaWiki isn't configured for Memcached usage", 1 ); + } + + # find out the longest server string to nicely align output later on + $maxSrvLen = $servers ? max( array_map( 'strlen', $servers ) ) : 0; + + foreach ( $servers as $server ) { + $this->output( + str_pad( $server, $maxSrvLen ), + $server # output channel + ); + + $mcc = new MemcachedClient( [ + 'persistant' => true, + 'timeout' => $wgMemCachedTimeout + ] ); + $mcc->set_servers( [ $server ] ); + $set = 0; + $incr = 0; + $get = 0; + $time_start = microtime( true ); + for ( $i = 1; $i <= $iterations; $i++ ) { + if ( $mcc->set( "test$i", $i ) ) { + $set++; + } + } + for ( $i = 1; $i <= $iterations; $i++ ) { + if ( !is_null( $mcc->incr( "test$i", $i ) ) ) { + $incr++; + } + } + for ( $i = 1; $i <= $iterations; $i++ ) { + $value = $mcc->get( "test$i" ); + if ( $value == $i * 2 ) { + $get++; + } + } + $exectime = microtime( true ) - $time_start; + + $this->output( " set: $set incr: $incr get: $get time: $exectime", $server ); + } + } +} + +$maintClass = "McTest"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/mergeMessageFileList.php b/www/wiki/maintenance/mergeMessageFileList.php new file mode 100644 index 00000000..bb476313 --- /dev/null +++ b/www/wiki/maintenance/mergeMessageFileList.php @@ -0,0 +1,209 @@ +<?php +/** + * Merge $wgExtensionMessagesFiles from various extensions to produce a + * single array containing all message files. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +# Start from scratch +define( 'MW_NO_EXTENSION_MESSAGES', 1 ); + +require_once __DIR__ . '/Maintenance.php'; +$maintClass = 'MergeMessageFileList'; +$mmfl = false; + +/** + * Maintenance script that merges $wgExtensionMessagesFiles from various + * extensions to produce a single array containing all message files. + * + * @ingroup Maintenance + */ +class MergeMessageFileList extends Maintenance { + function __construct() { + parent::__construct(); + $this->addOption( + 'list-file', + 'A file containing a list of extension setup files, one per line.', + false, + true + ); + $this->addOption( 'extensions-dir', 'Path where extensions can be found.', false, true ); + $this->addOption( 'output', 'Send output to this file (omit for stdout)', false, true ); + $this->addDescription( 'Merge $wgExtensionMessagesFiles and $wgMessagesDirs from ' . + ' various extensions to produce a single file listing all message files and dirs.' + ); + } + + public function execute() { + // @codingStandardsIgnoreStart Ignore error: Global variable "$mmfl" is lacking 'wg' prefix + global $mmfl; + // @codingStandardsIgnoreEnd + global $wgExtensionEntryPointListFiles; + + if ( !count( $wgExtensionEntryPointListFiles ) + && !$this->hasOption( 'list-file' ) + && !$this->hasOption( 'extensions-dir' ) + ) { + $this->error( "Either --list-file or --extensions-dir must be provided if " . + "\$wgExtensionEntryPointListFiles is not set", 1 ); + } + + $mmfl = [ 'setupFiles' => [] ]; + + # Add setup files contained in file passed to --list-file + if ( $this->hasOption( 'list-file' ) ) { + $extensionPaths = $this->readFile( $this->getOption( 'list-file' ) ); + $mmfl['setupFiles'] = array_merge( $mmfl['setupFiles'], $extensionPaths ); + } + + # Now find out files in a directory + if ( $this->hasOption( 'extensions-dir' ) ) { + $extdir = $this->getOption( 'extensions-dir' ); + # Allow multiple directories to be passed with ":" as delimiter + $extdirs = explode( ':', $extdir ); + $entries = []; + foreach ( $extdirs as $extdir ) { + $entries = array_merge( $entries, scandir( $extdir ) ); + } + foreach ( $entries as $extname ) { + if ( $extname == '.' || $extname == '..' || !is_dir( "$extdir/$extname" ) ) { + continue; + } + $possibilities = [ + "$extdir/$extname/extension.json", + "$extdir/$extname/skin.json", + "$extdir/$extname/$extname.php" + ]; + $found = false; + foreach ( $possibilities as $extfile ) { + if ( file_exists( $extfile ) ) { + $mmfl['setupFiles'][] = $extfile; + $found = true; + break; + } + } + + if ( !$found ) { + $this->error( "Extension {$extname} in {$extdir} lacks expected entry point: " . + "extension.json, skin.json, or {$extname}.php." ); + } + } + } + + # Add setup files defined via configuration + foreach ( $wgExtensionEntryPointListFiles as $points ) { + $extensionPaths = $this->readFile( $points ); + $mmfl['setupFiles'] = array_merge( $mmfl['setupFiles'], $extensionPaths ); + } + + if ( $this->hasOption( 'output' ) ) { + $mmfl['output'] = $this->getOption( 'output' ); + } + if ( $this->hasOption( 'quiet' ) ) { + $mmfl['quiet'] = true; + } + } + + /** + * @param string $fileName + * @return array List of absolute extension paths + */ + private function readFile( $fileName ) { + global $IP; + + $files = []; + $fileLines = file( $fileName ); + if ( $fileLines === false ) { + $this->hasError = true; + $this->error( "Unable to open list file $fileName." ); + + return $files; + } + # Strip comments, discard empty lines, and trim leading and trailing + # whitespace. Comments start with '#' and extend to the end of the line. + foreach ( $fileLines as $extension ) { + $extension = trim( preg_replace( '/#.*/', '', $extension ) ); + if ( $extension !== '' ) { + # Paths may use the string $IP to be substituted by the actual value + $extension = str_replace( '$IP', $IP, $extension ); + if ( file_exists( $extension ) ) { + $files[] = $extension; + } else { + $this->hasError = true; + $this->error( "Extension {$extension} doesn't exist" ); + } + } + } + + return $files; + } +} + +require_once RUN_MAINTENANCE_IF_MAIN; + +$queue = []; +foreach ( $mmfl['setupFiles'] as $fileName ) { + if ( strval( $fileName ) === '' ) { + continue; + } + if ( empty( $mmfl['quiet'] ) ) { + fwrite( STDERR, "Loading data from $fileName\n" ); + } + // Using extension.json or skin.json + if ( substr( $fileName, -strlen( '.json' ) ) === '.json' ) { + $queue[$fileName] = 1; + } else { + require_once $fileName; + } +} + +if ( $queue ) { + $registry = new ExtensionRegistry(); + $data = $registry->readFromQueue( $queue ); + foreach ( [ 'wgExtensionMessagesFiles', 'wgMessagesDirs' ] as $var ) { + if ( isset( $data['globals'][$var] ) ) { + $GLOBALS[$var] = array_merge( $data['globals'][$var], $GLOBALS[$var] ); + } + } +} + +fwrite( STDERR, "\n" ); +$s = + "<" . "?php\n" . + "## This file is generated by mergeMessageFileList.php. Do not edit it directly.\n\n" . + "if ( defined( 'MW_NO_EXTENSION_MESSAGES' ) ) return;\n\n" . + '$wgExtensionMessagesFiles = ' . var_export( $wgExtensionMessagesFiles, true ) . ";\n\n" . + '$wgMessagesDirs = ' . var_export( $wgMessagesDirs, true ) . ";\n\n"; + +$dirs = [ + $IP, + dirname( __DIR__ ), + realpath( $IP ) +]; + +foreach ( $dirs as $dir ) { + $s = preg_replace( "/'" . preg_quote( $dir, '/' ) . "([^']*)'/", '"$IP\1"', $s ); +} + +if ( isset( $mmfl['output'] ) ) { + file_put_contents( $mmfl['output'], $s ); +} else { + echo $s; +} diff --git a/www/wiki/maintenance/migrateComments.php b/www/wiki/maintenance/migrateComments.php new file mode 100644 index 00000000..4af6a2ad --- /dev/null +++ b/www/wiki/maintenance/migrateComments.php @@ -0,0 +1,295 @@ +<?php +/** + * Migrate comments from pre-1.30 columns to the 'comment' table + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +use Wikimedia\Rdbms\IDatabase; + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that migrates comments from pre-1.30 columns to the + * 'comment' table + * + * @ingroup Maintenance + */ +class MigrateComments extends LoggedUpdateMaintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Migrates comments from pre-1.30 columns to the \'comment\' table' ); + $this->setBatchSize( 100 ); + } + + protected function getUpdateKey() { + return __CLASS__; + } + + protected function updateSkippedMessage() { + return 'comments already migrated.'; + } + + protected function doDBUpdates() { + global $wgCommentTableSchemaMigrationStage; + + if ( $wgCommentTableSchemaMigrationStage < MIGRATION_WRITE_NEW ) { + $this->output( + "...cannot update while \$wgCommentTableSchemaMigrationStage < MIGRATION_WRITE_NEW\n" + ); + return false; + } + + $this->migrateToTemp( + 'revision', 'rev_id', 'rev_comment', 'revcomment_rev', 'revcomment_comment_id' + ); + $this->migrate( 'archive', 'ar_id', 'ar_comment' ); + $this->migrate( 'ipblocks', 'ipb_id', 'ipb_reason' ); + $this->migrateToTemp( + 'image', 'img_name', 'img_description', 'imgcomment_name', 'imgcomment_description_id' + ); + $this->migrate( 'oldimage', [ 'oi_name', 'oi_timestamp' ], 'oi_description' ); + $this->migrate( 'filearchive', 'fa_id', 'fa_deleted_reason' ); + $this->migrate( 'filearchive', 'fa_id', 'fa_description' ); + $this->migrate( 'recentchanges', 'rc_id', 'rc_comment' ); + $this->migrate( 'logging', 'log_id', 'log_comment' ); + $this->migrate( 'protected_titles', [ 'pt_namespace', 'pt_title' ], 'pt_reason' ); + return true; + } + + /** + * Fetch comment IDs for a set of comments + * @param IDatabase $dbw + * @param array &$comments Keys are comment names, values will be set to IDs. + * @return int Count of added comments + */ + private function loadCommentIDs( IDatabase $dbw, array &$comments ) { + $count = 0; + $needComments = $comments; + + while ( true ) { + $where = []; + foreach ( $needComments as $need => $dummy ) { + $where[] = $dbw->makeList( + [ + 'comment_hash' => CommentStore::hash( $need, null ), + 'comment_text' => $need, + ], + LIST_AND + ); + } + + $res = $dbw->select( + 'comment', + [ 'comment_id', 'comment_text' ], + [ + $dbw->makeList( $where, LIST_OR ), + 'comment_data' => null, + ], + __METHOD__ + ); + foreach ( $res as $row ) { + $comments[$row->comment_text] = $row->comment_id; + unset( $needComments[$row->comment_text] ); + } + + if ( !$needComments ) { + break; + } + + $dbw->insert( + 'comment', + array_map( function ( $v ) { + return [ + 'comment_hash' => CommentStore::hash( $v, null ), + 'comment_text' => $v, + ]; + }, array_keys( $needComments ) ), + __METHOD__ + ); + $count += $dbw->affectedRows(); + } + return $count; + } + + /** + * Migrate comments in a table. + * + * Assumes any row with the ID field non-zero have already been migrated. + * Assumes the new field name is the same as the old with '_id' appended. + * Blanks the old fields while migrating. + * + * @param string $table Table to migrate + * @param string|string[] $primaryKey Primary key of the table. + * @param string $oldField Old comment field name + */ + protected function migrate( $table, $primaryKey, $oldField ) { + $newField = $oldField . '_id'; + $primaryKey = (array)$primaryKey; + $pkFilter = array_flip( $primaryKey ); + $this->output( "Beginning migration of $table.$oldField to $table.$newField\n" ); + wfWaitForSlaves(); + + $dbw = $this->getDB( DB_MASTER ); + $next = '1=1'; + $countUpdated = 0; + $countComments = 0; + while ( true ) { + // Fetch the rows needing update + $res = $dbw->select( + $table, + array_merge( $primaryKey, [ $oldField ] ), + [ + $newField => 0, + $next, + ], + __METHOD__, + [ + 'ORDER BY' => $primaryKey, + 'LIMIT' => $this->mBatchSize, + ] + ); + if ( !$res->numRows() ) { + break; + } + + // Collect the distinct comments from those rows + $comments = []; + foreach ( $res as $row ) { + $comments[$row->$oldField] = 0; + } + $countComments += $this->loadCommentIDs( $dbw, $comments ); + + // Update the existing rows + foreach ( $res as $row ) { + $dbw->update( + $table, + [ + $newField => $comments[$row->$oldField], + $oldField => '', + ], + array_intersect_key( (array)$row, $pkFilter ) + [ + $newField => 0 + ], + __METHOD__ + ); + $countUpdated += $dbw->affectedRows(); + } + + // Calculate the "next" condition + $next = ''; + $prompt = []; + for ( $i = count( $primaryKey ) - 1; $i >= 0; $i-- ) { + $field = $primaryKey[$i]; + $prompt[] = $row->$field; + $value = $dbw->addQuotes( $row->$field ); + if ( $next === '' ) { + $next = "$field > $value"; + } else { + $next = "$field > $value OR $field = $value AND ($next)"; + } + } + $prompt = join( ' ', array_reverse( $prompt ) ); + $this->output( "... $prompt\n" ); + wfWaitForSlaves(); + } + + $this->output( + "Completed migration, updated $countUpdated row(s) with $countComments new comment(s)\n" + ); + } + + /** + * Migrate comments in a table to a temporary table. + * + * Assumes any row with the ID field non-zero have already been migrated. + * Assumes the new table is named "{$table}_comment_temp", and it has two + * columns, in order, being the primary key of the original table and the + * comment ID field. + * Blanks the old fields while migrating. + * + * @param string $table Table to migrate + * @param string $primaryKey Primary key of the table. + * @param string $oldField Old comment field name + * @param string $newPrimaryKey Primary key of the new table. + * @param string $newField New comment field name + */ + protected function migrateToTemp( $table, $primaryKey, $oldField, $newPrimaryKey, $newField ) { + $newTable = $table . '_comment_temp'; + $this->output( "Beginning migration of $table.$oldField to $newTable.$newField\n" ); + wfWaitForSlaves(); + + $dbw = $this->getDB( DB_MASTER ); + $next = []; + $countUpdated = 0; + $countComments = 0; + while ( true ) { + // Fetch the rows needing update + $res = $dbw->select( + [ $table, $newTable ], + [ $primaryKey, $oldField ], + [ $newPrimaryKey => null ] + $next, + __METHOD__, + [ + 'ORDER BY' => $primaryKey, + 'LIMIT' => $this->mBatchSize, + ], + [ $newTable => [ 'LEFT JOIN', "{$primaryKey}={$newPrimaryKey}" ] ] + ); + if ( !$res->numRows() ) { + break; + } + + // Collect the distinct comments from those rows + $comments = []; + foreach ( $res as $row ) { + $comments[$row->$oldField] = 0; + } + $countComments += $this->loadCommentIDs( $dbw, $comments ); + + // Update rows + $inserts = []; + $updates = []; + foreach ( $res as $row ) { + $inserts[] = [ + $newPrimaryKey => $row->$primaryKey, + $newField => $comments[$row->$oldField] + ]; + $updates[] = $row->$primaryKey; + } + $this->beginTransaction( $dbw, __METHOD__ ); + $dbw->insert( $newTable, $inserts, __METHOD__ ); + $dbw->update( $table, [ $oldField => '' ], [ $primaryKey => $updates ], __METHOD__ ); + $countUpdated += $dbw->affectedRows(); + $this->commitTransaction( $dbw, __METHOD__ ); + + // Calculate the "next" condition + $next = [ $primaryKey . ' > ' . $dbw->addQuotes( $row->$primaryKey ) ]; + $this->output( "... {$row->$primaryKey}\n" ); + wfWaitForSlaves(); + } + + $this->output( + "Completed migration, updated $countUpdated row(s) with $countComments new comment(s)\n" + ); + } +} + +$maintClass = "MigrateComments"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/migrateFileRepoLayout.php b/www/wiki/maintenance/migrateFileRepoLayout.php new file mode 100644 index 00000000..f771fff7 --- /dev/null +++ b/www/wiki/maintenance/migrateFileRepoLayout.php @@ -0,0 +1,238 @@ +<?php +/** + * Copy all files in FileRepo to an originals container using SHA1 paths. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Copy all files in FileRepo to an originals container using SHA1 paths. + * + * This script should be run while the repo is still set to the old layout. + * + * @ingroup Maintenance + */ +class MigrateFileRepoLayout extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Copy files in repo to a different layout.' ); + $this->addOption( 'oldlayout', "Old layout; one of 'name' or 'sha1'", true, true ); + $this->addOption( 'newlayout', "New layout; one of 'name' or 'sha1'", true, true ); + $this->addOption( 'since', "Copy only files from after this timestamp", false, true ); + $this->setBatchSize( 50 ); + } + + public function execute() { + $oldLayout = $this->getOption( 'oldlayout' ); + if ( !in_array( $oldLayout, [ 'name', 'sha1' ] ) ) { + $this->error( "Invalid old layout.", 1 ); + } + $newLayout = $this->getOption( 'newlayout' ); + if ( !in_array( $newLayout, [ 'name', 'sha1' ] ) ) { + $this->error( "Invalid new layout.", 1 ); + } + $since = $this->getOption( 'since' ); + + $repo = $this->getRepo(); + + $be = $repo->getBackend(); + if ( $be instanceof FileBackendDBRepoWrapper ) { + $be = $be->getInternalBackend(); // avoid path translations for this script + } + + $dbw = $repo->getMasterDB(); + + $origBase = $be->getContainerStoragePath( "{$repo->getName()}-original" ); + $startTime = wfTimestampNow(); + + // Do current and archived versions... + $conds = []; + if ( $since ) { + $conds[] = 'img_timestamp >= ' . $dbw->addQuotes( $dbw->timestamp( $since ) ); + } + + $batch = []; + $lastName = ''; + do { + $res = $dbw->select( 'image', + [ 'img_name', 'img_sha1' ], + array_merge( [ 'img_name > ' . $dbw->addQuotes( $lastName ) ], $conds ), + __METHOD__, + [ 'LIMIT' => $this->mBatchSize, 'ORDER BY' => 'img_name' ] + ); + + foreach ( $res as $row ) { + $lastName = $row->img_name; + /** @var LocalFile $file */ + $file = $repo->newFile( $row->img_name ); + // Check in case SHA1 rows are not populated for some files + $sha1 = strlen( $row->img_sha1 ) ? $row->img_sha1 : $file->getSha1(); + + if ( !strlen( $sha1 ) ) { + $this->error( "Image SHA-1 not known for {$row->img_name}." ); + } else { + if ( $oldLayout === 'sha1' ) { + $spath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}"; + } else { + $spath = $file->getPath(); + } + + if ( $newLayout === 'sha1' ) { + $dpath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}"; + } else { + $dpath = $file->getPath(); + } + + $status = $be->prepare( [ + 'dir' => dirname( $dpath ), 'bypassReadOnly' => 1 ] ); + if ( !$status->isOK() ) { + $this->error( print_r( $status->getErrors(), true ) ); + } + + $batch[] = [ 'op' => 'copy', 'overwrite' => true, + 'src' => $spath, 'dst' => $dpath, 'img' => $row->img_name ]; + } + + foreach ( $file->getHistory() as $ofile ) { + $sha1 = $ofile->getSha1(); + if ( !strlen( $sha1 ) ) { + $this->error( "Image SHA-1 not set for {$ofile->getArchiveName()}." ); + continue; + } + + if ( $oldLayout === 'sha1' ) { + $spath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}"; + } elseif ( $ofile->isDeleted( File::DELETED_FILE ) ) { + $spath = $be->getContainerStoragePath( "{$repo->getName()}-deleted" ) . + '/' . $repo->getDeletedHashPath( $sha1 ) . + $sha1 . '.' . $ofile->getExtension(); + } else { + $spath = $ofile->getPath(); + } + + if ( $newLayout === 'sha1' ) { + $dpath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}"; + } else { + $dpath = $ofile->getPath(); + } + + $status = $be->prepare( [ + 'dir' => dirname( $dpath ), 'bypassReadOnly' => 1 ] ); + if ( !$status->isOK() ) { + $this->error( print_r( $status->getErrors(), true ) ); + } + $batch[] = [ 'op' => 'copy', 'overwrite' => true, + 'src' => $spath, 'dst' => $dpath, 'img' => $ofile->getArchiveName() ]; + } + + if ( count( $batch ) >= $this->mBatchSize ) { + $this->runBatch( $batch, $be ); + $batch = []; + } + } + } while ( $res->numRows() ); + + if ( count( $batch ) ) { + $this->runBatch( $batch, $be ); + } + + // Do deleted versions... + $conds = []; + if ( $since ) { + $conds[] = 'fa_deleted_timestamp >= ' . $dbw->addQuotes( $dbw->timestamp( $since ) ); + } + + $batch = []; + $lastId = 0; + do { + $res = $dbw->select( 'filearchive', [ 'fa_storage_key', 'fa_id', 'fa_name' ], + array_merge( [ 'fa_id > ' . $dbw->addQuotes( $lastId ) ], $conds ), + __METHOD__, + [ 'LIMIT' => $this->mBatchSize, 'ORDER BY' => 'fa_id' ] + ); + + foreach ( $res as $row ) { + $lastId = $row->fa_id; + $sha1Key = $row->fa_storage_key; + if ( !strlen( $sha1Key ) ) { + $this->error( "Image SHA-1 not set for file #{$row->fa_id} (deleted)." ); + continue; + } + $sha1 = substr( $sha1Key, 0, strpos( $sha1Key, '.' ) ); + + if ( $oldLayout === 'sha1' ) { + $spath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}"; + } else { + $spath = $be->getContainerStoragePath( "{$repo->getName()}-deleted" ) . + '/' . $repo->getDeletedHashPath( $sha1Key ) . $sha1Key; + } + + if ( $newLayout === 'sha1' ) { + $dpath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}"; + } else { + $dpath = $be->getContainerStoragePath( "{$repo->getName()}-deleted" ) . + '/' . $repo->getDeletedHashPath( $sha1Key ) . $sha1Key; + } + + $status = $be->prepare( [ + 'dir' => dirname( $dpath ), 'bypassReadOnly' => 1 ] ); + if ( !$status->isOK() ) { + $this->error( print_r( $status->getErrors(), true ) ); + } + + $batch[] = [ 'op' => 'copy', 'src' => $spath, 'dst' => $dpath, + 'overwriteSame' => true, 'img' => "(ID {$row->fa_id}) {$row->fa_name}" ]; + + if ( count( $batch ) >= $this->mBatchSize ) { + $this->runBatch( $batch, $be ); + $batch = []; + } + } + } while ( $res->numRows() ); + + if ( count( $batch ) ) { + $this->runBatch( $batch, $be ); + } + + $this->output( "Done (started $startTime)\n" ); + } + + protected function getRepo() { + return RepoGroup::singleton()->getLocalRepo(); + } + + protected function runBatch( array $ops, FileBackend $be ) { + $this->output( "Migrating file batch:\n" ); + foreach ( $ops as $op ) { + $this->output( "\"{$op['img']}\" (dest: {$op['dst']})\n" ); + } + + $status = $be->doOperations( $ops, [ 'bypassReadOnly' => 1 ] ); + if ( !$status->isOK() ) { + $this->output( print_r( $status->getErrors(), true ) ); + } + + $this->output( "Batch done\n\n" ); + } +} + +$maintClass = 'MigrateFileRepoLayout'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/migrateUserGroup.php b/www/wiki/maintenance/migrateUserGroup.php new file mode 100644 index 00000000..597a876d --- /dev/null +++ b/www/wiki/maintenance/migrateUserGroup.php @@ -0,0 +1,109 @@ +<?php +/** + * Re-assign users from an old group to a new one + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that re-assigns users from an old group to a new one. + * + * @ingroup Maintenance + */ +class MigrateUserGroup extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Re-assign users from an old group to a new one' ); + $this->addArg( 'oldgroup', 'Old user group key', true ); + $this->addArg( 'newgroup', 'New user group key', true ); + $this->setBatchSize( 200 ); + } + + public function execute() { + $count = 0; + $oldGroup = $this->getArg( 0 ); + $newGroup = $this->getArg( 1 ); + $dbw = $this->getDB( DB_MASTER ); + $start = $dbw->selectField( 'user_groups', 'MIN(ug_user)', + [ 'ug_group' => $oldGroup ], __FUNCTION__ ); + $end = $dbw->selectField( 'user_groups', 'MAX(ug_user)', + [ 'ug_group' => $oldGroup ], __FUNCTION__ ); + if ( $start === null ) { + $this->error( "Nothing to do - no users in the '$oldGroup' group", true ); + } + # Do remaining chunk + $end += $this->mBatchSize - 1; + $blockStart = $start; + $blockEnd = $start + $this->mBatchSize - 1; + // Migrate users over in batches... + while ( $blockEnd <= $end ) { + $affected = 0; + $this->output( "Doing users $blockStart to $blockEnd\n" ); + + $this->beginTransaction( $dbw, __METHOD__ ); + $dbw->update( 'user_groups', + [ 'ug_group' => $newGroup ], + [ 'ug_group' => $oldGroup, + "ug_user BETWEEN $blockStart AND $blockEnd" ], + __METHOD__, + [ 'IGNORE' ] + ); + $affected += $dbw->affectedRows(); + // Delete rows that the UPDATE operation above had to ignore. + // This happens when a user is in both the old and new group. + // Updating the row for the old group membership failed since + // user/group is UNIQUE. + $dbw->delete( 'user_groups', + [ 'ug_group' => $oldGroup, + "ug_user BETWEEN $blockStart AND $blockEnd" ], + __METHOD__ + ); + $affected += $dbw->affectedRows(); + $this->commitTransaction( $dbw, __METHOD__ ); + + // Clear cache for the affected users (T42340) + if ( $affected > 0 ) { + // XXX: This also invalidates cache of unaffected users that + // were in the new group and not in the group. + $res = $dbw->select( 'user_groups', 'ug_user', + [ 'ug_group' => $newGroup, + "ug_user BETWEEN $blockStart AND $blockEnd" ], + __METHOD__ + ); + if ( $res !== false ) { + foreach ( $res as $row ) { + $user = User::newFromId( $row->ug_user ); + $user->invalidateCache(); + } + } + } + + $count += $affected; + $blockStart += $this->mBatchSize; + $blockEnd += $this->mBatchSize; + wfWaitForSlaves(); + } + $this->output( "Done! $count users in group '$oldGroup' are now in '$newGroup' instead.\n" ); + } +} + +$maintClass = "MigrateUserGroup"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/minify.php b/www/wiki/maintenance/minify.php new file mode 100644 index 00000000..16e4d1c9 --- /dev/null +++ b/www/wiki/maintenance/minify.php @@ -0,0 +1,138 @@ +<?php +/** + * Minify a file or set of files + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that minifies a file or set of files. + * + * @ingroup Maintenance + */ +class MinifyScript extends Maintenance { + public $outDir; + + public function __construct() { + parent::__construct(); + $this->addOption( 'outfile', + 'File for output. Only a single file may be specified for input.', + false, true ); + $this->addOption( 'outdir', + "Directory for output. If this is not specified, and neither is --outfile, then the\n" . + "output files will be sent to the same directories as the input files.", + false, true ); + $this->addDescription( "Minify a file or set of files.\n\n" . + "If --outfile is not specified, then the output file names will have a .min extension\n" . + "added, e.g. jquery.js -> jquery.min.js." + ); + } + + public function execute() { + if ( !count( $this->mArgs ) ) { + $this->error( "minify.php: At least one input file must be specified." ); + exit( 1 ); + } + + if ( $this->hasOption( 'outfile' ) ) { + if ( count( $this->mArgs ) > 1 ) { + $this->error( '--outfile may only be used with a single input file.' ); + exit( 1 ); + } + + // Minify one file + $this->minify( $this->getArg( 0 ), $this->getOption( 'outfile' ) ); + + return; + } + + $outDir = $this->getOption( 'outdir', false ); + + foreach ( $this->mArgs as $arg ) { + $inPath = realpath( $arg ); + $inName = basename( $inPath ); + $inDir = dirname( $inPath ); + + if ( strpos( $inName, '.min.' ) !== false ) { + $this->error( "Skipping $inName\n" ); + continue; + } + + if ( !file_exists( $inPath ) ) { + $this->error( "File does not exist: $arg", true ); + } + + $extension = $this->getExtension( $inName ); + $outName = substr( $inName, 0, -strlen( $extension ) ) . 'min.' . $extension; + if ( $outDir === false ) { + $outPath = $inDir . '/' . $outName; + } else { + $outPath = $outDir . '/' . $outName; + } + + $this->minify( $inPath, $outPath ); + } + } + + public function getExtension( $fileName ) { + $dotPos = strrpos( $fileName, '.' ); + if ( $dotPos === false ) { + $this->error( "No file extension, cannot determine type: $fileName" ); + exit( 1 ); + } + + return substr( $fileName, $dotPos + 1 ); + } + + public function minify( $inPath, $outPath ) { + $extension = $this->getExtension( $inPath ); + $this->output( basename( $inPath ) . ' -> ' . basename( $outPath ) . '...' ); + + $inText = file_get_contents( $inPath ); + if ( $inText === false ) { + $this->error( "Unable to open file $inPath for reading." ); + exit( 1 ); + } + $outFile = fopen( $outPath, 'w' ); + if ( !$outFile ) { + $this->error( "Unable to open file $outPath for writing." ); + exit( 1 ); + } + + switch ( $extension ) { + case 'js': + $outText = JavaScriptMinifier::minify( $inText ); + break; + case 'css': + $outText = CSSMin::minify( $inText ); + break; + default: + $this->error( "No minifier defined for extension \"$extension\"" ); + } + + fwrite( $outFile, $outText ); + fclose( $outFile ); + $this->output( " ok\n" ); + } +} + +$maintClass = 'MinifyScript'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/moveBatch.php b/www/wiki/maintenance/moveBatch.php new file mode 100644 index 00000000..d578a496 --- /dev/null +++ b/www/wiki/maintenance/moveBatch.php @@ -0,0 +1,127 @@ +<?php +/** + * Move a batch of pages. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @author Tim Starling + * + * USAGE: php moveBatch.php [-u <user>] [-r <reason>] [-i <interval>] [-noredirects] [listfile] + * + * [listfile] - file with two titles per line, separated with pipe characters; + * the first title is the source, the second is the destination. + * Standard input is used if listfile is not given. + * <user> - username to perform moves as + * <reason> - reason to be given for moves + * <interval> - number of seconds to sleep after each move + * <noredirects> - suppress creation of redirects + * + * This will print out error codes from Title::moveTo() if something goes wrong, + * e.g. immobile_namespace for namespaces which can't be moved + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to move a batch of pages. + * + * @ingroup Maintenance + */ +class MoveBatch extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Moves a batch of pages' ); + $this->addOption( 'u', "User to perform move", false, true ); + $this->addOption( 'r', "Reason to move page", false, true ); + $this->addOption( 'i', "Interval to sleep between moves" ); + $this->addOption( 'noredirects', "Suppress creation of redirects" ); + $this->addArg( 'listfile', 'List of pages to move, newline delimited', false ); + } + + public function execute() { + global $wgUser; + + # Change to current working directory + $oldCwd = getcwd(); + chdir( $oldCwd ); + + # Options processing + $user = $this->getOption( 'u', false ); + $reason = $this->getOption( 'r', '' ); + $interval = $this->getOption( 'i', 0 ); + $noredirects = $this->hasOption( 'noredirects' ); + if ( $this->hasArg() ) { + $file = fopen( $this->getArg(), 'r' ); + } else { + $file = $this->getStdin(); + } + + # Setup + if ( !$file ) { + $this->error( "Unable to read file, exiting", true ); + } + if ( $user === false ) { + $wgUser = User::newSystemUser( 'Move page script', [ 'steal' => true ] ); + } else { + $wgUser = User::newFromName( $user ); + } + if ( !$wgUser ) { + $this->error( "Invalid username", true ); + } + + # Setup complete, now start + $dbw = $this->getDB( DB_MASTER ); + // @codingStandardsIgnoreStart Ignore avoid function calls in a FOR loop test part warning + for ( $linenum = 1; !feof( $file ); $linenum++ ) { + // @codingStandardsIgnoreEnd + $line = fgets( $file ); + if ( $line === false ) { + break; + } + $parts = array_map( 'trim', explode( '|', $line ) ); + if ( count( $parts ) != 2 ) { + $this->error( "Error on line $linenum, no pipe character" ); + continue; + } + $source = Title::newFromText( $parts[0] ); + $dest = Title::newFromText( $parts[1] ); + if ( is_null( $source ) || is_null( $dest ) ) { + $this->error( "Invalid title on line $linenum" ); + continue; + } + + $this->output( $source->getPrefixedText() . ' --> ' . $dest->getPrefixedText() ); + $this->beginTransaction( $dbw, __METHOD__ ); + $mp = new MovePage( $source, $dest ); + $status = $mp->move( $wgUser, $reason, !$noredirects ); + if ( !$status->isOK() ) { + $this->output( "\nFAILED: " . $status->getWikiText( false, false, 'en' ) ); + } + $this->commitTransaction( $dbw, __METHOD__ ); + $this->output( "\n" ); + + if ( $interval ) { + sleep( $interval ); + } + wfWaitForSlaves(); + } + } +} + +$maintClass = "MoveBatch"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/mssql/archives/patch-add-3d.sql b/www/wiki/maintenance/mssql/archives/patch-add-3d.sql new file mode 100644 index 00000000..51d2775f --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-add-3d.sql @@ -0,0 +1,27 @@ +ALTER TABLE /*$wgDBprefix*/image + DROP CONSTRAINT img_media_type_ckc; + +ALTER TABLE /*$wgDBprefix*/image + ADD CONSTRAINT img_media_type_ckc + CHECK (img_media_type IN("UNKNOWN", "BITMAP", "DRAWING", "AUDIO", "VIDEO", "MULTIMEDIA", "OFFICE", "TEXT", "EXECUTABLE", "ARCHIVE", "3D")); + +ALTER TABLE /*$wgDBprefix*/oldimage + DROP CONSTRAINT oi_media_type_ckc; + +ALTER TABLE /*$wgDBprefix*/oldimage + ADD CONSTRAINT oi_media_type_ckc + CHECK (oi_media_type IN("UNKNOWN", "BITMAP", "DRAWING", "AUDIO", "VIDEO", "MULTIMEDIA", "OFFICE", "TEXT", "EXECUTABLE", "ARCHIVE", "3D")); + +ALTER TABLE /*$wgDBprefix*/filearchive + DROP CONSTRAINT fa_media_type_ckc; + +ALTER TABLE /*$wgDBprefix*/filearchive + ADD CONSTRAINT fa_media_type_ckc + CHECK (fa_media_type IN("UNKNOWN", "BITMAP", "DRAWING", "AUDIO", "VIDEO", "MULTIMEDIA", "OFFICE", "TEXT", "EXECUTABLE", "ARCHIVE", "3D")); + +ALTER TABLE /*$wgDBprefix*/uploadstash + DROP CONSTRAINT us_media_type_ckc; + +ALTER TABLE /*$wgDBprefix*/uploadstash + ADD CONSTRAINT us_media_type_ckc + CHECK (us_media_type IN("UNKNOWN", "BITMAP", "DRAWING", "AUDIO", "VIDEO", "MULTIMEDIA", "OFFICE", "TEXT", "EXECUTABLE", "ARCHIVE", "3D")); diff --git a/www/wiki/maintenance/mssql/archives/patch-add-cl_collation_ext_index.sql b/www/wiki/maintenance/mssql/archives/patch-add-cl_collation_ext_index.sql new file mode 100644 index 00000000..8137dc64 --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-add-cl_collation_ext_index.sql @@ -0,0 +1,2 @@ +-- @since 1.27 +CREATE INDEX /*i*/cl_collation_ext ON /*_*/categorylinks (cl_collation, cl_to, cl_type, cl_from); diff --git a/www/wiki/maintenance/mssql/archives/patch-alter-table-oldimage.sql b/www/wiki/maintenance/mssql/archives/patch-alter-table-oldimage.sql new file mode 100644 index 00000000..fb31d6ae --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-alter-table-oldimage.sql @@ -0,0 +1 @@ +DROP INDEX /*i*/oi_name_archive_name ON /*_*/oldimage; diff --git a/www/wiki/maintenance/mssql/archives/patch-archive-drop-fks.sql b/www/wiki/maintenance/mssql/archives/patch-archive-drop-fks.sql new file mode 100644 index 00000000..3055ac98 --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-archive-drop-fks.sql @@ -0,0 +1,59 @@ +DECLARE @base nvarchar(max), + @SQL nvarchar(max), + @id sysname;-- + +SET @base = 'ALTER TABLE /*_*/archive DROP CONSTRAINT ';-- + +SELECT @id = fk.name +FROM sys.foreign_keys fk +JOIN sys.foreign_key_columns fkc + ON fkc.constraint_object_id = fk.object_id +JOIN sys.columns c + ON c.column_id = fkc.parent_column_id + AND c.object_id = fkc.parent_object_id +WHERE + fk.parent_object_id = OBJECT_ID('/*_*/archive') + AND fk.referenced_object_id = OBJECT_ID('/*_*/revision') + AND c.name = 'ar_parent_id';-- + +SET @SQL = @base + @id;-- + +EXEC sp_executesql @SQL;-- + +-- while we're at it, let's fix up the other foreign key constraints on archive +-- as future patches touch constraints on other tables, they'll take the time to update constraint names there as well +SELECT @id = fk.name +FROM sys.foreign_keys fk +JOIN sys.foreign_key_columns fkc + ON fkc.constraint_object_id = fk.object_id +JOIN sys.columns c + ON c.column_id = fkc.parent_column_id + AND c.object_id = fkc.parent_object_id +WHERE + fk.parent_object_id = OBJECT_ID('/*_*/archive') + AND fk.referenced_object_id = OBJECT_ID('/*_*/mwuser') + AND c.name = 'ar_user';-- + +SET @SQL = @base + @id;-- + +EXEC sp_executesql @SQL;-- + +ALTER TABLE /*_*/archive ADD CONSTRAINT ar_user__user_id__fk FOREIGN KEY (ar_user) REFERENCES /*_*/mwuser(user_id);-- + +SELECT @id = fk.name +FROM sys.foreign_keys fk +JOIN sys.foreign_key_columns fkc + ON fkc.constraint_object_id = fk.object_id +JOIN sys.columns c + ON c.column_id = fkc.parent_column_id + AND c.object_id = fkc.parent_object_id +WHERE + fk.parent_object_id = OBJECT_ID('/*_*/archive') + AND fk.referenced_object_id = OBJECT_ID('/*_*/text') + AND c.name = 'ar_text_id';-- + +SET @SQL = @base + @id;-- + +EXEC sp_executesql @SQL;-- + +ALTER TABLE /*_*/archive ADD CONSTRAINT ar_text_id__old_id__fk FOREIGN KEY (ar_text_id) REFERENCES /*_*/text(old_id) ON DELETE CASCADE; diff --git a/www/wiki/maintenance/mssql/archives/patch-bot_passwords.sql b/www/wiki/maintenance/mssql/archives/patch-bot_passwords.sql new file mode 100644 index 00000000..7718ffaa --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-bot_passwords.sql @@ -0,0 +1,13 @@ +-- +-- This table contains a user's bot passwords: passwords that allow access to +-- the account via the API with limited rights. +-- +CREATE TABLE /*_*/bot_passwords ( + bp_user int NOT NULL REFERENCES /*_*/mwuser(user_id) ON DELETE CASCADE, + bp_app_id nvarchar(32) NOT NULL, + bp_password nvarchar(255) NOT NULL, + bp_token nvarchar(255) NOT NULL, + bp_restrictions nvarchar(max) NOT NULL, + bp_grants nvarchar(max) NOT NULL, + PRIMARY KEY (bp_user, bp_app_id) +); diff --git a/www/wiki/maintenance/mssql/archives/patch-categorylinks-constraints.sql b/www/wiki/maintenance/mssql/archives/patch-categorylinks-constraints.sql new file mode 100644 index 00000000..cf9b5658 --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-categorylinks-constraints.sql @@ -0,0 +1,20 @@ +DECLARE @baseSQL nvarchar(max), + @SQL nvarchar(max), + @id sysname;-- + +SET @baseSQL = 'ALTER TABLE /*_*/categorylinks DROP CONSTRAINT ';-- + +SELECT @id = cc.name +FROM sys.check_constraints cc +JOIN sys.columns c + ON c.object_id = cc.parent_object_id + AND c.column_id = cc.parent_column_id +WHERE + cc.parent_object_id = OBJECT_ID('/*_*/categorylinks') + AND c.name = 'cl_type';-- + +SET @SQL = @baseSQL + @id;-- + +EXEC sp_executesql @SQL;-- + +ALTER TABLE /*_*/categorylinks ADD CONSTRAINT cl_type_ckc CHECK (cl_type IN('page', 'subcat', 'file')); diff --git a/www/wiki/maintenance/mssql/archives/patch-change_tag-ct_id.sql b/www/wiki/maintenance/mssql/archives/patch-change_tag-ct_id.sql new file mode 100644 index 00000000..94cb9d14 --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-change_tag-ct_id.sql @@ -0,0 +1,4 @@ +-- Primary key in change_tag table + +ALTER TABLE /*_*/change_tag ADD ct_id INT IDENTITY; +ALTER TABLE /*_*/change_tag ADD CONSTRAINT pk_change_tag PRIMARY KEY(ct_id) diff --git a/www/wiki/maintenance/mssql/archives/patch-drop-page_counter.sql b/www/wiki/maintenance/mssql/archives/patch-drop-page_counter.sql new file mode 100644 index 00000000..54ab9f7a --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-drop-page_counter.sql @@ -0,0 +1,19 @@ +DECLARE @sql nvarchar(max), + @id sysname;-- + +SET @sql = 'ALTER TABLE /*_*/page DROP CONSTRAINT ';-- + +SELECT @id = df.name +FROM sys.default_constraints df +JOIN sys.columns c + ON c.object_id = df.parent_object_id + AND c.column_id = df.parent_column_id +WHERE + df.parent_object_id = OBJECT_ID('/*_*/page') + AND c.name = 'page_counter';-- + +SET @sql = @sql + @id;-- + +EXEC sp_executesql @sql;-- + +ALTER TABLE /*_*/page DROP COLUMN page_counter; diff --git a/www/wiki/maintenance/mssql/archives/patch-drop-rc_cur_time.sql b/www/wiki/maintenance/mssql/archives/patch-drop-rc_cur_time.sql new file mode 100644 index 00000000..01c46d31 --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-drop-rc_cur_time.sql @@ -0,0 +1,19 @@ +DECLARE @sql nvarchar(max), + @id sysname;-- + +SET @sql = 'ALTER TABLE /*_*/recentchanges DROP CONSTRAINT ';-- + +SELECT @id = df.name +FROM sys.default_constraints df +JOIN sys.columns c + ON c.object_id = df.parent_object_id + AND c.column_id = df.parent_column_id +WHERE + df.parent_object_id = OBJECT_ID('/*_*/recentchanges') + AND c.name = 'rc_cur_time';-- + +SET @sql = @sql + @id;-- + +EXEC sp_executesql @sql;-- + +ALTER TABLE /*_*/recentchanges DROP COLUMN rc_cur_time; diff --git a/www/wiki/maintenance/mssql/archives/patch-drop-ss_total_views.sql b/www/wiki/maintenance/mssql/archives/patch-drop-ss_total_views.sql new file mode 100644 index 00000000..7525ed57 --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-drop-ss_total_views.sql @@ -0,0 +1,19 @@ +DECLARE @sql nvarchar(max), + @id sysname;-- + +SET @sql = 'ALTER TABLE /*_*/site_stats DROP CONSTRAINT ';-- + +SELECT @id = df.name +FROM sys.default_constraints df +JOIN sys.columns c + ON c.object_id = df.parent_object_id + AND c.column_id = df.parent_column_id +WHERE + df.parent_object_id = OBJECT_ID('/*_*/site_stats') + AND c.name = 'ss_total_views';-- + +SET @sql = @sql + @id;-- + +EXEC sp_executesql @sql;-- + +ALTER TABLE /*_*/site_stats DROP COLUMN ss_total_views; diff --git a/www/wiki/maintenance/mssql/archives/patch-drop-user_options.sql b/www/wiki/maintenance/mssql/archives/patch-drop-user_options.sql new file mode 100644 index 00000000..ab379567 --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-drop-user_options.sql @@ -0,0 +1,19 @@ +DECLARE @sql nvarchar(max), + @id sysname;-- + +SET @sql = 'ALTER TABLE /*_*/mwuser DROP CONSTRAINT ';-- + +SELECT @id = df.name +FROM sys.default_constraints df +JOIN sys.columns c + ON c.object_id = df.parent_object_id + AND c.column_id = df.parent_column_id +WHERE + df.parent_object_id = OBJECT_ID('/*_*/mwuser') + AND c.name = 'user_options';-- + +SET @sql = @sql + @id;-- + +EXEC sp_executesql @sql;-- + +ALTER TABLE /*_*/mwuser DROP COLUMN user_options; diff --git a/www/wiki/maintenance/mssql/archives/patch-fa_major_mime-chemical.sql b/www/wiki/maintenance/mssql/archives/patch-fa_major_mime-chemical.sql new file mode 100644 index 00000000..18368087 --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-fa_major_mime-chemical.sql @@ -0,0 +1,4 @@ +ALTER TABLE /*_*/filearchive +DROP CONSTRAINT fa_major_mime_ckc; +ALTER TABLE /*_*/filearchive +WITH NOCHECK ADD CONSTRAINT fa_major_mime_ckc CHECK (fa_major_mime IN('unknown', 'application', 'audio', 'image', 'text', 'video', 'message', 'model', 'multipart', 'chemical'));
\ No newline at end of file diff --git a/www/wiki/maintenance/mssql/archives/patch-filearchive-constraints.sql b/www/wiki/maintenance/mssql/archives/patch-filearchive-constraints.sql new file mode 100644 index 00000000..cefead54 --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-filearchive-constraints.sql @@ -0,0 +1,34 @@ +DECLARE @baseSQL nvarchar(max), + @SQL nvarchar(max), + @id sysname;-- + +SET @baseSQL = 'ALTER TABLE /*_*/filearchive DROP CONSTRAINT ';-- + +SELECT @id = cc.name +FROM sys.check_constraints cc +JOIN sys.columns c + ON c.object_id = cc.parent_object_id + AND c.column_id = cc.parent_column_id +WHERE + cc.parent_object_id = OBJECT_ID('/*_*/filearchive') + AND c.name = 'fa_major_mime';-- + +SET @SQL = @baseSQL + @id;-- + +EXEC sp_executesql @SQL;-- + +SELECT @id = cc.name +FROM sys.check_constraints cc +JOIN sys.columns c + ON c.object_id = cc.parent_object_id + AND c.column_id = cc.parent_column_id +WHERE + cc.parent_object_id = OBJECT_ID('/*_*/filearchive') + AND c.name = 'fa_media_type';-- + +SET @SQL = @baseSQL + @id;-- + +EXEC sp_executesql @SQL;-- + +ALTER TABLE /*_*/filearchive ADD CONSTRAINT fa_major_mime_ckc check (fa_major_mime IN('unknown', 'application', 'audio', 'image', 'text', 'video', 'message', 'model', 'multipart'));-- +ALTER TABLE /*_*/filearchive ADD CONSTRAINT fa_media_type_ckc check (fa_media_type in('UNKNOWN', 'BITMAP', 'DRAWING', 'AUDIO', 'VIDEO', 'MULTIMEDIA', 'OFFICE', 'TEXT', 'EXECUTABLE', 'ARCHIVE')); diff --git a/www/wiki/maintenance/mssql/archives/patch-filearchive-schema.sql b/www/wiki/maintenance/mssql/archives/patch-filearchive-schema.sql new file mode 100644 index 00000000..cf1c01fb --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-filearchive-schema.sql @@ -0,0 +1,120 @@ +-- MediaWiki looks for lines ending with semicolons and sends them as separate queries +-- However here we *really* need this all to be sent as a single batch. As such, DO NOT +-- remove the -- from the end of each statement. + +DECLARE @temp table ( + fa_id int, + fa_name nvarchar(255), + fa_archive_name nvarchar(255), + fa_storage_group nvarchar(16), + fa_storage_key nvarchar(64), + fa_deleted_user int, + fa_deleted_timestamp varchar(14), + fa_deleted_reason nvarchar(max), + fa_size int, + fa_width int, + fa_height int, + fa_metadata nvarchar(max), + fa_bits int, + fa_media_type varchar(16), + fa_major_mime varchar(16), + fa_minor_mime nvarchar(100), + fa_description nvarchar(255), + fa_user int, + fa_user_text nvarchar(255), + fa_timestamp varchar(14), + fa_deleted tinyint, + fa_sha1 nvarchar(32) +);-- + +INSERT INTO @temp +SELECT * FROM /*_*/filearchive;-- + +DROP TABLE /*_*/filearchive;-- + +CREATE TABLE /*_*/filearchive ( + fa_id int NOT NULL PRIMARY KEY IDENTITY, + fa_name nvarchar(255) NOT NULL default '', + fa_archive_name nvarchar(255) default '', + fa_storage_group nvarchar(16), + fa_storage_key nvarchar(64) default '', + fa_deleted_user int, + fa_deleted_timestamp varchar(14) default '', + fa_deleted_reason nvarchar(max), + fa_size int default 0, + fa_width int default 0, + fa_height int default 0, + fa_metadata varbinary(max), + fa_bits int default 0, + fa_media_type varchar(16) default null, + fa_major_mime varchar(16) not null default 'unknown', + fa_minor_mime nvarchar(100) default 'unknown', + fa_description nvarchar(255), + fa_user int default 0 REFERENCES /*_*/mwuser(user_id) ON DELETE SET NULL, + fa_user_text nvarchar(255), + fa_timestamp varchar(14) default '', + fa_deleted tinyint NOT NULL default 0, + fa_sha1 nvarchar(32) NOT NULL default '', + CONSTRAINT fa_major_mime_ckc check (fa_major_mime in('unknown', 'application', 'audio', 'image', 'text', 'video', 'message', 'model', 'multipart', 'chemical')), + CONSTRAINT fa_media_type_ckc check (fa_media_type in('UNKNOWN', 'BITMAP', 'DRAWING', 'AUDIO', 'VIDEO', 'MULTIMEDIA', 'OFFICE', 'TEXT', 'EXECUTABLE', 'ARCHIVE')) +);-- + +CREATE INDEX /*i*/fa_name ON /*_*/filearchive (fa_name, fa_timestamp);-- +CREATE INDEX /*i*/fa_storage_group ON /*_*/filearchive (fa_storage_group, fa_storage_key);-- +CREATE INDEX /*i*/fa_deleted_timestamp ON /*_*/filearchive (fa_deleted_timestamp);-- +CREATE INDEX /*i*/fa_user_timestamp ON /*_*/filearchive (fa_user_text,fa_timestamp);-- +CREATE INDEX /*i*/fa_sha1 ON /*_*/filearchive (fa_sha1);-- + +SET IDENTITY_INSERT /*_*/filearchive ON;-- + +INSERT INTO /*_*/filearchive +( + fa_id, + fa_name, + fa_archive_name, + fa_storage_group, + fa_storage_key, + fa_deleted_user, + fa_deleted_timestamp, + fa_deleted_reason, + fa_size, + fa_width, + fa_height, + fa_metadata, + fa_bits, + fa_media_type, + fa_major_mime, + fa_minor_mime, + fa_description, + fa_user, + fa_user_text, + fa_timestamp, + fa_deleted, + fa_sha1 +) +SELECT + fa_id, + fa_name, + fa_archive_name, + fa_storage_group, + fa_storage_key, + fa_deleted_user, + fa_deleted_timestamp, + fa_deleted_reason, + fa_size, + fa_width, + fa_height, + CONVERT(varbinary(max), fa_metadata, 0), + fa_bits, + fa_media_type, + fa_major_mime, + fa_minor_mime, + fa_description, + fa_user, + fa_user_text, + fa_timestamp, + fa_deleted, + fa_sha1 +FROM @temp t;-- + +SET IDENTITY_INSERT /*_*/filearchive OFF; diff --git a/www/wiki/maintenance/mssql/archives/patch-il_from_namespace.sql b/www/wiki/maintenance/mssql/archives/patch-il_from_namespace.sql new file mode 100644 index 00000000..e4ac98f2 --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-il_from_namespace.sql @@ -0,0 +1,4 @@ +ALTER TABLE /*_*/imagelinks + ADD il_from_namespace int NOT NULL default 0; + +CREATE INDEX /*i*/il_backlinks_namespace ON /*_*/imagelinks (il_from_namespace,il_to,il_from);
\ No newline at end of file diff --git a/www/wiki/maintenance/mssql/archives/patch-image-constraints.sql b/www/wiki/maintenance/mssql/archives/patch-image-constraints.sql new file mode 100644 index 00000000..0aeb627d --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-image-constraints.sql @@ -0,0 +1,34 @@ +DECLARE @baseSQL nvarchar(max), + @SQL nvarchar(max), + @id sysname;-- + +SET @baseSQL = 'ALTER TABLE /*_*/image DROP CONSTRAINT ';-- + +SELECT @id = cc.name +FROM sys.check_constraints cc +JOIN sys.columns c + ON c.object_id = cc.parent_object_id + AND c.column_id = cc.parent_column_id +WHERE + cc.parent_object_id = OBJECT_ID('/*_*/image') + AND c.name = 'img_major_mime';-- + +SET @SQL = @baseSQL + @id;-- + +EXEC sp_executesql @SQL;-- + +SELECT @id = cc.name +FROM sys.check_constraints cc +JOIN sys.columns c + ON c.object_id = cc.parent_object_id + AND c.column_id = cc.parent_column_id +WHERE + cc.parent_object_id = OBJECT_ID('/*_*/image') + AND c.name = 'img_media_type';-- + +SET @SQL = @baseSQL + @id;-- + +EXEC sp_executesql @SQL;-- + +ALTER TABLE /*_*/image ADD CONSTRAINT img_major_mime_ckc check (img_major_mime IN('unknown', 'application', 'audio', 'image', 'text', 'video', 'message', 'model', 'multipart'));-- +ALTER TABLE /*_*/image ADD CONSTRAINT img_media_type_ckc check (img_media_type in('UNKNOWN', 'BITMAP', 'DRAWING', 'AUDIO', 'VIDEO', 'MULTIMEDIA', 'OFFICE', 'TEXT', 'EXECUTABLE', 'ARCHIVE')); diff --git a/www/wiki/maintenance/mssql/archives/patch-image-schema.sql b/www/wiki/maintenance/mssql/archives/patch-image-schema.sql new file mode 100644 index 00000000..213b4381 --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-image-schema.sql @@ -0,0 +1,84 @@ +-- MediaWiki looks for lines ending with semicolons and sends them as separate queries +-- However here we *really* need this all to be sent as a single batch. As such, DO NOT +-- remove the -- from the end of each statement. + +DECLARE @temp table ( + img_name varbinary(255), + img_size int, + img_width int, + img_height int, + img_metadata varbinary(max), + img_bits int, + img_media_type varchar(16), + img_major_mime varchar(16), + img_minor_mime nvarchar(100), + img_description nvarchar(255), + img_user int, + img_user_text nvarchar(255), + img_timestamp nvarchar(14), + img_sha1 nvarchar(32) +);-- + +INSERT INTO @temp +SELECT * FROM /*_*/image;-- + +DROP TABLE /*_*/image;-- + +CREATE TABLE /*_*/image ( + img_name nvarchar(255) NOT NULL default '' PRIMARY KEY, + img_size int NOT NULL default 0, + img_width int NOT NULL default 0, + img_height int NOT NULL default 0, + img_metadata varbinary(max) NOT NULL, + img_bits int NOT NULL default 0, + img_media_type varchar(16) default null, + img_major_mime varchar(16) not null default 'unknown', + img_minor_mime nvarchar(100) NOT NULL default 'unknown', + img_description nvarchar(255) NOT NULL, + img_user int REFERENCES /*_*/mwuser(user_id) ON DELETE SET NULL, + img_user_text nvarchar(255) NOT NULL, + img_timestamp nvarchar(14) NOT NULL default '', + img_sha1 nvarchar(32) NOT NULL default '', + CONSTRAINT img_major_mime_ckc check (img_major_mime IN('unknown', 'application', 'audio', 'image', 'text', 'video', 'message', 'model', 'multipart', 'chemical')), + CONSTRAINT img_media_type_ckc check (img_media_type in('UNKNOWN', 'BITMAP', 'DRAWING', 'AUDIO', 'VIDEO', 'MULTIMEDIA', 'OFFICE', 'TEXT', 'EXECUTABLE', 'ARCHIVE')) +);-- + +CREATE INDEX /*i*/img_usertext_timestamp ON /*_*/image (img_user_text,img_timestamp);-- +CREATE INDEX /*i*/img_size ON /*_*/image (img_size);-- +CREATE INDEX /*i*/img_timestamp ON /*_*/image (img_timestamp);-- +CREATE INDEX /*i*/img_sha1 ON /*_*/image (img_sha1);-- +CREATE INDEX /*i*/img_media_mime ON /*_*/image (img_media_type,img_major_mime,img_minor_mime);-- + +INSERT INTO /*_*/image +( + img_name, + img_size, + img_width, + img_height, + img_metadata, + img_bits, + img_media_type, + img_major_mime, + img_minor_mime, + img_description, + img_user, + img_user_text, + img_timestamp, + img_sha1 +) +SELECT + img_name, + img_size, + img_width, + img_height, + img_metadata, + img_bits, + img_media_type, + img_major_mime, + img_minor_mime, + img_description, + img_user, + img_user_text, + img_timestamp, + img_sha1 +FROM @temp t; diff --git a/www/wiki/maintenance/mssql/archives/patch-img_major_mime-chemical.sql b/www/wiki/maintenance/mssql/archives/patch-img_major_mime-chemical.sql new file mode 100644 index 00000000..eed07869 --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-img_major_mime-chemical.sql @@ -0,0 +1,4 @@ +ALTER TABLE /*_*/image +DROP CONSTRAINT img_major_mime_ckc; +ALTER TABLE /*_*/image +WITH NOCHECK ADD CONSTRAINT img_major_mime_ckc CHECK (img_major_mime IN('unknown', 'application', 'audio', 'image', 'text', 'video', 'message', 'model', 'multipart', 'chemical'));
\ No newline at end of file diff --git a/www/wiki/maintenance/mssql/archives/patch-kill-cl_collation_index.sql b/www/wiki/maintenance/mssql/archives/patch-kill-cl_collation_index.sql new file mode 100644 index 00000000..7f75a623 --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-kill-cl_collation_index.sql @@ -0,0 +1,7 @@ +-- +-- Kill cl_collation index. +-- @since 1.27 +-- + +DROP INDEX /*i*/cl_collation ON /*_*/categorylinks; + diff --git a/www/wiki/maintenance/mssql/archives/patch-logging-drop-fks.sql b/www/wiki/maintenance/mssql/archives/patch-logging-drop-fks.sql new file mode 100644 index 00000000..c9cbca35 --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-logging-drop-fks.sql @@ -0,0 +1,37 @@ +DECLARE @base nvarchar(max), + @SQL nvarchar(max), + @id sysname;-- + +SET @base = 'ALTER TABLE /*_*/logging DROP CONSTRAINT ';-- + +SELECT @id = fk.name +FROM sys.foreign_keys fk +JOIN sys.foreign_key_columns fkc + ON fkc.constraint_object_id = fk.object_id +JOIN sys.columns c + ON c.column_id = fkc.parent_column_id + AND c.object_id = fkc.parent_object_id +WHERE + fk.parent_object_id = OBJECT_ID('/*_*/logging') + AND fk.referenced_object_id = OBJECT_ID('/*_*/mwuser') + AND c.name = 'log_user';-- + +SET @SQL = @base + @id;-- + +EXEC sp_executesql @SQL;-- + +SELECT @id = fk.name +FROM sys.foreign_keys fk +JOIN sys.foreign_key_columns fkc + ON fkc.constraint_object_id = fk.object_id +JOIN sys.columns c + ON c.column_id = fkc.parent_column_id + AND c.object_id = fkc.parent_object_id +WHERE + fk.parent_object_id = OBJECT_ID('/*_*/logging') + AND fk.referenced_object_id = OBJECT_ID('/*_*/page') + AND c.name = 'log_page';-- + +SET @SQL = @base + @id;-- + +EXEC sp_executesql @SQL; diff --git a/www/wiki/maintenance/mssql/archives/patch-oi_major_mime-chemical.sql b/www/wiki/maintenance/mssql/archives/patch-oi_major_mime-chemical.sql new file mode 100644 index 00000000..35482edc --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-oi_major_mime-chemical.sql @@ -0,0 +1,4 @@ +ALTER TABLE /*_*/oldimage +DROP CONSTRAINT oi_major_mime_ckc; +ALTER TABLE /*_*/oldimage +WITH NOCHECK ADD CONSTRAINT oi_major_mime_ckc CHECK (oi_major_mime IN('unknown', 'application', 'audio', 'image', 'text', 'video', 'message', 'model', 'multipart', 'chemical'));
\ No newline at end of file diff --git a/www/wiki/maintenance/mssql/archives/patch-oldimage-constraints.sql b/www/wiki/maintenance/mssql/archives/patch-oldimage-constraints.sql new file mode 100644 index 00000000..69ede2c4 --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-oldimage-constraints.sql @@ -0,0 +1,34 @@ +DECLARE @baseSQL nvarchar(max), + @SQL nvarchar(max), + @id sysname;-- + +SET @baseSQL = 'ALTER TABLE /*_*/oldimage DROP CONSTRAINT ';-- + +SELECT @id = cc.name +FROM sys.check_constraints cc +JOIN sys.columns c + ON c.object_id = cc.parent_object_id + AND c.column_id = cc.parent_column_id +WHERE + cc.parent_object_id = OBJECT_ID('/*_*/oldimage') + AND c.name = 'oi_major_mime';-- + +SET @SQL = @baseSQL + @id;-- + +EXEC sp_executesql @SQL;-- + +SELECT @id = cc.name +FROM sys.check_constraints cc +JOIN sys.columns c + ON c.object_id = cc.parent_object_id + AND c.column_id = cc.parent_column_id +WHERE + cc.parent_object_id = OBJECT_ID('/*_*/oldimage') + AND c.name = 'oi_media_type';-- + +SET @SQL = @baseSQL + @id;-- + +EXEC sp_executesql @SQL;-- + +ALTER TABLE /*_*/oldimage ADD CONSTRAINT oi_major_mime_ckc check (oi_major_mime IN('unknown', 'application', 'audio', 'image', 'text', 'video', 'message', 'model', 'multipart'));-- +ALTER TABLE /*_*/oldimage ADD CONSTRAINT oi_media_type_ckc check (oi_media_type in('UNKNOWN', 'BITMAP', 'DRAWING', 'AUDIO', 'VIDEO', 'MULTIMEDIA', 'OFFICE', 'TEXT', 'EXECUTABLE', 'ARCHIVE')); diff --git a/www/wiki/maintenance/mssql/archives/patch-oldimage-schema.sql b/www/wiki/maintenance/mssql/archives/patch-oldimage-schema.sql new file mode 100644 index 00000000..3391c1bf --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-oldimage-schema.sql @@ -0,0 +1,91 @@ +-- MediaWiki looks for lines ending with semicolons and sends them as separate queries +-- However here we *really* need this all to be sent as a single batch. As such, DO NOT +-- remove the -- from the end of each statement. + +DECLARE @temp table ( + oi_name varbinary(255), + oi_archive_name varbinary(255), + oi_size int, + oi_width int, + oi_height int, + oi_bits int, + oi_description nvarchar(255), + oi_user int, + oi_user_text nvarchar(255), + oi_timestamp varchar(14), + oi_metadata nvarchar(max), + oi_media_type varchar(16), + oi_major_mime varchar(16), + oi_minor_mime nvarchar(100), + oi_deleted tinyint, + oi_sha1 nvarchar(32) +);-- + +INSERT INTO @temp +SELECT * FROM /*_*/oldimage;-- + +DROP TABLE /*_*/oldimage;-- + +CREATE TABLE /*_*/oldimage ( + oi_name nvarchar(255) NOT NULL default '', + oi_archive_name nvarchar(255) NOT NULL default '', + oi_size int NOT NULL default 0, + oi_width int NOT NULL default 0, + oi_height int NOT NULL default 0, + oi_bits int NOT NULL default 0, + oi_description nvarchar(255) NOT NULL, + oi_user int REFERENCES /*_*/mwuser(user_id), + oi_user_text nvarchar(255) NOT NULL, + oi_timestamp varchar(14) NOT NULL default '', + oi_metadata varbinary(max) NOT NULL, + oi_media_type varchar(16) default null, + oi_major_mime varchar(16) not null default 'unknown', + oi_minor_mime nvarchar(100) NOT NULL default 'unknown', + oi_deleted tinyint NOT NULL default 0, + oi_sha1 nvarchar(32) NOT NULL default '', + CONSTRAINT oi_major_mime_ckc check (oi_major_mime IN('unknown', 'application', 'audio', 'image', 'text', 'video', 'message', 'model', 'multipart', 'chemical')), + CONSTRAINT oi_media_type_ckc check (oi_media_type IN('UNKNOWN', 'BITMAP', 'DRAWING', 'AUDIO', 'VIDEO', 'MULTIMEDIA', 'OFFICE', 'TEXT', 'EXECUTABLE', 'ARCHIVE')) +);-- + +CREATE INDEX /*i*/oi_usertext_timestamp ON /*_*/oldimage (oi_user_text, oi_timestamp);-- +CREATE INDEX /*i*/oi_name_timestamp ON /*_*/oldimage (oi_name, oi_timestamp);-- +CREATE INDEX /*i*/oi_name_archive_name ON /*_*/oldimage (oi_name, oi_archive_name);-- +CREATE INDEX /*i*/oi_sha1 ON /*_*/oldimage (oi_sha1);-- + +INSERT INTO /*_*/oldimage +( + oi_name, + oi_archive_name, + oi_size, + oi_width, + oi_height, + oi_bits, + oi_description, + oi_user, + oi_user_text, + oi_timestamp, + oi_metadata, + oi_media_type, + oi_major_mime, + oi_minor_mime, + oi_deleted, + oi_sha1 +) +SELECT + oi_name, + oi_archive_name, + oi_size, + oi_width, + oi_height, + oi_bits, + oi_description, + oi_user, + oi_user_text, + oi_timestamp, + CONVERT(varbinary(max), oi_metadata, 0), + oi_media_type, + oi_major_mime, + oi_minor_mime, + oi_deleted, + oi_sha1 +FROM @temp t; diff --git a/www/wiki/maintenance/mssql/archives/patch-page_page_lang.sql b/www/wiki/maintenance/mssql/archives/patch-page_page_lang.sql new file mode 100644 index 00000000..d2f537b0 --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-page_page_lang.sql @@ -0,0 +1 @@ +ALTER TABLE /*_*/page ADD page_lang VARBINARY(35) DEFAULT NULL diff --git a/www/wiki/maintenance/mssql/archives/patch-pl_from_namespace.sql b/www/wiki/maintenance/mssql/archives/patch-pl_from_namespace.sql new file mode 100644 index 00000000..b3bbd78d --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-pl_from_namespace.sql @@ -0,0 +1,4 @@ +ALTER TABLE /*_*/pagelinks + ADD pl_from_namespace int NOT NULL default 0; + +CREATE INDEX /*i*/pl_backlinks_namespace ON /*_*/pagelinks (pl_from_namespace,pl_namespace,pl_title,pl_from); diff --git a/www/wiki/maintenance/mssql/archives/patch-pp_sortkey.sql b/www/wiki/maintenance/mssql/archives/patch-pp_sortkey.sql new file mode 100644 index 00000000..b13b6055 --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-pp_sortkey.sql @@ -0,0 +1,8 @@ +-- Add a 'sortkey' field to page_props so pages can be efficiently +-- queried by the numeric value of a property. + +ALTER TABLE /*_*/page_props + ADD pp_sortkey float DEFAULT NULL; + +CREATE UNIQUE INDEX /*i*/pp_propname_sortkey_page + ON /*_*/page_props ( pp_propname, pp_sortkey, pp_page ); diff --git a/www/wiki/maintenance/mssql/archives/patch-recentchanges-drop-fks.sql b/www/wiki/maintenance/mssql/archives/patch-recentchanges-drop-fks.sql new file mode 100644 index 00000000..24f78f68 --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-recentchanges-drop-fks.sql @@ -0,0 +1,76 @@ +DECLARE @base nvarchar(max), + @SQL nvarchar(max), + @id sysname;-- + +SET @base = 'ALTER TABLE /*_*/recentchanges DROP CONSTRAINT ';-- + +SELECT @id = fk.name +FROM sys.foreign_keys fk +JOIN sys.foreign_key_columns fkc + ON fkc.constraint_object_id = fk.object_id +JOIN sys.columns c + ON c.column_id = fkc.parent_column_id + AND c.object_id = fkc.parent_object_id +WHERE + fk.parent_object_id = OBJECT_ID('/*_*/recentchanges') + AND fk.referenced_object_id = OBJECT_ID('/*_*/page') + AND c.name = 'rc_cur_id';-- + +SET @SQL = @base + @id;-- + +EXEC sp_executesql @SQL;-- + +SELECT @id = fk.name +FROM sys.foreign_keys fk +JOIN sys.foreign_key_columns fkc + ON fkc.constraint_object_id = fk.object_id +JOIN sys.columns c + ON c.column_id = fkc.parent_column_id + AND c.object_id = fkc.parent_object_id +WHERE + fk.parent_object_id = OBJECT_ID('/*_*/recentchanges') + AND fk.referenced_object_id = OBJECT_ID('/*_*/revision') + AND c.name = 'rc_this_oldid';-- + +SET @SQL = @base + @id;-- + +EXEC sp_executesql @SQL;-- + +SELECT @id = fk.name +FROM sys.foreign_keys fk +JOIN sys.foreign_key_columns fkc + ON fkc.constraint_object_id = fk.object_id +JOIN sys.columns c + ON c.column_id = fkc.parent_column_id + AND c.object_id = fkc.parent_object_id +WHERE + fk.parent_object_id = OBJECT_ID('/*_*/recentchanges') + AND fk.referenced_object_id = OBJECT_ID('/*_*/revision') + AND c.name = 'rc_last_oldid';-- + +SET @SQL = @base + @id;-- + +EXEC sp_executesql @SQL;-- + +-- while we're at it, let's fix up the other foreign key constraints on recentchanges +-- as future patches touch constraints on other tables, they'll take the time to update constraint names there as well +ALTER TABLE /*_*/recentchanges DROP CONSTRAINT FK_rc_logid_log_id;-- +ALTER TABLE /*_*/recentchanges ADD CONSTRAINT rc_logid__log_id__fk FOREIGN KEY (rc_logid) REFERENCES /*_*/logging(log_id) ON DELETE CASCADE;-- + +SELECT @id = fk.name +FROM sys.foreign_keys fk +JOIN sys.foreign_key_columns fkc + ON fkc.constraint_object_id = fk.object_id +JOIN sys.columns c + ON c.column_id = fkc.parent_column_id + AND c.object_id = fkc.parent_object_id +WHERE + fk.parent_object_id = OBJECT_ID('/*_*/recentchanges') + AND fk.referenced_object_id = OBJECT_ID('/*_*/mwuser') + AND c.name = 'rc_user';-- + +SET @SQL = @base + @id;-- + +EXEC sp_executesql @SQL;-- + +ALTER TABLE /*_*/recentchanges ADD CONSTRAINT rc_user__user_id__fk FOREIGN KEY (rc_user) REFERENCES /*_*/mwuser(user_id); diff --git a/www/wiki/maintenance/mssql/archives/patch-site_stats-pk.sql b/www/wiki/maintenance/mssql/archives/patch-site_stats-pk.sql new file mode 100644 index 00000000..7533719d --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-site_stats-pk.sql @@ -0,0 +1,2 @@ +DROP INDEX ss_row_id ON site_stats; +ALTER TABLE /*_*/site_stats ADD CONSTRAINT /*i*/ss_row_id PRIMARY KEY (ss_row_id); diff --git a/www/wiki/maintenance/mssql/archives/patch-tag_summary-ts_id.sql b/www/wiki/maintenance/mssql/archives/patch-tag_summary-ts_id.sql new file mode 100644 index 00000000..d62bd357 --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-tag_summary-ts_id.sql @@ -0,0 +1,4 @@ +-- Primary key in tag_summary table + +ALTER TABLE /*_*/tag_summary ADD ts_id INT IDENTITY; +ALTER TABLE /*_*/tag_summary ADD CONSTRAINT pk_tag_summary PRIMARY KEY(ts_id) diff --git a/www/wiki/maintenance/mssql/archives/patch-tl_from_namespace.sql b/www/wiki/maintenance/mssql/archives/patch-tl_from_namespace.sql new file mode 100644 index 00000000..9655165a --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-tl_from_namespace.sql @@ -0,0 +1,4 @@ +ALTER TABLE /*_*/templatelinks + ADD tl_from_namespace int NOT NULL default 0; + +CREATE INDEX /*i*/tl_backlinks_namespace ON /*_*/templatelinks (tl_from_namespace,tl_namespace,tl_title,tl_from); diff --git a/www/wiki/maintenance/mssql/archives/patch-uploadstash-constraints.sql b/www/wiki/maintenance/mssql/archives/patch-uploadstash-constraints.sql new file mode 100644 index 00000000..1cd668c5 --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-uploadstash-constraints.sql @@ -0,0 +1,20 @@ +DECLARE @baseSQL nvarchar(max), + @SQL nvarchar(max), + @id sysname;-- + +SET @baseSQL = 'ALTER TABLE /*_*/uploadstash DROP CONSTRAINT ';-- + +SELECT @id = cc.name +FROM sys.check_constraints cc +JOIN sys.columns c + ON c.object_id = cc.parent_object_id + AND c.column_id = cc.parent_column_id +WHERE + cc.parent_object_id = OBJECT_ID('/*_*/uploadstash') + AND c.name = 'us_media_type';-- + +SET @SQL = @baseSQL + @id;-- + +EXEC sp_executesql @SQL;-- + +ALTER TABLE /*_*/uploadstash ADD CONSTRAINT us_media_type_ckc check (us_media_type in('UNKNOWN', 'BITMAP', 'DRAWING', 'AUDIO', 'VIDEO', 'MULTIMEDIA', 'OFFICE', 'TEXT', 'EXECUTABLE', 'ARCHIVE')); diff --git a/www/wiki/maintenance/mssql/archives/patch-user_groups-ug_expiry.sql b/www/wiki/maintenance/mssql/archives/patch-user_groups-ug_expiry.sql new file mode 100644 index 00000000..371d80b2 --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-user_groups-ug_expiry.sql @@ -0,0 +1,6 @@ +-- Primary key and expiry column in user_groups table + +DROP INDEX IF EXISTS /*i*/ug_user_group ON /*_*/user_groups; +ALTER TABLE /*_*/tag_summary ADD CONSTRAINT pk_user_groups PRIMARY KEY(ug_user, ug_group); +ALTER TABLE /*_*/tag_summary ADD ug_expiry varchar(14) DEFAULT NULL; +CREATE INDEX /*i*/ug_expiry ON /*_*/user_groups(ug_expiry); diff --git a/www/wiki/maintenance/mssql/archives/patch-user_password_expires.sql b/www/wiki/maintenance/mssql/archives/patch-user_password_expires.sql new file mode 100644 index 00000000..c22b10c7 --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-user_password_expires.sql @@ -0,0 +1 @@ +ALTER TABLE /*_*/mwuser ADD user_password_expires VARCHAR(14) DEFAULT NULL
\ No newline at end of file diff --git a/www/wiki/maintenance/mssql/archives/patch-watchlist-wl_id.sql b/www/wiki/maintenance/mssql/archives/patch-watchlist-wl_id.sql new file mode 100644 index 00000000..b71f817a --- /dev/null +++ b/www/wiki/maintenance/mssql/archives/patch-watchlist-wl_id.sql @@ -0,0 +1,2 @@ +ALTER TABLE /*_*/watchlist ADD wl_id INT IDENTITY; +ALTER TABLE /*_*/watchlist ADD CONSTRAINT pk_watchlist PRIMARY KEY(wl_id) diff --git a/www/wiki/maintenance/mssql/tables.sql b/www/wiki/maintenance/mssql/tables.sql new file mode 100644 index 00000000..119cd5b8 --- /dev/null +++ b/www/wiki/maintenance/mssql/tables.sql @@ -0,0 +1,1331 @@ +-- Experimental table definitions for Microsoft SQL Server with +-- content-holding fields switched to explicit BINARY charset. +-- ------------------------------------------------------------ + +-- SQL to create the initial tables for the MediaWiki database. +-- This is read and executed by the install script; you should +-- not have to run it by itself unless doing a manual install. + +-- +-- General notes: +-- +-- The comments in this and other files are +-- replaced with the defined table prefix by the installer +-- and updater scripts. If you are installing or running +-- updates manually, you will need to manually insert the +-- table prefix if any when running these scripts. +-- + + +-- +-- The user table contains basic account information, +-- authentication keys, etc. +-- +-- Some multi-wiki sites may share a single central user table +-- between separate wikis using the $wgSharedDB setting. +-- +-- Note that when a external authentication plugin is used, +-- user table entries still need to be created to store +-- preferences and to key tracking information in the other +-- tables. + +-- LINE:53 +CREATE TABLE /*_*/mwuser ( + user_id INT NOT NULL PRIMARY KEY IDENTITY(0,1), + user_name NVARCHAR(255) NOT NULL UNIQUE DEFAULT '', + user_real_name NVARCHAR(255) NOT NULL DEFAULT '', + user_password NVARCHAR(255) NOT NULL DEFAULT '', + user_newpassword NVARCHAR(255) NOT NULL DEFAULT '', + user_newpass_time varchar(14) NULL DEFAULT NULL, + user_email NVARCHAR(255) NOT NULL DEFAULT '', + user_touched varchar(14) NOT NULL DEFAULT '', + user_token NCHAR(32) NOT NULL DEFAULT '', + user_email_authenticated varchar(14) DEFAULT NULL, + user_email_token NCHAR(32) DEFAULT '', + user_email_token_expires varchar(14) DEFAULT NULL, + user_registration varchar(14) DEFAULT NULL, + user_editcount INT NULL DEFAULT NULL, + user_password_expires varchar(14) DEFAULT NULL +); +CREATE UNIQUE INDEX /*i*/user_name ON /*_*/mwuser (user_name); +CREATE INDEX /*i*/user_email_token ON /*_*/mwuser (user_email_token); +CREATE INDEX /*i*/user_email ON /*_*/mwuser (user_email); + +-- Insert a dummy user to represent anons +INSERT INTO /*_*/mwuser (user_name) VALUES ('##Anonymous##'); + +-- +-- User permissions have been broken out to a separate table; +-- this allows sites with a shared user table to have different +-- permissions assigned to a user in each project. +-- +-- This table replaces the old user_rights field which used a +-- comma-separated nvarchar(max). +CREATE TABLE /*_*/user_groups ( + ug_user INT NOT NULL REFERENCES /*_*/mwuser(user_id) ON DELETE CASCADE, + ug_group NVARCHAR(255) NOT NULL DEFAULT '', + ug_expiry varchar(14) DEFAULT NULL, + PRIMARY KEY(ug_user, ug_group) +); +CREATE INDEX /*i*/ug_group ON /*_*/user_groups(ug_group); +CREATE INDEX /*i*/ug_expiry ON /*_*/user_groups(ug_expiry); + +-- Stores the groups the user has once belonged to. +-- The user may still belong to these groups (check user_groups). +-- Users are not autopromoted to groups from which they were removed. +CREATE TABLE /*_*/user_former_groups ( + ufg_user INT NOT NULL REFERENCES /*_*/mwuser(user_id) ON DELETE CASCADE, + ufg_group nvarchar(255) NOT NULL default '' +); +CREATE UNIQUE INDEX /*i*/ufg_user_group ON /*_*/user_former_groups (ufg_user,ufg_group); + +-- Stores notifications of user talk page changes, for the display +-- of the "you have new messages" box +-- Changed user_id column to user_id to avoid clashing with user_id function +CREATE TABLE /*_*/user_newtalk ( + user_id INT NOT NULL REFERENCES /*_*/mwuser(user_id) ON DELETE CASCADE, + user_ip NVARCHAR(40) NOT NULL DEFAULT '', + user_last_timestamp varchar(14) DEFAULT NULL, +); +CREATE INDEX /*i*/un_user_id ON /*_*/user_newtalk (user_id); +CREATE INDEX /*i*/un_user_ip ON /*_*/user_newtalk (user_ip); + +-- +-- User preferences and other fun stuff +-- replaces old user.user_options nvarchar(max) +-- +CREATE TABLE /*_*/user_properties ( + up_user INT NOT NULL REFERENCES /*_*/mwuser(user_id) ON DELETE CASCADE, + up_property NVARCHAR(255) NOT NULL, + up_value NVARCHAR(MAX), +); +CREATE UNIQUE CLUSTERED INDEX /*i*/user_properties_user_property ON /*_*/user_properties (up_user,up_property); +CREATE INDEX /*i*/user_properties_property ON /*_*/user_properties (up_property); + +-- +-- This table contains a user's bot passwords: passwords that allow access to +-- the account via the API with limited rights. +-- +CREATE TABLE /*_*/bot_passwords ( + bp_user int NOT NULL REFERENCES /*_*/mwuser(user_id) ON DELETE CASCADE, + bp_app_id nvarchar(32) NOT NULL, + bp_password nvarchar(255) NOT NULL, + bp_token nvarchar(255) NOT NULL, + bp_restrictions nvarchar(max) NOT NULL, + bp_grants nvarchar(max) NOT NULL, + PRIMARY KEY (bp_user, bp_app_id) +); + + +-- +-- Core of the wiki: each page has an entry here which identifies +-- it by title and contains some essential metadata. +-- +CREATE TABLE /*_*/page ( + page_id INT NOT NULL PRIMARY KEY IDENTITY(0,1), + page_namespace INT NOT NULL, + page_title NVARCHAR(255) NOT NULL, + page_restrictions NVARCHAR(255) NOT NULL, + page_is_redirect BIT NOT NULL DEFAULT 0, + page_is_new BIT NOT NULL DEFAULT 0, + page_random real NOT NULL DEFAULT RAND(), + page_touched varchar(14) NOT NULL default '', + page_links_updated varchar(14) DEFAULT NULL, + page_latest INT, -- FK inserted later + page_len INT NOT NULL, + page_content_model nvarchar(32) default null, + page_lang VARBINARY(35) DEFAULT NULL +); +CREATE UNIQUE INDEX /*i*/name_title ON /*_*/page (page_namespace,page_title); +CREATE INDEX /*i*/page_random ON /*_*/page (page_random); +CREATE INDEX /*i*/page_len ON /*_*/page (page_len); +CREATE INDEX /*i*/page_redirect_namespace_len ON /*_*/page (page_is_redirect, page_namespace, page_len); + +-- insert a dummy page +INSERT INTO /*_*/page (page_namespace, page_title, page_restrictions, page_latest, page_len) VALUES (-1,'','',0,0); + +-- +-- Every edit of a page creates also a revision row. +-- This stores metadata about the revision, and a reference +-- to the TEXT storage backend. +-- +CREATE TABLE /*_*/revision ( + rev_id INT NOT NULL UNIQUE IDENTITY(0,1), + rev_page INT NOT NULL REFERENCES /*_*/page(page_id) ON DELETE CASCADE, + rev_text_id INT NOT NULL, -- FK added later + rev_comment NVARCHAR(255) NOT NULL, + rev_user INT REFERENCES /*_*/mwuser(user_id) ON DELETE SET NULL, + rev_user_text NVARCHAR(255) NOT NULL DEFAULT '', + rev_timestamp varchar(14) NOT NULL default '', + rev_minor_edit BIT NOT NULL DEFAULT 0, + rev_deleted TINYINT NOT NULL DEFAULT 0, + rev_len INT, + rev_parent_id INT DEFAULT NULL REFERENCES /*_*/revision(rev_id), + rev_sha1 nvarchar(32) not null default '', + rev_content_model nvarchar(32) default null, + rev_content_format nvarchar(64) default null +); +CREATE UNIQUE CLUSTERED INDEX /*i*/rev_page_id ON /*_*/revision (rev_page, rev_id); +CREATE INDEX /*i*/rev_timestamp ON /*_*/revision (rev_timestamp); +CREATE INDEX /*i*/page_timestamp ON /*_*/revision (rev_page,rev_timestamp); +CREATE INDEX /*i*/user_timestamp ON /*_*/revision (rev_user,rev_timestamp); +CREATE INDEX /*i*/usertext_timestamp ON /*_*/revision (rev_user_text,rev_timestamp); +CREATE INDEX /*i*/page_user_timestamp ON /*_*/revision (rev_page,rev_user,rev_timestamp); + +-- insert a dummy revision +INSERT INTO /*_*/revision (rev_page,rev_text_id,rev_comment,rev_user,rev_len) VALUES (0,0,'',0,0); + +ALTER TABLE /*_*/page ADD CONSTRAINT FK_page_latest_page_id FOREIGN KEY (page_latest) REFERENCES /*_*/revision(rev_id); + +-- +-- Holds TEXT of individual page revisions. +-- +-- Field names are a holdover from the 'old' revisions table in +-- MediaWiki 1.4 and earlier: an upgrade will transform that +-- table INTo the 'text' table to minimize unnecessary churning +-- and downtime. If upgrading, the other fields will be left unused. +CREATE TABLE /*_*/text ( + old_id INT NOT NULL PRIMARY KEY IDENTITY(0,1), + old_text nvarchar(max) NOT NULL, + old_flags NVARCHAR(255) NOT NULL, +); + +-- insert a dummy text +INSERT INTO /*_*/text (old_text,old_flags) VALUES ('',''); + +ALTER TABLE /*_*/revision ADD CONSTRAINT FK_rev_text_id_old_id FOREIGN KEY (rev_text_id) REFERENCES /*_*/text(old_id) ON DELETE CASCADE; + +-- +-- Holding area for deleted articles, which may be viewed +-- or restored by admins through the Special:Undelete interface. +-- The fields generally correspond to the page, revision, and text +-- fields, with several caveats. +-- Cannot reasonably create views on this table, due to the presence of TEXT +-- columns. +CREATE TABLE /*_*/archive ( + ar_id int NOT NULL PRIMARY KEY IDENTITY, + ar_namespace SMALLINT NOT NULL DEFAULT 0, + ar_title NVARCHAR(255) NOT NULL DEFAULT '', + ar_text NVARCHAR(MAX) NOT NULL, + ar_comment NVARCHAR(255) NOT NULL, + ar_user INT CONSTRAINT ar_user__user_id__fk FOREIGN KEY REFERENCES /*_*/mwuser(user_id), + ar_user_text NVARCHAR(255) NOT NULL, + ar_timestamp varchar(14) NOT NULL default '', + ar_minor_edit BIT NOT NULL DEFAULT 0, + ar_flags NVARCHAR(255) NOT NULL, + ar_rev_id INT NULL, -- NOT a FK, the row gets deleted from revision and moved here + ar_text_id INT CONSTRAINT ar_text_id__old_id__fk FOREIGN KEY REFERENCES /*_*/text(old_id) ON DELETE CASCADE, + ar_deleted TINYINT NOT NULL DEFAULT 0, + ar_len INT, + ar_page_id INT NULL, -- NOT a FK, the row gets deleted from page and moved here + ar_parent_id INT NULL, -- NOT FK + ar_sha1 nvarchar(32) default null, + ar_content_model nvarchar(32) DEFAULT NULL, + ar_content_format nvarchar(64) DEFAULT NULL +); +CREATE INDEX /*i*/name_title_timestamp ON /*_*/archive (ar_namespace,ar_title,ar_timestamp); +CREATE INDEX /*i*/ar_usertext_timestamp ON /*_*/archive (ar_user_text,ar_timestamp); +CREATE INDEX /*i*/ar_revid ON /*_*/archive (ar_rev_id); + + +-- +-- Track page-to-page hyperlinks within the wiki. +-- +CREATE TABLE /*_*/pagelinks ( + pl_from INT NOT NULL REFERENCES /*_*/page(page_id) ON DELETE CASCADE, + pl_from_namespace int NOT NULL DEFAULT 0, + pl_namespace INT NOT NULL DEFAULT 0, + pl_title NVARCHAR(255) NOT NULL DEFAULT '', +); +CREATE UNIQUE INDEX /*i*/pl_from ON /*_*/pagelinks (pl_from,pl_namespace,pl_title); +CREATE UNIQUE INDEX /*i*/pl_namespace ON /*_*/pagelinks (pl_namespace,pl_title,pl_from); +CREATE INDEX /*i*/pl_backlinks_namespace ON /*_*/pagelinks (pl_from_namespace,pl_namespace,pl_title,pl_from); + + +-- +-- Track template inclusions. +-- +CREATE TABLE /*_*/templatelinks ( + tl_from int NOT NULL REFERENCES /*_*/page(page_id) ON DELETE CASCADE, + tl_from_namespace int NOT NULL default 0, + tl_namespace int NOT NULL default 0, + tl_title nvarchar(255) NOT NULL default '' +); + +CREATE UNIQUE INDEX /*i*/tl_from ON /*_*/templatelinks (tl_from,tl_namespace,tl_title); +CREATE UNIQUE INDEX /*i*/tl_namespace ON /*_*/templatelinks (tl_namespace,tl_title,tl_from); +CREATE INDEX /*i*/tl_backlinks_namespace ON /*_*/templatelinks (tl_from_namespace,tl_namespace,tl_title,tl_from); + + +-- +-- Track links to images *used inline* +-- We don't distinguish live from broken links here, so +-- they do not need to be changed on upload/removal. +-- +CREATE TABLE /*_*/imagelinks ( + -- Key to page_id of the page containing the image / media link. + il_from int NOT NULL REFERENCES /*_*/page(page_id) ON DELETE CASCADE, + il_from_namespace int NOT NULL default 0, + + -- Filename of target image. + -- This is also the page_title of the file's description page; + -- all such pages are in namespace 6 (NS_FILE). + il_to nvarchar(255) NOT NULL default '' +); + +CREATE UNIQUE INDEX /*i*/il_from ON /*_*/imagelinks (il_from,il_to); +CREATE UNIQUE INDEX /*i*/il_to ON /*_*/imagelinks (il_to,il_from); +CREATE INDEX /*i*/il_backlinks_namespace ON /*_*/imagelinks (il_from_namespace,il_to,il_from); + +-- +-- Track category inclusions *used inline* +-- This tracks a single level of category membership +-- +CREATE TABLE /*_*/categorylinks ( + -- Key to page_id of the page defined as a category member. + cl_from int NOT NULL REFERENCES /*_*/page(page_id) ON DELETE CASCADE, + + -- Name of the category. + -- This is also the page_title of the category's description page; + -- all such pages are in namespace 14 (NS_CATEGORY). + cl_to nvarchar(255) NOT NULL default '', + + -- A binary string obtained by applying a sortkey generation algorithm + -- (Collation::getSortKey()) to page_title, or cl_sortkey_prefix . "\n" + -- . page_title if cl_sortkey_prefix is nonempty. + cl_sortkey varbinary(230) NOT NULL default 0x, + + -- A prefix for the raw sortkey manually specified by the user, either via + -- [[Category:Foo|prefix]] or {{defaultsort:prefix}}. If nonempty, it's + -- concatenated with a line break followed by the page title before the sortkey + -- conversion algorithm is run. We store this so that we can update + -- collations without reparsing all pages. + -- Note: If you change the length of this field, you also need to change + -- code in LinksUpdate.php. See T27254. + cl_sortkey_prefix varbinary(255) NOT NULL default 0x, + + -- This isn't really used at present. Provided for an optional + -- sorting method by approximate addition time. + cl_timestamp varchar(14) NOT NULL, + + -- Stores $wgCategoryCollation at the time cl_sortkey was generated. This + -- can be used to install new collation versions, tracking which rows are not + -- yet updated. '' means no collation, this is a legacy row that needs to be + -- updated by updateCollation.php. In the future, it might be possible to + -- specify different collations per category. + cl_collation nvarchar(32) NOT NULL default '', + + -- Stores whether cl_from is a category, file, or other page, so we can + -- paginate the three categories separately. This never has to be updated + -- after the page is created, since none of these page types can be moved to + -- any other. + cl_type varchar(10) NOT NULL default 'page', + -- SQL server doesn't have enums, so we approximate with this + CONSTRAINT cl_type_ckc CHECK (cl_type IN('page', 'subcat', 'file')) +); + +CREATE UNIQUE INDEX /*i*/cl_from ON /*_*/categorylinks (cl_from,cl_to); + +-- We always sort within a given category, and within a given type. FIXME: +-- Formerly this index didn't cover cl_type (since that didn't exist), so old +-- callers won't be using an index: fix this? +CREATE INDEX /*i*/cl_sortkey ON /*_*/categorylinks (cl_to,cl_type,cl_sortkey,cl_from); + +-- Used by the API (and some extensions) +CREATE INDEX /*i*/cl_timestamp ON /*_*/categorylinks (cl_to,cl_timestamp); + +-- Used when updating collation (e.g. updateCollation.php) +CREATE INDEX /*i*/cl_collation_ext ON /*_*/categorylinks (cl_collation, cl_to, cl_type, cl_from); + +-- +-- Track all existing categories. Something is a category if 1) it has an entry +-- somewhere in categorylinks, or 2) it has a description page. Categories +-- might not have corresponding pages, so they need to be tracked separately. +-- +CREATE TABLE /*_*/category ( + -- Primary key + cat_id int NOT NULL PRIMARY KEY IDENTITY, + + -- Name of the category, in the same form as page_title (with underscores). + -- If there is a category page corresponding to this category, by definition, + -- it has this name (in the Category namespace). + cat_title nvarchar(255) NOT NULL, + + -- The numbers of member pages (including categories and media), subcatego- + -- ries, and Image: namespace members, respectively. These are signed to + -- make underflow more obvious. We make the first number include the second + -- two for better sorting: subtracting for display is easy, adding for order- + -- ing is not. + cat_pages int NOT NULL default 0, + cat_subcats int NOT NULL default 0, + cat_files int NOT NULL default 0 +); + +CREATE UNIQUE INDEX /*i*/cat_title ON /*_*/category (cat_title); + +-- For Special:Mostlinkedcategories +CREATE INDEX /*i*/cat_pages ON /*_*/category (cat_pages); + + +-- +-- Track links to external URLs +-- +CREATE TABLE /*_*/externallinks ( + -- Primary key + el_id int NOT NULL PRIMARY KEY IDENTITY, + + -- page_id of the referring page + el_from int NOT NULL REFERENCES /*_*/page(page_id) ON DELETE CASCADE, + + -- The URL + el_to nvarchar(max) NOT NULL, + + -- In the case of HTTP URLs, this is the URL with any username or password + -- removed, and with the labels in the hostname reversed and converted to + -- lower case. An extra dot is added to allow for matching of either + -- example.com or *.example.com in a single scan. + -- Example: + -- http://user:password@sub.example.com/page.html + -- becomes + -- http://com.example.sub./page.html + -- which allows for fast searching for all pages under example.com with the + -- clause: + -- WHERE el_index LIKE 'http://com.example.%' + el_index nvarchar(450) NOT NULL, + + -- This is el_index truncated to 60 bytes to allow for sortable queries that + -- aren't supported by a partial index. + -- @todo Drop the default once this is deployed everywhere and code is populating it. + el_index_60 varbinary(60) NOT NULL default '' +); + +CREATE INDEX /*i*/el_from ON /*_*/externallinks (el_from); +CREATE INDEX /*i*/el_index ON /*_*/externallinks (el_index); +CREATE INDEX /*i*/el_index_60 ON /*_*/externallinks (el_index_60, el_id); +CREATE INDEX /*i*/el_from_index_60 ON /*_*/externallinks (el_from, el_index_60, el_id); +-- el_to index intentionally not added; we cannot index nvarchar(max) columns, +-- but we also cannot restrict el_to to a smaller column size as the external +-- link may be larger. + +-- +-- Track interlanguage links +-- +CREATE TABLE /*_*/langlinks ( + -- page_id of the referring page + ll_from int NOT NULL REFERENCES /*_*/page(page_id) ON DELETE CASCADE, + + -- Language code of the target + ll_lang nvarchar(20) NOT NULL default '', + + -- Title of the target, including namespace + ll_title nvarchar(255) NOT NULL default '' +); + +CREATE UNIQUE INDEX /*i*/ll_from ON /*_*/langlinks (ll_from, ll_lang); +CREATE INDEX /*i*/ll_lang ON /*_*/langlinks (ll_lang, ll_title); + + +-- +-- Track inline interwiki links +-- +CREATE TABLE /*_*/iwlinks ( + -- page_id of the referring page + iwl_from int NOT NULL REFERENCES /*_*/page(page_id) ON DELETE CASCADE, + + -- Interwiki prefix code of the target + iwl_prefix nvarchar(20) NOT NULL default '', + + -- Title of the target, including namespace + iwl_title nvarchar(255) NOT NULL default '' +); + +CREATE UNIQUE INDEX /*i*/iwl_from ON /*_*/iwlinks (iwl_from, iwl_prefix, iwl_title); +CREATE INDEX /*i*/iwl_prefix_title_from ON /*_*/iwlinks (iwl_prefix, iwl_title, iwl_from); +CREATE INDEX /*i*/iwl_prefix_from_title ON /*_*/iwlinks (iwl_prefix, iwl_from, iwl_title); + + +-- +-- Contains a single row with some aggregate info +-- on the state of the site. +-- +CREATE TABLE /*_*/site_stats ( + -- The single row should contain 1 here. + ss_row_id int NOT NULL CONSTRAINT /*i*/ss_row_id PRIMARY KEY, + + -- Total number of edits performed. + ss_total_edits bigint default 0, + + -- An approximate count of pages matching the following criteria: + -- * in namespace 0 + -- * not a redirect + -- * contains the text '[[' + -- See Article::isCountable() in includes/Article.php + ss_good_articles bigint default 0, + + -- Total pages, theoretically equal to SELECT COUNT(*) FROM page; except faster + ss_total_pages bigint default '-1', + + -- Number of users, theoretically equal to SELECT COUNT(*) FROM user; + ss_users bigint default '-1', + + -- Number of users that still edit + ss_active_users bigint default '-1', + + -- Number of images, equivalent to SELECT COUNT(*) FROM image + ss_images int default 0 +); + + +-- +-- The internet is full of jerks, alas. Sometimes it's handy +-- to block a vandal or troll account. +-- +CREATE TABLE /*_*/ipblocks ( + -- Primary key, introduced for privacy. + ipb_id int NOT NULL PRIMARY KEY IDENTITY, + + -- Blocked IP address in dotted-quad form or user name. + ipb_address nvarchar(255) NOT NULL, + + -- Blocked user ID or 0 for IP blocks. + ipb_user int REFERENCES /*_*/mwuser(user_id), + + -- User ID who made the block. + ipb_by int REFERENCES /*_*/mwuser(user_id) ON DELETE CASCADE, + + -- User name of blocker + ipb_by_text nvarchar(255) NOT NULL default '', + + -- Text comment made by blocker. + ipb_reason nvarchar(255) NOT NULL, + + -- Creation (or refresh) date in standard YMDHMS form. + -- IP blocks expire automatically. + ipb_timestamp varchar(14) NOT NULL default '', + + -- Indicates that the IP address was banned because a banned + -- user accessed a page through it. If this is 1, ipb_address + -- will be hidden, and the block identified by block ID number. + ipb_auto bit NOT NULL default 0, + + -- If set to 1, block applies only to logged-out users + ipb_anon_only bit NOT NULL default 0, + + -- Block prevents account creation from matching IP addresses + ipb_create_account bit NOT NULL default 1, + + -- Block triggers autoblocks + ipb_enable_autoblock bit NOT NULL default 1, + + -- Time at which the block will expire. + -- May be "infinity" + ipb_expiry varchar(14) NOT NULL, + + -- Start and end of an address range, in hexadecimal + -- Size chosen to allow IPv6 + -- FIXME: these fields were originally blank for single-IP blocks, + -- but now they are populated. No migration was ever done. They + -- should be fixed to be blank again for such blocks (T51504). + ipb_range_start varchar(255) NOT NULL, + ipb_range_end varchar(255) NOT NULL, + + -- Flag for entries hidden from users and Sysops + ipb_deleted bit NOT NULL default 0, + + -- Block prevents user from accessing Special:Emailuser + ipb_block_email bit NOT NULL default 0, + + -- Block allows user to edit their own talk page + ipb_allow_usertalk bit NOT NULL default 0, + + -- ID of the block that caused this block to exist + -- Autoblocks set this to the original block + -- so that the original block being deleted also + -- deletes the autoblocks + ipb_parent_block_id int default NULL REFERENCES /*_*/ipblocks(ipb_id) + +); + +-- Unique index to support "user already blocked" messages +-- Any new options which prevent collisions should be included +CREATE UNIQUE INDEX /*i*/ipb_address ON /*_*/ipblocks (ipb_address, ipb_user, ipb_auto, ipb_anon_only); + +CREATE INDEX /*i*/ipb_user ON /*_*/ipblocks (ipb_user); +CREATE INDEX /*i*/ipb_range ON /*_*/ipblocks (ipb_range_start, ipb_range_end); +CREATE INDEX /*i*/ipb_timestamp ON /*_*/ipblocks (ipb_timestamp); +CREATE INDEX /*i*/ipb_expiry ON /*_*/ipblocks (ipb_expiry); +CREATE INDEX /*i*/ipb_parent_block_id ON /*_*/ipblocks (ipb_parent_block_id); + + +-- +-- Uploaded images and other files. +-- +CREATE TABLE /*_*/image ( + -- Filename. + -- This is also the title of the associated description page, + -- which will be in namespace 6 (NS_FILE). + img_name nvarchar(255) NOT NULL default '' PRIMARY KEY, + + -- File size in bytes. + img_size int NOT NULL default 0, + + -- For images, size in pixels. + img_width int NOT NULL default 0, + img_height int NOT NULL default 0, + + -- Extracted Exif metadata stored as a serialized PHP array. + img_metadata varbinary(max) NOT NULL, + + -- For images, bits per pixel if known. + img_bits int NOT NULL default 0, + + -- Media type as defined by the MEDIATYPE_xxx constants + img_media_type varchar(16) default null, + + -- major part of a MIME media type as defined by IANA + -- see https://www.iana.org/assignments/media-types/ + img_major_mime varchar(16) not null default 'unknown', + + -- minor part of a MIME media type as defined by IANA + -- the minor parts are not required to adher to any standard + -- but should be consistent throughout the database + -- see https://www.iana.org/assignments/media-types/ + img_minor_mime nvarchar(100) NOT NULL default 'unknown', + + -- Description field as entered by the uploader. + -- This is displayed in image upload history and logs. + img_description nvarchar(255) NOT NULL, + + -- user_id and user_name of uploader. + img_user int REFERENCES /*_*/mwuser(user_id) ON DELETE SET NULL, + img_user_text nvarchar(255) NOT NULL, + + -- Time of the upload. + img_timestamp nvarchar(14) NOT NULL default '', + + -- SHA-1 content hash in base-36 + img_sha1 nvarchar(32) NOT NULL default '', + + CONSTRAINT img_major_mime_ckc check (img_major_mime IN('unknown', 'application', 'audio', 'image', 'text', 'video', 'message', 'model', 'multipart', 'chemical')), + CONSTRAINT img_media_type_ckc check (img_media_type in('UNKNOWN', 'BITMAP', 'DRAWING', 'AUDIO', 'VIDEO', 'MULTIMEDIA', 'OFFICE', 'TEXT', 'EXECUTABLE', 'ARCHIVE','3D')) +); + +CREATE INDEX /*i*/img_usertext_timestamp ON /*_*/image (img_user_text,img_timestamp); +-- Used by Special:ListFiles for sort-by-size +CREATE INDEX /*i*/img_size ON /*_*/image (img_size); +-- Used by Special:Newimages and Special:ListFiles +CREATE INDEX /*i*/img_timestamp ON /*_*/image (img_timestamp); +-- Used in API and duplicate search +CREATE INDEX /*i*/img_sha1 ON /*_*/image (img_sha1); +-- Used to get media of one type +CREATE INDEX /*i*/img_media_mime ON /*_*/image (img_media_type,img_major_mime,img_minor_mime); + + +-- +-- Previous revisions of uploaded files. +-- Awkwardly, image rows have to be moved into +-- this table at re-upload time. +-- +CREATE TABLE /*_*/oldimage ( + -- Base filename: key to image.img_name + -- Not a FK because deleting images removes them from image + oi_name nvarchar(255) NOT NULL default '', + + -- Filename of the archived file. + -- This is generally a timestamp and '!' prepended to the base name. + oi_archive_name nvarchar(255) NOT NULL default '', + + -- Other fields as in image... + oi_size int NOT NULL default 0, + oi_width int NOT NULL default 0, + oi_height int NOT NULL default 0, + oi_bits int NOT NULL default 0, + oi_description nvarchar(255) NOT NULL, + oi_user int REFERENCES /*_*/mwuser(user_id), + oi_user_text nvarchar(255) NOT NULL, + oi_timestamp varchar(14) NOT NULL default '', + + oi_metadata varbinary(max) NOT NULL, + oi_media_type varchar(16) default null, + oi_major_mime varchar(16) not null default 'unknown', + oi_minor_mime nvarchar(100) NOT NULL default 'unknown', + oi_deleted tinyint NOT NULL default 0, + oi_sha1 nvarchar(32) NOT NULL default '', + + CONSTRAINT oi_major_mime_ckc check (oi_major_mime IN('unknown', 'application', 'audio', 'image', 'text', 'video', 'message', 'model', 'multipart', 'chemical')), + CONSTRAINT oi_media_type_ckc check (oi_media_type IN('UNKNOWN', 'BITMAP', 'DRAWING', 'AUDIO', 'VIDEO', 'MULTIMEDIA', 'OFFICE', 'TEXT', 'EXECUTABLE', 'ARCHIVE','3D')) +); + +CREATE INDEX /*i*/oi_usertext_timestamp ON /*_*/oldimage (oi_user_text,oi_timestamp); +CREATE INDEX /*i*/oi_name_timestamp ON /*_*/oldimage (oi_name,oi_timestamp); +CREATE INDEX /*i*/oi_sha1 ON /*_*/oldimage (oi_sha1); + + +-- +-- Record of deleted file data +-- +CREATE TABLE /*_*/filearchive ( + -- Unique row id + fa_id int NOT NULL PRIMARY KEY IDENTITY, + + -- Original base filename; key to image.img_name, page.page_title, etc + fa_name nvarchar(255) NOT NULL default '', + + -- Filename of archived file, if an old revision + fa_archive_name nvarchar(255) default '', + + -- Which storage bin (directory tree or object store) the file data + -- is stored in. Should be 'deleted' for files that have been deleted; + -- any other bin is not yet in use. + fa_storage_group nvarchar(16), + + -- SHA-1 of the file contents plus extension, used as a key for storage. + -- eg 8f8a562add37052a1848ff7771a2c515db94baa9.jpg + -- + -- If NULL, the file was missing at deletion time or has been purged + -- from the archival storage. + fa_storage_key nvarchar(64) default '', + + -- Deletion information, if this file is deleted. + fa_deleted_user int, + fa_deleted_timestamp varchar(14) default '', + fa_deleted_reason nvarchar(max), + + -- Duped fields from image + fa_size int default 0, + fa_width int default 0, + fa_height int default 0, + fa_metadata varbinary(max), + fa_bits int default 0, + fa_media_type varchar(16) default null, + fa_major_mime varchar(16) not null default 'unknown', + fa_minor_mime nvarchar(100) default 'unknown', + fa_description nvarchar(255), + fa_user int default 0 REFERENCES /*_*/mwuser(user_id) ON DELETE SET NULL, + fa_user_text nvarchar(255), + fa_timestamp varchar(14) default '', + + -- Visibility of deleted revisions, bitfield + fa_deleted tinyint NOT NULL default 0, + + -- sha1 hash of file content + fa_sha1 nvarchar(32) NOT NULL default '', + + CONSTRAINT fa_major_mime_ckc check (fa_major_mime in('unknown', 'application', 'audio', 'image', 'text', 'video', 'message', 'model', 'multipart', 'chemical')), + CONSTRAINT fa_media_type_ckc check (fa_media_type in('UNKNOWN', 'BITMAP', 'DRAWING', 'AUDIO', 'VIDEO', 'MULTIMEDIA', 'OFFICE', 'TEXT', 'EXECUTABLE', 'ARCHIVE','3D')) +); + +-- pick out by image name +CREATE INDEX /*i*/fa_name ON /*_*/filearchive (fa_name, fa_timestamp); +-- pick out dupe files +CREATE INDEX /*i*/fa_storage_group ON /*_*/filearchive (fa_storage_group, fa_storage_key); +-- sort by deletion time +CREATE INDEX /*i*/fa_deleted_timestamp ON /*_*/filearchive (fa_deleted_timestamp); +-- sort by uploader +CREATE INDEX /*i*/fa_user_timestamp ON /*_*/filearchive (fa_user_text,fa_timestamp); +-- find file by sha1, 10 bytes will be enough for hashes to be indexed +CREATE INDEX /*i*/fa_sha1 ON /*_*/filearchive (fa_sha1); + + +-- +-- Store information about newly uploaded files before they're +-- moved into the actual filestore +-- +CREATE TABLE /*_*/uploadstash ( + us_id int NOT NULL PRIMARY KEY IDENTITY, + + -- the user who uploaded the file. + us_user int REFERENCES /*_*/mwuser(user_id) ON DELETE SET NULL, + + -- file key. this is how applications actually search for the file. + -- this might go away, or become the primary key. + us_key nvarchar(255) NOT NULL, + + -- the original path + us_orig_path nvarchar(255) NOT NULL, + + -- the temporary path at which the file is actually stored + us_path nvarchar(255) NOT NULL, + + -- which type of upload the file came from (sometimes) + us_source_type nvarchar(50), + + -- the date/time on which the file was added + us_timestamp varchar(14) NOT NULL, + + us_status nvarchar(50) NOT NULL, + + -- chunk counter starts at 0, current offset is stored in us_size + us_chunk_inx int NULL, + + -- Serialized file properties from FSFile::getProps() + us_props nvarchar(max), + + -- file size in bytes + us_size int NOT NULL, + -- this hash comes from FSFile::getSha1Base36(), and is 31 characters + us_sha1 nvarchar(31) NOT NULL, + us_mime nvarchar(255), + -- Media type as defined by the MEDIATYPE_xxx constants, should duplicate definition in the image table + us_media_type varchar(16) default null, + -- image-specific properties + us_image_width int, + us_image_height int, + us_image_bits smallint, + + CONSTRAINT us_media_type_ckc check (us_media_type in('UNKNOWN', 'BITMAP', 'DRAWING', 'AUDIO', 'VIDEO', 'MULTIMEDIA', 'OFFICE', 'TEXT', 'EXECUTABLE', 'ARCHIVE', '3D')) +); + +-- sometimes there's a delete for all of a user's stuff. +CREATE INDEX /*i*/us_user ON /*_*/uploadstash (us_user); +-- pick out files by key, enforce key uniqueness +CREATE UNIQUE INDEX /*i*/us_key ON /*_*/uploadstash (us_key); +-- the abandoned upload cleanup script needs this +CREATE INDEX /*i*/us_timestamp ON /*_*/uploadstash (us_timestamp); + + +-- +-- Primarily a summary table for Special:Recentchanges, +-- this table contains some additional info on edits from +-- the last few days, see Article::editUpdates() +-- +CREATE TABLE /*_*/recentchanges ( + rc_id int NOT NULL CONSTRAINT recentchanges__pk PRIMARY KEY IDENTITY, + rc_timestamp varchar(14) not null default '', + + -- As in revision + rc_user int NOT NULL default 0 CONSTRAINT rc_user__user_id__fk FOREIGN KEY REFERENCES /*_*/mwuser(user_id), + rc_user_text nvarchar(255) NOT NULL, + + -- When pages are renamed, their RC entries do _not_ change. + rc_namespace int NOT NULL default 0, + rc_title nvarchar(255) NOT NULL default '', + + -- as in revision... + rc_comment nvarchar(255) NOT NULL default '', + rc_minor bit NOT NULL default 0, + + -- Edits by user accounts with the 'bot' rights key are + -- marked with a 1 here, and will be hidden from the + -- default view. + rc_bot bit NOT NULL default 0, + + -- Set if this change corresponds to a page creation + rc_new bit NOT NULL default 0, + + -- Key to page_id (was cur_id prior to 1.5). + -- This will keep links working after moves while + -- retaining the at-the-time name in the changes list. + rc_cur_id int, -- NOT FK + + -- rev_id of the given revision + rc_this_oldid int, -- NOT FK + + -- rev_id of the prior revision, for generating diff links. + rc_last_oldid int, -- NOT FK + + -- The type of change entry (RC_EDIT,RC_NEW,RC_LOG,RC_EXTERNAL) + rc_type tinyint NOT NULL default 0, + + -- The source of the change entry (replaces rc_type) + -- default of '' is temporary, needed for initial migration + rc_source nvarchar(16) not null default '', + + -- If the Recent Changes Patrol option is enabled, + -- users may mark edits as having been reviewed to + -- remove a warning flag on the RC list. + -- A value of 1 indicates the page has been reviewed. + rc_patrolled bit NOT NULL default 0, + + -- Recorded IP address the edit was made from, if the + -- $wgPutIPinRC option is enabled. + rc_ip nvarchar(40) NOT NULL default '', + + -- Text length in characters before + -- and after the edit + rc_old_len int, + rc_new_len int, + + -- Visibility of recent changes items, bitfield + rc_deleted tinyint NOT NULL default 0, + + -- Value corresponding to log_id, specific log entries + rc_logid int, -- FK added later + -- Store log type info here, or null + rc_log_type nvarchar(255) NULL default NULL, + -- Store log action or null + rc_log_action nvarchar(255) NULL default NULL, + -- Log params + rc_params nvarchar(max) NULL +); + +CREATE INDEX /*i*/rc_timestamp ON /*_*/recentchanges (rc_timestamp); +CREATE INDEX /*i*/rc_namespace_title ON /*_*/recentchanges (rc_namespace, rc_title); +CREATE INDEX /*i*/rc_cur_id ON /*_*/recentchanges (rc_cur_id); +CREATE INDEX /*i*/new_name_timestamp ON /*_*/recentchanges (rc_new,rc_namespace,rc_timestamp); +CREATE INDEX /*i*/rc_ip ON /*_*/recentchanges (rc_ip); +CREATE INDEX /*i*/rc_ns_usertext ON /*_*/recentchanges (rc_namespace, rc_user_text); +CREATE INDEX /*i*/rc_user_text ON /*_*/recentchanges (rc_user_text, rc_timestamp); +CREATE INDEX /*i*/rc_name_type_patrolled_timestamp ON /*_*/recentchanges (rc_namespace, rc_type, rc_patrolled, rc_timestamp); + + +CREATE TABLE /*_*/watchlist ( + wl_id int NOT NULL PRIMARY KEY IDENTITY, + -- Key to user.user_id + wl_user int NOT NULL REFERENCES /*_*/mwuser(user_id) ON DELETE CASCADE, + + -- Key to page_namespace/page_title + -- Note that users may watch pages which do not exist yet, + -- or existed in the past but have been deleted. + wl_namespace int NOT NULL default 0, + wl_title nvarchar(255) NOT NULL default '', + + -- Timestamp used to send notification e-mails and show "updated since last visit" markers on + -- history and recent changes / watchlist. Set to NULL when the user visits the latest revision + -- of the page, which means that they should be sent an e-mail on the next change. + wl_notificationtimestamp varchar(14) + +); + +CREATE UNIQUE INDEX /*i*/wl_user ON /*_*/watchlist (wl_user, wl_namespace, wl_title); +CREATE INDEX /*i*/namespace_title ON /*_*/watchlist (wl_namespace, wl_title); + + +-- +-- Our search index for the builtin MediaWiki search +-- +CREATE TABLE /*_*/searchindex ( + -- Key to page_id + si_page int NOT NULL REFERENCES /*_*/page(page_id) ON DELETE CASCADE, + + -- Munged version of title + si_title nvarchar(255) NOT NULL default '', + + -- Munged version of body text + si_text nvarchar(max) NOT NULL +); + +CREATE UNIQUE INDEX /*i*/si_page ON /*_*/searchindex (si_page); +-- Fulltext index is defined in MssqlInstaller.php + +-- +-- Recognized interwiki link prefixes +-- +CREATE TABLE /*_*/interwiki ( + -- The interwiki prefix, (e.g. "Meatball", or the language prefix "de") + iw_prefix nvarchar(32) NOT NULL, + + -- The URL of the wiki, with "$1" as a placeholder for an article name. + -- Any spaces in the name will be transformed to underscores before + -- insertion. + iw_url nvarchar(max) NOT NULL, + + -- The URL of the file api.php + iw_api nvarchar(max) NOT NULL, + + -- The name of the database (for a connection to be established with wfGetLB( 'wikiid' )) + iw_wikiid nvarchar(64) NOT NULL, + + -- A boolean value indicating whether the wiki is in this project + -- (used, for example, to detect redirect loops) + iw_local bit NOT NULL, + + -- Boolean value indicating whether interwiki transclusions are allowed. + iw_trans bit NOT NULL default 0 +); + +CREATE UNIQUE INDEX /*i*/iw_prefix ON /*_*/interwiki (iw_prefix); + + +-- +-- Used for caching expensive grouped queries +-- +CREATE TABLE /*_*/querycache ( + -- A key name, generally the base name of of the special page. + qc_type nvarchar(32) NOT NULL, + + -- Some sort of stored value. Sizes, counts... + qc_value int NOT NULL default 0, + + -- Target namespace+title + qc_namespace int NOT NULL default 0, + qc_title nvarchar(255) NOT NULL default '' +); + +CREATE INDEX /*i*/qc_type ON /*_*/querycache (qc_type,qc_value); + + +-- +-- For a few generic cache operations if not using Memcached +-- +CREATE TABLE /*_*/objectcache ( + keyname nvarchar(255) NOT NULL default '' PRIMARY KEY, + value varbinary(max), + exptime varchar(14) +); +CREATE INDEX /*i*/exptime ON /*_*/objectcache (exptime); + + +-- +-- Cache of interwiki transclusion +-- +CREATE TABLE /*_*/transcache ( + tc_url nvarchar(255) NOT NULL, + tc_contents nvarchar(max), + tc_time varchar(14) NOT NULL +); + +CREATE UNIQUE INDEX /*i*/tc_url_idx ON /*_*/transcache (tc_url); + + +CREATE TABLE /*_*/logging ( + -- Log ID, for referring to this specific log entry, probably for deletion and such. + log_id int NOT NULL PRIMARY KEY IDENTITY(0,1), + + -- Symbolic keys for the general log type and the action type + -- within the log. The output format will be controlled by the + -- action field, but only the type controls categorization. + log_type nvarchar(32) NOT NULL default '', + log_action nvarchar(32) NOT NULL default '', + + -- Timestamp. Duh. + log_timestamp varchar(14) NOT NULL default '', + + -- The user who performed this action; key to user_id + log_user int, -- NOT an FK, if a user is deleted we still want to maintain a record of who did a thing + + -- Name of the user who performed this action + log_user_text nvarchar(255) NOT NULL default '', + + -- Key to the page affected. Where a user is the target, + -- this will point to the user page. + log_namespace int NOT NULL default 0, + log_title nvarchar(255) NOT NULL default '', + log_page int NULL, -- NOT an FK, logging entries are inserted for deleted pages which still reference the deleted page ids + + -- Freeform text. Interpreted as edit history comments. + log_comment nvarchar(255) NOT NULL default '', + + -- miscellaneous parameters: + -- LF separated list (old system) or serialized PHP array (new system) + log_params nvarchar(max) NOT NULL, + + -- rev_deleted for logs + log_deleted tinyint NOT NULL default 0 +); + +CREATE INDEX /*i*/type_time ON /*_*/logging (log_type, log_timestamp); +CREATE INDEX /*i*/user_time ON /*_*/logging (log_user, log_timestamp); +CREATE INDEX /*i*/page_time ON /*_*/logging (log_namespace, log_title, log_timestamp); +CREATE INDEX /*i*/times ON /*_*/logging (log_timestamp); +CREATE INDEX /*i*/log_user_type_time ON /*_*/logging (log_user, log_type, log_timestamp); +CREATE INDEX /*i*/log_page_id_time ON /*_*/logging (log_page,log_timestamp); +CREATE INDEX /*i*/type_action ON /*_*/logging (log_type, log_action, log_timestamp); +CREATE INDEX /*i*/log_user_text_type_time ON /*_*/logging (log_user_text, log_type, log_timestamp); +CREATE INDEX /*i*/log_user_text_time ON /*_*/logging (log_user_text, log_timestamp); + +INSERT INTO /*_*/logging (log_user,log_page,log_params) VALUES(0,0,''); + +ALTER TABLE /*_*/recentchanges ADD CONSTRAINT rc_logid__log_id__fk FOREIGN KEY (rc_logid) REFERENCES /*_*/logging(log_id) ON DELETE CASCADE; + +CREATE TABLE /*_*/log_search ( + -- The type of ID (rev ID, log ID, rev timestamp, username) + ls_field nvarchar(32) NOT NULL, + -- The value of the ID + ls_value nvarchar(255) NOT NULL, + -- Key to log_id + ls_log_id int REFERENCES /*_*/logging(log_id) ON DELETE CASCADE +); +CREATE UNIQUE INDEX /*i*/ls_field_val ON /*_*/log_search (ls_field,ls_value,ls_log_id); +CREATE INDEX /*i*/ls_log_id ON /*_*/log_search (ls_log_id); + + +-- Jobs performed by parallel apache threads or a command-line daemon +CREATE TABLE /*_*/job ( + job_id int NOT NULL PRIMARY KEY IDENTITY, + + -- Command name + -- Limited to 60 to prevent key length overflow + job_cmd nvarchar(60) NOT NULL default '', + + -- Namespace and title to act on + -- Should be 0 and '' if the command does not operate on a title + job_namespace int NOT NULL, + job_title nvarchar(255) NOT NULL, + + -- Timestamp of when the job was inserted + -- NULL for jobs added before addition of the timestamp + job_timestamp nvarchar(14) NULL default NULL, + + -- Any other parameters to the command + -- Stored as a PHP serialized array, or an empty string if there are no parameters + job_params nvarchar(max) NOT NULL, + + -- Random, non-unique, number used for job acquisition (for lock concurrency) + job_random int NOT NULL default 0, + + -- The number of times this job has been locked + job_attempts int NOT NULL default 0, + + -- Field that conveys process locks on rows via process UUIDs + job_token nvarchar(32) NOT NULL default '', + + -- Timestamp when the job was locked + job_token_timestamp varchar(14) NULL default NULL, + + -- Base 36 SHA1 of the job parameters relevant to detecting duplicates + job_sha1 nvarchar(32) NOT NULL default '' +); + +CREATE INDEX /*i*/job_sha1 ON /*_*/job (job_sha1); +CREATE INDEX /*i*/job_cmd_token ON /*_*/job (job_cmd,job_token,job_random); +CREATE INDEX /*i*/job_cmd_token_id ON /*_*/job (job_cmd,job_token,job_id); +CREATE INDEX /*i*/job_cmd ON /*_*/job (job_cmd, job_namespace, job_title); +CREATE INDEX /*i*/job_timestamp ON /*_*/job (job_timestamp); + + +-- Details of updates to cached special pages +CREATE TABLE /*_*/querycache_info ( + -- Special page name + -- Corresponds to a qc_type value + qci_type nvarchar(32) NOT NULL default '', + + -- Timestamp of last update + qci_timestamp varchar(14) NOT NULL default '' +); + +CREATE UNIQUE INDEX /*i*/qci_type ON /*_*/querycache_info (qci_type); + + +-- For each redirect, this table contains exactly one row defining its target +CREATE TABLE /*_*/redirect ( + -- Key to the page_id of the redirect page + rd_from int NOT NULL REFERENCES /*_*/page(page_id) ON DELETE CASCADE, + + -- Key to page_namespace/page_title of the target page. + -- The target page may or may not exist, and due to renames + -- and deletions may refer to different page records as time + -- goes by. + rd_namespace int NOT NULL default 0, + rd_title nvarchar(255) NOT NULL default '', + rd_interwiki nvarchar(32) default NULL, + rd_fragment nvarchar(255) default NULL +); + +CREATE INDEX /*i*/rd_ns_title ON /*_*/redirect (rd_namespace,rd_title,rd_from); + + +-- Used for caching expensive grouped queries that need two links (for example double-redirects) +CREATE TABLE /*_*/querycachetwo ( + -- A key name, generally the base name of of the special page. + qcc_type nvarchar(32) NOT NULL, + + -- Some sort of stored value. Sizes, counts... + qcc_value int NOT NULL default 0, + + -- Target namespace+title + qcc_namespace int NOT NULL default 0, + qcc_title nvarchar(255) NOT NULL default '', + + -- Target namespace+title2 + qcc_namespacetwo int NOT NULL default 0, + qcc_titletwo nvarchar(255) NOT NULL default '' +); + +CREATE INDEX /*i*/qcc_type ON /*_*/querycachetwo (qcc_type,qcc_value); +CREATE INDEX /*i*/qcc_title ON /*_*/querycachetwo (qcc_type,qcc_namespace,qcc_title); +CREATE INDEX /*i*/qcc_titletwo ON /*_*/querycachetwo (qcc_type,qcc_namespacetwo,qcc_titletwo); + + +-- Used for storing page restrictions (i.e. protection levels) +CREATE TABLE /*_*/page_restrictions ( + -- Field for an ID for this restrictions row (sort-key for Special:ProtectedPages) + pr_id int NOT NULL PRIMARY KEY IDENTITY, + -- Page to apply restrictions to (Foreign Key to page). + pr_page int NOT NULL REFERENCES /*_*/page(page_id) ON DELETE CASCADE, + -- The protection type (edit, move, etc) + pr_type nvarchar(60) NOT NULL, + -- The protection level (Sysop, autoconfirmed, etc) + pr_level nvarchar(60) NOT NULL, + -- Whether or not to cascade the protection down to pages transcluded. + pr_cascade bit NOT NULL, + -- Field for future support of per-user restriction. + pr_user int NULL, + -- Field for time-limited protection. + pr_expiry varchar(14) NULL +); + +CREATE UNIQUE INDEX /*i*/pr_pagetype ON /*_*/page_restrictions (pr_page,pr_type); +CREATE INDEX /*i*/pr_typelevel ON /*_*/page_restrictions (pr_type,pr_level); +CREATE INDEX /*i*/pr_level ON /*_*/page_restrictions (pr_level); +CREATE INDEX /*i*/pr_cascade ON /*_*/page_restrictions (pr_cascade); + + +-- Protected titles - nonexistent pages that have been protected +CREATE TABLE /*_*/protected_titles ( + pt_namespace int NOT NULL, + pt_title nvarchar(255) NOT NULL, + pt_user int REFERENCES /*_*/mwuser(user_id) ON DELETE SET NULL, + pt_reason nvarchar(255), + pt_timestamp varchar(14) NOT NULL, + pt_expiry varchar(14) NOT NULL, + pt_create_perm nvarchar(60) NOT NULL +); + +CREATE UNIQUE INDEX /*i*/pt_namespace_title ON /*_*/protected_titles (pt_namespace,pt_title); +CREATE INDEX /*i*/pt_timestamp ON /*_*/protected_titles (pt_timestamp); + + +-- Name/value pairs indexed by page_id +CREATE TABLE /*_*/page_props ( + pp_page int NOT NULL REFERENCES /*_*/page(page_id) ON DELETE CASCADE, + pp_propname nvarchar(60) NOT NULL, + pp_value nvarchar(max) NOT NULL, + pp_sortkey float DEFAULT NULL +); + +CREATE UNIQUE INDEX /*i*/pp_page_propname ON /*_*/page_props (pp_page,pp_propname); +CREATE UNIQUE INDEX /*i*/pp_propname_page ON /*_*/page_props (pp_propname,pp_page); +CREATE UNIQUE INDEX /*i*/pp_propname_sortkey_page ON /*_*/page_props (pp_propname,pp_sortkey,pp_page); + + +-- A table to log updates, one text key row per update. +CREATE TABLE /*_*/updatelog ( + ul_key nvarchar(255) NOT NULL PRIMARY KEY, + ul_value nvarchar(max) +); + + +-- A table to track tags for revisions, logs and recent changes. +CREATE TABLE /*_*/change_tag ( + ct_id int NOT NULL PRIMARY KEY IDENTITY, + -- RCID for the change + ct_rc_id int NULL REFERENCES /*_*/recentchanges(rc_id), + -- LOGID for the change + ct_log_id int NULL REFERENCES /*_*/logging(log_id), + -- REVID for the change + ct_rev_id int NULL REFERENCES /*_*/revision(rev_id), + -- Tag applied + ct_tag nvarchar(255) NOT NULL, + -- Parameters for the tag, presently unused + ct_params nvarchar(max) NULL +); + +CREATE UNIQUE INDEX /*i*/change_tag_rc_tag ON /*_*/change_tag (ct_rc_id,ct_tag); +CREATE UNIQUE INDEX /*i*/change_tag_log_tag ON /*_*/change_tag (ct_log_id,ct_tag); +CREATE UNIQUE INDEX /*i*/change_tag_rev_tag ON /*_*/change_tag (ct_rev_id,ct_tag); +-- Covering index, so we can pull all the info only out of the index. +CREATE INDEX /*i*/change_tag_tag_id ON /*_*/change_tag (ct_tag,ct_rc_id,ct_rev_id,ct_log_id); + + +-- Rollup table to pull a LIST of tags simply without ugly GROUP_CONCAT +-- that only works on MySQL 4.1+ +CREATE TABLE /*_*/tag_summary ( + ts_id int NOT NULL PRIMARY KEY IDENTITY, + -- RCID for the change + ts_rc_id int NULL REFERENCES /*_*/recentchanges(rc_id), + -- LOGID for the change + ts_log_id int NULL REFERENCES /*_*/logging(log_id), + -- REVID for the change + ts_rev_id int NULL REFERENCES /*_*/revision(rev_id), + -- Comma-separated list of tags + ts_tags nvarchar(max) NOT NULL +); + +CREATE UNIQUE INDEX /*i*/tag_summary_rc_id ON /*_*/tag_summary (ts_rc_id); +CREATE UNIQUE INDEX /*i*/tag_summary_log_id ON /*_*/tag_summary (ts_log_id); +CREATE UNIQUE INDEX /*i*/tag_summary_rev_id ON /*_*/tag_summary (ts_rev_id); + + +CREATE TABLE /*_*/valid_tag ( + vt_tag nvarchar(255) NOT NULL PRIMARY KEY +); + +-- Table for storing localisation data +CREATE TABLE /*_*/l10n_cache ( + -- Language code + lc_lang nvarchar(32) NOT NULL, + -- Cache key + lc_key nvarchar(255) NOT NULL, + -- Value + lc_value varbinary(max) NOT NULL +); +CREATE INDEX /*i*/lc_lang_key ON /*_*/l10n_cache (lc_lang, lc_key); + +-- Table caching which local files a module depends on that aren't +-- registered directly, used for fast retrieval of file dependency. +-- Currently only used for tracking images that CSS depends on +CREATE TABLE /*_*/module_deps ( + -- Module name + md_module nvarchar(255) NOT NULL, + -- Skin name + md_skin nvarchar(32) NOT NULL, + -- JSON nvarchar(max) with file dependencies + md_deps nvarchar(max) NOT NULL +); +CREATE UNIQUE INDEX /*i*/md_module_skin ON /*_*/module_deps (md_module, md_skin); + +-- Holds all the sites known to the wiki. +CREATE TABLE /*_*/sites ( + -- Numeric id of the site + site_id int NOT NULL PRIMARY KEY IDENTITY, + + -- Global identifier for the site, ie 'enwiktionary' + site_global_key nvarchar(32) NOT NULL, + + -- Type of the site, ie 'mediawiki' + site_type nvarchar(32) NOT NULL, + + -- Group of the site, ie 'wikipedia' + site_group nvarchar(32) NOT NULL, + + -- Source of the site data, ie 'local', 'wikidata', 'my-magical-repo' + site_source nvarchar(32) NOT NULL, + + -- Language code of the sites primary language. + site_language nvarchar(32) NOT NULL, + + -- Protocol of the site, ie 'http://', 'irc://', '//' + -- This field is an index for lookups and is build from type specific data in site_data. + site_protocol nvarchar(32) NOT NULL, + + -- Domain of the site in reverse order, ie 'org.mediawiki.www.' + -- This field is an index for lookups and is build from type specific data in site_data. + site_domain NVARCHAR(255) NOT NULL, + + -- Type dependent site data. + site_data nvarchar(max) NOT NULL, + + -- If site.tld/path/key:pageTitle should forward users to the page on + -- the actual site, where "key" is the local identifier. + site_forward bit NOT NULL, + + -- Type dependent site config. + -- For instance if template transclusion should be allowed if it's a MediaWiki. + site_config nvarchar(max) NOT NULL +); + +CREATE UNIQUE INDEX /*i*/sites_global_key ON /*_*/sites (site_global_key); +CREATE INDEX /*i*/sites_type ON /*_*/sites (site_type); +CREATE INDEX /*i*/sites_group ON /*_*/sites (site_group); +CREATE INDEX /*i*/sites_source ON /*_*/sites (site_source); +CREATE INDEX /*i*/sites_language ON /*_*/sites (site_language); +CREATE INDEX /*i*/sites_protocol ON /*_*/sites (site_protocol); +CREATE INDEX /*i*/sites_domain ON /*_*/sites (site_domain); +CREATE INDEX /*i*/sites_forward ON /*_*/sites (site_forward); + +-- Links local site identifiers to their corresponding site. +CREATE TABLE /*_*/site_identifiers ( + -- Key on site.site_id + si_site int NOT NULL REFERENCES /*_*/sites(site_id) ON DELETE CASCADE, + + -- local key type, ie 'interwiki' or 'langlink' + si_type nvarchar(32) NOT NULL, + + -- local key value, ie 'en' or 'wiktionary' + si_key nvarchar(32) NOT NULL +); + +CREATE UNIQUE INDEX /*i*/site_ids_type ON /*_*/site_identifiers (si_type, si_key); +CREATE INDEX /*i*/site_ids_site ON /*_*/site_identifiers (si_site); +CREATE INDEX /*i*/site_ids_key ON /*_*/site_identifiers (si_key); diff --git a/www/wiki/maintenance/mssql/update-keys.sql b/www/wiki/maintenance/mssql/update-keys.sql new file mode 100644 index 00000000..4d2c1c12 --- /dev/null +++ b/www/wiki/maintenance/mssql/update-keys.sql @@ -0,0 +1,31 @@ +-- Update keys for Microsoft SQL Server +-- SQL to insert update keys into the initial tables after a +-- fresh installation of MediaWiki's database. +-- This is read and executed by the install script; you should +-- not have to run it by itself unless doing a manual install. +-- Insert keys here if either the unnecessary would cause heavy +-- processing or could potentially cause trouble by lowering field +-- sizes, adding constraints, etc. +-- When adjusting field sizes, it is recommended removing old +-- patches but to play safe, update keys should also inserted here. + +-- +-- The /*_*/ comments in this and other files are +-- replaced with the defined table prefix by the installer +-- and updater scripts. If you are installing or running +-- updates manually, you will need to manually insert the +-- table prefix if any when running these scripts. +-- + +INSERT INTO /*_*/updatelog + SELECT 'filearchive-fa_major_mime-patch-fa_major_mime-chemical.sql' AS ul_key, null as ul_value + UNION SELECT 'image-img_major_mime-patch-img_major_mime-chemical.sql', null + UNION SELECT 'oldimage-oi_major_mime-patch-oi_major_mime-chemical.sql', null + UNION SELECT 'cl_type-category_types-ck', null + UNION SELECT 'fa_major_mime-major_mime-ck', null + UNION SELECT 'fa_media_type-media_type-ck', null + UNION SELECT 'img_major_mime-major_mime-ck', null + UNION SELECT 'img_media_type-media_type-ck', null + UNION SELECT 'oi_major_mime-major_mime-ck', null + UNION SELECT 'oi_media_type-media_type-ck', null + UNION SELECT 'us_media_type-media_type-ck', null;
\ No newline at end of file diff --git a/www/wiki/maintenance/mwdoc-filter.php b/www/wiki/maintenance/mwdoc-filter.php new file mode 100644 index 00000000..feaad12a --- /dev/null +++ b/www/wiki/maintenance/mwdoc-filter.php @@ -0,0 +1,101 @@ +<?php +/** + * Doxygen filter to show correct member variable types in documentation. + * + * Should be set in Doxygen INPUT_FILTER as "php mwdoc-filter.php" + * + * Based on + * <https://virtualtee.blogspot.co.uk/2012/03/tip-for-using-doxygen-for-php-code.html> + * + * Improved to resolve various bugs and better MediaWiki PHPDoc conventions: + * + * - Insert variable name after typehint instead of at end of line so that + * documentation text may follow after "@var Type". + * - Insert typehint into source code before $variable instead of inside the comment + * so that Doxygen interprets it. + * - Strip the text after @var from the output to avoid Doxygen warnings aboug bogus + * symbols being documented but not declared or defined. + * + * Copyright (C) 2012 Tamas Imrei <tamas.imrei@gmail.com> https://virtualtee.blogspot.com/ + * Copyright (C) 2015 Timo Tijhof + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +// Warning: Converting this to a Maintenance script may reduce performance. +if ( PHP_SAPI != 'cli' ) { + die( "This filter can only be run from the command line.\n" ); +} + +$source = file_get_contents( $argv[1] ); +$tokens = token_get_all( $source ); + +$buffer = $bufferType = null; +foreach ( $tokens as $token ) { + if ( is_string( $token ) ) { + if ( $buffer !== null && $token === ';' ) { + // If we still have a buffer and the statement has ended, + // flush it and move on. + echo $buffer; + $buffer = $bufferType = null; + } + echo $token; + continue; + } + list( $id, $content ) = $token; + switch ( $id ) { + case T_DOC_COMMENT: + // Escape slashes so that references to namespaces are not + // wrongly interpreted as a Doxygen "\command". + $content = addcslashes( $content, '\\' ); + // Look for instances of "@var Type" not followed by $name. + if ( preg_match( '#@var\s+([^\s]+)\s+([^\$]+)#s', $content ) ) { + $buffer = preg_replace_callback( + // Strip the "@var Type" part and remember the type + '#(@var\s+)([^\s]+)#s', + function ( $matches ) use ( &$bufferType ) { + $bufferType = $matches[2]; + return ''; + }, + $content + ); + } else { + echo $content; + } + break; + + case T_VARIABLE: + if ( $buffer !== null ) { + echo $buffer; + echo "$bufferType $content"; + $buffer = $bufferType = null; + } else { + echo $content; + } + break; + + default: + if ( $buffer !== null ) { + $buffer .= $content; + } else { + echo $content; + } + break; + } +} diff --git a/www/wiki/maintenance/mwdocgen.php b/www/wiki/maintenance/mwdocgen.php new file mode 100644 index 00000000..43041a43 --- /dev/null +++ b/www/wiki/maintenance/mwdocgen.php @@ -0,0 +1,171 @@ +<?php +/** + * Generate class and file reference documentation for MediaWiki using doxygen. + * + * If the dot DOT language processor is available, attempt call graph + * generation. + * + * Usage: + * php mwdocgen.php + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @todo document + * @ingroup Maintenance + * + * @author Antoine Musso <hashar at free dot fr> + * @author Brion Vibber + * @author Alexandre Emsenhuber + * @version first release + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that builds doxygen documentation. + * @ingroup Maintenance + */ +class MWDocGen extends Maintenance { + + /** + * Prepare Maintenance class + */ + public function __construct() { + parent::__construct(); + $this->addDescription( 'Build doxygen documentation' ); + + $this->addOption( 'doxygen', + 'Path to doxygen', + false, true ); + $this->addOption( 'version', + 'Pass a MediaWiki version', + false, true ); + $this->addOption( 'generate-man', + 'Whether to generate man files' ); + $this->addOption( 'file', + "Only process given file or directory. Multiple values " . + "accepted with comma separation. Path relative to \$IP.", + false, true ); + $this->addOption( 'output', + 'Path to write doc to', + false, true ); + $this->addOption( 'no-extensions', + 'Ignore extensions' ); + } + + public function getDbType() { + return Maintenance::DB_NONE; + } + + protected function init() { + global $wgPhpCli, $IP; + + $this->doxygen = $this->getOption( 'doxygen', 'doxygen' ); + $this->mwVersion = $this->getOption( 'version', 'master' ); + + $this->input = ''; + $inputs = explode( ',', $this->getOption( 'file', '' ) ); + foreach ( $inputs as $input ) { + # Doxygen inputs are space separted and double quoted + $this->input .= " \"$IP/$input\""; + } + + $this->output = $this->getOption( 'output', "$IP/docs" ); + + // Do not use wfShellWikiCmd, because mwdoc-filter.php is not + // a Maintenance script. + $this->inputFilter = wfEscapeShellArg( [ + $wgPhpCli, + $IP . '/maintenance/mwdoc-filter.php' + ] ); + + $this->template = $IP . '/maintenance/Doxyfile'; + $this->excludes = [ + 'vendor', + 'node_modules', + 'images', + 'static', + ]; + $this->excludePatterns = []; + if ( $this->hasOption( 'no-extensions' ) ) { + $this->excludePatterns[] = 'extensions'; + } + + $this->doDot = shell_exec( 'which dot' ); + $this->doMan = $this->hasOption( 'generate-man' ); + } + + public function execute() { + global $IP; + + $this->init(); + + # Build out directories we want to exclude + $exclude = ''; + foreach ( $this->excludes as $item ) { + $exclude .= " $IP/$item"; + } + + $excludePatterns = implode( ' ', $this->excludePatterns ); + + $conf = strtr( file_get_contents( $this->template ), + [ + '{{OUTPUT_DIRECTORY}}' => $this->output, + '{{STRIP_FROM_PATH}}' => $IP, + '{{CURRENT_VERSION}}' => $this->mwVersion, + '{{INPUT}}' => $this->input, + '{{EXCLUDE}}' => $exclude, + '{{EXCLUDE_PATTERNS}}' => $excludePatterns, + '{{HAVE_DOT}}' => $this->doDot ? 'YES' : 'NO', + '{{GENERATE_MAN}}' => $this->doMan ? 'YES' : 'NO', + '{{INPUT_FILTER}}' => $this->inputFilter, + ] + ); + + $tmpFile = tempnam( wfTempDir(), 'MWDocGen-' ); + if ( file_put_contents( $tmpFile, $conf ) === false ) { + $this->error( "Could not write doxygen configuration to file $tmpFile\n", + /** exit code: */ 1 ); + } + + $command = $this->doxygen . ' ' . $tmpFile; + $this->output( "Executing command:\n$command\n" ); + + $exitcode = 1; + system( $command, $exitcode ); + + $this->output( <<<TEXT +--------------------------------------------------- +Doxygen execution finished. +Check above for possible errors. + +You might want to delete the temporary file: + $tmpFile +--------------------------------------------------- + +TEXT + ); + + if ( $exitcode !== 0 ) { + $this->error( "Something went wrong (exit: $exitcode)\n", + $exitcode ); + } + } +} + +$maintClass = 'MWDocGen'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/mwjsduck-gen b/www/wiki/maintenance/mwjsduck-gen new file mode 100755 index 00000000..6b7c77b6 --- /dev/null +++ b/www/wiki/maintenance/mwjsduck-gen @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -e +cd $(dirname $0)/.. +jsduck diff --git a/www/wiki/maintenance/namespaceDupes.php b/www/wiki/maintenance/namespaceDupes.php new file mode 100644 index 00000000..84d45335 --- /dev/null +++ b/www/wiki/maintenance/namespaceDupes.php @@ -0,0 +1,620 @@ +<?php +/** + * Check for articles to fix after adding/deleting namespaces + * + * Copyright © 2005-2007 Brion Vibber <brion@pobox.com> + * https://www.mediawiki.org/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +use MediaWiki\Linker\LinkTarget; +use MediaWiki\MediaWikiServices; +use Wikimedia\Rdbms\ResultWrapper; +use Wikimedia\Rdbms\IMaintainableDatabase; + +/** + * Maintenance script that checks for articles to fix after + * adding/deleting namespaces. + * + * @ingroup Maintenance + */ +class NamespaceConflictChecker extends Maintenance { + + /** + * @var IMaintainableDatabase + */ + protected $db; + + private $resolvablePages = 0; + private $totalPages = 0; + + private $resolvableLinks = 0; + private $totalLinks = 0; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Find and fix pages affected by namespace addition/removal' ); + $this->addOption( 'fix', 'Attempt to automatically fix errors' ); + $this->addOption( 'merge', "Instead of renaming conflicts, do a history merge with " . + "the correct title" ); + $this->addOption( 'add-suffix', "Dupes will be renamed with correct namespace with " . + "<text> appended after the article name", false, true ); + $this->addOption( 'add-prefix', "Dupes will be renamed with correct namespace with " . + "<text> prepended before the article name", false, true ); + $this->addOption( 'source-pseudo-namespace', "Move all pages with the given source " . + "prefix (with an implied colon following it). If --dest-namespace is not specified, " . + "the colon will be replaced with a hyphen.", + false, true ); + $this->addOption( 'dest-namespace', "In combination with --source-pseudo-namespace, " . + "specify the namespace ID of the destination.", false, true ); + $this->addOption( 'move-talk', "If this is specified, pages in the Talk namespace that " . + "begin with a conflicting prefix will be renamed, for example " . + "Talk:File:Foo -> File_Talk:Foo" ); + } + + public function execute() { + $this->db = $this->getDB( DB_MASTER ); + + $options = [ + 'fix' => $this->hasOption( 'fix' ), + 'merge' => $this->hasOption( 'merge' ), + 'add-suffix' => $this->getOption( 'add-suffix', '' ), + 'add-prefix' => $this->getOption( 'add-prefix', '' ), + 'move-talk' => $this->hasOption( 'move-talk' ), + 'source-pseudo-namespace' => $this->getOption( 'source-pseudo-namespace', '' ), + 'dest-namespace' => intval( $this->getOption( 'dest-namespace', 0 ) ) ]; + + if ( $options['source-pseudo-namespace'] !== '' ) { + $retval = $this->checkPrefix( $options ); + } else { + $retval = $this->checkAll( $options ); + } + + if ( $retval ) { + $this->output( "\nLooks good!\n" ); + } else { + $this->output( "\nOh noeees\n" ); + } + } + + /** + * Check all namespaces + * + * @param array $options Associative array of validated command-line options + * + * @return bool + */ + private function checkAll( $options ) { + global $wgContLang, $wgNamespaceAliases, $wgCapitalLinks; + + $spaces = []; + + // List interwikis first, so they'll be overridden + // by any conflicting local namespaces. + foreach ( $this->getInterwikiList() as $prefix ) { + $name = $wgContLang->ucfirst( $prefix ); + $spaces[$name] = 0; + } + + // Now pull in all canonical and alias namespaces... + foreach ( MWNamespace::getCanonicalNamespaces() as $ns => $name ) { + // This includes $wgExtraNamespaces + if ( $name !== '' ) { + $spaces[$name] = $ns; + } + } + foreach ( $wgContLang->getNamespaces() as $ns => $name ) { + if ( $name !== '' ) { + $spaces[$name] = $ns; + } + } + foreach ( $wgNamespaceAliases as $name => $ns ) { + $spaces[$name] = $ns; + } + foreach ( $wgContLang->getNamespaceAliases() as $name => $ns ) { + $spaces[$name] = $ns; + } + + // We'll need to check for lowercase keys as well, + // since we're doing case-sensitive searches in the db. + foreach ( $spaces as $name => $ns ) { + $moreNames = []; + $moreNames[] = $wgContLang->uc( $name ); + $moreNames[] = $wgContLang->ucfirst( $wgContLang->lc( $name ) ); + $moreNames[] = $wgContLang->ucwords( $name ); + $moreNames[] = $wgContLang->ucwords( $wgContLang->lc( $name ) ); + $moreNames[] = $wgContLang->ucwordbreaks( $name ); + $moreNames[] = $wgContLang->ucwordbreaks( $wgContLang->lc( $name ) ); + if ( !$wgCapitalLinks ) { + foreach ( $moreNames as $altName ) { + $moreNames[] = $wgContLang->lcfirst( $altName ); + } + $moreNames[] = $wgContLang->lcfirst( $name ); + } + foreach ( array_unique( $moreNames ) as $altName ) { + if ( $altName !== $name ) { + $spaces[$altName] = $ns; + } + } + } + + // Sort by namespace index, and if there are two with the same index, + // break the tie by sorting by name + $origSpaces = $spaces; + uksort( $spaces, function ( $a, $b ) use ( $origSpaces ) { + if ( $origSpaces[$a] < $origSpaces[$b] ) { + return -1; + } elseif ( $origSpaces[$a] > $origSpaces[$b] ) { + return 1; + } elseif ( $a < $b ) { + return -1; + } elseif ( $a > $b ) { + return 1; + } else { + return 0; + } + } ); + + $ok = true; + foreach ( $spaces as $name => $ns ) { + $ok = $this->checkNamespace( $ns, $name, $options ) && $ok; + } + + $this->output( "{$this->totalPages} pages to fix, " . + "{$this->resolvablePages} were resolvable.\n\n" ); + + foreach ( $spaces as $name => $ns ) { + if ( $ns != 0 ) { + /* Fix up link destinations for non-interwiki links only. + * + * For example if a page has [[Foo:Bar]] and then a Foo namespace + * is introduced, pagelinks needs to be updated to have + * page_namespace = NS_FOO. + * + * If instead an interwiki prefix was introduced called "Foo", + * the link should instead be moved to the iwlinks table. If a new + * language is introduced called "Foo", or if there is a pagelink + * [[fr:Bar]] when interlanguage magic links are turned on, the + * link would have to be moved to the langlinks table. Let's put + * those cases in the too-hard basket for now. The consequences are + * not especially severe. + * @fixme Handle interwiki links, and pagelinks to Category:, File: + * which probably need reparsing. + */ + + $this->checkLinkTable( 'pagelinks', 'pl', $ns, $name, $options ); + $this->checkLinkTable( 'templatelinks', 'tl', $ns, $name, $options ); + + // The redirect table has interwiki links randomly mixed in, we + // need to filter those out. For example [[w:Foo:Bar]] would + // have rd_interwiki=w and rd_namespace=0, which would match the + // query for a conflicting namespace "Foo" if filtering wasn't done. + $this->checkLinkTable( 'redirect', 'rd', $ns, $name, $options, + [ 'rd_interwiki' => null ] ); + $this->checkLinkTable( 'redirect', 'rd', $ns, $name, $options, + [ 'rd_interwiki' => '' ] ); + } + } + + $this->output( "{$this->totalLinks} links to fix, " . + "{$this->resolvableLinks} were resolvable.\n" ); + + return $ok; + } + + /** + * Get the interwiki list + * + * @return array + */ + private function getInterwikiList() { + $result = MediaWikiServices::getInstance()->getInterwikiLookup()->getAllPrefixes(); + $prefixes = []; + foreach ( $result as $row ) { + $prefixes[] = $row['iw_prefix']; + } + + return $prefixes; + } + + /** + * Check a given prefix and try to move it into the given destination namespace + * + * @param int $ns Destination namespace id + * @param string $name + * @param array $options Associative array of validated command-line options + * @return bool + */ + private function checkNamespace( $ns, $name, $options ) { + $targets = $this->getTargetList( $ns, $name, $options ); + $count = $targets->numRows(); + $this->totalPages += $count; + if ( $count == 0 ) { + return true; + } + + $dryRunNote = $options['fix'] ? '' : ' DRY RUN ONLY'; + + $ok = true; + foreach ( $targets as $row ) { + // Find the new title and determine the action to take + + $newTitle = $this->getDestinationTitle( $ns, $name, + $row->page_namespace, $row->page_title, $options ); + $logStatus = false; + if ( !$newTitle ) { + $logStatus = 'invalid title'; + $action = 'abort'; + } elseif ( $newTitle->exists() ) { + if ( $options['merge'] ) { + if ( $this->canMerge( $row->page_id, $newTitle, $logStatus ) ) { + $action = 'merge'; + } else { + $action = 'abort'; + } + } elseif ( $options['add-prefix'] == '' && $options['add-suffix'] == '' ) { + $action = 'abort'; + $logStatus = 'dest title exists and --add-prefix not specified'; + } else { + $newTitle = $this->getAlternateTitle( $newTitle, $options ); + if ( !$newTitle ) { + $action = 'abort'; + $logStatus = 'alternate title is invalid'; + } elseif ( $newTitle->exists() ) { + $action = 'abort'; + $logStatus = 'title conflict'; + } else { + $action = 'move'; + $logStatus = 'alternate'; + } + } + } else { + $action = 'move'; + $logStatus = 'no conflict'; + } + + // Take the action or log a dry run message + + $logTitle = "id={$row->page_id} ns={$row->page_namespace} dbk={$row->page_title}"; + $pageOK = true; + + switch ( $action ) { + case 'abort': + $this->output( "$logTitle *** $logStatus\n" ); + $pageOK = false; + break; + case 'move': + $this->output( "$logTitle -> " . + $newTitle->getPrefixedDBkey() . " ($logStatus)$dryRunNote\n" ); + + if ( $options['fix'] ) { + $pageOK = $this->movePage( $row->page_id, $newTitle ); + } + break; + case 'merge': + $this->output( "$logTitle => " . + $newTitle->getPrefixedDBkey() . " (merge)$dryRunNote\n" ); + + if ( $options['fix'] ) { + $pageOK = $this->mergePage( $row, $newTitle ); + } + break; + } + + if ( $pageOK ) { + $this->resolvablePages++; + } else { + $ok = false; + } + } + + return $ok; + } + + /** + * Check and repair the destination fields in a link table + * @param string $table The link table name + * @param string $fieldPrefix The field prefix in the link table + * @param int $ns Destination namespace id + * @param string $name + * @param array $options Associative array of validated command-line options + * @param array $extraConds Extra conditions for the SQL query + */ + private function checkLinkTable( $table, $fieldPrefix, $ns, $name, $options, + $extraConds = [] + ) { + $batchConds = []; + $fromField = "{$fieldPrefix}_from"; + $namespaceField = "{$fieldPrefix}_namespace"; + $titleField = "{$fieldPrefix}_title"; + $batchSize = 500; + while ( true ) { + $res = $this->db->select( + $table, + [ $fromField, $namespaceField, $titleField ], + array_merge( $batchConds, $extraConds, [ + $namespaceField => 0, + $titleField . $this->db->buildLike( "$name:", $this->db->anyString() ) + ] ), + __METHOD__, + [ + 'ORDER BY' => [ $titleField, $fromField ], + 'LIMIT' => $batchSize + ] + ); + + if ( $res->numRows() == 0 ) { + break; + } + foreach ( $res as $row ) { + $logTitle = "from={$row->$fromField} ns={$row->$namespaceField} " . + "dbk={$row->$titleField}"; + $destTitle = $this->getDestinationTitle( $ns, $name, + $row->$namespaceField, $row->$titleField, $options ); + $this->totalLinks++; + if ( !$destTitle ) { + $this->output( "$table $logTitle *** INVALID\n" ); + continue; + } + $this->resolvableLinks++; + if ( !$options['fix'] ) { + $this->output( "$table $logTitle -> " . + $destTitle->getPrefixedDBkey() . " DRY RUN\n" ); + continue; + } + + $this->db->update( $table, + // SET + [ + $namespaceField => $destTitle->getNamespace(), + $titleField => $destTitle->getDBkey() + ], + // WHERE + [ + $namespaceField => 0, + $titleField => $row->$titleField, + $fromField => $row->$fromField + ], + __METHOD__, + [ 'IGNORE' ] + ); + $this->output( "$table $logTitle -> " . + $destTitle->getPrefixedDBkey() . "\n" ); + } + $encLastTitle = $this->db->addQuotes( $row->$titleField ); + $encLastFrom = $this->db->addQuotes( $row->$fromField ); + + $batchConds = [ + "$titleField > $encLastTitle " . + "OR ($titleField = $encLastTitle AND $fromField > $encLastFrom)" ]; + + wfWaitForSlaves(); + } + } + + /** + * Move the given pseudo-namespace, either replacing the colon with a hyphen + * (useful for pseudo-namespaces that conflict with interwiki links) or move + * them to another namespace if specified. + * @param array $options Associative array of validated command-line options + * @return bool + */ + private function checkPrefix( $options ) { + $prefix = $options['source-pseudo-namespace']; + $ns = $options['dest-namespace']; + $this->output( "Checking prefix \"$prefix\" vs namespace $ns\n" ); + + return $this->checkNamespace( $ns, $prefix, $options ); + } + + /** + * Find pages in main and talk namespaces that have a prefix of the new + * namespace so we know titles that will need migrating + * + * @param int $ns Destination namespace id + * @param string $name Prefix that is being made a namespace + * @param array $options Associative array of validated command-line options + * + * @return ResultWrapper + */ + private function getTargetList( $ns, $name, $options ) { + if ( $options['move-talk'] && MWNamespace::isSubject( $ns ) ) { + $checkNamespaces = [ NS_MAIN, NS_TALK ]; + } else { + $checkNamespaces = NS_MAIN; + } + + return $this->db->select( 'page', + [ + 'page_id', + 'page_title', + 'page_namespace', + ], + [ + 'page_namespace' => $checkNamespaces, + 'page_title' . $this->db->buildLike( "$name:", $this->db->anyString() ), + ], + __METHOD__ + ); + } + + /** + * Get the preferred destination title for a given target page. + * @param int $ns The destination namespace ID + * @param string $name The conflicting prefix + * @param int $sourceNs The source namespace + * @param int $sourceDbk The source DB key (i.e. page_title) + * @param array $options Associative array of validated command-line options + * @return Title|false + */ + private function getDestinationTitle( $ns, $name, $sourceNs, $sourceDbk, $options ) { + $dbk = substr( $sourceDbk, strlen( "$name:" ) ); + if ( $ns == 0 ) { + // An interwiki; try an alternate encoding with '-' for ':' + $dbk = "$name-" . $dbk; + } + $destNS = $ns; + if ( $sourceNs == NS_TALK && MWNamespace::isSubject( $ns ) ) { + // This is an associated talk page moved with the --move-talk feature. + $destNS = MWNamespace::getTalk( $destNS ); + } + $newTitle = Title::makeTitleSafe( $destNS, $dbk ); + if ( !$newTitle || !$newTitle->canExist() ) { + return false; + } + return $newTitle; + } + + /** + * Get an alternative title to move a page to. This is used if the + * preferred destination title already exists. + * + * @param LinkTarget $linkTarget + * @param array $options Associative array of validated command-line options + * @return Title|bool + */ + private function getAlternateTitle( LinkTarget $linkTarget, $options ) { + $prefix = $options['add-prefix']; + $suffix = $options['add-suffix']; + if ( $prefix == '' && $suffix == '' ) { + return false; + } + while ( true ) { + $dbk = $prefix . $linkTarget->getDBkey() . $suffix; + $title = Title::makeTitleSafe( $linkTarget->getNamespace(), $dbk ); + if ( !$title ) { + return false; + } + if ( !$title->exists() ) { + return $title; + } + } + } + + /** + * Move a page + * + * @param integer $id The page_id + * @param LinkTarget $newLinkTarget The new title link target + * @return bool + */ + private function movePage( $id, LinkTarget $newLinkTarget ) { + $this->db->update( 'page', + [ + "page_namespace" => $newLinkTarget->getNamespace(), + "page_title" => $newLinkTarget->getDBkey(), + ], + [ + "page_id" => $id, + ], + __METHOD__ ); + + // Update *_from_namespace in links tables + $fromNamespaceTables = [ + [ 'pagelinks', 'pl' ], + [ 'templatelinks', 'tl' ], + [ 'imagelinks', 'il' ] ]; + foreach ( $fromNamespaceTables as $tableInfo ) { + list( $table, $fieldPrefix ) = $tableInfo; + $this->db->update( $table, + // SET + [ "{$fieldPrefix}_from_namespace" => $newLinkTarget->getNamespace() ], + // WHERE + [ "{$fieldPrefix}_from" => $id ], + __METHOD__ ); + } + + return true; + } + + /** + * Determine if we can merge a page. + * We check if an inaccessible revision would become the latest and + * deny the merge if so -- it's theoretically possible to update the + * latest revision, but opens a can of worms -- search engine updates, + * recentchanges review, etc. + * + * @param integer $id The page_id + * @param LinkTarget $linkTarget The new link target + * @param string $logStatus This is set to the log status message on failure + * @return bool + */ + private function canMerge( $id, LinkTarget $linkTarget, &$logStatus ) { + $latestDest = Revision::newFromTitle( $linkTarget, 0, Revision::READ_LATEST ); + $latestSource = Revision::newFromPageId( $id, 0, Revision::READ_LATEST ); + if ( $latestSource->getTimestamp() > $latestDest->getTimestamp() ) { + $logStatus = 'cannot merge since source is later'; + return false; + } else { + return true; + } + } + + /** + * Merge page histories + * + * @param stdClass $row Page row + * @param Title $newTitle The new title + * @return bool + */ + private function mergePage( $row, Title $newTitle ) { + $id = $row->page_id; + + // Construct the WikiPage object we will need later, while the + // page_id still exists. Note that this cannot use makeTitleSafe(), + // we are deliberately constructing an invalid title. + $sourceTitle = Title::makeTitle( $row->page_namespace, $row->page_title ); + $sourceTitle->resetArticleID( $id ); + $wikiPage = new WikiPage( $sourceTitle ); + $wikiPage->loadPageData( 'fromdbmaster' ); + + $destId = $newTitle->getArticleID(); + $this->beginTransaction( $this->db, __METHOD__ ); + $this->db->update( 'revision', + // SET + [ 'rev_page' => $destId ], + // WHERE + [ 'rev_page' => $id ], + __METHOD__ ); + + $this->db->delete( 'page', [ 'page_id' => $id ], __METHOD__ ); + + $this->commitTransaction( $this->db, __METHOD__ ); + + /* Call LinksDeletionUpdate to delete outgoing links from the old title, + * and update category counts. + * + * Calling external code with a fake broken Title is a fairly dubious + * idea. It's necessary because it's quite a lot of code to duplicate, + * but that also makes it fragile since it would be easy for someone to + * accidentally introduce an assumption of title validity to the code we + * are calling. + */ + DeferredUpdates::addUpdate( new LinksDeletionUpdate( $wikiPage ) ); + DeferredUpdates::doUpdates(); + + return true; + } +} + +$maintClass = "NamespaceConflictChecker"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/nukeNS.php b/www/wiki/maintenance/nukeNS.php new file mode 100644 index 00000000..e735aed0 --- /dev/null +++ b/www/wiki/maintenance/nukeNS.php @@ -0,0 +1,122 @@ +<?php +/** + * Remove pages with only 1 revision from the MediaWiki namespace, without + * flooding recent changes, delete logs, etc. + * Irreversible (can't use standard undelete) and does not update link tables + * + * This is mainly useful to run before maintenance/update.php when upgrading + * to 1.9, to prevent flooding recent changes/deletion logs. It's intended + * to be conservative, so it's possible that a few entries will be left for + * deletion by the upgrade script. It's also possible that it hasn't been + * tested thouroughly enough, and will delete something it shouldn't; so + * back up your DB if there's anything in the MediaWiki that is important to + * you. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @author Steve Sanbeg + * based on nukePage by Rob Church + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that removes pages with only one revision from the + * MediaWiki namespace. + * + * @ingroup Maintenance + */ +class NukeNS extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Remove pages with only 1 revision from any namespace' ); + $this->addOption( 'delete', "Actually delete the page" ); + $this->addOption( 'ns', 'Namespace to delete from, default NS_MEDIAWIKI', false, true ); + $this->addOption( 'all', 'Delete everything regardless of revision count' ); + } + + public function execute() { + $ns = $this->getOption( 'ns', NS_MEDIAWIKI ); + $delete = $this->hasOption( 'delete' ); + $all = $this->hasOption( 'all' ); + $dbw = $this->getDB( DB_MASTER ); + $this->beginTransaction( $dbw, __METHOD__ ); + + $tbl_pag = $dbw->tableName( 'page' ); + $tbl_rev = $dbw->tableName( 'revision' ); + $res = $dbw->query( "SELECT page_title FROM $tbl_pag WHERE page_namespace = $ns" ); + + $n_deleted = 0; + + foreach ( $res as $row ) { + // echo "$ns_name:".$row->page_title, "\n"; + $title = Title::makeTitle( $ns, $row->page_title ); + $id = $title->getArticleID(); + + // Get corresponding revisions + $res2 = $dbw->query( "SELECT rev_id FROM $tbl_rev WHERE rev_page = $id" ); + $revs = []; + + foreach ( $res2 as $row2 ) { + $revs[] = $row2->rev_id; + } + $count = count( $revs ); + + // skip anything that looks modified (i.e. multiple revs) + if ( $all || $count == 1 ) { + # echo $title->getPrefixedText(), "\t", $count, "\n"; + $this->output( "delete: " . $title->getPrefixedText() . "\n" ); + + // as much as I hate to cut & paste this, it's a little different, and + // I already have the id & revs + if ( $delete ) { + $dbw->query( "DELETE FROM $tbl_pag WHERE page_id = $id" ); + $this->commitTransaction( $dbw, __METHOD__ ); + // Delete revisions as appropriate + $child = $this->runChild( 'NukePage', 'nukePage.php' ); + $child->deleteRevisions( $revs ); + $this->purgeRedundantText( true ); + $n_deleted++; + } + } else { + $this->output( "skip: " . $title->getPrefixedText() . "\n" ); + } + } + $this->commitTransaction( $dbw, __METHOD__ ); + + if ( $n_deleted > 0 ) { + # update statistics - better to decrement existing count, or just count + # the page table? + $pages = $dbw->selectField( 'site_stats', 'ss_total_pages' ); + $pages -= $n_deleted; + $dbw->update( + 'site_stats', + [ 'ss_total_pages' => $pages ], + [ 'ss_row_id' => 1 ], + __METHOD__ + ); + } + + if ( !$delete ) { + $this->output( "To update the database, run the script with the --delete option.\n" ); + } + } +} + +$maintClass = "NukeNS"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/nukePage.php b/www/wiki/maintenance/nukePage.php new file mode 100644 index 00000000..e27324a7 --- /dev/null +++ b/www/wiki/maintenance/nukePage.php @@ -0,0 +1,119 @@ +<?php +/** + * Erase a page record from the database + * Irreversible (can't use standard undelete) and does not update link tables + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @author Rob Church <robchur@gmail.com> + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that erases a page record from the database. + * + * @ingroup Maintenance + */ +class NukePage extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Remove a page record from the database' ); + $this->addOption( 'delete', "Actually delete the page" ); + $this->addArg( 'title', 'Title to delete' ); + } + + public function execute() { + $name = $this->getArg(); + $delete = $this->hasOption( 'delete' ); + + $dbw = $this->getDB( DB_MASTER ); + $this->beginTransaction( $dbw, __METHOD__ ); + + $tbl_pag = $dbw->tableName( 'page' ); + $tbl_rec = $dbw->tableName( 'recentchanges' ); + $tbl_rev = $dbw->tableName( 'revision' ); + + # Get page ID + $this->output( "Searching for \"$name\"..." ); + $title = Title::newFromText( $name ); + if ( $title ) { + $id = $title->getArticleID(); + $real = $title->getPrefixedText(); + $isGoodArticle = $title->isContentPage(); + $this->output( "found \"$real\" with ID $id.\n" ); + + # Get corresponding revisions + $this->output( "Searching for revisions..." ); + $res = $dbw->query( "SELECT rev_id FROM $tbl_rev WHERE rev_page = $id" ); + $revs = []; + foreach ( $res as $row ) { + $revs[] = $row->rev_id; + } + $count = count( $revs ); + $this->output( "found $count.\n" ); + + # Delete the page record and associated recent changes entries + if ( $delete ) { + $this->output( "Deleting page record..." ); + $dbw->query( "DELETE FROM $tbl_pag WHERE page_id = $id" ); + $this->output( "done.\n" ); + $this->output( "Cleaning up recent changes..." ); + $dbw->query( "DELETE FROM $tbl_rec WHERE rc_cur_id = $id" ); + $this->output( "done.\n" ); + } + + $this->commitTransaction( $dbw, __METHOD__ ); + + # Delete revisions as appropriate + if ( $delete && $count ) { + $this->output( "Deleting revisions..." ); + $this->deleteRevisions( $revs ); + $this->output( "done.\n" ); + $this->purgeRedundantText( true ); + } + + # Update stats as appropriate + if ( $delete ) { + $this->output( "Updating site stats..." ); + $ga = $isGoodArticle ? -1 : 0; // if it was good, decrement that too + $stats = new SiteStatsUpdate( 0, -$count, $ga, -1 ); + $stats->doUpdate(); + $this->output( "done.\n" ); + } + } else { + $this->output( "not found in database.\n" ); + $this->commitTransaction( $dbw, __METHOD__ ); + } + } + + public function deleteRevisions( $ids ) { + $dbw = $this->getDB( DB_MASTER ); + $this->beginTransaction( $dbw, __METHOD__ ); + + $tbl_rev = $dbw->tableName( 'revision' ); + + $set = implode( ', ', $ids ); + $dbw->query( "DELETE FROM $tbl_rev WHERE rev_id IN ( $set )" ); + + $this->commitTransaction( $dbw, __METHOD__ ); + } +} + +$maintClass = "NukePage"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/oracle/alterSharedConstraints.php b/www/wiki/maintenance/oracle/alterSharedConstraints.php new file mode 100644 index 00000000..ed412dae --- /dev/null +++ b/www/wiki/maintenance/oracle/alterSharedConstraints.php @@ -0,0 +1,97 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +use Wikimedia\Rdbms\DBQueryError; + +/** + * When using shared tables that are referenced by foreign keys on local + * tables you have to change the constraints on local tables. + * + * The shared tables have to have GRANT REFERENCE on shared tables to local schema + * i.e.: GRANT REFERENCES (user_id) ON mwuser TO hubclient; + */ + +require_once __DIR__ . '/../Maintenance.php'; + +class AlterSharedConstraints extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Alter foreign key to reference master tables in shared database setup.' ); + } + + public function getDbType() { + return Maintenance::DB_ADMIN; + } + + public function execute() { + global $wgSharedDB, $wgSharedTables, $wgSharedPrefix, $wgDBprefix; + + if ( $wgSharedDB == null ) { + $this->output( "Database sharing is not enabled\n" ); + + return; + } + + $dbw = $this->getDB( DB_MASTER ); + foreach ( $wgSharedTables as $table ) { + $stable = $dbw->tableNameInternal( $table ); + if ( $wgSharedPrefix != null ) { + $ltable = preg_replace( "/^$wgSharedPrefix(.*)/i", "$wgDBprefix\\1", $stable ); + } else { + $ltable = "{$wgDBprefix}{$stable}"; + } + + $result = $dbw->query( "SELECT uc.constraint_name, uc.table_name, ucc.column_name, + uccpk.table_name pk_table_name, uccpk.column_name pk_column_name, + uc.delete_rule, uc.deferrable, uc.deferred + FROM user_constraints uc, user_cons_columns ucc, user_cons_columns uccpk + WHERE uc.constraint_type = 'R' + AND ucc.constraint_name = uc.constraint_name + AND uccpk.constraint_name = uc.r_constraint_name + AND uccpk.table_name = '$ltable'" ); + + while ( ( $row = $result->fetchRow() ) !== false ) { + $this->output( "Altering {$row['constraint_name']} ..." ); + + try { + $dbw->query( "ALTER TABLE {$row['table_name']} + DROP CONSTRAINT {$wgDBprefix}{$row['constraint_name']}" ); + } catch ( DBQueryError $exdb ) { + if ( $exdb->errno != 2443 ) { + throw $exdb; + } + } + + $deleteRule = $row['delete_rule'] == 'NO ACTION' ? '' : "ON DELETE {$row['delete_rule']}"; + $dbw->query( "ALTER TABLE {$row['table_name']} + ADD CONSTRAINT {$wgDBprefix}{$row['constraint_name']} + FOREIGN KEY ({$row['column_name']}) + REFERENCES {$wgSharedDB}.$stable({$row['pk_column_name']}) + {$deleteRule} {$row['deferrable']} INITIALLY {$row['deferred']}" ); + + $this->output( "DONE\n" ); + } + } + } +} + +$maintClass = "AlterSharedConstraints"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/oracle/archives/patch-add-rc_name_type_patrolled_timestamp_index.sql b/www/wiki/maintenance/oracle/archives/patch-add-rc_name_type_patrolled_timestamp_index.sql new file mode 100644 index 00000000..cd0d3968 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-add-rc_name_type_patrolled_timestamp_index.sql @@ -0,0 +1,4 @@ +define mw_prefix='{$wgDBprefix}'; + +CREATE INDEX &mw_prefix.recentchanges_i08 ON &mw_prefix.recentchanges (rc_namespace, rc_type, rc_patrolled, rc_timestamp); + diff --git a/www/wiki/maintenance/oracle/archives/patch-ar_sha1_field.sql b/www/wiki/maintenance/oracle/archives/patch-ar_sha1_field.sql new file mode 100644 index 00000000..de723ce7 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-ar_sha1_field.sql @@ -0,0 +1,3 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.archive ADD ar_sha1 VARCHAR2(32); diff --git a/www/wiki/maintenance/oracle/archives/patch-archive-ar_content_format.sql b/www/wiki/maintenance/oracle/archives/patch-archive-ar_content_format.sql new file mode 100644 index 00000000..0c0c0d94 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-archive-ar_content_format.sql @@ -0,0 +1,3 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.archive ADD ar_content_format VARCHAR2(64); diff --git a/www/wiki/maintenance/oracle/archives/patch-archive-ar_content_model.sql b/www/wiki/maintenance/oracle/archives/patch-archive-ar_content_model.sql new file mode 100644 index 00000000..d18fc9e4 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-archive-ar_content_model.sql @@ -0,0 +1,3 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.archive ADD ar_content_model VARCHAR2(32); diff --git a/www/wiki/maintenance/oracle/archives/patch-archive-ar_id.sql b/www/wiki/maintenance/oracle/archives/patch-archive-ar_id.sql new file mode 100644 index 00000000..a43f7602 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-archive-ar_id.sql @@ -0,0 +1,6 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.archive ADD ( +ar_id NUMBER NOT NULL, +); +ALTER TABLE &mw_prefix.archive ADD CONSTRAINT &mw_prefix.archive_pk PRIMARY KEY (ar_id); diff --git a/www/wiki/maintenance/oracle/archives/patch-auto_increment_triggers.sql b/www/wiki/maintenance/oracle/archives/patch-auto_increment_triggers.sql new file mode 100644 index 00000000..6b471b04 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-auto_increment_triggers.sql @@ -0,0 +1,144 @@ +define mw_prefix='{$wgDBprefix}'; + +-- Package to help with making Oracle more like other DBs with respect to +-- auto-incrementing columns. +/*$mw$*/ +CREATE PACKAGE &mw_prefix.lastval_pkg IS + lastval NUMBER; + PROCEDURE setLastval(val IN NUMBER, field OUT NUMBER); + FUNCTION getLastval RETURN NUMBER; +END; +/*$mw$*/ + +/*$mw$*/ +CREATE PACKAGE BODY &mw_prefix.lastval_pkg IS + PROCEDURE setLastval(val IN NUMBER, field OUT NUMBER) IS BEGIN + lastval := val; + field := val; + END; + + FUNCTION getLastval RETURN NUMBER IS BEGIN + RETURN lastval; + END; +END; +/*$mw$*/ + +/*$mw$*/ +CREATE TRIGGER &mw_prefix.mwuser_default_user_id BEFORE INSERT ON &mw_prefix.mwuser + FOR EACH ROW WHEN (new.user_id IS NULL) +BEGIN + &mw_prefix.lastval_pkg.setLastval(user_user_id_seq.nextval, :new.user_id); +END; +/*$mw$*/ + +/*$mw$*/ +CREATE TRIGGER &mw_prefix.page_default_page_id BEFORE INSERT ON &mw_prefix.page + FOR EACH ROW WHEN (new.page_id IS NULL) +BEGIN + &mw_prefix.lastval_pkg.setLastval(page_page_id_seq.nextval, :new.page_id); +END; +/*$mw$*/ + +/*$mw$*/ +CREATE TRIGGER &mw_prefix.revision_default_rev_id BEFORE INSERT ON &mw_prefix.revision + FOR EACH ROW WHEN (new.rev_id IS NULL) +BEGIN + &mw_prefix.lastval_pkg.setLastval(revision_rev_id_seq.nextval, :new.rev_id); +END; +/*$mw$*/ + +/*$mw$*/ +CREATE TRIGGER &mw_prefix.text_default_old_id BEFORE INSERT ON &mw_prefix.text + FOR EACH ROW WHEN (new.old_id IS NULL) +BEGIN + &mw_prefix.lastval_pkg.setLastval(text_old_id_seq.nextval, :new.old_id); +END; +/*$mw$*/ + +/*$mw$*/ +CREATE TRIGGER &mw_prefix.archive_default_ar_id BEFORE INSERT ON &mw_prefix.archive + FOR EACH ROW WHEN (new.ar_id IS NULL) +BEGIN + &mw_prefix.lastval_pkg.setLastval(archive_ar_id_seq.nextval, :new.ar_id); +END; +/*$mw$*/ + +/*$mw$*/ +CREATE TRIGGER &mw_prefix.category_default_cat_id BEFORE INSERT ON &mw_prefix.category + FOR EACH ROW WHEN (new.cat_id IS NULL) +BEGIN + &mw_prefix.lastval_pkg.setLastval(category_cat_id_seq.nextval, :new.cat_id); +END; +/*$mw$*/ + +/*$mw$*/ +CREATE TRIGGER &mw_prefix.externallinks_default_el_id BEFORE INSERT ON &mw_prefix.externallinks + FOR EACH ROW WHEN (new.el_id IS NULL) +BEGIN + &mw_prefix.lastval_pkg.setLastval(externallinks_el_id_seq.nextval, :new.el_id); +END; +/*$mw$*/ + +/*$mw$*/ +CREATE TRIGGER &mw_prefix.ipblocks_default_ipb_id BEFORE INSERT ON &mw_prefix.ipblocks + FOR EACH ROW WHEN (new.ipb_id IS NULL) +BEGIN + &mw_prefix.lastval_pkg.setLastval(ipblocks_ipb_id_seq.nextval, :new.ipb_id); +END; +/*$mw$*/ + +/*$mw$*/ +CREATE TRIGGER &mw_prefix.filearchive_default_fa_id BEFORE INSERT ON &mw_prefix.filearchive + FOR EACH ROW WHEN (new.fa_id IS NULL) +BEGIN + &mw_prefix.lastval_pkg.setLastval(filearchive_fa_id_seq.nextval, :new.fa_id); +END; +/*$mw$*/ + +/*$mw$*/ +CREATE TRIGGER &mw_prefix.uploadstash_default_us_id BEFORE INSERT ON &mw_prefix.uploadstash + FOR EACH ROW WHEN (new.us_id IS NULL) +BEGIN + &mw_prefix.lastval_pkg.setLastval(uploadstash_us_id_seq.nextval, :new.us_id); +END; +/*$mw$*/ + +/*$mw$*/ +CREATE TRIGGER &mw_prefix.recentchanges_default_rc_id BEFORE INSERT ON &mw_prefix.recentchanges + FOR EACH ROW WHEN (new.rc_id IS NULL) +BEGIN + &mw_prefix.lastval_pkg.setLastval(recentchanges_rc_id_seq.nextval, :new.rc_id); +END; +/*$mw$*/ + +/*$mw$*/ +CREATE TRIGGER &mw_prefix.logging_default_log_id BEFORE INSERT ON &mw_prefix.logging + FOR EACH ROW WHEN (new.log_id IS NULL) +BEGIN + &mw_prefix.lastval_pkg.setLastval(logging_log_id_seq.nextval, :new.log_id); +END; +/*$mw$*/ + +/*$mw$*/ +CREATE TRIGGER &mw_prefix.job_default_job_id BEFORE INSERT ON &mw_prefix.job + FOR EACH ROW WHEN (new.job_id IS NULL) +BEGIN + &mw_prefix.lastval_pkg.setLastval(job_job_id_seq.nextval, :new.job_id); +END; +/*$mw$*/ + +/*$mw$*/ +CREATE TRIGGER &mw_prefix.page_restrictions_default_pr_id BEFORE INSERT ON &mw_prefix.page_restrictions + FOR EACH ROW WHEN (new.pr_id IS NULL) +BEGIN + &mw_prefix.lastval_pkg.setLastval(page_restrictions_pr_id_seq.nextval, :new.pr_id); +END; +/*$mw$*/ + +/*$mw$*/ +CREATE TRIGGER &mw_prefix.sites_default_site_id BEFORE INSERT ON &mw_prefix.sites + FOR EACH ROW WHEN (new.site_id IS NULL) +BEGIN + &mw_prefix.lastval_pkg.setLastval(sites_site_id_seq.nextval, :new.site_id); +END; +/*$mw$*/ diff --git a/www/wiki/maintenance/oracle/archives/patch-cat_hidden.sql b/www/wiki/maintenance/oracle/archives/patch-cat_hidden.sql new file mode 100644 index 00000000..d1649c7c --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-cat_hidden.sql @@ -0,0 +1,4 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.category DROP COLUMN cat_hidden; + diff --git a/www/wiki/maintenance/oracle/archives/patch-change_tag-ct_id.sql b/www/wiki/maintenance/oracle/archives/patch-change_tag-ct_id.sql new file mode 100644 index 00000000..6672872f --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-change_tag-ct_id.sql @@ -0,0 +1,6 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.change_tag ADD ( +ct_id NUMBER NOT NULL, +); +ALTER TABLE &mw_prefix.change_tag ADD CONSTRAINT &mw_prefix.change_tag_pk PRIMARY KEY (ct_id); diff --git a/www/wiki/maintenance/oracle/archives/patch-externallinks-el_id.sql b/www/wiki/maintenance/oracle/archives/patch-externallinks-el_id.sql new file mode 100644 index 00000000..a8c443f4 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-externallinks-el_id.sql @@ -0,0 +1,4 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.externallinks ADD el_id NUMBER NOT NULL; +ALTER TABLE &mw_prefix.externallinks ADD CONSTRAINT &mw_prefix.externallinks_pk PRIMARY KEY (el_id);
\ No newline at end of file diff --git a/www/wiki/maintenance/oracle/archives/patch-externallinks-el_index_60.sql b/www/wiki/maintenance/oracle/archives/patch-externallinks-el_index_60.sql new file mode 100644 index 00000000..c4b906d1 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-externallinks-el_index_60.sql @@ -0,0 +1,5 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.externallinks ADD el_index_60 VARBINARY(60) NOT NULL DEFAULT ''; +CREATE INDEX &mw_prefix.externallinks_i04 ON &mw_prefix.externallinks (el_index_60, el_id); +CREATE INDEX &mw_prefix.externallinks_i05 ON &mw_prefix.externallinks (el_from, el_index_60, el_id); diff --git a/www/wiki/maintenance/oracle/archives/patch-fa_sha1.sql b/www/wiki/maintenance/oracle/archives/patch-fa_sha1.sql new file mode 100644 index 00000000..70c9e60c --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-fa_sha1.sql @@ -0,0 +1,5 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.filearchive ADD fa_sha1 VARCHAR2(32); +CREATE INDEX &mw_prefix.filearchive_i05 ON &mw_prefix.filearchive (fa_sha1); + diff --git a/www/wiki/maintenance/oracle/archives/patch-ipblocks_i05_index.sql b/www/wiki/maintenance/oracle/archives/patch-ipblocks_i05_index.sql new file mode 100644 index 00000000..14275383 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-ipblocks_i05_index.sql @@ -0,0 +1,4 @@ +define mw_prefix='{$wgDBprefix}'; + +CREATE INDEX &mw_prefix.ipblocks_i05 ON &mw_prefix.ipblocks (ipb_parent_block_id); + diff --git a/www/wiki/maintenance/oracle/archives/patch-job_attempts.sql b/www/wiki/maintenance/oracle/archives/patch-job_attempts.sql new file mode 100644 index 00000000..b05c8779 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-job_attempts.sql @@ -0,0 +1,4 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.job ADD job_attempts NUMBER DEFAULT 0 NOT NULL; +CREATE INDEX &mw_prefix.job_i05 ON &mw_prefix.job (job_attempts); diff --git a/www/wiki/maintenance/oracle/archives/patch-job_timestamp_field.sql b/www/wiki/maintenance/oracle/archives/patch-job_timestamp_field.sql new file mode 100644 index 00000000..4901c87c --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-job_timestamp_field.sql @@ -0,0 +1,4 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.job ADD job_timestamp TIMESTAMP(6) WITH TIME ZONE NULL; + diff --git a/www/wiki/maintenance/oracle/archives/patch-job_timestamp_index.sql b/www/wiki/maintenance/oracle/archives/patch-job_timestamp_index.sql new file mode 100644 index 00000000..6db43046 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-job_timestamp_index.sql @@ -0,0 +1,4 @@ +define mw_prefix='{$wgDBprefix}'; + +CREATE INDEX &mw_prefix.job_i02 ON &mw_prefix.job (job_timestamp); + diff --git a/www/wiki/maintenance/oracle/archives/patch-job_token.sql b/www/wiki/maintenance/oracle/archives/patch-job_token.sql new file mode 100644 index 00000000..1a730e95 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-job_token.sql @@ -0,0 +1,12 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.job ADD ( + job_random NUMBER DEFAULT 0 NOT NULL, + job_token VARCHAR2(32), + job_token_timestamp TIMESTAMP(6) WITH TIME ZONE, + job_sha1 VARCHAR2(32) +); + +CREATE INDEX &mw_prefix.job_i03 ON &mw_prefix.job (job_sha1); +CREATE INDEX &mw_prefix.job_i04 ON &mw_prefix.job (job_cmd,job_token,job_random); + diff --git a/www/wiki/maintenance/oracle/archives/patch-logging_type_action_index.sql b/www/wiki/maintenance/oracle/archives/patch-logging_type_action_index.sql new file mode 100644 index 00000000..d30e0cfc --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-logging_type_action_index.sql @@ -0,0 +1,4 @@ +define mw_prefix='{$wgDBprefix}'; + +CREATE INDEX &mw_prefix.logging_i05 ON &mw_prefix.logging (log_type, log_action, log_timestamp); + diff --git a/www/wiki/maintenance/oracle/archives/patch-logging_user_text_time_index.sql b/www/wiki/maintenance/oracle/archives/patch-logging_user_text_time_index.sql new file mode 100644 index 00000000..e04abf5f --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-logging_user_text_time_index.sql @@ -0,0 +1,4 @@ +define mw_prefix='{$wgDBprefix}'; + +CREATE INDEX &mw_prefix.logging_i07 ON &mw_prefix.logging (log_user_text, log_timestamp); + diff --git a/www/wiki/maintenance/oracle/archives/patch-logging_user_text_type_time_index.sql b/www/wiki/maintenance/oracle/archives/patch-logging_user_text_type_time_index.sql new file mode 100644 index 00000000..c1c0d4f2 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-logging_user_text_type_time_index.sql @@ -0,0 +1,4 @@ +define mw_prefix='{$wgDBprefix}'; + +CREATE INDEX &mw_prefix.logging_i06 ON &mw_prefix.logging (log_user_text, log_type, log_timestamp); + diff --git a/www/wiki/maintenance/oracle/archives/patch-page-page_content_model.sql b/www/wiki/maintenance/oracle/archives/patch-page-page_content_model.sql new file mode 100644 index 00000000..e5839d9a --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-page-page_content_model.sql @@ -0,0 +1,3 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.page ADD page_content_model VARCHAR2(32); diff --git a/www/wiki/maintenance/oracle/archives/patch-page-page_lang.sql b/www/wiki/maintenance/oracle/archives/patch-page-page_lang.sql new file mode 100644 index 00000000..cae7cf90 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-page-page_lang.sql @@ -0,0 +1,3 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.page ADD page_lang VARCHAR2(35); diff --git a/www/wiki/maintenance/oracle/archives/patch-page_links_updated.sql b/www/wiki/maintenance/oracle/archives/patch-page_links_updated.sql new file mode 100644 index 00000000..53603294 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-page_links_updated.sql @@ -0,0 +1,4 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.page ADD page_links_updated TIMESTAMP(6) WITH TIME ZONE; + diff --git a/www/wiki/maintenance/oracle/archives/patch-page_redirect_namespace_len.sql b/www/wiki/maintenance/oracle/archives/patch-page_redirect_namespace_len.sql new file mode 100644 index 00000000..1f8b9d9a --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-page_redirect_namespace_len.sql @@ -0,0 +1,4 @@ +define mw_prefix='{$wgDBprefix}'; + +CREATE INDEX &mw_prefix.page_i03 ON &mw_prefix.page (page_is_redirect, page_namespace, page_len); + diff --git a/www/wiki/maintenance/oracle/archives/patch-page_restrictions_pkuk_fix.sql b/www/wiki/maintenance/oracle/archives/patch-page_restrictions_pkuk_fix.sql new file mode 100644 index 00000000..56c392c1 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-page_restrictions_pkuk_fix.sql @@ -0,0 +1,7 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.page_restrictions DROP CONSTRAINT &mw_prefix.page_restrictions_pk; + +ALTER TABLE &mw_prefix.page_restrictions ADD CONSTRAINT &mw_prefix.page_restrictions_pk PRIMARY KEY (pr_id); + +CREATE UNIQUE INDEX &mw_prefix.page_restrictions_u01 ON &mw_prefix.page_restrictions (pr_page,pr_type); diff --git a/www/wiki/maintenance/oracle/archives/patch-rc_moved.sql b/www/wiki/maintenance/oracle/archives/patch-rc_moved.sql new file mode 100644 index 00000000..2a71315d --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-rc_moved.sql @@ -0,0 +1,4 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.recentchanges DROP ( rc_moved_to_ns, rc_moved_to_title ); + diff --git a/www/wiki/maintenance/oracle/archives/patch-rc_source.sql b/www/wiki/maintenance/oracle/archives/patch-rc_source.sql new file mode 100644 index 00000000..0c80afab --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-rc_source.sql @@ -0,0 +1,3 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.recentchanges ADD rc_source VARCHAR2(16); diff --git a/www/wiki/maintenance/oracle/archives/patch-rev_sha1_field.sql b/www/wiki/maintenance/oracle/archives/patch-rev_sha1_field.sql new file mode 100644 index 00000000..80544e89 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-rev_sha1_field.sql @@ -0,0 +1,4 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.revision ADD rev_sha1 VARCHAR2(32); + diff --git a/www/wiki/maintenance/oracle/archives/patch-revision-rev_content_format.sql b/www/wiki/maintenance/oracle/archives/patch-revision-rev_content_format.sql new file mode 100644 index 00000000..ebde71c9 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-revision-rev_content_format.sql @@ -0,0 +1,3 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.revision ADD rev_content_format VARCHAR2(64); diff --git a/www/wiki/maintenance/oracle/archives/patch-revision-rev_content_model.sql b/www/wiki/maintenance/oracle/archives/patch-revision-rev_content_model.sql new file mode 100644 index 00000000..dd226423 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-revision-rev_content_model.sql @@ -0,0 +1,3 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.revision ADD rev_content_model VARCHAR2(32); diff --git a/www/wiki/maintenance/oracle/archives/patch-revision_i05_index.sql b/www/wiki/maintenance/oracle/archives/patch-revision_i05_index.sql new file mode 100644 index 00000000..929c7b31 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-revision_i05_index.sql @@ -0,0 +1,4 @@ +define mw_prefix='{$wgDBprefix}'; + +CREATE INDEX &mw_prefix.revision_i05 ON &mw_prefix.revision (rev_page,rev_user,rev_timestamp); + diff --git a/www/wiki/maintenance/oracle/archives/patch-site_stats-pk.sql b/www/wiki/maintenance/oracle/archives/patch-site_stats-pk.sql new file mode 100644 index 00000000..a288c08d --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-site_stats-pk.sql @@ -0,0 +1,4 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.site_stats DROP CONSTRAINT &mw_prefix.site_stats_u01; +ALTER TABLE &mw_prefix.site_stats ADD CONSTRAINT &mw_prefix.site_stats_pk PRIMARY KEY(ss_row_id); diff --git a/www/wiki/maintenance/oracle/archives/patch-sites.sql b/www/wiki/maintenance/oracle/archives/patch-sites.sql new file mode 100644 index 00000000..868b210f --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-sites.sql @@ -0,0 +1,34 @@ +define mw_prefix='{$wgDBprefix}'; + +CREATE SEQUENCE sites_site_id_seq MINVALUE 0 START WITH 0; +CREATE TABLE &mw_prefix.sites ( + site_id NUMBER NOT NULL, + site_global_key VARCHAR2(32) NOT NULL, + site_type VARCHAR2(32) NOT NULL, + site_group VARCHAR2(32) NOT NULL, + site_source VARCHAR2(32) NOT NULL, + site_language VARCHAR2(32) NOT NULL, + site_protocol VARCHAR2(32) NOT NULL, + site_domain VARCHAR2(255) NOT NULL, + site_data BLOB NOT NULL, + site_forward NUMBER(1) NOT NULL, + site_config BLOB NOT NULL +); +ALTER TABLE &mw_prefix.sites ADD CONSTRAINT &mw_prefix.sites_pk PRIMARY KEY (site_id); +CREATE UNIQUE INDEX &mw_prefix.sites_u01 ON &mw_prefix.sites (site_global_key); +CREATE INDEX &mw_prefix.sites_i01 ON &mw_prefix.sites (site_type); +CREATE INDEX &mw_prefix.sites_i02 ON &mw_prefix.sites (site_group); +CREATE INDEX &mw_prefix.sites_i03 ON &mw_prefix.sites (site_source); +CREATE INDEX &mw_prefix.sites_i04 ON &mw_prefix.sites (site_language); +CREATE INDEX &mw_prefix.sites_i05 ON &mw_prefix.sites (site_protocol); +CREATE INDEX &mw_prefix.sites_i06 ON &mw_prefix.sites (site_domain); +CREATE INDEX &mw_prefix.sites_i07 ON &mw_prefix.sites (site_forward); + +CREATE TABLE &mw_prefix.site_identifiers ( + si_site NUMBER NOT NULL, + si_type VARCHAR2(32) NOT NULL, + si_key VARCHAR2(32) NOT NULL +); +CREATE UNIQUE INDEX &mw_prefix.site_identifiers_u01 ON &mw_prefix.site_identifiers (si_type, si_key); +CREATE INDEX &mw_prefix.site_identifiers_i01 ON &mw_prefix.site_identifiers (si_site); +CREATE INDEX &mw_prefix.site_identifiers_i02 ON &mw_prefix.site_identifiers (si_key); diff --git a/www/wiki/maintenance/oracle/archives/patch-ss_admins.sql b/www/wiki/maintenance/oracle/archives/patch-ss_admins.sql new file mode 100644 index 00000000..c2e9242e --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-ss_admins.sql @@ -0,0 +1,4 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.site_stats DROP COLUMN ss_admins; + diff --git a/www/wiki/maintenance/oracle/archives/patch-tag_summary-ts_id.sql b/www/wiki/maintenance/oracle/archives/patch-tag_summary-ts_id.sql new file mode 100644 index 00000000..91c33383 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-tag_summary-ts_id.sql @@ -0,0 +1,6 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.tag_summary ADD ( +ts_id NUMBER NOT NULL, +); +ALTER TABLE &mw_prefix.tag_summary ADD CONSTRAINT &mw_prefix.tag_summary_pk PRIMARY KEY (ts_id); diff --git a/www/wiki/maintenance/oracle/archives/patch-testrun.sql b/www/wiki/maintenance/oracle/archives/patch-testrun.sql new file mode 100644 index 00000000..84facabc --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-testrun.sql @@ -0,0 +1,37 @@ +-- +-- Optional tables for parserTests recording mode +-- With --record option, success data will be saved to these tables, +-- and comparisons of what's changed from the previous run will be +-- displayed at the end of each run. +-- +-- defines must comply with ^define\s*([^\s=]*)\s*=\s?'\{\$([^\}]*)\}'; +define mw_prefix='{$wgDBprefix}'; + +DROP TABLE &mw_prefix.testitem CASCADE CONSTRAINTS; +DROP TABLE &mw_prefix.testrun CASCADE CONSTRAINTS; + +CREATE SEQUENCE testrun_tr_id_seq; +CREATE TABLE &mw_prefix.testrun ( + tr_id NUMBER NOT NULL, + tr_date DATE, + tr_mw_version BLOB, + tr_php_version BLOB, + tr_db_version BLOB, + tr_uname BLOB, +); +ALTER TABLE &mw_prefix.testrun ADD CONSTRAINT &mw_prefix.testrun_pk PRIMARY KEY (tr_id); +CREATE OR REPLACE TRIGGER &mw_prefix.testrun_bir +BEFORE UPDATE FOR EACH ROW +ON &mw_prefix.testrun +BEGIN + SELECT testrun_tr_id_seq.NEXTVAL into :NEW.tr_id FROM dual; +END; + +CREATE TABLE /*$wgDBprefix*/testitem ( + ti_run NUMBER NOT NULL REFERENCES &mw_prefix.testrun (tr_id) ON DELETE CASCADE, + ti_name VARCHAR22(255), + ti_success NUMBER(1) +); +CREATE UNIQUE INDEX &mw_prefix.testitem_u01 ON &mw_prefix.testitem (ti_run, ti_name); +CREATE UNIQUE INDEX &mw_prefix.testitem_u01 ON &mw_prefix.testitem (ti_run, ti_success); + diff --git a/www/wiki/maintenance/oracle/archives/patch-ufg_group-length-increase-255.sql b/www/wiki/maintenance/oracle/archives/patch-ufg_group-length-increase-255.sql new file mode 100644 index 00000000..6a4a7517 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-ufg_group-length-increase-255.sql @@ -0,0 +1,9 @@ +define mw_prefix='{$wgDBprefix}'; + +/*$mw$*/ +BEGIN + EXECUTE IMMEDIATE 'ALTER TABLE &mw_prefix.user_former_groups MODIFY ufg_group VARCHAR2(255) NOT NULL'; +EXCEPTION WHEN OTHERS THEN + IF (SQLCODE = -01442) THEN NULL; ELSE RAISE; END IF; +END; +/*$mw$*/ diff --git a/www/wiki/maintenance/oracle/archives/patch-ug_group-length-increase-255.sql b/www/wiki/maintenance/oracle/archives/patch-ug_group-length-increase-255.sql new file mode 100644 index 00000000..00a5e7b2 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-ug_group-length-increase-255.sql @@ -0,0 +1,9 @@ +define mw_prefix='{$wgDBprefix}'; + +/*$mw$*/ +BEGIN + EXECUTE IMMEDIATE 'ALTER TABLE &mw_prefix.user_groups MODIFY ug_group VARCHAR2(255) NOT NULL'; +EXCEPTION WHEN OTHERS THEN + IF (SQLCODE = -01442) THEN NULL; ELSE RAISE; END IF; +END; +/*$mw$*/ diff --git a/www/wiki/maintenance/oracle/archives/patch-up_property.sql b/www/wiki/maintenance/oracle/archives/patch-up_property.sql new file mode 100644 index 00000000..c8e2dd95 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-up_property.sql @@ -0,0 +1,3 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.user_properties MODIFY up_property varchar2(255); diff --git a/www/wiki/maintenance/oracle/archives/patch-uploadstash-us_props.sql b/www/wiki/maintenance/oracle/archives/patch-uploadstash-us_props.sql new file mode 100644 index 00000000..8962dc7c --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-uploadstash-us_props.sql @@ -0,0 +1,4 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.uploadstash ADD us_props BLOB; + diff --git a/www/wiki/maintenance/oracle/archives/patch-uploadstash.sql b/www/wiki/maintenance/oracle/archives/patch-uploadstash.sql new file mode 100644 index 00000000..3e37ceff --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-uploadstash.sql @@ -0,0 +1,25 @@ +define mw_prefix='{$wgDBprefix}'; + +CREATE SEQUENCE uploadstash_us_id_seq; +CREATE TABLE &mw_prefix.uploadstash ( + us_id NUMBER NOT NULL, + us_user NUMBER DEFAULT 0 NOT NULL, + us_key VARCHAR2(255) NOT NULL, + us_orig_path VARCHAR2(255) NOT NULL, + us_path VARCHAR2(255) NOT NULL, + us_source_type VARCHAR2(50), + us_timestamp TIMESTAMP(6) WITH TIME ZONE, + us_status VARCHAR2(50) NOT NULL, + us_size NUMBER NOT NULL, + us_sha1 VARCHAR2(32) NOT NULL, + us_mime VARCHAR2(255), + us_media_type VARCHAR2(32) DEFAULT NULL, + us_image_width NUMBER, + us_image_height NUMBER, + us_image_bits NUMBER +); +ALTER TABLE &mw_prefix.uploadstash ADD CONSTRAINT &mw_prefix.uploadstash_pk PRIMARY KEY (us_id); +ALTER TABLE &mw_prefix.uploadstash ADD CONSTRAINT &mw_prefix.uploadstash_fk1 FOREIGN KEY (us_user) REFERENCES &mw_prefix.mwuser(user_id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED; +CREATE INDEX &mw_prefix.uploadstash_i01 ON &mw_prefix.uploadstash (us_user); +CREATE INDEX &mw_prefix.uploadstash_i02 ON &mw_prefix.uploadstash (us_timestamp); +CREATE UNIQUE INDEX &mw_prefix.uploadstash_u01 ON &mw_prefix.uploadstash (us_key); diff --git a/www/wiki/maintenance/oracle/archives/patch-us_chunk_inx_field.sql b/www/wiki/maintenance/oracle/archives/patch-us_chunk_inx_field.sql new file mode 100644 index 00000000..43ee16ec --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-us_chunk_inx_field.sql @@ -0,0 +1,4 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.uploadstash ADD us_chunk_inx NUMBER; + diff --git a/www/wiki/maintenance/oracle/archives/patch-user_email_index.sql b/www/wiki/maintenance/oracle/archives/patch-user_email_index.sql new file mode 100644 index 00000000..e34d8656 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-user_email_index.sql @@ -0,0 +1,4 @@ +define mw_prefix='{$wgDBprefix}'; + +CREATE INDEX &mw_prefix.mwuser_i02 ON &mw_prefix.mwuser (user_email); + diff --git a/www/wiki/maintenance/oracle/archives/patch-user_former_groups.sql b/www/wiki/maintenance/oracle/archives/patch-user_former_groups.sql new file mode 100644 index 00000000..c14824eb --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-user_former_groups.sql @@ -0,0 +1,9 @@ +define mw_prefix='{$wgDBprefix}'; + +CREATE TABLE &mw_prefix.user_former_groups ( + ufg_user NUMBER DEFAULT 0 NOT NULL, + ufg_group VARCHAR2(255) NOT NULL +); +ALTER TABLE &mw_prefix.user_former_groups ADD CONSTRAINT &mw_prefix.user_former_groups_fk1 FOREIGN KEY (ufg_user) REFERENCES &mw_prefix.mwuser(user_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +CREATE UNIQUE INDEX &mw_prefix.user_former_groups_u01 ON &mw_prefix.user_former_groups (ufg_user,ufg_group); + diff --git a/www/wiki/maintenance/oracle/archives/patch-user_groups-ug_expiry.sql b/www/wiki/maintenance/oracle/archives/patch-user_groups-ug_expiry.sql new file mode 100644 index 00000000..d5376a31 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-user_groups-ug_expiry.sql @@ -0,0 +1,8 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.user_groups ADD ( +ug_expiry TIMESTAMP(6) WITH TIME ZONE NULL +); +DROP INDEX IF EXISTS &mw_prefix.user_groups_u01; +ALTER TABLE &mw_prefix.user_groups ADD CONSTRAINT &mw_prefix.user_groups_pk PRIMARY KEY (ug_user,ug_group); +CREATE INDEX &mw_prefix.user_groups_i02 ON &mw_prefix.user_groups (ug_expiry); diff --git a/www/wiki/maintenance/oracle/archives/patch-user_password_expire.sql b/www/wiki/maintenance/oracle/archives/patch-user_password_expire.sql new file mode 100644 index 00000000..824cc820 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-user_password_expire.sql @@ -0,0 +1,3 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.mwuser ADD user_password_expires TIMESTAMP(6) WITH TIME ZONE; diff --git a/www/wiki/maintenance/oracle/archives/patch-watchlist-wl_id.sql b/www/wiki/maintenance/oracle/archives/patch-watchlist-wl_id.sql new file mode 100644 index 00000000..4f7180d5 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch-watchlist-wl_id.sql @@ -0,0 +1,6 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.watchlist ADD ( +wl_id NUMBER NOT NULL, +); +ALTER TABLE &mw_prefix.watchlist ADD CONSTRAINT &mw_prefix.watchlist_pk PRIMARY KEY (wl_id); diff --git a/www/wiki/maintenance/oracle/archives/patch_16_17_schema_changes.sql b/www/wiki/maintenance/oracle/archives/patch_16_17_schema_changes.sql new file mode 100644 index 00000000..dfaaf5cb --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch_16_17_schema_changes.sql @@ -0,0 +1,84 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.archive MODIFY ar_user DEFAULT 0 NOT NULL; +ALTER TABLE &mw_prefix.archive MODIFY ar_deleted CHAR(1); +CREATE INDEX &mw_prefix.archive_i03 ON &mw_prefix.archive (ar_rev_id); + +ALTER TABLE &mw_prefix.page MODIFY page_is_redirect default '0'; +ALTER TABLE &mw_prefix.page MODIFY page_is_new default '0'; +ALTER TABLE &mw_prefix.page MODIFY page_latest default 0; +ALTER TABLE &mw_prefix.page MODIFY page_len default 0; + +ALTER TABLE &mw_prefix.categorylinks MODIFY cl_sortkey VARCHAR2(230); +ALTER TABLE &mw_prefix.categorylinks ADD cl_sortkey_prefix VARCHAR2(255) DEFAULT '' NOT NULL; +ALTER TABLE &mw_prefix.categorylinks ADD cl_collation VARCHAR2(32) DEFAULT '' NOT NULL; +ALTER TABLE &mw_prefix.categorylinks ADD cl_type VARCHAR2(6) DEFAULT 'page' NOT NULL; +DROP INDEX &mw_prefix.categorylinks_i01; +CREATE INDEX &mw_prefix.categorylinks_i01 ON &mw_prefix.categorylinks (cl_to,cl_type,cl_sortkey,cl_from); +CREATE INDEX &mw_prefix.categorylinks_i03 ON &mw_prefix.categorylinks (cl_collation); + +ALTER TABLE &mw_prefix.filearchive MODIFY fa_deleted_user DEFAULT 0 NOT NULL; +ALTER TABLE &mw_prefix.filearchive MODIFY fa_size DEFAULT 0; +ALTER TABLE &mw_prefix.filearchive MODIFY fa_width DEFAULT 0; +ALTER TABLE &mw_prefix.filearchive MODIFY fa_height DEFAULT 0; +ALTER TABLE &mw_prefix.filearchive MODIFY fa_bits DEFAULT 0 NOT NULL; +ALTER TABLE &mw_prefix.filearchive MODIFY fa_user DEFAULT 0 NOT NULL; +ALTER TABLE &mw_prefix.filearchive MODIFY fa_deleted DEFAULT 0; + +ALTER TABLE &mw_prefix.image MODIFY img_size DEFAULT 0; +ALTER TABLE &mw_prefix.image MODIFY img_width DEFAULT 0; +ALTER TABLE &mw_prefix.image MODIFY img_height DEFAULT 0; +ALTER TABLE &mw_prefix.image MODIFY img_bits DEFAULT 0 NOT NULL; +ALTER TABLE &mw_prefix.image MODIFY img_user DEFAULT 0 NOT NULL; + +ALTER TABLE &mw_prefix.interwiki ADD iw_api BLOB DEFAULT EMPTY_BLOB(); +ALTER TABLE &mw_prefix.interwiki MODIFY iw_api DEFAULT NULL NOT NULL; +ALTER TABLE &mw_prefix.interwiki ADD iw_wikiid VARCHAR2(64); + +ALTER TABLE &mw_prefix.ipblocks MODIFY ipb_user DEFAULT 0 NOT NULL; +ALTER TABLE &mw_prefix.ipblocks MODIFY ipb_by DEFAULT 0; + +CREATE TABLE &mw_prefix.iwlinks ( + iwl_from NUMBER DEFAULT 0 NOT NULL, + iwl_prefix VARCHAR2(20) DEFAULT '' NOT NULL, + iwl_title VARCHAR2(255) DEFAULT '' NOT NULL +); +CREATE UNIQUE INDEX &mw_prefix.iwlinks_ui01 ON &mw_prefix.iwlinks (iwl_from, iwl_prefix, iwl_title); +CREATE UNIQUE INDEX &mw_prefix.iwlinks_ui02 ON &mw_prefix.iwlinks (iwl_prefix, iwl_title, iwl_from); + +ALTER TABLE &mw_prefix.logging MODIFY log_user DEFAULT 0 NOT NULL; +ALTER TABLE &mw_prefix.logging MODIFY log_deleted CHAR(1); + +CREATE TABLE &mw_prefix.module_deps ( + md_module VARCHAR2(255) NOT NULL, + md_skin VARCHAR2(32) NOT NULL, + md_deps BLOB NOT NULL +); +CREATE UNIQUE INDEX &mw_prefix.module_deps_u01 ON &mw_prefix.module_deps (md_module, md_skin); + +ALTER TABLE &mw_prefix.oldimage MODIFY oi_name DEFAULT 0; +ALTER TABLE &mw_prefix.oldimage MODIFY oi_size DEFAULT 0; +ALTER TABLE &mw_prefix.oldimage MODIFY oi_width DEFAULT 0; +ALTER TABLE &mw_prefix.oldimage MODIFY oi_height DEFAULT 0; +ALTER TABLE &mw_prefix.oldimage MODIFY oi_bits DEFAULT 0; +ALTER TABLE &mw_prefix.oldimage MODIFY oi_user DEFAULT 0 NOT NULL; + +ALTER TABLE &mw_prefix.querycache MODIFY qc_value DEFAULT 0; + +ALTER TABLE &mw_prefix.recentchanges MODIFY rc_user DEFAULT 0 NOT NULL; +ALTER TABLE &mw_prefix.recentchanges MODIFY rc_cur_id DEFAULT 0 NOT NULL; +ALTER TABLE &mw_prefix.recentchanges MODIFY rc_this_oldid DEFAULT 0; +ALTER TABLE &mw_prefix.recentchanges MODIFY rc_last_oldid DEFAULT 0; +ALTER TABLE &mw_prefix.recentchanges MODIFY rc_moved_to_ns DEFAULT 0 NOT NULL; +ALTER TABLE &mw_prefix.recentchanges MODIFY rc_deleted CHAR(1); +ALTER TABLE &mw_prefix.recentchanges MODIFY rc_logid DEFAULT 0; + +ALTER TABLE &mw_prefix.revision MODIFY rev_page NOT NULL; +ALTER TABLE &mw_prefix.revision MODIFY rev_user DEFAULT 0; + +ALTER TABLE &mw_prefix.updatelog ADD ul_value BLOB; + +ALTER TABLE &mw_prefix.user_groups MODIFY ug_user DEFAULT 0 NOT NULL; + +ALTER TABLE &mw_prefix.user_newtalk MODIFY user_id DEFAULT 0; + diff --git a/www/wiki/maintenance/oracle/archives/patch_create_17_functions.sql b/www/wiki/maintenance/oracle/archives/patch_create_17_functions.sql new file mode 100644 index 00000000..6c9c9542 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch_create_17_functions.sql @@ -0,0 +1,125 @@ +define mw_prefix='{$wgDBprefix}'; + +/*$mw$*/ +CREATE OR REPLACE PROCEDURE duplicate_table(p_tabname IN VARCHAR2, + p_oldprefix IN VARCHAR2, + p_newprefix IN VARCHAR2, + p_temporary IN BOOLEAN) IS + e_table_not_exist EXCEPTION; + PRAGMA EXCEPTION_INIT(e_table_not_exist, -00942); + l_temp_ei_sql VARCHAR2(2000); +BEGIN + BEGIN + EXECUTE IMMEDIATE 'DROP TABLE ' || p_newprefix || p_tabname || + ' CASCADE CONSTRAINTS'; + EXCEPTION + WHEN e_table_not_exist THEN + NULL; + END; + IF (p_temporary) THEN + EXECUTE IMMEDIATE 'CREATE GLOBAL TEMPORARY TABLE ' || p_newprefix || + p_tabname || ' AS SELECT * FROM ' || p_oldprefix || + p_tabname || ' WHERE ROWNUM = 0'; + ELSE + EXECUTE IMMEDIATE 'CREATE TABLE ' || p_newprefix || p_tabname || + ' AS SELECT * FROM ' || p_oldprefix || p_tabname || + ' WHERE ROWNUM = 0'; + END IF; + FOR rc IN (SELECT column_name, data_default + FROM user_tab_columns + WHERE table_name = p_oldprefix || p_tabname + AND data_default IS NOT NULL) LOOP + EXECUTE IMMEDIATE 'ALTER TABLE ' || p_newprefix || p_tabname || + ' MODIFY ' || rc.column_name || ' DEFAULT ' || + SUBSTR(rc.data_default, 1, 2000); + END LOOP; + FOR rc IN (SELECT REPLACE(REPLACE(DBMS_LOB.SUBSTR(DBMS_METADATA.get_ddl('CONSTRAINT', + constraint_name), + 32767, + 1), + USER || '"."' || p_oldprefix, + USER || '"."' || p_newprefix), + '"' || constraint_name || '"', + '"' || p_newprefix || constraint_name || '"') DDLVC2, + constraint_name + FROM user_constraints uc + WHERE table_name = p_oldprefix || p_tabname + AND constraint_type = 'P') LOOP + l_temp_ei_sql := SUBSTR(rc.ddlvc2, 1, INSTR(rc.ddlvc2, 'PCTFREE') - 1); + l_temp_ei_sql := SUBSTR(l_temp_ei_sql, 1, INSTR(l_temp_ei_sql, ')', INSTR(l_temp_ei_sql, 'PRIMARY KEY')+1)+1); + EXECUTE IMMEDIATE l_temp_ei_sql; + END LOOP; + IF (NOT p_temporary) THEN + FOR rc IN (SELECT REPLACE(DBMS_LOB.SUBSTR(DBMS_METADATA.get_ddl('REF_CONSTRAINT', + constraint_name), + 32767, + 1), + USER || '"."' || p_oldprefix, + USER || '"."' || p_newprefix) DDLVC2, + constraint_name + FROM user_constraints uc + WHERE table_name = p_oldprefix || p_tabname + AND constraint_type = 'R') LOOP + EXECUTE IMMEDIATE rc.ddlvc2; + END LOOP; + END IF; + FOR rc IN (SELECT REPLACE(REPLACE(DBMS_LOB.SUBSTR(DBMS_METADATA.get_ddl('INDEX', + index_name), + 32767, + 1), + USER || '"."' || p_oldprefix, + USER || '"."' || p_newprefix), + '"' || index_name || '"', + '"' || p_newprefix || index_name || '"') DDLVC2, + index_name, + index_type + FROM user_indexes ui + WHERE table_name = p_oldprefix || p_tabname + AND index_type NOT IN ('LOB', 'DOMAIN') + AND NOT EXISTS + (SELECT NULL + FROM user_constraints + WHERE table_name = ui.table_name + AND constraint_name = ui.index_name)) LOOP + l_temp_ei_sql := SUBSTR(rc.ddlvc2, 1, INSTR(rc.ddlvc2, 'PCTFREE') - 1); + l_temp_ei_sql := SUBSTR(l_temp_ei_sql, 1, INSTR(l_temp_ei_sql, ')', INSTR(l_temp_ei_sql, '"' || USER || '"."' || p_newprefix || '"')+1)+1); + EXECUTE IMMEDIATE l_temp_ei_sql; + END LOOP; + FOR rc IN (SELECT REPLACE(REPLACE(UPPER(DBMS_LOB.SUBSTR(DBMS_METADATA.get_ddl('TRIGGER', + trigger_name), + 32767, + 1)), + USER || '"."' || p_oldprefix, + USER || '"."' || p_newprefix), + ' ON ' || p_oldprefix || p_tabname, + ' ON ' || p_newprefix || p_tabname) DDLVC2, + trigger_name + FROM user_triggers + WHERE table_name = p_oldprefix || p_tabname) LOOP + l_temp_ei_sql := SUBSTR(rc.ddlvc2, 1, INSTR(rc.ddlvc2, 'ALTER ') - 1); + dbms_output.put_line(l_temp_ei_sql); + EXECUTE IMMEDIATE l_temp_ei_sql; + END LOOP; +END; +/*$mw$*/ + +CREATE OR REPLACE TYPE GET_OUTPUT_TYPE IS TABLE OF VARCHAR2(255); + +/*$mw$*/ +CREATE OR REPLACE FUNCTION GET_OUTPUT_LINES RETURN GET_OUTPUT_TYPE PIPELINED AS + v_line VARCHAR2(255); + v_status INTEGER := 0; +BEGIN + + LOOP + DBMS_OUTPUT.GET_LINE(v_line, v_status); + IF (v_status = 0) THEN RETURN; END IF; + PIPE ROW (v_line); + END LOOP; + RETURN; +EXCEPTION + WHEN OTHERS THEN + RETURN; +END; +/*$mw$*/ + diff --git a/www/wiki/maintenance/oracle/archives/patch_fk_rename_deferred.sql b/www/wiki/maintenance/oracle/archives/patch_fk_rename_deferred.sql new file mode 100644 index 00000000..ca9c997f --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch_fk_rename_deferred.sql @@ -0,0 +1,40 @@ +define mw_prefix='{$wgDBprefix}'; + +/*$mw$*/ +BEGIN +-- drop all, recreate manual in case anyone was missing + FOR cc1 IN (SELECT uc.table_name, + uc.constraint_name + FROM user_constraints uc + WHERE uc.constraint_type = 'R') LOOP + EXECUTE IMMEDIATE 'ALTER TABLE &mw_prefix.' || cc1.table_name || + ' DROP CONSTRAINT ' || cc1.constraint_name; + END LOOP; +END; +/*$mw$*/ + +ALTER TABLE &mw_prefix.user_groups ADD CONSTRAINT &mw_prefix.user_groups_fk1 FOREIGN KEY (ug_user) REFERENCES &mw_prefix.mwuser(user_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE &mw_prefix.user_newtalk ADD CONSTRAINT &mw_prefix.user_newtalk_fk1 FOREIGN KEY (user_id) REFERENCES &mw_prefix.mwuser(user_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE &mw_prefix.revision ADD CONSTRAINT &mw_prefix.revision_fk1 FOREIGN KEY (rev_page) REFERENCES &mw_prefix.page(page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE &mw_prefix.revision ADD CONSTRAINT &mw_prefix.revision_fk2 FOREIGN KEY (rev_user) REFERENCES &mw_prefix.mwuser(user_id) DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE &mw_prefix.archive ADD CONSTRAINT &mw_prefix.archive_fk1 FOREIGN KEY (ar_user) REFERENCES &mw_prefix.mwuser(user_id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE &mw_prefix.pagelinks ADD CONSTRAINT &mw_prefix.pagelinks_fk1 FOREIGN KEY (pl_from) REFERENCES &mw_prefix.page(page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE &mw_prefix.templatelinks ADD CONSTRAINT &mw_prefix.templatelinks_fk1 FOREIGN KEY (tl_from) REFERENCES &mw_prefix.page(page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE &mw_prefix.imagelinks ADD CONSTRAINT &mw_prefix.imagelinks_fk1 FOREIGN KEY (il_from) REFERENCES &mw_prefix.page(page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE &mw_prefix.categorylinks ADD CONSTRAINT &mw_prefix.categorylinks_fk1 FOREIGN KEY (cl_from) REFERENCES &mw_prefix.page(page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE &mw_prefix.externallinks ADD CONSTRAINT &mw_prefix.externallinks_fk1 FOREIGN KEY (el_from) REFERENCES &mw_prefix.page(page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE &mw_prefix.langlinks ADD CONSTRAINT &mw_prefix.langlinks_fk1 FOREIGN KEY (ll_from) REFERENCES &mw_prefix.page(page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE &mw_prefix.ipblocks ADD CONSTRAINT &mw_prefix.ipblocks_fk1 FOREIGN KEY (ipb_user) REFERENCES &mw_prefix.mwuser(user_id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE &mw_prefix.ipblocks ADD CONSTRAINT &mw_prefix.ipblocks_fk2 FOREIGN KEY (ipb_by) REFERENCES &mw_prefix.mwuser(user_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE &mw_prefix.image ADD CONSTRAINT &mw_prefix.image_fk1 FOREIGN KEY (img_user) REFERENCES &mw_prefix.mwuser(user_id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE &mw_prefix.oldimage ADD CONSTRAINT &mw_prefix.oldimage_fk1 FOREIGN KEY (oi_name) REFERENCES &mw_prefix.image(img_name) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE &mw_prefix.oldimage ADD CONSTRAINT &mw_prefix.oldimage_fk2 FOREIGN KEY (oi_user) REFERENCES &mw_prefix.mwuser(user_id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE &mw_prefix.filearchive ADD CONSTRAINT &mw_prefix.filearchive_fk1 FOREIGN KEY (fa_deleted_user) REFERENCES &mw_prefix.mwuser(user_id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE &mw_prefix.filearchive ADD CONSTRAINT &mw_prefix.filearchive_fk2 FOREIGN KEY (fa_user) REFERENCES &mw_prefix.mwuser(user_id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE &mw_prefix.recentchanges ADD CONSTRAINT &mw_prefix.recentchanges_fk1 FOREIGN KEY (rc_user) REFERENCES &mw_prefix.mwuser(user_id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE &mw_prefix.recentchanges ADD CONSTRAINT &mw_prefix.recentchanges_fk2 FOREIGN KEY (rc_cur_id) REFERENCES &mw_prefix.page(page_id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE &mw_prefix.watchlist ADD CONSTRAINT &mw_prefix.watchlist_fk1 FOREIGN KEY (wl_user) REFERENCES &mw_prefix.mwuser(user_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE &mw_prefix.logging ADD CONSTRAINT &mw_prefix.logging_fk1 FOREIGN KEY (log_user) REFERENCES &mw_prefix.mwuser(user_id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE &mw_prefix.redirect ADD CONSTRAINT &mw_prefix.redirect_fk1 FOREIGN KEY (rd_from) REFERENCES &mw_prefix.page(page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE &mw_prefix.page_restrictions ADD CONSTRAINT &mw_prefix.page_restrictions_fk1 FOREIGN KEY (pr_page) REFERENCES &mw_prefix.page(page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; + diff --git a/www/wiki/maintenance/oracle/archives/patch_namespace_defaults.sql b/www/wiki/maintenance/oracle/archives/patch_namespace_defaults.sql new file mode 100644 index 00000000..24c95643 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch_namespace_defaults.sql @@ -0,0 +1,17 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.page MODIFY page_namespace DEFAULT 0; +ALTER TABLE &mw_prefix.archive MODIFY ar_namespace DEFAULT 0; +ALTER TABLE &mw_prefix.pagelinks MODIFY pl_namespace DEFAULT 0; +ALTER TABLE &mw_prefix.templatelinks MODIFY tl_namespace DEFAULT 0; +ALTER TABLE &mw_prefix.recentchanges MODIFY rc_namespace DEFAULT 0; +ALTER TABLE &mw_prefix.querycache MODIFY qc_namespace DEFAULT 0; +ALTER TABLE &mw_prefix.logging MODIFY log_namespace DEFAULT 0; +ALTER TABLE &mw_prefix.job MODIFY job_namespace DEFAULT 0; +ALTER TABLE &mw_prefix.redirect MODIFY rd_namespace DEFAULT 0; +ALTER TABLE &mw_prefix.protected_titles MODIFY pt_namespace DEFAULT 0; +ALTER TABLE &mw_prefix.archive MODIFY ar_namespace DEFAULT 0; +ALTER TABLE &mw_prefix.archive MODIFY ar_namespace DEFAULT 0; +ALTER TABLE &mw_prefix.archive MODIFY ar_namespace DEFAULT 0; +ALTER TABLE &mw_prefix.archive MODIFY ar_namespace DEFAULT 0; + diff --git a/www/wiki/maintenance/oracle/archives/patch_rebuild_dupfunc.sql b/www/wiki/maintenance/oracle/archives/patch_rebuild_dupfunc.sql new file mode 100644 index 00000000..56ee5b3e --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch_rebuild_dupfunc.sql @@ -0,0 +1,149 @@ +/*$mw$*/ +CREATE OR REPLACE PROCEDURE duplicate_table(p_tabname IN VARCHAR2, + p_oldprefix IN VARCHAR2, + p_newprefix IN VARCHAR2, + p_temporary IN BOOLEAN) IS + e_table_not_exist EXCEPTION; + PRAGMA EXCEPTION_INIT(e_table_not_exist, -00942); + l_temp_ei_sql VARCHAR2(2000); + l_temporary BOOLEAN := p_temporary; +BEGIN + BEGIN + EXECUTE IMMEDIATE 'DROP TABLE ' || p_newprefix || p_tabname || + ' CASCADE CONSTRAINTS PURGE'; + EXCEPTION + WHEN e_table_not_exist THEN + NULL; + END; + IF (p_tabname = 'SEARCHINDEX') THEN + l_temporary := FALSE; + END IF; + IF (l_temporary) THEN + EXECUTE IMMEDIATE 'CREATE GLOBAL TEMPORARY TABLE ' || p_newprefix || + p_tabname || + ' ON COMMIT PRESERVE ROWS AS SELECT * FROM ' || + p_oldprefix || p_tabname || ' WHERE ROWNUM = 0'; + ELSE + EXECUTE IMMEDIATE 'CREATE TABLE ' || p_newprefix || p_tabname || + ' AS SELECT * FROM ' || p_oldprefix || p_tabname || + ' WHERE ROWNUM = 0'; + END IF; + FOR rc IN (SELECT column_name, data_default + FROM user_tab_columns + WHERE table_name = p_oldprefix || p_tabname + AND data_default IS NOT NULL) LOOP + EXECUTE IMMEDIATE 'ALTER TABLE ' || p_newprefix || p_tabname || + ' MODIFY ' || rc.column_name || ' DEFAULT ' || + SUBSTR(rc.data_default, 1, 2000); + END LOOP; + FOR rc IN (SELECT REPLACE(REPLACE(DBMS_LOB.SUBSTR(DBMS_METADATA.get_ddl('CONSTRAINT', + constraint_name), + 32767, + 1), + USER || '"."' || p_oldprefix, + USER || '"."' || p_newprefix), + '"' || constraint_name || '"', + '"' || p_newprefix || constraint_name || '"') DDLVC2, + constraint_name + FROM user_constraints uc + WHERE table_name = p_oldprefix || p_tabname + AND constraint_type = 'P') LOOP + l_temp_ei_sql := SUBSTR(rc.ddlvc2, 1, INSTR(rc.ddlvc2, 'PCTFREE') - 1); + l_temp_ei_sql := SUBSTR(l_temp_ei_sql, + 1, + INSTR(l_temp_ei_sql, + ')', + INSTR(l_temp_ei_sql, 'PRIMARY KEY') + 1) + 1); + IF nvl(length(l_temp_ei_sql), 0) > 0 THEN + EXECUTE IMMEDIATE l_temp_ei_sql; + END IF; + END LOOP; + IF (NOT l_temporary) THEN + FOR rc IN (SELECT REPLACE(DBMS_LOB.SUBSTR(DBMS_METADATA.get_ddl('REF_CONSTRAINT', + constraint_name), + 32767, + 1), + USER || '"."' || p_oldprefix, + USER || '"."' || p_newprefix) DDLVC2, + constraint_name + FROM user_constraints uc + WHERE table_name = p_oldprefix || p_tabname + AND constraint_type = 'R') LOOP + IF nvl(length(l_temp_ei_sql), 0) > 0 AND + INSTR(l_temp_ei_sql, 'PRIMARY KEY') = 0 THEN + EXECUTE IMMEDIATE l_temp_ei_sql; + END IF; + END LOOP; + END IF; + FOR rc IN (SELECT REPLACE(REPLACE(DBMS_LOB.SUBSTR(DBMS_METADATA.get_ddl('INDEX', + index_name), + 32767, + 1), + USER || '"."' || p_oldprefix, + USER || '"."' || p_newprefix), + '"' || index_name || '"', + '"' || p_newprefix || index_name || '"') DDLVC2, + index_name, + index_type + FROM user_indexes ui + WHERE table_name = p_oldprefix || p_tabname + AND index_type NOT IN ('LOB', 'DOMAIN') + AND NOT EXISTS + (SELECT NULL + FROM user_constraints + WHERE table_name = ui.table_name + AND constraint_name = ui.index_name)) LOOP + l_temp_ei_sql := SUBSTR(rc.ddlvc2, 1, INSTR(rc.ddlvc2, 'PCTFREE') - 1); + l_temp_ei_sql := SUBSTR(l_temp_ei_sql, + 1, + INSTR(l_temp_ei_sql, + ')', + INSTR(l_temp_ei_sql, + '"' || USER || '"."' || p_newprefix || '"') + 1) + 1); + IF nvl(length(l_temp_ei_sql), 0) > 0 THEN + EXECUTE IMMEDIATE l_temp_ei_sql; + END IF; + END LOOP; + FOR rc IN (SELECT REPLACE(REPLACE(DBMS_LOB.SUBSTR(DBMS_METADATA.get_ddl('INDEX', + index_name), + 32767, + 1), + USER || '"."' || p_oldprefix, + USER || '"."' || p_newprefix), + '"' || index_name || '"', + '"' || p_newprefix || index_name || '"') DDLVC2, + index_name, + index_type + FROM user_indexes ui + WHERE table_name = p_oldprefix || p_tabname + AND index_type = 'DOMAIN' + AND NOT EXISTS + (SELECT NULL + FROM user_constraints + WHERE table_name = ui.table_name + AND constraint_name = ui.index_name)) LOOP + l_temp_ei_sql := rc.ddlvc2; + IF nvl(length(l_temp_ei_sql), 0) > 0 THEN + EXECUTE IMMEDIATE l_temp_ei_sql; + END IF; + END LOOP; + FOR rc IN (SELECT REPLACE(REPLACE(UPPER(DBMS_LOB.SUBSTR(DBMS_METADATA.get_ddl('TRIGGER', + trigger_name), + 32767, + 1)), + USER || '"."' || p_oldprefix, + USER || '"."' || p_newprefix), + ' ON ' || p_oldprefix || p_tabname, + ' ON ' || p_newprefix || p_tabname) DDLVC2, + trigger_name + FROM user_triggers + WHERE table_name = p_oldprefix || p_tabname) LOOP + l_temp_ei_sql := SUBSTR(rc.ddlvc2, 1, INSTR(rc.ddlvc2, 'ALTER ') - 1); + IF nvl(length(l_temp_ei_sql), 0) > 0 THEN + EXECUTE IMMEDIATE l_temp_ei_sql; + END IF; + END LOOP; +END; + +/*$mw$*/ + diff --git a/www/wiki/maintenance/oracle/archives/patch_recentchanges_fk2_cascade.sql b/www/wiki/maintenance/oracle/archives/patch_recentchanges_fk2_cascade.sql new file mode 100644 index 00000000..45509518 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch_recentchanges_fk2_cascade.sql @@ -0,0 +1,5 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.recentchanges DROP CONSTRAINT &mw_prefix.recentchanges_fk2; +ALTER TABLE &mw_prefix.recentchanges ADD CONSTRAINT &mw_prefix.recentchanges_fk2 FOREIGN KEY (rc_cur_id) REFERENCES &mw_prefix.page(page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; + diff --git a/www/wiki/maintenance/oracle/archives/patch_remove_not_null_empty_defs.sql b/www/wiki/maintenance/oracle/archives/patch_remove_not_null_empty_defs.sql new file mode 100644 index 00000000..76e50a0a --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch_remove_not_null_empty_defs.sql @@ -0,0 +1,9 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.categorylinks MODIFY cl_sortkey_prefix DEFAULT NULL NULL; +ALTER TABLE &mw_prefix.categorylinks MODIFY cl_collation DEFAULT NULL NULL; +ALTER TABLE &mw_prefix.iwlinks MODIFY iwl_prefix DEFAULT NULL NULL; +ALTER TABLE &mw_prefix.iwlinks MODIFY iwl_title DEFAULT NULL NULL; +ALTER TABLE &mw_prefix.searchindex MODIFY si_title DEFAULT NULL NULL; +ALTER TABLE &mw_prefix.querycachetwo MODIFY qcc_title DEFAULT NULL NULL; +ALTER TABLE &mw_prefix.querycachetwo MODIFY qcc_titletwo DEFAULT NULL NULL; diff --git a/www/wiki/maintenance/oracle/archives/patch_remove_not_null_empty_defs2.sql b/www/wiki/maintenance/oracle/archives/patch_remove_not_null_empty_defs2.sql new file mode 100644 index 00000000..f7a38a05 --- /dev/null +++ b/www/wiki/maintenance/oracle/archives/patch_remove_not_null_empty_defs2.sql @@ -0,0 +1,3 @@ +define mw_prefix='{$wgDBprefix}'; + +ALTER TABLE &mw_prefix.ipblocks MODIFY ipb_by_text DEFAULT NULL NULL; diff --git a/www/wiki/maintenance/oracle/patch_seq_names_pre1.16.sql b/www/wiki/maintenance/oracle/patch_seq_names_pre1.16.sql new file mode 100644 index 00000000..5346b141 --- /dev/null +++ b/www/wiki/maintenance/oracle/patch_seq_names_pre1.16.sql @@ -0,0 +1,8 @@ +-- script for renameing sequence names to conform with <table>_<field>_seq format +RENAME rev_rev_id_val TO revision_rev_id_seq; +RENAME text_old_id_val TO text_old_id_seq; +RENAME category_id_seq TO category_cat_id_seq; +RENAME ipblocks_ipb_id_val TO ipblocks_ipb_id_seq; +RENAME rc_rc_id_seq TO recentchanges_rc_id_seq; +RENAME log_log_id_seq TO logging_log_id_seq; +RENAME pr_id_val TO page_restrictions_pr_id_seq;
\ No newline at end of file diff --git a/www/wiki/maintenance/oracle/tables.sql b/www/wiki/maintenance/oracle/tables.sql new file mode 100644 index 00000000..e6e2e565 --- /dev/null +++ b/www/wiki/maintenance/oracle/tables.sql @@ -0,0 +1,1095 @@ +-- defines must comply with ^define\s*([^\s=]*)\s*=\s?'\{\$([^\}]*)\}'; +define mw_prefix='{$wgDBprefix}'; + +-- Package to help with making Oracle more like other DBs with respect to +-- auto-incrementing columns. +/*$mw$*/ +CREATE PACKAGE &mw_prefix.lastval_pkg IS + lastval NUMBER; + PROCEDURE setLastval(val IN NUMBER, field OUT NUMBER); + FUNCTION getLastval RETURN NUMBER; +END; +/*$mw$*/ + +/*$mw$*/ +CREATE PACKAGE BODY &mw_prefix.lastval_pkg IS + PROCEDURE setLastval(val IN NUMBER, field OUT NUMBER) IS BEGIN + lastval := val; + field := val; + END; + + FUNCTION getLastval RETURN NUMBER IS BEGIN + RETURN lastval; + END; +END; +/*$mw$*/ + +CREATE SEQUENCE user_user_id_seq; +CREATE TABLE &mw_prefix.mwuser ( -- replace reserved word 'user' + user_id NUMBER NOT NULL, + user_name VARCHAR2(255) NOT NULL, + user_real_name VARCHAR2(512), + user_password VARCHAR2(255), + user_newpassword VARCHAR2(255), + user_newpass_time TIMESTAMP(6) WITH TIME ZONE, + user_token VARCHAR2(32), + user_email VARCHAR2(255), + user_email_token VARCHAR2(32), + user_email_token_expires TIMESTAMP(6) WITH TIME ZONE, + user_email_authenticated TIMESTAMP(6) WITH TIME ZONE, + user_options CLOB, + user_touched TIMESTAMP(6) WITH TIME ZONE, + user_registration TIMESTAMP(6) WITH TIME ZONE, + user_editcount NUMBER, + user_password_expires TIMESTAMP(6) WITH TIME ZONE +); +ALTER TABLE &mw_prefix.mwuser ADD CONSTRAINT &mw_prefix.mwuser_pk PRIMARY KEY (user_id); +CREATE UNIQUE INDEX &mw_prefix.mwuser_u01 ON &mw_prefix.mwuser (user_name); +CREATE INDEX &mw_prefix.mwuser_i01 ON &mw_prefix.mwuser (user_email_token); +CREATE INDEX &mw_prefix.mwuser_i02 ON &mw_prefix.mwuser (user_email, user_name); +/*$mw$*/ +CREATE TRIGGER &mw_prefix.mwuser_default_user_id BEFORE INSERT ON &mw_prefix.mwuser + FOR EACH ROW WHEN (new.user_id IS NULL) +BEGIN + &mw_prefix.lastval_pkg.setLastval(user_user_id_seq.nextval, :new.user_id); +END; +/*$mw$*/ + +-- Create a dummy user to satisfy fk contraints especially with revisions +INSERT INTO &mw_prefix.mwuser + (user_id, user_name, user_options, user_touched, user_registration, user_editcount) + VALUES (0,'Anonymous','', current_timestamp, current_timestamp,0); + +CREATE TABLE &mw_prefix.user_groups ( + ug_user NUMBER DEFAULT 0 NOT NULL, + ug_group VARCHAR2(255) NOT NULL, + ug_expiry TIMESTAMP(6) WITH TIME ZONE NULL +); +ALTER TABLE &mw_prefix.user_groups ADD CONSTRAINT &mw_prefix.user_groups_pk PRIMARY KEY (ug_user,ug_group); +ALTER TABLE &mw_prefix.user_groups ADD CONSTRAINT &mw_prefix.user_groups_fk1 FOREIGN KEY (ug_user) REFERENCES &mw_prefix.mwuser(user_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +CREATE INDEX &mw_prefix.user_groups_i01 ON &mw_prefix.user_groups (ug_group); +CREATE INDEX &mw_prefix.user_groups_i02 ON &mw_prefix.user_groups (ug_expiry); + +CREATE TABLE &mw_prefix.user_former_groups ( + ufg_user NUMBER DEFAULT 0 NOT NULL, + ufg_group VARCHAR2(255) NOT NULL +); +ALTER TABLE &mw_prefix.user_former_groups ADD CONSTRAINT &mw_prefix.user_former_groups_fk1 FOREIGN KEY (ufg_user) REFERENCES &mw_prefix.mwuser(user_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +CREATE UNIQUE INDEX &mw_prefix.user_former_groups_u01 ON &mw_prefix.user_former_groups (ufg_user,ufg_group); + +CREATE TABLE &mw_prefix.user_newtalk ( + user_id NUMBER DEFAULT 0 NOT NULL, + user_ip VARCHAR2(40) NULL, + user_last_timestamp TIMESTAMP(6) WITH TIME ZONE +); +ALTER TABLE &mw_prefix.user_newtalk ADD CONSTRAINT &mw_prefix.user_newtalk_fk1 FOREIGN KEY (user_id) REFERENCES &mw_prefix.mwuser(user_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +CREATE INDEX &mw_prefix.user_newtalk_i01 ON &mw_prefix.user_newtalk (user_id); +CREATE INDEX &mw_prefix.user_newtalk_i02 ON &mw_prefix.user_newtalk (user_ip); + +CREATE TABLE &mw_prefix.user_properties ( + up_user NUMBER NOT NULL, + up_property VARCHAR2(255) NOT NULL, + up_value CLOB +); +CREATE UNIQUE INDEX &mw_prefix.user_properties_u01 on &mw_prefix.user_properties (up_user,up_property); +CREATE INDEX &mw_prefix.user_properties_i01 on &mw_prefix.user_properties (up_property); + +CREATE SEQUENCE page_page_id_seq; +CREATE TABLE &mw_prefix.page ( + page_id NUMBER NOT NULL, + page_namespace NUMBER DEFAULT 0 NOT NULL, + page_title VARCHAR2(255) NOT NULL, + page_restrictions VARCHAR2(255), + page_is_redirect CHAR(1) DEFAULT '0' NOT NULL, + page_is_new CHAR(1) DEFAULT '0' NOT NULL, + page_random NUMBER(15,14) NOT NULL, + page_touched TIMESTAMP(6) WITH TIME ZONE, + page_links_updated TIMESTAMP(6) WITH TIME ZONE, + page_latest NUMBER DEFAULT 0 NOT NULL, -- FK? + page_len NUMBER DEFAULT 0 NOT NULL, + page_content_model VARCHAR2(32), + page_lang VARCHAR2(35) DEFAULT NULL +); +ALTER TABLE &mw_prefix.page ADD CONSTRAINT &mw_prefix.page_pk PRIMARY KEY (page_id); +CREATE UNIQUE INDEX &mw_prefix.page_u01 ON &mw_prefix.page (page_namespace,page_title); +CREATE INDEX &mw_prefix.page_i01 ON &mw_prefix.page (page_random); +CREATE INDEX &mw_prefix.page_i02 ON &mw_prefix.page (page_len); +CREATE INDEX &mw_prefix.page_i03 ON &mw_prefix.page (page_is_redirect, page_namespace, page_len); +/*$mw$*/ +CREATE TRIGGER &mw_prefix.page_default_page_id BEFORE INSERT ON &mw_prefix.page + FOR EACH ROW WHEN (new.page_id IS NULL) +BEGIN + &mw_prefix.lastval_pkg.setLastval(page_page_id_seq.nextval, :new.page_id); +END; +/*$mw$*/ + +-- Create a dummy page to satisfy fk contraints especially with revisions +INSERT INTO &mw_prefix.page + VALUES (0, 0, ' ', NULL, 0, 0, 0, current_timestamp, NULL, 0, 0, NULL, NULL); + +/*$mw$*/ +CREATE TRIGGER &mw_prefix.page_set_random BEFORE INSERT ON &mw_prefix.page + FOR EACH ROW WHEN (new.page_random IS NULL) +BEGIN + SELECT dbms_random.value INTO :NEW.page_random FROM dual; +END; +/*$mw$*/ + +CREATE SEQUENCE revision_rev_id_seq; +CREATE TABLE &mw_prefix.revision ( + rev_id NUMBER NOT NULL, + rev_page NUMBER NOT NULL, + rev_text_id NUMBER NULL, + rev_comment VARCHAR2(255), + rev_user NUMBER DEFAULT 0 NOT NULL, + rev_user_text VARCHAR2(255) NOT NULL, + rev_timestamp TIMESTAMP(6) WITH TIME ZONE NOT NULL, + rev_minor_edit CHAR(1) DEFAULT '0' NOT NULL, + rev_deleted CHAR(1) DEFAULT '0' NOT NULL, + rev_len NUMBER NULL, + rev_parent_id NUMBER DEFAULT NULL, + rev_sha1 VARCHAR2(32) NULL, + rev_content_model VARCHAR2(32), + rev_content_format VARCHAR2(64) +); +ALTER TABLE &mw_prefix.revision ADD CONSTRAINT &mw_prefix.revision_pk PRIMARY KEY (rev_id); +ALTER TABLE &mw_prefix.revision ADD CONSTRAINT &mw_prefix.revision_fk1 FOREIGN KEY (rev_page) REFERENCES &mw_prefix.page(page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE &mw_prefix.revision ADD CONSTRAINT &mw_prefix.revision_fk2 FOREIGN KEY (rev_user) REFERENCES &mw_prefix.mwuser(user_id) DEFERRABLE INITIALLY DEFERRED; +CREATE UNIQUE INDEX &mw_prefix.revision_u01 ON &mw_prefix.revision (rev_page, rev_id); +CREATE INDEX &mw_prefix.revision_i01 ON &mw_prefix.revision (rev_timestamp); +CREATE INDEX &mw_prefix.revision_i02 ON &mw_prefix.revision (rev_page,rev_timestamp); +CREATE INDEX &mw_prefix.revision_i03 ON &mw_prefix.revision (rev_user,rev_timestamp); +CREATE INDEX &mw_prefix.revision_i04 ON &mw_prefix.revision (rev_user_text,rev_timestamp); +CREATE INDEX &mw_prefix.revision_i05 ON &mw_prefix.revision (rev_page,rev_user,rev_timestamp); +/*$mw$*/ +CREATE TRIGGER &mw_prefix.revision_default_rev_id BEFORE INSERT ON &mw_prefix.revision + FOR EACH ROW WHEN (new.rev_id IS NULL) +BEGIN + &mw_prefix.lastval_pkg.setLastval(revision_rev_id_seq.nextval, :new.rev_id); +END; +/*$mw$*/ + +CREATE SEQUENCE text_old_id_seq; +CREATE TABLE &mw_prefix.pagecontent ( -- replaces reserved word 'text' + old_id NUMBER NOT NULL, + old_text CLOB, + old_flags VARCHAR2(255) +); +ALTER TABLE &mw_prefix.pagecontent ADD CONSTRAINT &mw_prefix.pagecontent_pk PRIMARY KEY (old_id); +/*$mw$*/ +CREATE TRIGGER &mw_prefix.text_default_old_id BEFORE INSERT ON &mw_prefix.text + FOR EACH ROW WHEN (new.old_id IS NULL) +BEGIN + &mw_prefix.lastval_pkg.setLastval(text_old_id_seq.nextval, :new.old_id); +END; +/*$mw$*/ + +CREATE SEQUENCE archive_ar_id_seq; +CREATE TABLE &mw_prefix.archive ( + ar_id NUMBER NOT NULL, + ar_namespace NUMBER DEFAULT 0 NOT NULL, + ar_title VARCHAR2(255) NOT NULL, + ar_text CLOB, + ar_comment VARCHAR2(255), + ar_user NUMBER DEFAULT 0 NOT NULL, + ar_user_text VARCHAR2(255) NOT NULL, + ar_timestamp TIMESTAMP(6) WITH TIME ZONE NOT NULL, + ar_minor_edit CHAR(1) DEFAULT '0' NOT NULL, + ar_flags VARCHAR2(255), + ar_rev_id NUMBER, + ar_text_id NUMBER, + ar_deleted CHAR(1) DEFAULT '0' NOT NULL, + ar_len NUMBER, + ar_page_id NUMBER, + ar_parent_id NUMBER, + ar_sha1 VARCHAR2(32), + ar_content_model VARCHAR2(32), + ar_content_format VARCHAR2(64) +); +ALTER TABLE &mw_prefix.archive ADD CONSTRAINT &mw_prefix.archive_pk PRIMARY KEY (ar_id); +ALTER TABLE &mw_prefix.archive ADD CONSTRAINT &mw_prefix.archive_fk1 FOREIGN KEY (ar_user) REFERENCES &mw_prefix.mwuser(user_id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED; +CREATE INDEX &mw_prefix.archive_i01 ON &mw_prefix.archive (ar_namespace,ar_title,ar_timestamp); +CREATE INDEX &mw_prefix.archive_i02 ON &mw_prefix.archive (ar_user_text,ar_timestamp); +CREATE INDEX &mw_prefix.archive_i03 ON &mw_prefix.archive (ar_rev_id); +/*$mw$*/ +CREATE TRIGGER &mw_prefix.archive_default_ar_id BEFORE INSERT ON &mw_prefix.archive + FOR EACH ROW WHEN (new.ar_id IS NULL) +BEGIN + &mw_prefix.lastval_pkg.setLastval(archive_ar_id_seq.nextval, :new.ar_id); +END; +/*$mw$*/ + +CREATE TABLE &mw_prefix.pagelinks ( + pl_from NUMBER NOT NULL, + pl_namespace NUMBER DEFAULT 0 NOT NULL, + pl_title VARCHAR2(255) NOT NULL +); +ALTER TABLE &mw_prefix.pagelinks ADD CONSTRAINT &mw_prefix.pagelinks_fk1 FOREIGN KEY (pl_from) REFERENCES &mw_prefix.page(page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +CREATE UNIQUE INDEX &mw_prefix.pagelinks_u01 ON &mw_prefix.pagelinks (pl_from,pl_namespace,pl_title); +CREATE UNIQUE INDEX &mw_prefix.pagelinks_u02 ON &mw_prefix.pagelinks (pl_namespace,pl_title,pl_from); + +CREATE TABLE &mw_prefix.templatelinks ( + tl_from NUMBER NOT NULL, + tl_namespace NUMBER DEFAULT 0 NOT NULL, + tl_title VARCHAR2(255) NOT NULL +); +ALTER TABLE &mw_prefix.templatelinks ADD CONSTRAINT &mw_prefix.templatelinks_fk1 FOREIGN KEY (tl_from) REFERENCES &mw_prefix.page(page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +CREATE UNIQUE INDEX &mw_prefix.templatelinks_u01 ON &mw_prefix.templatelinks (tl_from,tl_namespace,tl_title); +CREATE UNIQUE INDEX &mw_prefix.templatelinks_u02 ON &mw_prefix.templatelinks (tl_namespace,tl_title,tl_from); + +CREATE TABLE &mw_prefix.imagelinks ( + il_from NUMBER NOT NULL, + il_to VARCHAR2(255) NOT NULL +); +ALTER TABLE &mw_prefix.imagelinks ADD CONSTRAINT &mw_prefix.imagelinks_fk1 FOREIGN KEY (il_from) REFERENCES &mw_prefix.page(page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +CREATE UNIQUE INDEX &mw_prefix.imagelinks_u01 ON &mw_prefix.imagelinks (il_from,il_to); +CREATE UNIQUE INDEX &mw_prefix.imagelinks_u02 ON &mw_prefix.imagelinks (il_to,il_from); + + +CREATE TABLE &mw_prefix.categorylinks ( + cl_from NUMBER NOT NULL, + cl_to VARCHAR2(255) NOT NULL, + cl_sortkey VARCHAR2(230), + cl_sortkey_prefix VARCHAR2(255), + cl_timestamp TIMESTAMP(6) WITH TIME ZONE NOT NULL, + cl_collation VARCHAR2(32), + cl_type VARCHAR2(6) DEFAULT 'page' NOT NULL +); +ALTER TABLE &mw_prefix.categorylinks ADD CONSTRAINT &mw_prefix.categorylinks_fk1 FOREIGN KEY (cl_from) REFERENCES &mw_prefix.page(page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +CREATE UNIQUE INDEX &mw_prefix.categorylinks_u01 ON &mw_prefix.categorylinks (cl_from,cl_to); +CREATE INDEX &mw_prefix.categorylinks_i01 ON &mw_prefix.categorylinks (cl_to,cl_type,cl_sortkey,cl_from); +CREATE INDEX &mw_prefix.categorylinks_i02 ON &mw_prefix.categorylinks (cl_to,cl_timestamp); +CREATE INDEX &mw_prefix.categorylinks_i03 ON &mw_prefix.categorylinks (cl_collation); + +CREATE SEQUENCE category_cat_id_seq; +CREATE TABLE &mw_prefix.category ( + cat_id NUMBER NOT NULL, + cat_title VARCHAR2(255) NOT NULL, + cat_pages NUMBER DEFAULT 0 NOT NULL, + cat_subcats NUMBER DEFAULT 0 NOT NULL, + cat_files NUMBER DEFAULT 0 NOT NULL +); +ALTER TABLE &mw_prefix.category ADD CONSTRAINT &mw_prefix.category_pk PRIMARY KEY (cat_id); +CREATE UNIQUE INDEX &mw_prefix.category_u01 ON &mw_prefix.category (cat_title); +CREATE INDEX &mw_prefix.category_i01 ON &mw_prefix.category (cat_pages); +/*$mw$*/ +CREATE TRIGGER &mw_prefix.category_default_cat_id BEFORE INSERT ON &mw_prefix.category + FOR EACH ROW WHEN (new.cat_id IS NULL) +BEGIN + &mw_prefix.lastval_pkg.setLastval(category_cat_id_seq.nextval, :new.cat_id); +END; +/*$mw$*/ + +CREATE SEQUENCE externallinks_el_id_seq; +CREATE TABLE &mw_prefix.externallinks ( + el_id NUMBER NOT NULL, + el_from NUMBER NOT NULL, + el_to VARCHAR2(2048) NOT NULL, + el_index VARCHAR2(2048) NOT NULL, + el_index_60 VARBINARY(60) NOT NULL DEFAULT '' +); +ALTER TABLE &mw_prefix.externallinks ADD CONSTRAINT &mw_prefix.externallinks_pk PRIMARY KEY (el_id); +ALTER TABLE &mw_prefix.externallinks ADD CONSTRAINT &mw_prefix.externallinks_fk1 FOREIGN KEY (el_from) REFERENCES &mw_prefix.page(page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +CREATE INDEX &mw_prefix.externallinks_i01 ON &mw_prefix.externallinks (el_from, el_to); +CREATE INDEX &mw_prefix.externallinks_i02 ON &mw_prefix.externallinks (el_to, el_from); +CREATE INDEX &mw_prefix.externallinks_i03 ON &mw_prefix.externallinks (el_index); +CREATE INDEX &mw_prefix.externallinks_i04 ON &mw_prefix.externallinks (el_index_60, el_id); +CREATE INDEX &mw_prefix.externallinks_i05 ON &mw_prefix.externallinks (el_from, el_index_60, el_id); +/*$mw$*/ +CREATE TRIGGER &mw_prefix.externallinks_default_el_id BEFORE INSERT ON &mw_prefix.externallinks + FOR EACH ROW WHEN (new.el_id IS NULL) +BEGIN + &mw_prefix.lastval_pkg.setLastval(externallinks_el_id_seq.nextval, :new.el_id); +END; +/*$mw$*/ + +CREATE TABLE &mw_prefix.langlinks ( + ll_from NUMBER NOT NULL, + ll_lang VARCHAR2(20), + ll_title VARCHAR2(255) +); +ALTER TABLE &mw_prefix.langlinks ADD CONSTRAINT &mw_prefix.langlinks_fk1 FOREIGN KEY (ll_from) REFERENCES &mw_prefix.page(page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +CREATE UNIQUE INDEX &mw_prefix.langlinks_u01 ON &mw_prefix.langlinks (ll_from, ll_lang); +CREATE INDEX &mw_prefix.langlinks_i01 ON &mw_prefix.langlinks (ll_lang, ll_title); + +CREATE TABLE &mw_prefix.iwlinks ( + iwl_from NUMBER DEFAULT 0 NOT NULL, + iwl_prefix VARCHAR2(20), + iwl_title VARCHAR2(255) +); +CREATE UNIQUE INDEX &mw_prefix.iwlinks_ui01 ON &mw_prefix.iwlinks (iwl_from, iwl_prefix, iwl_title); +CREATE UNIQUE INDEX &mw_prefix.iwlinks_ui02 ON &mw_prefix.iwlinks (iwl_prefix, iwl_title, iwl_from); + +CREATE TABLE &mw_prefix.site_stats ( + ss_row_id NUMBER NOT NULL PRIMARY KEY, + ss_total_edits NUMBER DEFAULT 0, + ss_good_articles NUMBER DEFAULT 0, + ss_total_pages NUMBER DEFAULT -1, + ss_users NUMBER DEFAULT -1, + ss_active_users NUMBER DEFAULT -1, + ss_images NUMBER DEFAULT 0 +); + +CREATE SEQUENCE ipblocks_ipb_id_seq; +CREATE TABLE &mw_prefix.ipblocks ( + ipb_id NUMBER NOT NULL, + ipb_address VARCHAR2(255) NULL, + ipb_user NUMBER DEFAULT 0 NOT NULL, + ipb_by NUMBER DEFAULT 0 NOT NULL, + ipb_by_text VARCHAR2(255) NULL, + ipb_reason VARCHAR2(255) NOT NULL, + ipb_timestamp TIMESTAMP(6) WITH TIME ZONE NOT NULL, + ipb_auto CHAR(1) DEFAULT '0' NOT NULL, + ipb_anon_only CHAR(1) DEFAULT '0' NOT NULL, + ipb_create_account CHAR(1) DEFAULT '1' NOT NULL, + ipb_enable_autoblock CHAR(1) DEFAULT '1' NOT NULL, + ipb_expiry TIMESTAMP(6) WITH TIME ZONE NOT NULL, + ipb_range_start VARCHAR2(255), + ipb_range_end VARCHAR2(255), + ipb_deleted CHAR(1) DEFAULT '0' NOT NULL, + ipb_block_email CHAR(1) DEFAULT '0' NOT NULL, + ipb_allow_usertalk CHAR(1) DEFAULT '0' NOT NULL, + ipb_parent_block_id NUMBER DEFAULT NULL +); +ALTER TABLE &mw_prefix.ipblocks ADD CONSTRAINT &mw_prefix.ipblocks_pk PRIMARY KEY (ipb_id); +ALTER TABLE &mw_prefix.ipblocks ADD CONSTRAINT &mw_prefix.ipblocks_fk1 FOREIGN KEY (ipb_user) REFERENCES &mw_prefix.mwuser(user_id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE &mw_prefix.ipblocks ADD CONSTRAINT &mw_prefix.ipblocks_fk2 FOREIGN KEY (ipb_by) REFERENCES &mw_prefix.mwuser(user_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +CREATE UNIQUE INDEX &mw_prefix.ipblocks_u01 ON &mw_prefix.ipblocks (ipb_address, ipb_user, ipb_auto, ipb_anon_only); +CREATE INDEX &mw_prefix.ipblocks_i01 ON &mw_prefix.ipblocks (ipb_user); +CREATE INDEX &mw_prefix.ipblocks_i02 ON &mw_prefix.ipblocks (ipb_range_start, ipb_range_end); +CREATE INDEX &mw_prefix.ipblocks_i03 ON &mw_prefix.ipblocks (ipb_timestamp); +CREATE INDEX &mw_prefix.ipblocks_i04 ON &mw_prefix.ipblocks (ipb_expiry); +CREATE INDEX &mw_prefix.ipblocks_i05 ON &mw_prefix.ipblocks (ipb_parent_block_id); +/*$mw$*/ +CREATE TRIGGER &mw_prefix.ipblocks_default_ipb_id BEFORE INSERT ON &mw_prefix.ipblocks + FOR EACH ROW WHEN (new.ipb_id IS NULL) +BEGIN + &mw_prefix.lastval_pkg.setLastval(ipblocks_ipb_id_seq.nextval, :new.ipb_id); +END; +/*$mw$*/ + +CREATE TABLE &mw_prefix.image ( + img_name VARCHAR2(255) NOT NULL, + img_size NUMBER DEFAULT 0 NOT NULL, + img_width NUMBER DEFAULT 0 NOT NULL, + img_height NUMBER DEFAULT 0 NOT NULL, + img_metadata CLOB, + img_bits NUMBER DEFAULT 0 NOT NULL, + img_media_type VARCHAR2(32), + img_major_mime VARCHAR2(32) DEFAULT 'unknown', + img_minor_mime VARCHAR2(100) DEFAULT 'unknown', + img_description VARCHAR2(255), + img_user NUMBER DEFAULT 0 NOT NULL, + img_user_text VARCHAR2(255) NOT NULL, + img_timestamp TIMESTAMP(6) WITH TIME ZONE, + img_sha1 VARCHAR2(32) +); +ALTER TABLE &mw_prefix.image ADD CONSTRAINT &mw_prefix.image_pk PRIMARY KEY (img_name); +ALTER TABLE &mw_prefix.image ADD CONSTRAINT &mw_prefix.image_fk1 FOREIGN KEY (img_user) REFERENCES &mw_prefix.mwuser(user_id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED; +CREATE INDEX &mw_prefix.image_i01 ON &mw_prefix.image (img_user_text,img_timestamp); +CREATE INDEX &mw_prefix.image_i02 ON &mw_prefix.image (img_size); +CREATE INDEX &mw_prefix.image_i03 ON &mw_prefix.image (img_timestamp); +CREATE INDEX &mw_prefix.image_i04 ON &mw_prefix.image (img_sha1); + + +CREATE TABLE &mw_prefix.oldimage ( + oi_name VARCHAR2(255) DEFAULT 0 NOT NULL, + oi_archive_name VARCHAR2(255), + oi_size NUMBER DEFAULT 0 NOT NULL, + oi_width NUMBER DEFAULT 0 NOT NULL, + oi_height NUMBER DEFAULT 0 NOT NULL, + oi_bits NUMBER DEFAULT 0 NOT NULL, + oi_description VARCHAR2(255), + oi_user NUMBER DEFAULT 0 NOT NULL, + oi_user_text VARCHAR2(255) NOT NULL, + oi_timestamp TIMESTAMP(6) WITH TIME ZONE NOT NULL, + oi_metadata CLOB, + oi_media_type VARCHAR2(32) DEFAULT NULL, + oi_major_mime VARCHAR2(32) DEFAULT 'unknown', + oi_minor_mime VARCHAR2(100) DEFAULT 'unknown', + oi_deleted NUMBER DEFAULT 0 NOT NULL, + oi_sha1 VARCHAR2(32) +); +ALTER TABLE &mw_prefix.oldimage ADD CONSTRAINT &mw_prefix.oldimage_fk1 FOREIGN KEY (oi_name) REFERENCES &mw_prefix.image(img_name) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE &mw_prefix.oldimage ADD CONSTRAINT &mw_prefix.oldimage_fk2 FOREIGN KEY (oi_user) REFERENCES &mw_prefix.mwuser(user_id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED; +CREATE INDEX &mw_prefix.oldimage_i01 ON &mw_prefix.oldimage (oi_user_text,oi_timestamp); +CREATE INDEX &mw_prefix.oldimage_i02 ON &mw_prefix.oldimage (oi_name,oi_timestamp); +CREATE INDEX &mw_prefix.oldimage_i03 ON &mw_prefix.oldimage (oi_name,oi_archive_name); +CREATE INDEX &mw_prefix.oldimage_i04 ON &mw_prefix.oldimage (oi_sha1); + + +CREATE SEQUENCE filearchive_fa_id_seq; +CREATE TABLE &mw_prefix.filearchive ( + fa_id NUMBER NOT NULL, + fa_name VARCHAR2(255) NOT NULL, + fa_archive_name VARCHAR2(255), + fa_storage_group VARCHAR2(16), + fa_storage_key VARCHAR2(64), + fa_deleted_user NUMBER DEFAULT 0 NOT NULL, + fa_deleted_timestamp TIMESTAMP(6) WITH TIME ZONE NOT NULL, + fa_deleted_reason CLOB, + fa_size NUMBER DEFAULT 0 NOT NULL, + fa_width NUMBER DEFAULT 0 NOT NULL, + fa_height NUMBER DEFAULT 0 NOT NULL, + fa_metadata CLOB, + fa_bits NUMBER DEFAULT 0 NOT NULL, + fa_media_type VARCHAR2(32) DEFAULT NULL, + fa_major_mime VARCHAR2(32) DEFAULT 'unknown', + fa_minor_mime VARCHAR2(100) DEFAULT 'unknown', + fa_description VARCHAR2(255), + fa_user NUMBER DEFAULT 0 NOT NULL, + fa_user_text VARCHAR2(255) NOT NULL, + fa_timestamp TIMESTAMP(6) WITH TIME ZONE, + fa_deleted NUMBER DEFAULT 0 NOT NULL, + fa_sha1 VARCHAR2(32) +); +ALTER TABLE &mw_prefix.filearchive ADD CONSTRAINT &mw_prefix.filearchive_pk PRIMARY KEY (fa_id); +ALTER TABLE &mw_prefix.filearchive ADD CONSTRAINT &mw_prefix.filearchive_fk1 FOREIGN KEY (fa_deleted_user) REFERENCES &mw_prefix.mwuser(user_id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE &mw_prefix.filearchive ADD CONSTRAINT &mw_prefix.filearchive_fk2 FOREIGN KEY (fa_user) REFERENCES &mw_prefix.mwuser(user_id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED; +CREATE INDEX &mw_prefix.filearchive_i01 ON &mw_prefix.filearchive (fa_name, fa_timestamp); +CREATE INDEX &mw_prefix.filearchive_i02 ON &mw_prefix.filearchive (fa_storage_group, fa_storage_key); +CREATE INDEX &mw_prefix.filearchive_i03 ON &mw_prefix.filearchive (fa_deleted_timestamp); +CREATE INDEX &mw_prefix.filearchive_i04 ON &mw_prefix.filearchive (fa_user_text,fa_timestamp); +CREATE INDEX &mw_prefix.filearchive_i05 ON &mw_prefix.filearchive (fa_sha1); +/*$mw$*/ +CREATE TRIGGER &mw_prefix.filearchive_default_fa_id BEFORE INSERT ON &mw_prefix.filearchive + FOR EACH ROW WHEN (new.fa_id IS NULL) +BEGIN + &mw_prefix.lastval_pkg.setLastval(filearchive_fa_id_seq.nextval, :new.fa_id); +END; +/*$mw$*/ + +CREATE SEQUENCE uploadstash_us_id_seq; +CREATE TABLE &mw_prefix.uploadstash ( + us_id NUMBER NOT NULL, + us_user NUMBER DEFAULT 0 NOT NULL, + us_key VARCHAR2(255) NOT NULL, + us_orig_path VARCHAR2(255) NOT NULL, + us_path VARCHAR2(255) NOT NULL, + us_source_type VARCHAR2(50), + us_timestamp TIMESTAMP(6) WITH TIME ZONE, + us_status VARCHAR2(50) NOT NULL, + us_chunk_inx NUMBER, + us_size NUMBER NOT NULL, + us_sha1 VARCHAR2(32) NOT NULL, + us_mime VARCHAR2(255), + us_media_type VARCHAR2(32) DEFAULT NULL, + us_image_width NUMBER, + us_image_height NUMBER, + us_image_bits NUMBER, + us_props BLOB +); +ALTER TABLE &mw_prefix.uploadstash ADD CONSTRAINT &mw_prefix.uploadstash_pk PRIMARY KEY (us_id); +ALTER TABLE &mw_prefix.uploadstash ADD CONSTRAINT &mw_prefix.uploadstash_fk1 FOREIGN KEY (us_user) REFERENCES &mw_prefix.mwuser(user_id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED; +CREATE INDEX &mw_prefix.uploadstash_i01 ON &mw_prefix.uploadstash (us_user); +CREATE INDEX &mw_prefix.uploadstash_i02 ON &mw_prefix.uploadstash (us_timestamp); +CREATE UNIQUE INDEX &mw_prefix.uploadstash_u01 ON &mw_prefix.uploadstash (us_key); +/*$mw$*/ +CREATE TRIGGER &mw_prefix.uploadstash_default_us_id BEFORE INSERT ON &mw_prefix.uploadstash + FOR EACH ROW WHEN (new.us_id IS NULL) +BEGIN + &mw_prefix.lastval_pkg.setLastval(uploadstash_us_id_seq.nextval, :new.us_id); +END; +/*$mw$*/ + +CREATE SEQUENCE recentchanges_rc_id_seq; +CREATE TABLE &mw_prefix.recentchanges ( + rc_id NUMBER NOT NULL, + rc_timestamp TIMESTAMP(6) WITH TIME ZONE NOT NULL, + rc_cur_time TIMESTAMP(6) WITH TIME ZONE, + rc_user NUMBER DEFAULT 0 NOT NULL, + rc_user_text VARCHAR2(255) NOT NULL, + rc_namespace NUMBER DEFAULT 0 NOT NULL, + rc_title VARCHAR2(255) NOT NULL, + rc_comment VARCHAR2(255), + rc_minor CHAR(1) DEFAULT '0' NOT NULL, + rc_bot CHAR(1) DEFAULT '0' NOT NULL, + rc_new CHAR(1) DEFAULT '0' NOT NULL, + rc_cur_id NUMBER DEFAULT 0 NOT NULL, + rc_this_oldid NUMBER DEFAULT 0 NOT NULL, + rc_last_oldid NUMBER DEFAULT 0 NOT NULL, + rc_type CHAR(1) DEFAULT '0' NOT NULL, + rc_source VARCHAR2(16), + rc_patrolled CHAR(1) DEFAULT '0' NOT NULL, + rc_ip VARCHAR2(15), + rc_old_len NUMBER, + rc_new_len NUMBER, + rc_deleted CHAR(1) DEFAULT '0' NOT NULL, + rc_logid NUMBER DEFAULT 0 NOT NULL, + rc_log_type VARCHAR2(255), + rc_log_action VARCHAR2(255), + rc_params CLOB +); +ALTER TABLE &mw_prefix.recentchanges ADD CONSTRAINT &mw_prefix.recentchanges_pk PRIMARY KEY (rc_id); +ALTER TABLE &mw_prefix.recentchanges ADD CONSTRAINT &mw_prefix.recentchanges_fk1 FOREIGN KEY (rc_user) REFERENCES &mw_prefix.mwuser(user_id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE &mw_prefix.recentchanges ADD CONSTRAINT &mw_prefix.recentchanges_fk2 FOREIGN KEY (rc_cur_id) REFERENCES &mw_prefix.page(page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +CREATE INDEX &mw_prefix.recentchanges_i01 ON &mw_prefix.recentchanges (rc_timestamp); +CREATE INDEX &mw_prefix.recentchanges_i02 ON &mw_prefix.recentchanges (rc_namespace, rc_title); +CREATE INDEX &mw_prefix.recentchanges_i03 ON &mw_prefix.recentchanges (rc_cur_id); +CREATE INDEX &mw_prefix.recentchanges_i04 ON &mw_prefix.recentchanges (rc_new,rc_namespace,rc_timestamp); +CREATE INDEX &mw_prefix.recentchanges_i05 ON &mw_prefix.recentchanges (rc_ip); +CREATE INDEX &mw_prefix.recentchanges_i06 ON &mw_prefix.recentchanges (rc_namespace, rc_user_text); +CREATE INDEX &mw_prefix.recentchanges_i07 ON &mw_prefix.recentchanges (rc_user_text, rc_timestamp); +CREATE INDEX &mw_prefix.recentchanges_i08 ON &mw_prefix.recentchanges (rc_namespace, rc_type, rc_patrolled, rc_timestamp); +/*$mw$*/ +CREATE TRIGGER &mw_prefix.recentchanges_default_rc_id BEFORE INSERT ON &mw_prefix.recentchanges + FOR EACH ROW WHEN (new.rc_id IS NULL) +BEGIN + &mw_prefix.lastval_pkg.setLastval(recentchanges_rc_id_seq.nextval, :new.rc_id); +END; +/*$mw$*/ + +CREATE TABLE &mw_prefix.watchlist ( + wl_id NUMBER NOT NULL, + wl_user NUMBER NOT NULL, + wl_namespace NUMBER DEFAULT 0 NOT NULL, + wl_title VARCHAR2(255) NOT NULL, + wl_notificationtimestamp TIMESTAMP(6) WITH TIME ZONE +); +ALTER TABLE &mw_prefix.watchlist ADD CONSTRAINT &mw_prefix.watchlist_pk PRIMARY KEY (wl_id); +ALTER TABLE &mw_prefix.watchlist ADD CONSTRAINT &mw_prefix.watchlist_fk1 FOREIGN KEY (wl_user) REFERENCES &mw_prefix.mwuser(user_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +CREATE UNIQUE INDEX &mw_prefix.watchlist_u01 ON &mw_prefix.watchlist (wl_user, wl_namespace, wl_title); +CREATE INDEX &mw_prefix.watchlist_i01 ON &mw_prefix.watchlist (wl_namespace, wl_title); + + +CREATE TABLE &mw_prefix.searchindex ( + si_page NUMBER NOT NULL, + si_title VARCHAR2(255), + si_text CLOB NOT NULL +); +CREATE UNIQUE INDEX &mw_prefix.searchindex_u01 ON &mw_prefix.searchindex (si_page); + +CREATE TABLE &mw_prefix.interwiki ( + iw_prefix VARCHAR2(32) NOT NULL, + iw_url VARCHAR2(127) NOT NULL, + iw_api BLOB NOT NULL, + iw_wikiid VARCHAR2(64), + iw_local CHAR(1) NOT NULL, + iw_trans CHAR(1) DEFAULT '0' NOT NULL +); +CREATE UNIQUE INDEX &mw_prefix.interwiki_u01 ON &mw_prefix.interwiki (iw_prefix); + +CREATE TABLE &mw_prefix.querycache ( + qc_type VARCHAR2(32) NOT NULL, + qc_value NUMBER DEFAULT 0 NOT NULL, + qc_namespace NUMBER DEFAULT 0 NOT NULL, + qc_title VARCHAR2(255) NOT NULL +); +CREATE INDEX &mw_prefix.querycache_u01 ON &mw_prefix.querycache (qc_type,qc_value); + +CREATE TABLE &mw_prefix.objectcache ( + keyname VARCHAR2(255) , + value BLOB, + exptime TIMESTAMP(6) WITH TIME ZONE NOT NULL +); +CREATE INDEX &mw_prefix.objectcache_i01 ON &mw_prefix.objectcache (exptime); + +CREATE TABLE &mw_prefix.transcache ( + tc_url VARCHAR2(255) NOT NULL, + tc_contents CLOB NOT NULL, + tc_time TIMESTAMP(6) WITH TIME ZONE NOT NULL +); +CREATE UNIQUE INDEX &mw_prefix.transcache_u01 ON &mw_prefix.transcache (tc_url); + + +CREATE SEQUENCE logging_log_id_seq; +CREATE TABLE &mw_prefix.logging ( + log_id NUMBER NOT NULL, + log_type VARCHAR2(10) NOT NULL, + log_action VARCHAR2(10) NOT NULL, + log_timestamp TIMESTAMP(6) WITH TIME ZONE NOT NULL, + log_user NUMBER DEFAULT 0 NOT NULL, + log_user_text VARCHAR2(255), + log_namespace NUMBER DEFAULT 0 NOT NULL, + log_title VARCHAR2(255) NOT NULL, + log_page NUMBER, + log_comment VARCHAR2(255), + log_params CLOB, + log_deleted CHAR(1) DEFAULT '0' NOT NULL +); +ALTER TABLE &mw_prefix.logging ADD CONSTRAINT &mw_prefix.logging_pk PRIMARY KEY (log_id); +ALTER TABLE &mw_prefix.logging ADD CONSTRAINT &mw_prefix.logging_fk1 FOREIGN KEY (log_user) REFERENCES &mw_prefix.mwuser(user_id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED; +CREATE INDEX &mw_prefix.logging_i01 ON &mw_prefix.logging (log_type, log_timestamp); +CREATE INDEX &mw_prefix.logging_i02 ON &mw_prefix.logging (log_user, log_timestamp); +CREATE INDEX &mw_prefix.logging_i03 ON &mw_prefix.logging (log_namespace, log_title, log_timestamp); +CREATE INDEX &mw_prefix.logging_i04 ON &mw_prefix.logging (log_timestamp); +CREATE INDEX &mw_prefix.logging_i05 ON &mw_prefix.logging (log_type, log_action, log_timestamp); +CREATE INDEX &mw_prefix.logging_i06 ON &mw_prefix.logging (log_user_text, log_type, log_timestamp); +CREATE INDEX &mw_prefix.logging_i07 ON &mw_prefix.logging (log_user_text, log_timestamp); +/*$mw$*/ +CREATE TRIGGER &mw_prefix.logging_default_log_id BEFORE INSERT ON &mw_prefix.logging + FOR EACH ROW WHEN (new.log_id IS NULL) +BEGIN + &mw_prefix.lastval_pkg.setLastval(logging_log_id_seq.nextval, :new.log_id); +END; +/*$mw$*/ + +CREATE TABLE &mw_prefix.log_search ( + ls_field VARCHAR2(32) NOT NULL, + ls_value VARCHAR2(255) NOT NULL, + ls_log_id NuMBER DEFAULT 0 NOT NULL +); +ALTER TABLE &mw_prefix.log_search ADD CONSTRAINT log_search_pk PRIMARY KEY (ls_field,ls_value,ls_log_id); +CREATE INDEX &mw_prefix.log_search_i01 ON &mw_prefix.log_search (ls_log_id); + + +CREATE SEQUENCE job_job_id_seq; +CREATE TABLE &mw_prefix.job ( + job_id NUMBER NOT NULL, + job_cmd VARCHAR2(60) NOT NULL, + job_namespace NUMBER DEFAULT 0 NOT NULL, + job_title VARCHAR2(255) NOT NULL, + job_timestamp TIMESTAMP(6) WITH TIME ZONE NULL, + job_params CLOB NOT NULL, + job_random NUMBER DEFAULT 0 NOT NULL, + job_token VARCHAR2(32), + job_token_timestamp TIMESTAMP(6) WITH TIME ZONE, + job_sha1 VARCHAR2(32), + job_attempts NUMBER DEFAULT 0 NOT NULL +); +ALTER TABLE &mw_prefix.job ADD CONSTRAINT &mw_prefix.job_pk PRIMARY KEY (job_id); +CREATE INDEX &mw_prefix.job_i01 ON &mw_prefix.job (job_cmd, job_namespace, job_title); +CREATE INDEX &mw_prefix.job_i02 ON &mw_prefix.job (job_timestamp); +CREATE INDEX &mw_prefix.job_i03 ON &mw_prefix.job (job_sha1); +CREATE INDEX &mw_prefix.job_i04 ON &mw_prefix.job (job_cmd,job_token,job_random); +CREATE INDEX &mw_prefix.job_i05 ON &mw_prefix.job (job_attempts); +/*$mw$*/ +CREATE TRIGGER &mw_prefix.job_default_job_id BEFORE INSERT ON &mw_prefix.job + FOR EACH ROW WHEN (new.job_id IS NULL) +BEGIN + &mw_prefix.lastval_pkg.setLastval(job_job_id_seq.nextval, :new.job_id); +END; +/*$mw$*/ + +CREATE TABLE &mw_prefix.querycache_info ( + qci_type VARCHAR2(32) NOT NULL, + qci_timestamp TIMESTAMP(6) WITH TIME ZONE NULL +); +CREATE UNIQUE INDEX &mw_prefix.querycache_info_u01 ON &mw_prefix.querycache_info (qci_type); + +CREATE TABLE &mw_prefix.redirect ( + rd_from NUMBER NOT NULL, + rd_namespace NUMBER DEFAULT 0 NOT NULL, + rd_title VARCHAR2(255) NOT NULL, + rd_interwiki VARCHAR2(32), + rd_fragment VARCHAR2(255) +); +ALTER TABLE &mw_prefix.redirect ADD CONSTRAINT &mw_prefix.redirect_fk1 FOREIGN KEY (rd_from) REFERENCES &mw_prefix.page(page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +CREATE INDEX &mw_prefix.redirect_i01 ON &mw_prefix.redirect (rd_namespace,rd_title,rd_from); + +CREATE TABLE &mw_prefix.querycachetwo ( + qcc_type VARCHAR2(32) NOT NULL, + qcc_value NUMBER DEFAULT 0 NOT NULL, + qcc_namespace NUMBER DEFAULT 0 NOT NULL, + qcc_title VARCHAR2(255), + qcc_namespacetwo NUMBER DEFAULT 0 NOT NULL, + qcc_titletwo VARCHAR2(255) +); +CREATE INDEX &mw_prefix.querycachetwo_i01 ON &mw_prefix.querycachetwo (qcc_type,qcc_value); +CREATE INDEX &mw_prefix.querycachetwo_i02 ON &mw_prefix.querycachetwo (qcc_type,qcc_namespace,qcc_title); +CREATE INDEX &mw_prefix.querycachetwo_i03 ON &mw_prefix.querycachetwo (qcc_type,qcc_namespacetwo,qcc_titletwo); + +CREATE SEQUENCE page_restrictions_pr_id_seq; +CREATE TABLE &mw_prefix.page_restrictions ( + pr_id NUMBER NOT NULL, + pr_page NUMBER NOT NULL, + pr_type VARCHAR2(255) NOT NULL, + pr_level VARCHAR2(255) NOT NULL, + pr_cascade NUMBER NOT NULL, + pr_user NUMBER NULL, + pr_expiry TIMESTAMP(6) WITH TIME ZONE NULL +); +ALTER TABLE &mw_prefix.page_restrictions ADD CONSTRAINT &mw_prefix.page_restrictions_pk PRIMARY KEY (pr_id); +ALTER TABLE &mw_prefix.page_restrictions ADD CONSTRAINT &mw_prefix.page_restrictions_fk1 FOREIGN KEY (pr_page) REFERENCES &mw_prefix.page(page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +CREATE UNIQUE INDEX &mw_prefix.page_restrictions_u01 ON &mw_prefix.page_restrictions (pr_page,pr_type); +CREATE INDEX &mw_prefix.page_restrictions_i01 ON &mw_prefix.page_restrictions (pr_type,pr_level); +CREATE INDEX &mw_prefix.page_restrictions_i02 ON &mw_prefix.page_restrictions (pr_level); +CREATE INDEX &mw_prefix.page_restrictions_i03 ON &mw_prefix.page_restrictions (pr_cascade); +/*$mw$*/ +CREATE TRIGGER &mw_prefix.page_restrictions_default_pr_id BEFORE INSERT ON &mw_prefix.page_restrictions + FOR EACH ROW WHEN (new.pr_id IS NULL) +BEGIN + &mw_prefix.lastval_pkg.setLastval(page_restrictions_pr_id_seq.nextval, :new.pr_id); +END; +/*$mw$*/ + +CREATE TABLE &mw_prefix.protected_titles ( + pt_namespace NUMBER DEFAULT 0 NOT NULL, + pt_title VARCHAR2(255) NOT NULL, + pt_user NUMBER NOT NULL, + pt_reason VARCHAR2(255), + pt_timestamp TIMESTAMP(6) WITH TIME ZONE NOT NULL, + pt_expiry VARCHAR2(14) NOT NULL, + pt_create_perm VARCHAR2(60) NOT NULL +); +CREATE UNIQUE INDEX &mw_prefix.protected_titles_u01 ON &mw_prefix.protected_titles (pt_namespace,pt_title); +CREATE INDEX &mw_prefix.protected_titles_i01 ON &mw_prefix.protected_titles (pt_timestamp); + +CREATE TABLE &mw_prefix.page_props ( + pp_page NUMBER NOT NULL, + pp_propname VARCHAR2(60) NOT NULL, + pp_value BLOB NOT NULL +); +CREATE UNIQUE INDEX &mw_prefix.page_props_u01 ON &mw_prefix.page_props (pp_page,pp_propname); + + +CREATE TABLE &mw_prefix.updatelog ( + ul_key VARCHAR2(255) NOT NULL, + ul_value BLOB +); +ALTER TABLE &mw_prefix.updatelog ADD CONSTRAINT &mw_prefix.updatelog_pk PRIMARY KEY (ul_key); + +CREATE TABLE &mw_prefix.change_tag ( + ct_id NUMBER NOT NULL, + ct_rc_id NUMBER NULL, + ct_log_id NUMBER NULL, + ct_rev_id NUMBER NULL, + ct_tag VARCHAR2(255) NOT NULL, + ct_params BLOB NULL +); +ALTER TABLE &mw_prefix.change_tag ADD CONSTRAINT &mw_prefix.change_tag_pk PRIMARY KEY (ct_id); +CREATE UNIQUE INDEX &mw_prefix.change_tag_u01 ON &mw_prefix.change_tag (ct_rc_id,ct_tag); +CREATE UNIQUE INDEX &mw_prefix.change_tag_u02 ON &mw_prefix.change_tag (ct_log_id,ct_tag); +CREATE UNIQUE INDEX &mw_prefix.change_tag_u03 ON &mw_prefix.change_tag (ct_rev_id,ct_tag); +CREATE INDEX &mw_prefix.change_tag_i01 ON &mw_prefix.change_tag (ct_tag,ct_rc_id,ct_rev_id,ct_log_id); + +CREATE TABLE &mw_prefix.tag_summary ( + ts_id NUMBER NOT NULL, + ts_rc_id NUMBER NULL, + ts_log_id NUMBER NULL, + ts_rev_id NUMBER NULL, + ts_tags BLOB NOT NULL +); +ALTER TABLE &mw_prefix.tag_summary ADD CONSTRAINT &mw_prefix.tag_summary_pk PRIMARY KEY (ts_id); +CREATE UNIQUE INDEX &mw_prefix.tag_summary_u01 ON &mw_prefix.tag_summary (ts_rc_id); +CREATE UNIQUE INDEX &mw_prefix.tag_summary_u02 ON &mw_prefix.tag_summary (ts_log_id); +CREATE UNIQUE INDEX &mw_prefix.tag_summary_u03 ON &mw_prefix.tag_summary (ts_rev_id); + +CREATE TABLE &mw_prefix.valid_tag ( + vt_tag VARCHAR2(255) NOT NULL +); +ALTER TABLE &mw_prefix.valid_tag ADD CONSTRAINT &mw_prefix.valid_tag_pk PRIMARY KEY (vt_tag); + +-- This table is not used unless profiling is turned on +--CREATE TABLE &mw_prefix.profiling ( +-- pf_count NUMBER DEFAULT 0 NOT NULL, +-- pf_time NUMBER(18,10) DEFAULT 0 NOT NULL, +-- pf_memory NUMBER(18,10) DEFAULT 0 NOT NULL, +-- pf_name VARCHAR2(255), +-- pf_server VARCHAR2(30) +--); +--CREATE UNIQUE INDEX &mw_prefix.profiling_u01 ON &mw_prefix.profiling (pf_name, pf_server); + +CREATE INDEX &mw_prefix.si_title_idx ON &mw_prefix.searchindex(si_title) INDEXTYPE IS ctxsys.context; +CREATE INDEX &mw_prefix.si_text_idx ON &mw_prefix.searchindex(si_text) INDEXTYPE IS ctxsys.context; + +CREATE TABLE &mw_prefix.l10n_cache ( + lc_lang varchar2(32) NOT NULL, + lc_key varchar2(255) NOT NULL, + lc_value clob NOT NULL +); +CREATE INDEX &mw_prefix.l10n_cache_u01 ON &mw_prefix.l10n_cache (lc_lang, lc_key); + +CREATE TABLE &mw_prefix.module_deps ( + md_module VARCHAR2(255) NOT NULL, + md_skin VARCHAR2(32) NOT NULL, + md_deps BLOB NOT NULL +); +CREATE UNIQUE INDEX &mw_prefix.module_deps_u01 ON &mw_prefix.module_deps (md_module, md_skin); + +CREATE SEQUENCE sites_site_id_seq MINVALUE 0 START WITH 0; +CREATE TABLE &mw_prefix.sites ( + site_id NUMBER NOT NULL, + site_global_key VARCHAR2(32) NOT NULL, + site_type VARCHAR2(32) NOT NULL, + site_group VARCHAR2(32) NOT NULL, + site_source VARCHAR2(32) NOT NULL, + site_language VARCHAR2(32) NOT NULL, + site_protocol VARCHAR2(32) NOT NULL, + site_domain VARCHAR2(255) NOT NULL, + site_data BLOB NOT NULL, + site_forward NUMBER(1) NOT NULL, + site_config BLOB NOT NULL +); +ALTER TABLE &mw_prefix.sites ADD CONSTRAINT &mw_prefix.sites_pk PRIMARY KEY (site_id); +CREATE UNIQUE INDEX &mw_prefix.sites_u01 ON &mw_prefix.sites (site_global_key); +CREATE INDEX &mw_prefix.sites_i01 ON &mw_prefix.sites (site_type); +CREATE INDEX &mw_prefix.sites_i02 ON &mw_prefix.sites (site_group); +CREATE INDEX &mw_prefix.sites_i03 ON &mw_prefix.sites (site_source); +CREATE INDEX &mw_prefix.sites_i04 ON &mw_prefix.sites (site_language); +CREATE INDEX &mw_prefix.sites_i05 ON &mw_prefix.sites (site_protocol); +CREATE INDEX &mw_prefix.sites_i06 ON &mw_prefix.sites (site_domain); +CREATE INDEX &mw_prefix.sites_i07 ON &mw_prefix.sites (site_forward); +/*$mw$*/ +CREATE TRIGGER &mw_prefix.sites_default_site_id BEFORE INSERT ON &mw_prefix.sites + FOR EACH ROW WHEN (new.site_id IS NULL) +BEGIN + &mw_prefix.lastval_pkg.setLastval(sites_site_id_seq.nextval, :new.site_id); +END; +/*$mw$*/ + +CREATE TABLE &mw_prefix.site_identifiers ( + si_site NUMBER NOT NULL, + si_type VARCHAR2(32) NOT NULL, + si_key VARCHAR2(32) NOT NULL +); +CREATE UNIQUE INDEX &mw_prefix.site_identifiers_u01 ON &mw_prefix.site_identifiers (si_type, si_key); +CREATE INDEX &mw_prefix.site_identifiers_i01 ON &mw_prefix.site_identifiers (si_site); +CREATE INDEX &mw_prefix.site_identifiers_i02 ON &mw_prefix.site_identifiers (si_key); + +-- do not prefix this table as it breaks parserTests +CREATE TABLE wiki_field_info_full ( +table_name VARCHAR2(35) NOT NULL, +column_name VARCHAR2(35) NOT NULL, +data_default VARCHAR2(4000), +data_length NUMBER NOT NULL, +data_type VARCHAR2(106), +not_null CHAR(1) NOT NULL, +prim NUMBER(1), +uniq NUMBER(1), +nonuniq NUMBER(1) +); +ALTER TABLE wiki_field_info_full ADD CONSTRAINT wiki_field_info_full_pk PRIMARY KEY (table_name, column_name); + +/*$mw$*/ +CREATE PROCEDURE fill_wiki_info IS + BEGIN + DELETE wiki_field_info_full; + + FOR x_rec IN (SELECT t.table_name table_name, t.column_name, + t.data_default, t.data_length, t.data_type, + DECODE (t.nullable, 'Y', '1', 'N', '0') not_null, + (SELECT 1 + FROM user_cons_columns ucc, + user_constraints uc + WHERE ucc.table_name = t.table_name + AND ucc.column_name = t.column_name + AND uc.constraint_name = ucc.constraint_name + AND uc.constraint_type = 'P' + AND ROWNUM < 2) prim, + (SELECT 1 + FROM user_ind_columns uic, + user_indexes ui + WHERE uic.table_name = t.table_name + AND uic.column_name = t.column_name + AND ui.index_name = uic.index_name + AND ui.uniqueness = 'UNIQUE' + AND ROWNUM < 2) uniq, + (SELECT 1 + FROM user_ind_columns uic, + user_indexes ui + WHERE uic.table_name = t.table_name + AND uic.column_name = t.column_name + AND ui.index_name = uic.index_name + AND ui.uniqueness = 'NONUNIQUE' + AND ROWNUM < 2) nonuniq + FROM user_tab_columns t, user_tables ut + WHERE ut.table_name = t.table_name) + LOOP + INSERT INTO wiki_field_info_full + (table_name, column_name, + data_default, data_length, + data_type, not_null, prim, + uniq, nonuniq + ) + VALUES (x_rec.table_name, x_rec.column_name, + x_rec.data_default, x_rec.data_length, + x_rec.data_type, x_rec.not_null, x_rec.prim, + x_rec.uniq, x_rec.nonuniq + ); + END LOOP; + COMMIT; +END; +/*$mw$*/ + +/*$mw$*/ +CREATE OR REPLACE PROCEDURE duplicate_table(p_tabname IN VARCHAR2, + p_oldprefix IN VARCHAR2, + p_newprefix IN VARCHAR2, + p_temporary IN BOOLEAN) IS + e_table_not_exist EXCEPTION; + PRAGMA EXCEPTION_INIT(e_table_not_exist, -00942); + l_temp_ei_sql VARCHAR2(2000); + l_temporary BOOLEAN := p_temporary; +BEGIN + BEGIN + EXECUTE IMMEDIATE 'DROP TABLE ' || p_newprefix || p_tabname || + ' CASCADE CONSTRAINTS PURGE'; + EXCEPTION + WHEN e_table_not_exist THEN + NULL; + END; + IF (p_tabname = 'SEARCHINDEX') THEN + l_temporary := FALSE; + END IF; + IF (l_temporary) THEN + EXECUTE IMMEDIATE 'CREATE GLOBAL TEMPORARY TABLE ' || p_newprefix || + p_tabname || + ' ON COMMIT PRESERVE ROWS AS SELECT * FROM ' || + p_oldprefix || p_tabname || ' WHERE ROWNUM = 0'; + ELSE + EXECUTE IMMEDIATE 'CREATE TABLE ' || p_newprefix || p_tabname || + ' AS SELECT * FROM ' || p_oldprefix || p_tabname || + ' WHERE ROWNUM = 0'; + END IF; + FOR rc IN (SELECT column_name, data_default + FROM user_tab_columns + WHERE table_name = p_oldprefix || p_tabname + AND data_default IS NOT NULL) LOOP + EXECUTE IMMEDIATE 'ALTER TABLE ' || p_newprefix || p_tabname || + ' MODIFY ' || rc.column_name || ' DEFAULT ' || + SUBSTR(rc.data_default, 1, 2000); + END LOOP; + FOR rc IN (SELECT REPLACE(REPLACE(DBMS_LOB.SUBSTR(DBMS_METADATA.get_ddl('CONSTRAINT', + constraint_name), + 32767, + 1), + USER || '"."' || p_oldprefix, + USER || '"."' || p_newprefix), + '"' || constraint_name || '"', + '"' || p_newprefix || constraint_name || '"') DDLVC2, + constraint_name + FROM user_constraints uc + WHERE table_name = p_oldprefix || p_tabname + AND constraint_type = 'P') LOOP + l_temp_ei_sql := SUBSTR(rc.ddlvc2, 1, INSTR(rc.ddlvc2, 'PCTFREE') - 1); + l_temp_ei_sql := SUBSTR(l_temp_ei_sql, + 1, + INSTR(l_temp_ei_sql, + ')', + INSTR(l_temp_ei_sql, 'PRIMARY KEY') + 1) + 1); + IF nvl(length(l_temp_ei_sql), 0) > 0 THEN + EXECUTE IMMEDIATE l_temp_ei_sql; + END IF; + END LOOP; + IF (NOT l_temporary) THEN + FOR rc IN (SELECT REPLACE(DBMS_LOB.SUBSTR(DBMS_METADATA.get_ddl('REF_CONSTRAINT', + constraint_name), + 32767, + 1), + USER || '"."' || p_oldprefix, + USER || '"."' || p_newprefix) DDLVC2, + constraint_name + FROM user_constraints uc + WHERE table_name = p_oldprefix || p_tabname + AND constraint_type = 'R') LOOP + IF nvl(length(l_temp_ei_sql), 0) > 0 AND + INSTR(l_temp_ei_sql, 'PRIMARY KEY') = 0 THEN + EXECUTE IMMEDIATE l_temp_ei_sql; + END IF; + END LOOP; + END IF; + FOR rc IN (SELECT REPLACE(REPLACE(DBMS_LOB.SUBSTR(DBMS_METADATA.get_ddl('INDEX', + index_name), + 32767, + 1), + USER || '"."' || p_oldprefix, + USER || '"."' || p_newprefix), + '"' || index_name || '"', + '"' || p_newprefix || index_name || '"') DDLVC2, + index_name, + index_type + FROM user_indexes ui + WHERE table_name = p_oldprefix || p_tabname + AND index_type NOT IN ('LOB', 'DOMAIN') + AND NOT EXISTS + (SELECT NULL + FROM user_constraints + WHERE table_name = ui.table_name + AND constraint_name = ui.index_name)) LOOP + l_temp_ei_sql := SUBSTR(rc.ddlvc2, 1, INSTR(rc.ddlvc2, 'PCTFREE') - 1); + l_temp_ei_sql := SUBSTR(l_temp_ei_sql, + 1, + INSTR(l_temp_ei_sql, + ')', + INSTR(l_temp_ei_sql, + '"' || USER || '"."' || p_newprefix || '"') + 1) + 1); + IF nvl(length(l_temp_ei_sql), 0) > 0 THEN + EXECUTE IMMEDIATE l_temp_ei_sql; + END IF; + END LOOP; + FOR rc IN (SELECT REPLACE(REPLACE(DBMS_LOB.SUBSTR(DBMS_METADATA.get_ddl('INDEX', + index_name), + 32767, + 1), + USER || '"."' || p_oldprefix, + USER || '"."' || p_newprefix), + '"' || index_name || '"', + '"' || p_newprefix || index_name || '"') DDLVC2, + index_name, + index_type + FROM user_indexes ui + WHERE table_name = p_oldprefix || p_tabname + AND index_type = 'DOMAIN' + AND NOT EXISTS + (SELECT NULL + FROM user_constraints + WHERE table_name = ui.table_name + AND constraint_name = ui.index_name)) LOOP + l_temp_ei_sql := rc.ddlvc2; + IF nvl(length(l_temp_ei_sql), 0) > 0 THEN + EXECUTE IMMEDIATE l_temp_ei_sql; + END IF; + END LOOP; + FOR rc IN (SELECT REPLACE(REPLACE(UPPER(DBMS_LOB.SUBSTR(DBMS_METADATA.get_ddl('TRIGGER', + trigger_name), + 32767, + 1)), + USER || '"."' || p_oldprefix, + USER || '"."' || p_newprefix), + ' ON ' || p_oldprefix || p_tabname, + ' ON ' || p_newprefix || p_tabname) DDLVC2, + trigger_name + FROM user_triggers + WHERE table_name = p_oldprefix || p_tabname) LOOP + l_temp_ei_sql := SUBSTR(rc.ddlvc2, 1, INSTR(rc.ddlvc2, 'ALTER ') - 1); + IF nvl(length(l_temp_ei_sql), 0) > 0 THEN + EXECUTE IMMEDIATE l_temp_ei_sql; + END IF; + END LOOP; +END; + +/*$mw$*/ + +/*$mw$*/ +CREATE OR REPLACE FUNCTION BITOR (x IN NUMBER, y IN NUMBER) RETURN NUMBER AS +BEGIN + RETURN (x + y - BITAND(x, y)); +END; +/*$mw$*/ + +/*$mw$*/ +CREATE OR REPLACE FUNCTION BITNOT (x IN NUMBER) RETURN NUMBER AS +BEGIN + RETURN (4294967295 - x); +END; +/*$mw$*/ + +CREATE OR REPLACE TYPE GET_OUTPUT_TYPE IS TABLE OF VARCHAR2(255); + +/*$mw$*/ +CREATE OR REPLACE FUNCTION GET_OUTPUT_LINES RETURN GET_OUTPUT_TYPE PIPELINED AS + v_line VARCHAR2(255); + v_status INTEGER := 0; +BEGIN + + LOOP + DBMS_OUTPUT.GET_LINE(v_line, v_status); + IF (v_status = 0) THEN RETURN; END IF; + PIPE ROW (v_line); + END LOOP; + RETURN; +EXCEPTION + WHEN OTHERS THEN + RETURN; +END; +/*$mw$*/ + +/*$mw$*/ +CREATE OR REPLACE FUNCTION GET_SEQUENCE_VALUE(seq IN VARCHAR2) RETURN NUMBER AS + v_value NUMBER; +BEGIN + EXECUTE IMMEDIATE 'SELECT '||seq||'.NEXTVAL INTO :outVar FROM DUAL' INTO v_value; + RETURN v_value; +END; +/*$mw$*/ diff --git a/www/wiki/maintenance/oracle/update-keys.sql b/www/wiki/maintenance/oracle/update-keys.sql new file mode 100644 index 00000000..7761d0c5 --- /dev/null +++ b/www/wiki/maintenance/oracle/update-keys.sql @@ -0,0 +1,29 @@ +-- SQL to insert update keys into the initial tables after a +-- fresh installation of MediaWiki's database. +-- This is read and executed by the install script; you should +-- not have to run it by itself unless doing a manual install. +-- Insert keys here if either the unnecessary would cause heavy +-- processing or could potentially cause trouble by lowering field +-- sizes, adding constraints, etc. +-- When adjusting field sizes, it is recommended removing old +-- patches but to play safe, update keys should also inserted here. + +-- The /*_*/ comments in this and other files are +-- replaced with the defined table prefix by the installer +-- and updater scripts. If you are installing or running +-- updates manually, you will need to manually insert the +-- table prefix if any when running these scripts. +-- + +INSERT INTO /*_*/updatelog (ul_key, ul_value) + VALUES( 'filearchive-fa_major_mime-patch-fa_major_mime-chemical.sql', null ); +INSERT INTO /*_*/updatelog (ul_key, ul_value) + VALUES( 'image-img_major_mime-patch-img_major_mime-chemical.sql', null ); +INSERT INTO /*_*/updatelog (ul_key, ul_value) + VALUES( 'oldimage-oi_major_mime-patch-oi_major_mime-chemical.sql', null ); +INSERT INTO /*_*/updatelog (ul_key, ul_value) + VALUES( 'user_groups-ug_group-patch-ug_group-length-increase-255.sql', null ); +INSERT INTO /*_*/updatelog (ul_key, ul_value) + VALUES( 'user_former_groups-ufg_group-patch-ufg_group-length-increase-255.sql', null ); +INSERT INTO /*_*/updatelog (ul_key, ul_value) + VALUES( 'user_properties-up_property-patch-up_property.sql', null ); diff --git a/www/wiki/maintenance/oracle/user.sql b/www/wiki/maintenance/oracle/user.sql new file mode 100644 index 00000000..57688eae --- /dev/null +++ b/www/wiki/maintenance/oracle/user.sql @@ -0,0 +1,16 @@ +-- defines must comply with ^define\s*([^\s=]*)\s*=\s?'\{\$([^\}]*)\}'; +define wiki_user='{$wgDBuser}'; +define wiki_pass='{$wgDBpassword}'; +define def_ts='{$_OracleDefTS}'; +define temp_ts='{$_OracleTempTS}'; + +create user &wiki_user. identified by &wiki_pass. default tablespace &def_ts. temporary tablespace &temp_ts. quota unlimited on &def_ts.; +grant connect, resource to &wiki_user.; +grant alter session to &wiki_user.; +grant ctxapp to &wiki_user.; +grant execute on ctx_ddl to &wiki_user.; +grant create view to &wiki_user.; +grant create synonym to &wiki_user.; +grant create table to &wiki_user.; +grant create sequence to &wiki_user.; +grant create trigger to &wiki_user.; diff --git a/www/wiki/maintenance/orphans.php b/www/wiki/maintenance/orphans.php new file mode 100644 index 00000000..644fb958 --- /dev/null +++ b/www/wiki/maintenance/orphans.php @@ -0,0 +1,256 @@ +<?php +/** + * Look for 'orphan' revisions hooked to pages which don't exist and + * 'childless' pages with no revisions. + * Then, kill the poor widows and orphans. + * Man this is depressing. + * + * Copyright © 2005 Brion Vibber <brion@pobox.com> + * https://www.mediawiki.org/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @author <brion@pobox.com> + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +use Wikimedia\Rdbms\IMaintainableDatabase; + +/** + * Maintenance script that looks for 'orphan' revisions hooked to pages which + * don't exist and 'childless' pages with no revisions. + * + * @ingroup Maintenance + */ +class Orphans extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( "Look for 'orphan' revisions hooked to pages which don't exist\n" . + "and 'childless' pages with no revisions\n" . + "Then, kill the poor widows and orphans\n" . + "Man this is depressing" + ); + $this->addOption( 'fix', 'Actually fix broken entries' ); + } + + public function execute() { + $this->checkOrphans( $this->hasOption( 'fix' ) ); + $this->checkSeparation( $this->hasOption( 'fix' ) ); + # Does not work yet, do not use + # $this->checkWidows( $this->hasOption( 'fix' ) ); + } + + /** + * Lock the appropriate tables for the script + * @param IMaintainableDatabase $db + * @param string[] $extraTable The name of any extra tables to lock (eg: text) + */ + private function lockTables( $db, $extraTable = [] ) { + $tbls = [ 'page', 'revision', 'redirect' ]; + if ( $extraTable ) { + $tbls = array_merge( $tbls, $extraTable ); + } + $db->lockTables( [], $tbls, __METHOD__, false ); + } + + /** + * Check for orphan revisions + * @param bool $fix Whether to fix broken revisions when found + */ + private function checkOrphans( $fix ) { + $dbw = $this->getDB( DB_MASTER ); + $commentStore = new CommentStore( 'rev_comment' ); + + if ( $fix ) { + $this->lockTables( $dbw ); + } + + $commentQuery = $commentStore->getJoin(); + + $this->output( "Checking for orphan revision table entries... " + . "(this may take a while on a large wiki)\n" ); + $result = $dbw->select( + [ 'revision', 'page' ] + $commentQuery['tables'], + [ 'rev_id', 'rev_page', 'rev_timestamp', 'rev_user_text' ] + $commentQuery['fields'], + [ 'page_id' => null ], + __METHOD__, + [], + [ 'page' => [ 'LEFT JOIN', [ 'rev_page=page_id' ] ] ] + $commentQuery['joins'] + ); + $orphans = $result->numRows(); + if ( $orphans > 0 ) { + global $wgContLang; + + $this->output( "$orphans orphan revisions...\n" ); + $this->output( sprintf( + "%10s %10s %14s %20s %s\n", + 'rev_id', 'rev_page', 'rev_timestamp', 'rev_user_text', 'rev_comment' + ) ); + + foreach ( $result as $row ) { + $comment = $commentStore->getComment( $row )->text; + if ( $comment !== '' ) { + $comment = '(' . $wgContLang->truncate( $comment, 40 ) . ')'; + } + $this->output( sprintf( "%10d %10d %14s %20s %s\n", + $row->rev_id, + $row->rev_page, + $row->rev_timestamp, + $wgContLang->truncate( $row->rev_user_text, 17 ), + $comment ) ); + if ( $fix ) { + $dbw->delete( 'revision', [ 'rev_id' => $row->rev_id ] ); + } + } + if ( !$fix ) { + $this->output( "Run again with --fix to remove these entries automatically.\n" ); + } + } else { + $this->output( "No orphans! Yay!\n" ); + } + + if ( $fix ) { + $dbw->unlockTables( __METHOD__ ); + } + } + + /** + * @param bool $fix + * @todo DON'T USE THIS YET! It will remove entries which have children, + * but which aren't properly attached (eg if page_latest is bogus + * but valid revisions do exist) + */ + private function checkWidows( $fix ) { + $dbw = $this->getDB( DB_MASTER ); + $page = $dbw->tableName( 'page' ); + $revision = $dbw->tableName( 'revision' ); + + if ( $fix ) { + $this->lockTables( $dbw ); + } + + $this->output( "\nChecking for childless page table entries... " + . "(this may take a while on a large wiki)\n" ); + $result = $dbw->query( " + SELECT * + FROM $page LEFT OUTER JOIN $revision ON page_latest=rev_id + WHERE rev_id IS NULL + " ); + $widows = $result->numRows(); + if ( $widows > 0 ) { + $this->output( "$widows childless pages...\n" ); + $this->output( sprintf( "%10s %11s %2s %s\n", 'page_id', 'page_latest', 'ns', 'page_title' ) ); + foreach ( $result as $row ) { + printf( "%10d %11d %2d %s\n", + $row->page_id, + $row->page_latest, + $row->page_namespace, + $row->page_title ); + if ( $fix ) { + $dbw->delete( 'page', [ 'page_id' => $row->page_id ] ); + } + } + if ( !$fix ) { + $this->output( "Run again with --fix to remove these entries automatically.\n" ); + } + } else { + $this->output( "No childless pages! Yay!\n" ); + } + + if ( $fix ) { + $dbw->unlockTables( __METHOD__ ); + } + } + + /** + * Check for pages where page_latest is wrong + * @param bool $fix Whether to fix broken entries + */ + private function checkSeparation( $fix ) { + $dbw = $this->getDB( DB_MASTER ); + $page = $dbw->tableName( 'page' ); + $revision = $dbw->tableName( 'revision' ); + + if ( $fix ) { + $this->lockTables( $dbw, [ 'user', 'text' ] ); + } + + $this->output( "\nChecking for pages whose page_latest links are incorrect... " + . "(this may take a while on a large wiki)\n" ); + $result = $dbw->query( " + SELECT * + FROM $page LEFT OUTER JOIN $revision ON page_latest=rev_id + " ); + $found = 0; + foreach ( $result as $row ) { + $result2 = $dbw->query( " + SELECT MAX(rev_timestamp) as max_timestamp + FROM $revision + WHERE rev_page=$row->page_id + " ); + $row2 = $dbw->fetchObject( $result2 ); + if ( $row2 ) { + if ( $row->rev_timestamp != $row2->max_timestamp ) { + if ( $found == 0 ) { + $this->output( sprintf( "%10s %10s %14s %14s\n", + 'page_id', 'rev_id', 'timestamp', 'max timestamp' ) ); + } + ++$found; + $this->output( sprintf( "%10d %10d %14s %14s\n", + $row->page_id, + $row->page_latest, + $row->rev_timestamp, + $row2->max_timestamp ) ); + if ( $fix ) { + # ... + $maxId = $dbw->selectField( + 'revision', + 'rev_id', + [ + 'rev_page' => $row->page_id, + 'rev_timestamp' => $row2->max_timestamp ] ); + $this->output( "... updating to revision $maxId\n" ); + $maxRev = Revision::newFromId( $maxId ); + $title = Title::makeTitle( $row->page_namespace, $row->page_title ); + $article = WikiPage::factory( $title ); + $article->updateRevisionOn( $dbw, $maxRev ); + } + } + } else { + $this->output( "wtf\n" ); + } + } + + if ( $found ) { + $this->output( "Found $found pages with incorrect latest revision.\n" ); + } else { + $this->output( "No pages with incorrect latest revision. Yay!\n" ); + } + if ( !$fix && $found > 0 ) { + $this->output( "Run again with --fix to remove these entries automatically.\n" ); + } + + if ( $fix ) { + $dbw->unlockTables( __METHOD__ ); + } + } +} + +$maintClass = "Orphans"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/pageExists.php b/www/wiki/maintenance/pageExists.php new file mode 100644 index 00000000..b631005f --- /dev/null +++ b/www/wiki/maintenance/pageExists.php @@ -0,0 +1,53 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * @ingroup Maintenance + */ +class PageExists extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Report whether a specific page exists' ); + $this->addArg( 'title', 'Page title to check whether it exists' ); + } + + public function execute() { + $titleArg = $this->getArg(); + $title = Title::newFromText( $titleArg ); + $pageExists = $title && $title->exists(); + + $text = ''; + $code = 0; + if ( $pageExists ) { + $text = "{$title} exists."; + } else { + $text = "{$titleArg} doesn't exist."; + $code = 1; + } + $this->output( $text ); + $this->error( '', $code ); + } +} + +$maintClass = "PageExists"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/parse.php b/www/wiki/maintenance/parse.php new file mode 100644 index 00000000..6279a348 --- /dev/null +++ b/www/wiki/maintenance/parse.php @@ -0,0 +1,144 @@ +<?php +/** + * Parse some wikitext. + * + * Wikitext can be given by stdin or using a file. The wikitext will be parsed + * using 'CLIParser' as a title. This can be overridden with --title option. + * + * Example1: + * @code + * $ php parse.php --title foo + * ''[[foo]]''^D + * <p><i><strong class="selflink">foo</strong></i> + * </p> + * @endcode + * + * Example2: + * @code + * $ echo "'''bold'''" > /tmp/foo.txt + * $ php parse.php /tmp/foo.txt + * <p><b>bold</b> + * </p>$ + * @endcode + * + * Example3: + * @code + * $ cat /tmp/foo | php parse.php + * <p><b>bold</b> + * </p>$ + * @endcode + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @author Antoine Musso <hashar at free dot fr> + * @license GNU General Public License 2.0 or later + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to parse some wikitext. + * + * @ingroup Maintenance + */ +class CLIParser extends Maintenance { + protected $parser; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Parse a given wikitext' ); + $this->addOption( + 'title', + 'Title name for the given wikitext (Default: \'CLIParser\')', + false, + true + ); + $this->addOption( 'tidy', 'Tidy the output' ); + $this->addArg( 'file', 'File containing wikitext (Default: stdin)', false ); + } + + public function execute() { + $this->initParser(); + print $this->render( $this->Wikitext() ); + } + + /** + * @param string $wikitext Wikitext to get rendered + * @return string HTML Rendering + */ + public function render( $wikitext ) { + return $this->parse( $wikitext )->getText(); + } + + /** + * Get wikitext from a the file passed as argument or STDIN + * @return string Wikitext + */ + protected function Wikitext() { + $php_stdin = 'php://stdin'; + $input_file = $this->getArg( 0, $php_stdin ); + + if ( $input_file === $php_stdin && !$this->mQuiet ) { + $ctrl = wfIsWindows() ? 'CTRL+Z' : 'CTRL+D'; + $this->error( basename( __FILE__ ) + . ": warning: reading wikitext from STDIN. Press $ctrl to parse.\n" ); + } + + return file_get_contents( $input_file ); + } + + protected function initParser() { + global $wgParserConf; + $parserClass = $wgParserConf['class']; + $this->parser = new $parserClass(); + } + + /** + * Title object to use for CLI parsing. + * Default title is 'CLIParser', it can be overridden with the option + * --title <Your:Title> + * + * @return Title + */ + protected function getTitle() { + $title = $this->getOption( 'title' ) + ? $this->getOption( 'title' ) + : 'CLIParser'; + + return Title::newFromText( $title ); + } + + /** + * @param string $wikitext Wikitext to parse + * @return ParserOutput + */ + protected function parse( $wikitext ) { + $options = new ParserOptions; + if ( $this->getOption( 'tidy' ) ) { + $options->setTidy( true ); + } + return $this->parser->parse( + $wikitext, + $this->getTitle(), + $options + ); + } +} + +$maintClass = "CLIParser"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/patchSql.php b/www/wiki/maintenance/patchSql.php new file mode 100644 index 00000000..bc211409 --- /dev/null +++ b/www/wiki/maintenance/patchSql.php @@ -0,0 +1,70 @@ +<?php +/** + * Manually run an SQL patch outside of the general updaters. + * This ensures that the DB options (charset, prefix, engine) are correctly set. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that manually runs an SQL patch outside of the general updaters. + * + * @ingroup Maintenance + */ +class PatchSql extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Run an SQL file into the DB, replacing prefix and charset vars' ); + $this->addArg( + 'patch-name', + 'Name of the patch file, either full path or in maintenance/archives' + ); + } + + public function getDbType() { + return Maintenance::DB_ADMIN; + } + + public function execute() { + $dbw = $this->getDB( DB_MASTER ); + $updater = DatabaseUpdater::newForDB( $dbw, true, $this ); + + foreach ( $this->mArgs as $arg ) { + $files = [ + $arg, + $updater->patchPath( $dbw, $arg ), + $updater->patchPath( $dbw, "patch-$arg.sql" ), + ]; + foreach ( $files as $file ) { + if ( file_exists( $file ) ) { + $this->output( "$file ...\n" ); + $dbw->sourceFile( $file ); + continue 2; + } + } + $this->error( "Could not find $arg\n" ); + } + $this->output( "done.\n" ); + } +} + +$maintClass = "PatchSql"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/populateBacklinkNamespace.php b/www/wiki/maintenance/populateBacklinkNamespace.php new file mode 100644 index 00000000..295dacda --- /dev/null +++ b/www/wiki/maintenance/populateBacklinkNamespace.php @@ -0,0 +1,97 @@ +<?php +/** + * Optional upgrade script to populate *_from_namespace fields + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to populate *_from_namespace fields + * + * @ingroup Maintenance + */ +class PopulateBacklinkNamespace extends LoggedUpdateMaintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Populate the *_from_namespace fields' ); + $this->addOption( 'lastUpdatedId', "Highest page_id with updated links", false, true ); + } + + protected function getUpdateKey() { + return 'populate *_from_namespace'; + } + + protected function updateSkippedMessage() { + return '*_from_namespace column of backlink tables already populated.'; + } + + public function doDBUpdates() { + $force = $this->getOption( 'force' ); + + $db = $this->getDB( DB_MASTER ); + + $this->output( "Updating *_from_namespace fields in links tables.\n" ); + + $start = $this->getOption( 'lastUpdatedId' ); + if ( !$start ) { + $start = $db->selectField( 'page', 'MIN(page_id)', false, __METHOD__ ); + } + if ( !$start ) { + $this->output( "Nothing to do." ); + return false; + } + $end = $db->selectField( 'page', 'MAX(page_id)', false, __METHOD__ ); + + # Do remaining chunk + $end += $this->mBatchSize - 1; + $blockStart = $start; + $blockEnd = $start + $this->mBatchSize - 1; + while ( $blockEnd <= $end ) { + $this->output( "...doing page_id from $blockStart to $blockEnd\n" ); + $cond = "page_id BETWEEN $blockStart AND $blockEnd"; + $res = $db->select( 'page', [ 'page_id', 'page_namespace' ], $cond, __METHOD__ ); + foreach ( $res as $row ) { + $db->update( 'pagelinks', + [ 'pl_from_namespace' => $row->page_namespace ], + [ 'pl_from' => $row->page_id ], + __METHOD__ + ); + $db->update( 'templatelinks', + [ 'tl_from_namespace' => $row->page_namespace ], + [ 'tl_from' => $row->page_id ], + __METHOD__ + ); + $db->update( 'imagelinks', + [ 'il_from_namespace' => $row->page_namespace ], + [ 'il_from' => $row->page_id ], + __METHOD__ + ); + } + $blockStart += $this->mBatchSize - 1; + $blockEnd += $this->mBatchSize - 1; + wfWaitForSlaves(); + } + return true; + } +} + +$maintClass = "PopulateBacklinkNamespace"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/populateCategory.php b/www/wiki/maintenance/populateCategory.php new file mode 100644 index 00000000..5dccdd65 --- /dev/null +++ b/www/wiki/maintenance/populateCategory.php @@ -0,0 +1,154 @@ +<?php +/** + * Populate the category table. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @author Simetrical + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to populate the category table. + * + * @ingroup Maintenance + */ +class PopulateCategory extends Maintenance { + + const REPORTING_INTERVAL = 1000; + + public function __construct() { + parent::__construct(); + $this->addDescription( + <<<TEXT +This script will populate the category table, added in MediaWiki 1.13. It will +print out progress indicators every 1000 categories it adds to the table. The +script is perfectly safe to run on large, live wikis, and running it multiple +times is harmless. You may want to use the throttling options if it's causing +too much load; they will not affect correctness. + +If the script is stopped and later resumed, you can use the --begin option with +the last printed progress indicator to pick up where you left off. This is +safe, because any newly-added categories before this cutoff will have been +added after the software update and so will be populated anyway. + +When the script has finished, it will make a note of this in the database, and +will not run again without the --force option. +TEXT + ); + + $this->addOption( + 'begin', + 'Only do categories whose names are alphabetically after the provided name', + false, + true + ); + $this->addOption( + 'throttle', + 'Wait this many milliseconds after each category. Default: 0', + false, + true + ); + $this->addOption( 'force', 'Run regardless of whether the database says it\'s been run already' ); + } + + public function execute() { + $begin = $this->getOption( 'begin', '' ); + $throttle = $this->getOption( 'throttle', 0 ); + $force = $this->hasOption( 'force' ); + + $dbw = $this->getDB( DB_MASTER ); + + if ( !$force ) { + $row = $dbw->selectRow( + 'updatelog', + '1', + [ 'ul_key' => 'populate category' ], + __METHOD__ + ); + if ( $row ) { + $this->output( "Category table already populated. Use php " . + "maintenance/populateCategory.php\n--force from the command line " . + "to override.\n" ); + + return true; + } + } + + $throttle = intval( $throttle ); + if ( $begin !== '' ) { + $where = 'cl_to > ' . $dbw->addQuotes( $begin ); + } else { + $where = null; + } + $i = 0; + + while ( true ) { + # Find which category to update + $row = $dbw->selectRow( + 'categorylinks', + 'cl_to', + $where, + __METHOD__, + [ + 'ORDER BY' => 'cl_to' + ] + ); + if ( !$row ) { + # Done, hopefully. + break; + } + $name = $row->cl_to; + $where = 'cl_to > ' . $dbw->addQuotes( $name ); + + # Use the row to update the category count + $cat = Category::newFromName( $name ); + if ( !is_object( $cat ) ) { + $this->output( "The category named $name is not valid?!\n" ); + } else { + $cat->refreshCounts(); + } + + ++$i; + if ( !( $i % self::REPORTING_INTERVAL ) ) { + $this->output( "$name\n" ); + wfWaitForSlaves(); + } + usleep( $throttle * 1000 ); + } + + if ( $dbw->insert( + 'updatelog', + [ 'ul_key' => 'populate category' ], + __METHOD__, + 'IGNORE' + ) ) { + $this->output( "Category population complete.\n" ); + + return true; + } else { + $this->output( "Could not insert category population row.\n" ); + + return false; + } + } +} + +$maintClass = "PopulateCategory"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/populateContentModel.php b/www/wiki/maintenance/populateContentModel.php new file mode 100644 index 00000000..74a918ae --- /dev/null +++ b/www/wiki/maintenance/populateContentModel.php @@ -0,0 +1,252 @@ +<?php +/** + * Populate the page_content_model and {rev,ar}_content_{model,format} fields. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +use Wikimedia\Rdbms\IDatabase; +use MediaWiki\MediaWikiServices; + +/** + * Usage: + * populateContentModel.php --ns=1 --table=page + */ +class PopulateContentModel extends Maintenance { + protected $wikiId; + /** @var WANObjectCache */ + protected $wanCache; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Populate the various content_* fields' ); + $this->addOption( 'ns', 'Namespace to run in, or "all" for all namespaces', true, true ); + $this->addOption( 'table', 'Table to run in', true, true ); + $this->setBatchSize( 100 ); + } + + public function execute() { + $dbw = $this->getDB( DB_MASTER ); + + $this->wikiId = $dbw->getDomainID(); + $this->wanCache = MediaWikiServices::getInstance()->getMainWANObjectCache(); + + $ns = $this->getOption( 'ns' ); + if ( !ctype_digit( $ns ) && $ns !== 'all' ) { + $this->error( 'Invalid namespace', 1 ); + } + $ns = $ns === 'all' ? 'all' : (int)$ns; + $table = $this->getOption( 'table' ); + switch ( $table ) { + case 'revision': + case 'archive': + $this->populateRevisionOrArchive( $dbw, $table, $ns ); + break; + case 'page': + $this->populatePage( $dbw, $ns ); + break; + default: + $this->error( "Invalid table name: $table", 1 ); + } + } + + protected function clearCache( $page_id, $rev_id ) { + $contentModelKey = $this->wanCache->makeKey( 'page', 'content-model', $rev_id ); + $revisionKey = + $this->wanCache->makeGlobalKey( 'revision', $this->wikiId, $page_id, $rev_id ); + + // WikiPage content model cache + $this->wanCache->delete( $contentModelKey ); + + // Revision object cache, which contains a content model + $this->wanCache->delete( $revisionKey ); + } + + private function updatePageRows( IDatabase $dbw, $pageIds, $model ) { + $count = count( $pageIds ); + $this->output( "Setting $count rows to $model..." ); + $dbw->update( + 'page', + [ 'page_content_model' => $model ], + [ 'page_id' => $pageIds ], + __METHOD__ + ); + wfWaitForSlaves(); + $this->output( "done.\n" ); + } + + protected function populatePage( IDatabase $dbw, $ns ) { + $toSave = []; + $lastId = 0; + $nsCondition = $ns === 'all' ? [] : [ 'page_namespace' => $ns ]; + do { + $rows = $dbw->select( + 'page', + [ 'page_namespace', 'page_title', 'page_id' ], + [ + 'page_content_model' => null, + 'page_id > ' . $dbw->addQuotes( $lastId ), + ] + $nsCondition, + __METHOD__, + [ 'LIMIT' => $this->mBatchSize, 'ORDER BY' => 'page_id ASC' ] + ); + $this->output( "Fetched {$rows->numRows()} rows.\n" ); + foreach ( $rows as $row ) { + $title = Title::newFromRow( $row ); + $model = ContentHandler::getDefaultModelFor( $title ); + $toSave[$model][] = $row->page_id; + if ( count( $toSave[$model] ) >= $this->mBatchSize ) { + $this->updatePageRows( $dbw, $toSave[$model], $model ); + unset( $toSave[$model] ); + } + $lastId = $row->page_id; + } + } while ( $rows->numRows() >= $this->mBatchSize ); + foreach ( $toSave as $model => $pages ) { + $this->updatePageRows( $dbw, $pages, $model ); + } + } + + private function updateRevisionOrArchiveRows( IDatabase $dbw, $ids, $model, $table ) { + $prefix = $table === 'archive' ? 'ar' : 'rev'; + $model_column = "{$prefix}_content_model"; + $format_column = "{$prefix}_content_format"; + $key = "{$prefix}_id"; + + $count = count( $ids ); + $format = ContentHandler::getForModelID( $model )->getDefaultFormat(); + $this->output( "Setting $count rows to $model / $format..." ); + $dbw->update( + $table, + [ $model_column => $model, $format_column => $format ], + [ $key => $ids ], + __METHOD__ + ); + + $this->output( "done.\n" ); + } + + protected function populateRevisionOrArchive( IDatabase $dbw, $table, $ns ) { + $prefix = $table === 'archive' ? 'ar' : 'rev'; + $model_column = "{$prefix}_content_model"; + $format_column = "{$prefix}_content_format"; + $key = "{$prefix}_id"; + if ( $table === 'archive' ) { + $selectTables = 'archive'; + $fields = [ 'ar_namespace', 'ar_title' ]; + $join_conds = []; + $where = $ns === 'all' ? [] : [ 'ar_namespace' => $ns ]; + $page_id_column = 'ar_page_id'; + $rev_id_column = 'ar_rev_id'; + } else { // revision + $selectTables = [ 'revision', 'page' ]; + $fields = [ 'page_title', 'page_namespace' ]; + $join_conds = [ 'page' => [ 'INNER JOIN', 'rev_page=page_id' ] ]; + $where = $ns === 'all' ? [] : [ 'page_namespace' => $ns ]; + $page_id_column = 'rev_page'; + $rev_id_column = 'rev_id'; + } + + $toSave = []; + $idsToClear = []; + $lastId = 0; + do { + $rows = $dbw->select( + $selectTables, + array_merge( + $fields, + [ $model_column, $format_column, $key, $page_id_column, $rev_id_column ] + ), + // @todo support populating format if model is already set + [ + $model_column => null, + "$key > " . $dbw->addQuotes( $lastId ), + ] + $where, + __METHOD__, + [ 'LIMIT' => $this->mBatchSize, 'ORDER BY' => "$key ASC" ], + $join_conds + ); + $this->output( "Fetched {$rows->numRows()} rows.\n" ); + foreach ( $rows as $row ) { + if ( $table === 'archive' ) { + $title = Title::makeTitle( $row->ar_namespace, $row->ar_title ); + } else { + $title = Title::newFromRow( $row ); + } + $lastId = $row->{$key}; + try { + $handler = ContentHandler::getForTitle( $title ); + } catch ( MWException $e ) { + $this->error( "Invalid content model for $title" ); + continue; + } + $defaultModel = $handler->getModelID(); + $defaultFormat = $handler->getDefaultFormat(); + $dbModel = $row->{$model_column}; + $dbFormat = $row->{$format_column}; + $id = $row->{$key}; + if ( $dbModel === null && $dbFormat === null ) { + // Set the defaults + $toSave[$defaultModel][] = $row->{$key}; + $idsToClear[] = [ + 'page_id' => $row->{$page_id_column}, + 'rev_id' => $row->{$rev_id_column}, + ]; + } else { // $dbModel === null, $dbFormat set. + if ( $dbFormat === $defaultFormat ) { + $toSave[$defaultModel][] = $row->{$key}; + $idsToClear[] = [ + 'page_id' => $row->{$page_id_column}, + 'rev_id' => $row->{$rev_id_column}, + ]; + } else { // non-default format, just update now + $this->output( "Updating model to match format for $table $id of $title... " ); + $dbw->update( + $table, + [ $model_column => $defaultModel ], + [ $key => $id ], + __METHOD__ + ); + wfWaitForSlaves(); + $this->clearCache( $row->{$page_id_column}, $row->{$rev_id_column} ); + $this->output( "done.\n" ); + continue; + } + } + + if ( count( $toSave[$defaultModel] ) >= $this->mBatchSize ) { + $this->updateRevisionOrArchiveRows( $dbw, $toSave[$defaultModel], $defaultModel, $table ); + unset( $toSave[$defaultModel] ); + } + } + } while ( $rows->numRows() >= $this->mBatchSize ); + foreach ( $toSave as $model => $ids ) { + $this->updateRevisionOrArchiveRows( $dbw, $ids, $model, $table ); + } + + foreach ( $idsToClear as $idPair ) { + $this->clearCache( $idPair['page_id'], $idPair['rev_id'] ); + } + } +} + +$maintClass = 'PopulateContentModel'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/populateFilearchiveSha1.php b/www/wiki/maintenance/populateFilearchiveSha1.php new file mode 100644 index 00000000..7557a42f --- /dev/null +++ b/www/wiki/maintenance/populateFilearchiveSha1.php @@ -0,0 +1,108 @@ +<?php +/** + * Optional upgrade script to populate the fa_sha1 field + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to populate the fa_sha1 field. + * + * @ingroup Maintenance + * @since 1.21 + */ +class PopulateFilearchiveSha1 extends LoggedUpdateMaintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Populate the fa_sha1 field from fa_storage_key' ); + } + + protected function getUpdateKey() { + return 'populate fa_sha1'; + } + + protected function updateSkippedMessage() { + return 'fa_sha1 column of filearchive table already populated.'; + } + + public function doDBUpdates() { + $startTime = microtime( true ); + $dbw = $this->getDB( DB_MASTER ); + $table = 'filearchive'; + $conds = [ 'fa_sha1' => '', 'fa_storage_key IS NOT NULL' ]; + + if ( !$dbw->fieldExists( $table, 'fa_sha1', __METHOD__ ) ) { + $this->output( "fa_sha1 column does not exist\n\n", true ); + + return false; + } + + $this->output( "Populating fa_sha1 field from fa_storage_key\n" ); + $endId = $dbw->selectField( $table, 'MAX(fa_id)', false, __METHOD__ ); + + $batchSize = $this->mBatchSize; + $done = 0; + + do { + $res = $dbw->select( + $table, + [ 'fa_id', 'fa_storage_key' ], + $conds, + __METHOD__, + [ 'LIMIT' => $batchSize ] + ); + + $i = 0; + foreach ( $res as $row ) { + if ( $row->fa_storage_key == '' ) { + // Revision was missing pre-deletion + continue; + } + $sha1 = LocalRepo::getHashFromKey( $row->fa_storage_key ); + $dbw->update( $table, + [ 'fa_sha1' => $sha1 ], + [ 'fa_id' => $row->fa_id ], + __METHOD__ + ); + $lastId = $row->fa_id; + $i++; + } + + $done += $i; + if ( $i !== $batchSize ) { + break; + } + + // print status and let replica DBs catch up + $this->output( sprintf( + "id %d done (up to %d), %5.3f%% \r", $lastId, $endId, $lastId / $endId * 100 ) ); + wfWaitForSlaves(); + } while ( true ); + + $processingTime = microtime( true ) - $startTime; + $this->output( sprintf( "\nDone %d files in %.1f seconds\n", $done, $processingTime ) ); + + return true; // we only updated *some* files, don't log + } +} + +$maintClass = "PopulateFilearchiveSha1"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/populateImageSha1.php b/www/wiki/maintenance/populateImageSha1.php new file mode 100644 index 00000000..b581d661 --- /dev/null +++ b/www/wiki/maintenance/populateImageSha1.php @@ -0,0 +1,184 @@ +<?php +/** + * Optional upgrade script to populate the img_sha1 field + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to populate the img_sha1 field. + * + * @ingroup Maintenance + */ +class PopulateImageSha1 extends LoggedUpdateMaintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Populate the img_sha1 field' ); + $this->addOption( 'force', "Recalculate sha1 for rows that already have a value" ); + $this->addOption( 'multiversiononly', "Calculate only for files with several versions" ); + $this->addOption( 'method', "Use 'pipe' to pipe to mysql command line,\n" . + "\t\tdefault uses Database class", false, true ); + $this->addOption( + 'file', + 'Fix for a specific file, without File: namespace prefixed', + false, + true + ); + } + + protected function getUpdateKey() { + return 'populate img_sha1'; + } + + protected function updateSkippedMessage() { + return 'img_sha1 column of image table already populated.'; + } + + public function execute() { + if ( $this->getOption( 'file' ) || $this->hasOption( 'multiversiononly' ) ) { + $this->doDBUpdates(); // skip update log checks/saves + } else { + parent::execute(); + } + } + + public function doDBUpdates() { + $method = $this->getOption( 'method', 'normal' ); + $file = $this->getOption( 'file', '' ); + $force = $this->getOption( 'force' ); + $isRegen = ( $force || $file != '' ); // forced recalculation? + + $t = -microtime( true ); + $dbw = $this->getDB( DB_MASTER ); + if ( $file != '' ) { + $res = $dbw->select( + 'image', + [ 'img_name' ], + [ 'img_name' => $file ], + __METHOD__ + ); + if ( !$res ) { + $this->error( "No such file: $file", true ); + + return false; + } + $this->output( "Populating img_sha1 field for specified files\n" ); + } else { + if ( $this->hasOption( 'multiversiononly' ) ) { + $conds = []; + $this->output( "Populating and recalculating img_sha1 field for versioned files\n" ); + } elseif ( $force ) { + $conds = []; + $this->output( "Populating and recalculating img_sha1 field\n" ); + } else { + $conds = [ 'img_sha1' => '' ]; + $this->output( "Populating img_sha1 field\n" ); + } + if ( $this->hasOption( 'multiversiononly' ) ) { + $res = $dbw->select( 'oldimage', + [ 'img_name' => 'DISTINCT(oi_name)' ], $conds, __METHOD__ ); + } else { + $res = $dbw->select( 'image', [ 'img_name' ], $conds, __METHOD__ ); + } + } + + $imageTable = $dbw->tableName( 'image' ); + $oldImageTable = $dbw->tableName( 'oldimage' ); + + if ( $method == 'pipe' ) { + // Opening a pipe allows the SHA-1 operation to be done in parallel + // with the database write operation, because the writes are queued + // in the pipe buffer. This can improve performance by up to a + // factor of 2. + global $wgDBuser, $wgDBserver, $wgDBpassword, $wgDBname; + $cmd = 'mysql -u' . wfEscapeShellArg( $wgDBuser ) . + ' -h' . wfEscapeShellArg( $wgDBserver ) . + ' -p' . wfEscapeShellArg( $wgDBpassword, $wgDBname ); + $this->output( "Using pipe method\n" ); + $pipe = popen( $cmd, 'w' ); + } + + $numRows = $res->numRows(); + $i = 0; + foreach ( $res as $row ) { + if ( $i % $this->mBatchSize == 0 ) { + $this->output( sprintf( + "Done %d of %d, %5.3f%% \r", $i, $numRows, $i / $numRows * 100 ) ); + wfWaitForSlaves(); + } + + $file = wfLocalFile( $row->img_name ); + if ( !$file ) { + continue; + } + + // Upgrade the current file version... + $sha1 = $file->getRepo()->getFileSha1( $file->getPath() ); + if ( strval( $sha1 ) !== '' ) { // file on disk and hashed properly + if ( $isRegen && $file->getSha1() !== $sha1 ) { + // The population was probably done already. If the old SHA1 + // does not match, then both fix the SHA1 and the metadata. + $file->upgradeRow(); + } else { + $sql = "UPDATE $imageTable SET img_sha1=" . $dbw->addQuotes( $sha1 ) . + " WHERE img_name=" . $dbw->addQuotes( $file->getName() ); + if ( $method == 'pipe' ) { + fwrite( $pipe, "$sql;\n" ); + } else { + $dbw->query( $sql, __METHOD__ ); + } + } + } + // Upgrade the old file versions... + foreach ( $file->getHistory() as $oldFile ) { + $sha1 = $oldFile->getRepo()->getFileSha1( $oldFile->getPath() ); + if ( strval( $sha1 ) !== '' ) { // file on disk and hashed properly + if ( $isRegen && $oldFile->getSha1() !== $sha1 ) { + // The population was probably done already. If the old SHA1 + // does not match, then both fix the SHA1 and the metadata. + $oldFile->upgradeRow(); + } else { + $sql = "UPDATE $oldImageTable SET oi_sha1=" . $dbw->addQuotes( $sha1 ) . + " WHERE (oi_name=" . $dbw->addQuotes( $oldFile->getName() ) . " AND" . + " oi_archive_name=" . $dbw->addQuotes( $oldFile->getArchiveName() ) . ")"; + if ( $method == 'pipe' ) { + fwrite( $pipe, "$sql;\n" ); + } else { + $dbw->query( $sql, __METHOD__ ); + } + } + } + } + $i++; + } + if ( $method == 'pipe' ) { + fflush( $pipe ); + pclose( $pipe ); + } + $t += microtime( true ); + $this->output( sprintf( "\nDone %d files in %.1f seconds\n", $numRows, $t ) ); + + return !$file; // we only updated *some* files, don't log + } +} + +$maintClass = "PopulateImageSha1"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/populateInterwiki.php b/www/wiki/maintenance/populateInterwiki.php new file mode 100644 index 00000000..1b05e1ed --- /dev/null +++ b/www/wiki/maintenance/populateInterwiki.php @@ -0,0 +1,156 @@ +<?php + +/** + * Maintenance script that populates the interwiki table with list of sites from + * a source wiki, such as English Wikipedia. (the default source) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @author Katie Filbert < aude.wiki@gmail.com > + */ + +require_once __DIR__ . '/Maintenance.php'; + +class PopulateInterwiki extends Maintenance { + + /** + * @var string + */ + private $source; + + public function __construct() { + parent::__construct(); + + $this->addDescription( <<<TEXT +This script will populate the interwiki table, pulling in interwiki links that are used on Wikipedia +or another MediaWiki wiki. + +When the script has finished, it will make a note of this in the database, and will not run again +without the --force option. + +--source parameter is the url for the source wiki api, such as "https://en.wikipedia.org/w/api.php" +(the default) from which the script fetches the interwiki data and uses here to populate +the interwiki database table. +TEXT + ); + + $this->addOption( 'source', 'Source wiki for interwiki table, such as ' + . 'https://en.wikipedia.org/w/api.php (the default)', false, true ); + $this->addOption( 'force', 'Run regardless of whether the database says it has ' + . 'been run already.' ); + } + + public function execute() { + $force = $this->hasOption( 'force' ); + $this->source = $this->getOption( 'source', 'https://en.wikipedia.org/w/api.php' ); + + $data = $this->fetchLinks(); + + if ( $data === false ) { + $this->error( "Error during fetching data." ); + } else { + $this->doPopulate( $data, $force ); + } + } + + /** + * @return array[]|bool The 'interwikimap' sub-array or false on failure. + */ + protected function fetchLinks() { + $url = wfArrayToCgi( [ + 'action' => 'query', + 'meta' => 'siteinfo', + 'siprop' => 'interwikimap', + 'sifilteriw' => 'local', + 'format' => 'json' + ] ); + + if ( !empty( $this->source ) ) { + $url = rtrim( $this->source, '?' ) . '?' . $url; + } + + $json = Http::get( $url ); + $data = json_decode( $json, true ); + + if ( is_array( $data ) ) { + return $data['query']['interwikimap']; + } else { + return false; + } + } + + /** + * @param array[] $data + * @param bool $force + * + * @return bool + */ + protected function doPopulate( array $data, $force ) { + $dbw = wfGetDB( DB_MASTER ); + + if ( !$force ) { + $row = $dbw->selectRow( + 'updatelog', + '1', + [ 'ul_key' => 'populate interwiki' ], + __METHOD__ + ); + + if ( $row ) { + $this->output( "Interwiki table already populated. Use php " . + "maintenance/populateInterwiki.php\n--force from the command line " . + "to override.\n" ); + return true; + } + } + + foreach ( $data as $d ) { + $prefix = $d['prefix']; + + $row = $dbw->selectRow( + 'interwiki', + '1', + [ 'iw_prefix' => $prefix ], + __METHOD__ + ); + + if ( !$row ) { + $dbw->insert( + 'interwiki', + [ + 'iw_prefix' => $prefix, + 'iw_url' => $d['url'], + 'iw_local' => 1 + ], + __METHOD__, + 'IGNORE' + ); + } + + Interwiki::invalidateCache( $prefix ); + } + + $this->output( "Interwiki links are populated.\n" ); + + return true; + } + +} + +$maintClass = PopulateInterwiki::class; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/populateIpChanges.php b/www/wiki/maintenance/populateIpChanges.php new file mode 100644 index 00000000..ac456507 --- /dev/null +++ b/www/wiki/maintenance/populateIpChanges.php @@ -0,0 +1,147 @@ +<?php +/** + * Find all revisions by logged out users and copy the rev_id, + * rev_timestamp, and a hex representation of rev_user_text to the + * new ip_changes table. This table is used to efficiently query for + * contributions within an IP range. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +use MediaWiki\MediaWikiServices; + +/** + * Maintenance script that will find all rows in the revision table where + * rev_user = 0 (user is an IP), and copy relevant fields to ip_changes so + * that historical data will be available when querying for IP ranges. + * + * @ingroup Maintenance + */ +class PopulateIpChanges extends LoggedUpdateMaintenance { + public function __construct() { + parent::__construct(); + + $this->addDescription( <<<TEXT +This script will find all rows in the revision table where the user is an IP, +and copy relevant fields to the ip_changes table. This backfilled data will +then be available when querying for IP ranges at Special:Contributions. +TEXT + ); + $this->addOption( 'rev-id', 'The rev_id to start copying from. Default: 0', false, true ); + $this->addOption( + 'max-rev-id', + 'The rev_id to stop at. Default: result of MAX(rev_id)', + false, + true + ); + $this->addOption( + 'throttle', + 'Wait this many milliseconds after copying each batch of revisions. Default: 0', + false, + true + ); + $this->addOption( 'force', 'Run regardless of whether the database says it\'s been run already' ); + } + + public function doDBUpdates() { + $dbw = $this->getDB( DB_MASTER ); + + if ( !$dbw->tableExists( 'ip_changes' ) ) { + $this->error( 'ip_changes table does not exist', true ); + } + + $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory(); + $dbr = $this->getDB( DB_REPLICA, [ 'vslow' ] ); + $throttle = intval( $this->getOption( 'throttle', 0 ) ); + $maxRevId = intval( $this->getOption( 'max-rev-id', 0 ) ); + $start = $this->getOption( 'rev-id', 0 ); + $end = $maxRevId > 0 + ? $maxRevId + : $dbw->selectField( 'revision', 'MAX(rev_id)', false, __METHOD__ ); + + if ( empty( $end ) ) { + $this->output( "No revisions found, aborting.\n" ); + return true; + } + + $blockStart = $start; + $attempted = 0; + $inserted = 0; + + $this->output( "Copying IP revisions to ip_changes, from rev_id $start to rev_id $end\n" ); + + while ( $blockStart <= $end ) { + $blockEnd = min( $blockStart + $this->mBatchSize, $end ); + $rows = $dbr->select( + 'revision', + [ 'rev_id', 'rev_timestamp', 'rev_user_text' ], + [ "rev_id BETWEEN $blockStart AND $blockEnd", 'rev_user' => 0 ], + __METHOD__ + ); + + $numRows = $rows->numRows(); + + if ( !$rows || $numRows === 0 ) { + $blockStart = $blockEnd + 1; + continue; + } + + $this->output( "...checking $numRows revisions for IP edits that need copying, " . + "between rev_ids $blockStart and $blockEnd\n" ); + + $insertRows = []; + foreach ( $rows as $row ) { + // Make sure this is really an IP, e.g. not maintenance user or imported revision. + if ( IP::isValid( $row->rev_user_text ) ) { + $insertRows[] = [ + 'ipc_rev_id' => $row->rev_id, + 'ipc_rev_timestamp' => $row->rev_timestamp, + 'ipc_hex' => IP::toHex( $row->rev_user_text ), + ]; + + $attempted++; + } + } + + if ( $insertRows ) { + $dbw->insert( 'ip_changes', $insertRows, __METHOD__, 'IGNORE' ); + + $inserted += $dbw->affectedRows(); + } + + $lbFactory->waitForReplication(); + usleep( $throttle * 1000 ); + + $blockStart = $blockEnd + 1; + } + + $this->output( "Attempted to insert $attempted IP revisions, $inserted actually done.\n" ); + + return true; + } + + protected function getUpdateKey() { + return 'populate ip_changes'; + } +} + +$maintClass = "PopulateIpChanges"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/populateLogSearch.php b/www/wiki/maintenance/populateLogSearch.php new file mode 100644 index 00000000..1b07407e --- /dev/null +++ b/www/wiki/maintenance/populateLogSearch.php @@ -0,0 +1,170 @@ +<?php +/** + * Makes the required database updates for populating the + * log_search table retroactively + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that makes the required database updates for populating the + * log_search table retroactively + * + * @ingroup Maintenance + */ +class PopulateLogSearch extends LoggedUpdateMaintenance { + private static $tableMap = [ + 'rev' => 'revision', + 'fa' => 'filearchive', + 'oi' => 'oldimage', + 'ar' => 'archive' + ]; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Migrate log params to new table and index for searching' ); + $this->setBatchSize( 100 ); + } + + protected function getUpdateKey() { + return 'populate log_search'; + } + + protected function updateSkippedMessage() { + return 'log_search table already populated.'; + } + + protected function doDBUpdates() { + $db = $this->getDB( DB_MASTER ); + if ( !$db->tableExists( 'log_search' ) ) { + $this->error( "log_search does not exist" ); + + return false; + } + $start = $db->selectField( 'logging', 'MIN(log_id)', false, __FUNCTION__ ); + if ( !$start ) { + $this->output( "Nothing to do.\n" ); + + return true; + } + $end = $db->selectField( 'logging', 'MAX(log_id)', false, __FUNCTION__ ); + + # Do remaining chunk + $end += $this->mBatchSize - 1; + $blockStart = $start; + $blockEnd = $start + $this->mBatchSize - 1; + + $delTypes = [ 'delete', 'suppress' ]; // revisiondelete types + while ( $blockEnd <= $end ) { + $this->output( "...doing log_id from $blockStart to $blockEnd\n" ); + $cond = "log_id BETWEEN $blockStart AND $blockEnd"; + $res = $db->select( 'logging', '*', $cond, __FUNCTION__ ); + foreach ( $res as $row ) { + // RevisionDelete logs - revisions + if ( LogEventsList::typeAction( $row, $delTypes, 'revision' ) ) { + $params = LogPage::extractParams( $row->log_params ); + // Param format: <urlparam> <item CSV> [<ofield> <nfield>] + if ( count( $params ) < 2 ) { + continue; // bad row? + } + $field = RevisionDeleter::getRelationType( $params[0] ); + // B/C, the params may start with a title key (<title> <urlparam> <CSV>) + if ( $field == null ) { + array_shift( $params ); // remove title param + $field = RevisionDeleter::getRelationType( $params[0] ); + if ( $field == null ) { + $this->output( "Invalid param type for {$row->log_id}\n" ); + continue; // skip this row + } else { + // Clean up the row... + $db->update( 'logging', + [ 'log_params' => implode( ',', $params ) ], + [ 'log_id' => $row->log_id ] ); + } + } + $items = explode( ',', $params[1] ); + $log = new LogPage( $row->log_type ); + // Add item relations... + $log->addRelations( $field, $items, $row->log_id ); + // Determine what table to query... + $prefix = substr( $field, 0, strpos( $field, '_' ) ); // db prefix + if ( !isset( self::$tableMap[$prefix] ) ) { + continue; // bad row? + } + $table = self::$tableMap[$prefix]; + $userField = $prefix . '_user'; + $userTextField = $prefix . '_user_text'; + // Add item author relations... + $userIds = $userIPs = []; + $sres = $db->select( $table, + [ $userField, $userTextField ], + [ $field => $items ] + ); + foreach ( $sres as $srow ) { + if ( $srow->$userField > 0 ) { + $userIds[] = intval( $srow->$userField ); + } elseif ( $srow->$userTextField != '' ) { + $userIPs[] = $srow->$userTextField; + } + } + // Add item author relations... + $log->addRelations( 'target_author_id', $userIds, $row->log_id ); + $log->addRelations( 'target_author_ip', $userIPs, $row->log_id ); + } elseif ( LogEventsList::typeAction( $row, $delTypes, 'event' ) ) { + // RevisionDelete logs - log events + $params = LogPage::extractParams( $row->log_params ); + // Param format: <item CSV> [<ofield> <nfield>] + if ( count( $params ) < 1 ) { + continue; // bad row + } + $items = explode( ',', $params[0] ); + $log = new LogPage( $row->log_type ); + // Add item relations... + $log->addRelations( 'log_id', $items, $row->log_id ); + // Add item author relations... + $userIds = $userIPs = []; + $sres = $db->select( 'logging', + [ 'log_user', 'log_user_text' ], + [ 'log_id' => $items ] + ); + foreach ( $sres as $srow ) { + if ( $srow->log_user > 0 ) { + $userIds[] = intval( $srow->log_user ); + } elseif ( IP::isIPAddress( $srow->log_user_text ) ) { + $userIPs[] = $srow->log_user_text; + } + } + $log->addRelations( 'target_author_id', $userIds, $row->log_id ); + $log->addRelations( 'target_author_ip', $userIPs, $row->log_id ); + } + } + $blockStart += $this->mBatchSize; + $blockEnd += $this->mBatchSize; + wfWaitForSlaves(); + } + $this->output( "Done populating log_search table.\n" ); + + return true; + } +} + +$maintClass = "PopulateLogSearch"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/populateLogUsertext.php b/www/wiki/maintenance/populateLogUsertext.php new file mode 100644 index 00000000..dd120fe0 --- /dev/null +++ b/www/wiki/maintenance/populateLogUsertext.php @@ -0,0 +1,87 @@ +<?php +/** + * Makes the required database updates for Special:ProtectedPages + * to show all protected pages, even ones before the page restrictions + * schema change. All remaining page_restriction column values are moved + * to the new table. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that makes the required database updates for + * Special:ProtectedPages to show all protected pages. + * + * @ingroup Maintenance + */ +class PopulateLogUsertext extends LoggedUpdateMaintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Populates the log_user_text field' ); + $this->setBatchSize( 100 ); + } + + protected function getUpdateKey() { + return 'populate log_usertext'; + } + + protected function updateSkippedMessage() { + return 'log_user_text column of logging table already populated.'; + } + + protected function doDBUpdates() { + $db = $this->getDB( DB_MASTER ); + $start = $db->selectField( 'logging', 'MIN(log_id)', false, __METHOD__ ); + if ( !$start ) { + $this->output( "Nothing to do.\n" ); + + return true; + } + $end = $db->selectField( 'logging', 'MAX(log_id)', false, __METHOD__ ); + + # Do remaining chunk + $end += $this->mBatchSize - 1; + $blockStart = $start; + $blockEnd = $start + $this->mBatchSize - 1; + while ( $blockEnd <= $end ) { + $this->output( "...doing log_id from $blockStart to $blockEnd\n" ); + $cond = "log_id BETWEEN $blockStart AND $blockEnd AND log_user = user_id"; + $res = $db->select( [ 'logging', 'user' ], + [ 'log_id', 'user_name' ], $cond, __METHOD__ ); + + $this->beginTransaction( $db, __METHOD__ ); + foreach ( $res as $row ) { + $db->update( 'logging', [ 'log_user_text' => $row->user_name ], + [ 'log_id' => $row->log_id ], __METHOD__ ); + } + $this->commitTransaction( $db, __METHOD__ ); + $blockStart += $this->mBatchSize; + $blockEnd += $this->mBatchSize; + wfWaitForSlaves(); + } + $this->output( "Done populating log_user_text field.\n" ); + + return true; + } +} + +$maintClass = "PopulateLogUsertext"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/populatePPSortKey.php b/www/wiki/maintenance/populatePPSortKey.php new file mode 100644 index 00000000..7e3c2c3d --- /dev/null +++ b/www/wiki/maintenance/populatePPSortKey.php @@ -0,0 +1,106 @@ +<?php +/** + * Populate the pp_sortkey fields in the page_props table + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +use Wikimedia\Rdbms\IDatabase; + +/** + * Usage: + * populatePPSortKey.php + */ +class PopulatePPSortKey extends LoggedUpdateMaintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Populate the pp_sortkey field' ); + $this->setBatchSize( 100 ); + } + + protected function doDBUpdates() { + $dbw = $this->getDB( DB_MASTER ); + + $lastProp = null; + $lastPageValue = 0; + $editedRowCount = 0; + + $this->output( "Populating page_props.pp_sortkey...\n" ); + while ( true ) { + $conditions = [ 'pp_sortkey IS NULL' ]; + if ( $lastPageValue !== 0 ) { + $conditions[] = 'pp_page > ' . $dbw->addQuotes( $lastPageValue ) . ' OR ' . + '( pp_page = ' . $dbw->addQuotes( $lastPageValue ) . + ' AND pp_propname > ' . $dbw->addQuotes( $lastProp ) . ' )'; + } + + $res = $dbw->select( + 'page_props', + [ 'pp_propname', 'pp_page', 'pp_sortkey', 'pp_value' ], + $conditions, + __METHOD__, + [ + 'ORDER BY' => 'pp_page, pp_propname', + 'LIMIT' => $this->mBatchSize + ] + ); + + if ( $res->numRows() === 0 ) { + break; + } + + $this->beginTransaction( $dbw, __METHOD__ ); + + foreach ( $res as $row ) { + if ( !is_numeric( $row->pp_value ) ) { + continue; + } + $dbw->update( + 'page_props', + [ 'pp_sortkey' => $row->pp_value ], + [ + 'pp_page' => $row->pp_page, + 'pp_propname' => $row->pp_propname + ], + __METHOD__ + ); + $editedRowCount++; + } + + $this->output( "Updated " . $editedRowCount . " rows\n" ); + $this->commitTransaction( $dbw, __METHOD__ ); + + // We need to get the last element's page ID + $lastPageValue = $row->pp_page; + // And the propname... + $lastProp = $row->pp_propname; + } + + $this->output( "Populating page_props.pp_sortkey complete.\n" ); + } + + protected function getUpdateKey() { + return 'populate pp_sortkey'; + } +} + +$maintClass = 'PopulatePPSortKey'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/populateParentId.php b/www/wiki/maintenance/populateParentId.php new file mode 100644 index 00000000..457033a4 --- /dev/null +++ b/www/wiki/maintenance/populateParentId.php @@ -0,0 +1,130 @@ +<?php +/** + * Makes the required database updates for rev_parent_id + * to be of any use. It can be used for some simple tracking + * and to find new page edits by users. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that makes the required database updates for rev_parent_id + * to be of any use. + * + * @ingroup Maintenance + */ +class PopulateParentId extends LoggedUpdateMaintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Populates rev_parent_id' ); + } + + protected function getUpdateKey() { + return 'populate rev_parent_id'; + } + + protected function updateSkippedMessage() { + return 'rev_parent_id column of revision table already populated.'; + } + + protected function doDBUpdates() { + $db = $this->getDB( DB_MASTER ); + if ( !$db->tableExists( 'revision' ) ) { + $this->error( "revision table does not exist" ); + + return false; + } + $this->output( "Populating rev_parent_id column\n" ); + $start = $db->selectField( 'revision', 'MIN(rev_id)', false, __FUNCTION__ ); + $end = $db->selectField( 'revision', 'MAX(rev_id)', false, __FUNCTION__ ); + if ( is_null( $start ) || is_null( $end ) ) { + $this->output( "...revision table seems to be empty, nothing to do.\n" ); + + return true; + } + # Do remaining chunk + $blockStart = intval( $start ); + $blockEnd = intval( $start ) + $this->mBatchSize - 1; + $count = 0; + $changed = 0; + while ( $blockStart <= $end ) { + $this->output( "...doing rev_id from $blockStart to $blockEnd\n" ); + $cond = "rev_id BETWEEN $blockStart AND $blockEnd"; + $res = $db->select( 'revision', + [ 'rev_id', 'rev_page', 'rev_timestamp', 'rev_parent_id' ], + [ $cond, 'rev_parent_id' => null ], __METHOD__ ); + # Go through and update rev_parent_id from these rows. + # Assume that the previous revision of the title was + # the original previous revision of the title when the + # edit was made... + foreach ( $res as $row ) { + # First, check rows with the same timestamp other than this one + # with a smaller rev ID. The highest ID "wins". This avoids loops + # as timestamp can only decrease and never loops with IDs (from parent to parent) + $previousID = $db->selectField( 'revision', 'rev_id', + [ 'rev_page' => $row->rev_page, 'rev_timestamp' => $row->rev_timestamp, + "rev_id < " . intval( $row->rev_id ) ], + __METHOD__, + [ 'ORDER BY' => 'rev_id DESC' ] ); + # If there are none, check the highest ID with a lower timestamp + if ( !$previousID ) { + # Get the highest older timestamp + $lastTimestamp = $db->selectField( + 'revision', + 'rev_timestamp', + [ + 'rev_page' => $row->rev_page, + "rev_timestamp < " . $db->addQuotes( $row->rev_timestamp ) + ], + __METHOD__, + [ 'ORDER BY' => 'rev_timestamp DESC' ] + ); + # If there is one, let the highest rev ID win + if ( $lastTimestamp ) { + $previousID = $db->selectField( 'revision', 'rev_id', + [ 'rev_page' => $row->rev_page, 'rev_timestamp' => $lastTimestamp ], + __METHOD__, + [ 'ORDER BY' => 'rev_id DESC' ] ); + } + } + $previousID = intval( $previousID ); + if ( $previousID != $row->rev_parent_id ) { + $changed++; + } + # Update the row... + $db->update( 'revision', + [ 'rev_parent_id' => $previousID ], + [ 'rev_id' => $row->rev_id ], + __METHOD__ ); + $count++; + } + $blockStart += $this->mBatchSize; + $blockEnd += $this->mBatchSize; + wfWaitForSlaves(); + } + $this->output( "rev_parent_id population complete ... {$count} rows [{$changed} changed]\n" ); + + return true; + } +} + +$maintClass = "PopulateParentId"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/populateRecentChangesSource.php b/www/wiki/maintenance/populateRecentChangesSource.php new file mode 100644 index 00000000..5d5da89a --- /dev/null +++ b/www/wiki/maintenance/populateRecentChangesSource.php @@ -0,0 +1,109 @@ +<?php +/** + * Upgrade script to populate the rc_source field + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +use Wikimedia\Rdbms\IDatabase; + +/** + * Maintenance script to populate the rc_source field. + * + * @ingroup Maintenance + * @since 1.22 + */ +class PopulateRecentChangesSource extends LoggedUpdateMaintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( + 'Populates rc_source field of the recentchanges table with the data in rc_type.' ); + $this->setBatchSize( 100 ); + } + + protected function doDBUpdates() { + $dbw = $this->getDB( DB_MASTER ); + if ( !$dbw->fieldExists( 'recentchanges', 'rc_source' ) ) { + $this->error( 'rc_source field in recentchanges table does not exist.' ); + } + + $start = $dbw->selectField( 'recentchanges', 'MIN(rc_id)', false, __METHOD__ ); + if ( !$start ) { + $this->output( "Nothing to do.\n" ); + + return true; + } + $end = $dbw->selectField( 'recentchanges', 'MAX(rc_id)', false, __METHOD__ ); + $end += $this->mBatchSize - 1; + $blockStart = $start; + $blockEnd = $start + $this->mBatchSize - 1; + + $updatedValues = $this->buildUpdateCondition( $dbw ); + + while ( $blockEnd <= $end ) { + $cond = "rc_id BETWEEN $blockStart AND $blockEnd"; + + $dbw->update( + 'recentchanges', + [ $updatedValues ], + [ + "rc_source = ''", + "rc_id BETWEEN $blockStart AND $blockEnd" + ], + __METHOD__ + ); + + $this->output( "." ); + wfWaitForSlaves(); + + $blockStart += $this->mBatchSize; + $blockEnd += $this->mBatchSize; + } + + $this->output( "\nDone.\n" ); + } + + protected function getUpdateKey() { + return __CLASS__; + } + + protected function buildUpdateCondition( IDatabase $dbw ) { + $rcNew = $dbw->addQuotes( RC_NEW ); + $rcSrcNew = $dbw->addQuotes( RecentChange::SRC_NEW ); + $rcEdit = $dbw->addQuotes( RC_EDIT ); + $rcSrcEdit = $dbw->addQuotes( RecentChange::SRC_EDIT ); + $rcLog = $dbw->addQuotes( RC_LOG ); + $rcSrcLog = $dbw->addQuotes( RecentChange::SRC_LOG ); + $rcExternal = $dbw->addQuotes( RC_EXTERNAL ); + $rcSrcExternal = $dbw->addQuotes( RecentChange::SRC_EXTERNAL ); + + return "rc_source = CASE + WHEN rc_type = $rcNew THEN $rcSrcNew + WHEN rc_type = $rcEdit THEN $rcSrcEdit + WHEN rc_type = $rcLog THEN $rcSrcLog + WHEN rc_type = $rcExternal THEN $rcSrcExternal + ELSE '' + END"; + } +} + +$maintClass = "PopulateRecentChangesSource"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/populateRevisionLength.php b/www/wiki/maintenance/populateRevisionLength.php new file mode 100644 index 00000000..a9457c2a --- /dev/null +++ b/www/wiki/maintenance/populateRevisionLength.php @@ -0,0 +1,158 @@ +<?php +/** + * Populates the rev_len and ar_len fields when they are NULL. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that populates the rev_len and ar_len fields when they are NULL. + * This is the case for all revisions created before MW 1.10, as well as those affected + * by T18748 (MW 1.10-1.13) and those affected by T135414 (MW 1.21-1.24). + * + * @ingroup Maintenance + */ +class PopulateRevisionLength extends LoggedUpdateMaintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Populates the rev_len and ar_len fields' ); + $this->setBatchSize( 200 ); + } + + protected function getUpdateKey() { + return 'populate rev_len and ar_len'; + } + + public function doDBUpdates() { + $dbw = $this->getDB( DB_MASTER ); + if ( !$dbw->tableExists( 'revision' ) ) { + $this->error( "revision table does not exist", true ); + } elseif ( !$dbw->tableExists( 'archive' ) ) { + $this->error( "archive table does not exist", true ); + } elseif ( !$dbw->fieldExists( 'revision', 'rev_len', __METHOD__ ) ) { + $this->output( "rev_len column does not exist\n\n", true ); + + return false; + } + + $this->output( "Populating rev_len column\n" ); + $rev = $this->doLenUpdates( 'revision', 'rev_id', 'rev', Revision::selectFields() ); + + $this->output( "Populating ar_len column\n" ); + $ar = $this->doLenUpdates( 'archive', 'ar_id', 'ar', Revision::selectArchiveFields() ); + + $this->output( "rev_len and ar_len population complete " + . "[$rev revision rows, $ar archive rows].\n" ); + + return true; + } + + /** + * @param string $table + * @param string $idCol + * @param string $prefix + * @param array $fields + * @return int + */ + protected function doLenUpdates( $table, $idCol, $prefix, $fields ) { + $dbr = $this->getDB( DB_REPLICA ); + $dbw = $this->getDB( DB_MASTER ); + $start = $dbw->selectField( $table, "MIN($idCol)", false, __METHOD__ ); + $end = $dbw->selectField( $table, "MAX($idCol)", false, __METHOD__ ); + if ( !$start || !$end ) { + $this->output( "...$table table seems to be empty.\n" ); + + return 0; + } + + # Do remaining chunks + $blockStart = intval( $start ); + $blockEnd = intval( $start ) + $this->mBatchSize - 1; + $count = 0; + + while ( $blockStart <= $end ) { + $this->output( "...doing $idCol from $blockStart to $blockEnd\n" ); + $res = $dbr->select( + $table, + $fields, + [ + "$idCol >= $blockStart", + "$idCol <= $blockEnd", + "{$prefix}_len IS NULL" + ], + __METHOD__ + ); + + if ( $res->numRows() > 0 ) { + $this->beginTransaction( $dbw, __METHOD__ ); + # Go through and update rev_len from these rows. + foreach ( $res as $row ) { + if ( $this->upgradeRow( $row, $table, $idCol, $prefix ) ) { + $count++; + } + } + $this->commitTransaction( $dbw, __METHOD__ ); + } + + $blockStart += $this->mBatchSize; + $blockEnd += $this->mBatchSize; + wfWaitForSlaves(); + } + + return $count; + } + + /** + * @param stdClass $row + * @param string $table + * @param string $idCol + * @param string $prefix + * @return bool + */ + protected function upgradeRow( $row, $table, $idCol, $prefix ) { + $dbw = $this->getDB( DB_MASTER ); + + $rev = ( $table === 'archive' ) + ? Revision::newFromArchiveRow( $row ) + : new Revision( $row ); + + $content = $rev->getContent(); + if ( !$content ) { + # This should not happen, but sometimes does (T22757) + $id = $row->$idCol; + $this->output( "Content of $table $id unavailable!\n" ); + + return false; + } + + # Update the row... + $dbw->update( $table, + [ "{$prefix}_len" => $content->getSize() ], + [ $idCol => $row->$idCol ], + __METHOD__ + ); + + return true; + } +} + +$maintClass = "PopulateRevisionLength"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/populateRevisionSha1.php b/www/wiki/maintenance/populateRevisionSha1.php new file mode 100644 index 00000000..fb97e910 --- /dev/null +++ b/www/wiki/maintenance/populateRevisionSha1.php @@ -0,0 +1,216 @@ +<?php +/** + * Fills the rev_sha1 and ar_sha1 columns of revision + * and archive tables for revisions created before MW 1.19. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that fills the rev_sha1 and ar_sha1 columns of revision + * and archive tables for revisions created before MW 1.19. + * + * @ingroup Maintenance + */ +class PopulateRevisionSha1 extends LoggedUpdateMaintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Populates the rev_sha1 and ar_sha1 fields' ); + $this->setBatchSize( 200 ); + } + + protected function getUpdateKey() { + return 'populate rev_sha1'; + } + + protected function doDBUpdates() { + $db = $this->getDB( DB_MASTER ); + + if ( !$db->tableExists( 'revision' ) ) { + $this->error( "revision table does not exist", true ); + } elseif ( !$db->tableExists( 'archive' ) ) { + $this->error( "archive table does not exist", true ); + } elseif ( !$db->fieldExists( 'revision', 'rev_sha1', __METHOD__ ) ) { + $this->output( "rev_sha1 column does not exist\n\n", true ); + + return false; + } + + $this->output( "Populating rev_sha1 column\n" ); + $rc = $this->doSha1Updates( 'revision', 'rev_id', 'rev' ); + + $this->output( "Populating ar_sha1 column\n" ); + $ac = $this->doSha1Updates( 'archive', 'ar_rev_id', 'ar' ); + $this->output( "Populating ar_sha1 column legacy rows\n" ); + $ac += $this->doSha1LegacyUpdates(); + + $this->output( "rev_sha1 and ar_sha1 population complete " + . "[$rc revision rows, $ac archive rows].\n" ); + + return true; + } + + /** + * @param string $table + * @param string $idCol + * @param string $prefix + * @return int Rows changed + */ + protected function doSha1Updates( $table, $idCol, $prefix ) { + $db = $this->getDB( DB_MASTER ); + $start = $db->selectField( $table, "MIN($idCol)", false, __METHOD__ ); + $end = $db->selectField( $table, "MAX($idCol)", false, __METHOD__ ); + if ( !$start || !$end ) { + $this->output( "...$table table seems to be empty.\n" ); + + return 0; + } + + $count = 0; + # Do remaining chunk + $end += $this->mBatchSize - 1; + $blockStart = $start; + $blockEnd = $start + $this->mBatchSize - 1; + while ( $blockEnd <= $end ) { + $this->output( "...doing $idCol from $blockStart to $blockEnd\n" ); + $cond = "$idCol BETWEEN $blockStart AND $blockEnd + AND $idCol IS NOT NULL AND {$prefix}_sha1 = ''"; + $res = $db->select( $table, '*', $cond, __METHOD__ ); + + $this->beginTransaction( $db, __METHOD__ ); + foreach ( $res as $row ) { + if ( $this->upgradeRow( $row, $table, $idCol, $prefix ) ) { + $count++; + } + } + $this->commitTransaction( $db, __METHOD__ ); + + $blockStart += $this->mBatchSize; + $blockEnd += $this->mBatchSize; + wfWaitForSlaves(); + } + + return $count; + } + + /** + * @return int + */ + protected function doSha1LegacyUpdates() { + $count = 0; + $db = $this->getDB( DB_MASTER ); + $res = $db->select( 'archive', '*', + [ 'ar_rev_id IS NULL', 'ar_sha1' => '' ], __METHOD__ ); + + $updateSize = 0; + $this->beginTransaction( $db, __METHOD__ ); + foreach ( $res as $row ) { + if ( $this->upgradeLegacyArchiveRow( $row ) ) { + ++$count; + } + if ( ++$updateSize >= 100 ) { + $updateSize = 0; + $this->commitTransaction( $db, __METHOD__ ); + $this->output( "Commited row with ar_timestamp={$row->ar_timestamp}\n" ); + wfWaitForSlaves(); + $this->beginTransaction( $db, __METHOD__ ); + } + } + $this->commitTransaction( $db, __METHOD__ ); + + return $count; + } + + /** + * @param stdClass $row + * @param string $table + * @param string $idCol + * @param string $prefix + * @return bool + */ + protected function upgradeRow( $row, $table, $idCol, $prefix ) { + $db = $this->getDB( DB_MASTER ); + try { + $rev = ( $table === 'archive' ) + ? Revision::newFromArchiveRow( $row ) + : new Revision( $row ); + $text = $rev->getSerializedData(); + } catch ( Exception $e ) { + $this->output( "Data of revision with {$idCol}={$row->$idCol} unavailable!\n" ); + + return false; // T24624? + } + if ( !is_string( $text ) ) { + # This should not happen, but sometimes does (T22757) + $this->output( "Data of revision with {$idCol}={$row->$idCol} unavailable!\n" ); + + return false; + } else { + $db->update( $table, + [ "{$prefix}_sha1" => Revision::base36Sha1( $text ) ], + [ $idCol => $row->$idCol ], + __METHOD__ + ); + + return true; + } + } + + /** + * @param stdClass $row + * @return bool + */ + protected function upgradeLegacyArchiveRow( $row ) { + $db = $this->getDB( DB_MASTER ); + try { + $rev = Revision::newFromArchiveRow( $row ); + } catch ( Exception $e ) { + $this->output( "Text of revision with timestamp {$row->ar_timestamp} unavailable!\n" ); + + return false; // T24624? + } + $text = $rev->getSerializedData(); + if ( !is_string( $text ) ) { + # This should not happen, but sometimes does (T22757) + $this->output( "Data of revision with timestamp {$row->ar_timestamp} unavailable!\n" ); + + return false; + } else { + # Archive table as no PK, but (NS,title,time) should be near unique. + # Any duplicates on those should also have duplicated text anyway. + $db->update( 'archive', + [ 'ar_sha1' => Revision::base36Sha1( $text ) ], + [ + 'ar_namespace' => $row->ar_namespace, + 'ar_title' => $row->ar_title, + 'ar_timestamp' => $row->ar_timestamp, + 'ar_len' => $row->ar_len // extra sanity + ], + __METHOD__ + ); + + return true; + } + } +} + +$maintClass = "PopulateRevisionSha1"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/postgres/archives/patch-add_interwiki.sql b/www/wiki/maintenance/postgres/archives/patch-add_interwiki.sql new file mode 100644 index 00000000..6c08af7a --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-add_interwiki.sql @@ -0,0 +1,14 @@ +DROP FUNCTION IF EXISTS add_interwiki(TEXT,INT,CHARACTER) CASCADE; +CREATE OR REPLACE FUNCTION "add_interwiki" (TEXT,INT,SMALLINT) RETURNS INT LANGUAGE SQL AS +$mw$ + INSERT INTO interwiki (iw_prefix, iw_url, iw_local) VALUES ($1,$2,$3); + SELECT 1; +$mw$; + +DROP FUNCTION IF EXISTS add_interwiki(TEXT,INT,CHARACTER) CASCADE; +CREATE OR REPLACE FUNCTION "add_interwiki" (TEXT,INT,SMALLINT) RETURNS INT LANGUAGE SQL AS +$mw$ + INSERT INTO interwiki (iw_prefix, iw_url, iw_local) VALUES ($1,$2,$3); + SELECT 1; +$mw$; + diff --git a/www/wiki/maintenance/postgres/archives/patch-bot_passwords.sql b/www/wiki/maintenance/postgres/archives/patch-bot_passwords.sql new file mode 100644 index 00000000..8e8a794c --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-bot_passwords.sql @@ -0,0 +1,9 @@ +CREATE TABLE bot_passwords ( + bp_user INTEGER NOT NULL, + bp_app_id TEXT NOT NULL, + bp_password TEXT NOT NULL, + bp_token TEXT NOT NULL, + bp_restrictions TEXT NOT NULL, + bp_grants TEXT NOT NULL, + PRIMARY KEY ( bp_user, bp_app_id ) +); diff --git a/www/wiki/maintenance/postgres/archives/patch-category.sql b/www/wiki/maintenance/postgres/archives/patch-category.sql new file mode 100644 index 00000000..266b1d00 --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-category.sql @@ -0,0 +1,15 @@ + +CREATE SEQUENCE category_cat_id_seq; + +CREATE TABLE category ( + cat_id INTEGER NOT NULL PRIMARY KEY DEFAULT nextval('category_cat_id_seq'), + cat_title TEXT NOT NULL, + cat_pages INTEGER NOT NULL DEFAULT 0, + cat_subcats INTEGER NOT NULL DEFAULT 0, + cat_files INTEGER NOT NULL DEFAULT 0, + cat_hidden SMALLINT NOT NULL DEFAULT 0 +); + +CREATE UNIQUE INDEX category_title ON category(cat_title); +CREATE INDEX category_pages ON category(cat_pages); + diff --git a/www/wiki/maintenance/postgres/archives/patch-categorylinks-better-collation.sql b/www/wiki/maintenance/postgres/archives/patch-categorylinks-better-collation.sql new file mode 100644 index 00000000..b3fa6346 --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-categorylinks-better-collation.sql @@ -0,0 +1,8 @@ +CREATE TYPE link_type AS ENUM ('page', 'subcat', 'file'); +DROP INDEX cl_sortkey; +ALTER TABLE categorylinks + ADD COLUMN cl_sortkey_prefix TEXT NOT NULL DEFAULT '', + ADD COLUMN cl_collation SMALLINT NOT NULL DEFAULT 0, + ADD COLUMN cl_type link_type NOT NULL DEFAULT 'page'; +CREATE INDEX cl_collation ON categorylinks ( cl_collation ); +CREATE INDEX cl_sortkey ON categorylinks ( cl_to, cl_type, cl_sortkey, cl_from ); diff --git a/www/wiki/maintenance/postgres/archives/patch-change_tag.sql b/www/wiki/maintenance/postgres/archives/patch-change_tag.sql new file mode 100644 index 00000000..89d74b63 --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-change_tag.sql @@ -0,0 +1,11 @@ +CREATE TABLE change_tag ( + ct_rc_id INTEGER NULL, + ct_log_id INTEGER NULL, + ct_rev_id INTEGER NULL, + ct_tag TEXT NOT NULL, + ct_params TEXT NULL +); +CREATE UNIQUE INDEX change_tag_rc_tag ON change_tag(ct_rc_id,ct_tag); +CREATE UNIQUE INDEX change_tag_log_tag ON change_tag(ct_log_id,ct_tag); +CREATE UNIQUE INDEX change_tag_rev_tag ON change_tag(ct_rev_id,ct_tag); +CREATE INDEX change_tag_tag_id ON change_tag(ct_tag,ct_rc_id,ct_rev_id,ct_log_id); diff --git a/www/wiki/maintenance/postgres/archives/patch-comment-table.sql b/www/wiki/maintenance/postgres/archives/patch-comment-table.sql new file mode 100644 index 00000000..243a3b31 --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-comment-table.sql @@ -0,0 +1,27 @@ +-- +-- patch-comment-table.sql +-- +-- T166732. Add a `comment` table, and temporary tables to reference it. + +CREATE SEQUENCE comment_comment_id_seq; +CREATE TABLE comment ( + comment_id INTEGER NOT NULL PRIMARY KEY DEFAULT nextval('comment_comment_id_seq'), + comment_hash INTEGER NOT NULL, + comment_text TEXT NOT NULL, + comment_data TEXT +); +CREATE INDEX comment_hash ON comment (comment_hash); + +CREATE TABLE revision_comment_temp ( + revcomment_rev INTEGER NOT NULL, + revcomment_comment_id INTEGER NOT NULL, + PRIMARY KEY (revcomment_rev, revcomment_comment_id) +); +CREATE UNIQUE INDEX revcomment_rev ON revision_comment_temp (revcomment_rev); + +CREATE TABLE image_comment_temp ( + imgcomment_name TEXT NOT NULL, + imgcomment_description_id INTEGER NOT NULL, + PRIMARY KEY (imgcomment_name, imgcomment_description_id) +); +CREATE UNIQUE INDEX imgcomment_name ON image_comment_temp (imgcomment_name); diff --git a/www/wiki/maintenance/postgres/archives/patch-ip_changes.sql b/www/wiki/maintenance/postgres/archives/patch-ip_changes.sql new file mode 100644 index 00000000..64cc0d71 --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-ip_changes.sql @@ -0,0 +1,10 @@ +CREATE SEQUENCE ip_changes_ipc_rev_id_seq; + +CREATE TABLE ip_changes ( + ipc_rev_id INTEGER PRIMARY KEY NOT NULL DEFAULT nextval('ip_changes_ipc_rev_id_seq'), + ipc_rev_timestamp TIMESTAMPTZ NOT NULL, + ipc_hex BYTEA NOT NULL DEFAULT '' +); + +CREATE INDEX ipc_rev_timestamp ON ip_changes (ipc_rev_timestamp); +CREATE INDEX ipc_hex_time ON ip_changes (ipc_hex,ipc_rev_timestamp); diff --git a/www/wiki/maintenance/postgres/archives/patch-iwlinks.sql b/www/wiki/maintenance/postgres/archives/patch-iwlinks.sql new file mode 100644 index 00000000..db26eae4 --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-iwlinks.sql @@ -0,0 +1,8 @@ + +CREATE TABLE iwlinks ( + iwl_from INTEGER NOT NULL DEFAULT 0, + iwl_prefix TEXT NOT NULL DEFAULT '', + iwl_title TEXT NOT NULL DEFAULT '' +); +CREATE UNIQUE INDEX iwl_from ON iwlinks (iwl_from, iwl_prefix, iwl_title); +CREATE UNIQUE INDEX iwl_prefix_title_from ON iwlinks (iwl_prefix, iwl_title, iwl_from); diff --git a/www/wiki/maintenance/postgres/archives/patch-kill-iwl_prefix.sql b/www/wiki/maintenance/postgres/archives/patch-kill-iwl_prefix.sql new file mode 100644 index 00000000..8b6d1084 --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-kill-iwl_prefix.sql @@ -0,0 +1,7 @@ +-- +-- Kill the old iwl_prefix index, which may be present on some +-- installs if they ran update.php between it being added and being renamed +-- + +DROP INDEX iwl_prefix; + diff --git a/www/wiki/maintenance/postgres/archives/patch-l10n_cache.sql b/www/wiki/maintenance/postgres/archives/patch-l10n_cache.sql new file mode 100644 index 00000000..9b39b1b7 --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-l10n_cache.sql @@ -0,0 +1,8 @@ +CREATE TABLE l10n_cache ( + lc_lang TEXT NOT NULL, + lc_key TEXT NOT NULL, + lc_value TEXT NOT NULL +); +CREATE INDEX l10n_cache_lc_lang_key ON l10n_cache (lc_lang, lc_key); + + diff --git a/www/wiki/maintenance/postgres/archives/patch-log_search.sql b/www/wiki/maintenance/postgres/archives/patch-log_search.sql new file mode 100644 index 00000000..4c0b3c61 --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-log_search.sql @@ -0,0 +1,9 @@ + +CREATE TABLE log_search ( + ls_field TEXT NOT NULL, + ls_value TEXT NOT NULL, + ls_log_id INTEGER NOT NULL DEFAULT 0 +); + +ALTER TABLE log_search ADD CONSTRAINT log_search_pkey PRIMARY KEY(ls_field, ls_value, ls_log_id); +CREATE INDEX ls_log_id ON log_search (ls_log_id); diff --git a/www/wiki/maintenance/postgres/archives/patch-module_deps.sql b/www/wiki/maintenance/postgres/archives/patch-module_deps.sql new file mode 100644 index 00000000..bd7bb1f0 --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-module_deps.sql @@ -0,0 +1,7 @@ +CREATE TABLE module_deps ( + md_module TEXT NOT NULL, + md_skin TEXT NOT NULL, + md_deps TEXT NOT NULL +); + +CREATE UNIQUE INDEX md_module_skin ON module_deps (md_module, md_skin); diff --git a/www/wiki/maintenance/postgres/archives/patch-page.sql b/www/wiki/maintenance/postgres/archives/patch-page.sql new file mode 100644 index 00000000..cceef898 --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-page.sql @@ -0,0 +1,24 @@ +CREATE SEQUENCE page_page_id_seq; +CREATE TABLE page ( + page_id INTEGER NOT NULL PRIMARY KEY DEFAULT nextval('page_page_id_seq'), + page_namespace SMALLINT NOT NULL, + page_title TEXT NOT NULL, + page_restrictions TEXT, + page_counter BIGINT NOT NULL DEFAULT 0, + page_is_redirect SMALLINT NOT NULL DEFAULT 0, + page_is_new SMALLINT NOT NULL DEFAULT 0, + page_random NUMERIC(15,14) NOT NULL DEFAULT RANDOM(), + page_touched TIMESTAMPTZ, + page_latest INTEGER NOT NULL, + page_len INTEGER NOT NULL +); +CREATE UNIQUE INDEX page_unique_name ON page (page_namespace, page_title); +CREATE INDEX page_main_title ON page (page_title) WHERE page_namespace = 0; +CREATE INDEX page_talk_title ON page (page_title) WHERE page_namespace = 1; +CREATE INDEX page_user_title ON page (page_title) WHERE page_namespace = 2; +CREATE INDEX page_utalk_title ON page (page_title) WHERE page_namespace = 3; +CREATE INDEX page_project_title ON page (page_title) WHERE page_namespace = 4; +CREATE INDEX page_mediawiki_title ON page (page_title) WHERE page_namespace = 8; +CREATE INDEX page_random_idx ON page (page_random); +CREATE INDEX page_len_idx ON page (page_len); + diff --git a/www/wiki/maintenance/postgres/archives/patch-page_deleted.sql b/www/wiki/maintenance/postgres/archives/patch-page_deleted.sql new file mode 100644 index 00000000..5b0782cb --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-page_deleted.sql @@ -0,0 +1,11 @@ +CREATE FUNCTION page_deleted() RETURNS TRIGGER LANGUAGE plpgsql AS +$mw$ +BEGIN +DELETE FROM recentchanges WHERE rc_namespace = OLD.page_namespace AND rc_title = OLD.page_title; +RETURN NULL; +END; +$mw$; + +CREATE TRIGGER page_deleted AFTER DELETE ON page + FOR EACH ROW EXECUTE PROCEDURE page_deleted(); + diff --git a/www/wiki/maintenance/postgres/archives/patch-page_props.sql b/www/wiki/maintenance/postgres/archives/patch-page_props.sql new file mode 100644 index 00000000..ab707022 --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-page_props.sql @@ -0,0 +1,9 @@ + +CREATE TABLE page_props ( + pp_page INTEGER NOT NULL REFERENCES page (page_id) ON DELETE CASCADE, + pp_propname TEXT NOT NULL, + pp_value TEXT NOT NULL +); +ALTER TABLE page_props ADD CONSTRAINT page_props_pk PRIMARY KEY (pp_page,pp_propname); +CREATE INDEX page_props_propname ON page_props (pp_propname); + diff --git a/www/wiki/maintenance/postgres/archives/patch-page_restrictions.sql b/www/wiki/maintenance/postgres/archives/patch-page_restrictions.sql new file mode 100644 index 00000000..1faa14a9 --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-page_restrictions.sql @@ -0,0 +1,10 @@ +CREATE TABLE page_restrictions ( + pr_page INTEGER NULL REFERENCES page (page_id) ON DELETE CASCADE, + pr_type TEXT NOT NULL, + pr_level TEXT NOT NULL, + pr_cascade SMALLINT NOT NULL, + pr_user INTEGER NULL, + pr_expiry TIMESTAMPTZ NULL +); +ALTER TABLE page_restrictions ADD CONSTRAINT page_restrictions_pk PRIMARY KEY (pr_page,pr_type); + diff --git a/www/wiki/maintenance/postgres/archives/patch-profiling.sql b/www/wiki/maintenance/postgres/archives/patch-profiling.sql new file mode 100644 index 00000000..5a2710a8 --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-profiling.sql @@ -0,0 +1,8 @@ +CREATE TABLE profiling ( + pf_count INTEGER NOT NULL DEFAULT 0, + pf_time FLOAT NOT NULL DEFAULT 0, + pf_memory FLOAT NOT NULL DEFAULT 0, + pf_name TEXT NOT NULL, + pf_server TEXT NULL +); +CREATE UNIQUE INDEX pf_name_server ON profiling (pf_name, pf_server); diff --git a/www/wiki/maintenance/postgres/archives/patch-protected_titles.sql b/www/wiki/maintenance/postgres/archives/patch-protected_titles.sql new file mode 100644 index 00000000..93f10e44 --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-protected_titles.sql @@ -0,0 +1,10 @@ +CREATE TABLE protected_titles ( + pt_namespace SMALLINT NOT NULL, + pt_title TEXT NOT NULL, + pt_user INTEGER NULL REFERENCES mwuser(user_id) ON DELETE SET NULL, + pt_reason TEXT NULL, + pt_timestamp TIMESTAMPTZ NOT NULL, + pt_expiry TIMESTAMPTZ NULL, + pt_create_perm TEXT NOT NULL DEFAULT '' +); +CREATE UNIQUE INDEX protected_titles_unique ON protected_titles(pt_namespace, pt_title); diff --git a/www/wiki/maintenance/postgres/archives/patch-querycachetwo.sql b/www/wiki/maintenance/postgres/archives/patch-querycachetwo.sql new file mode 100644 index 00000000..cb70cd89 --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-querycachetwo.sql @@ -0,0 +1,12 @@ +CREATE TABLE querycachetwo ( + qcc_type TEXT NOT NULL, + qcc_value SMALLINT NOT NULL DEFAULT 0, + qcc_namespace INTEGER NOT NULL DEFAULT 0, + qcc_title TEXT NOT NULL DEFAULT '', + qcc_namespacetwo INTEGER NOT NULL DEFAULT 0, + qcc_titletwo TEXT NOT NULL DEFAULT '' +); +CREATE INDEX querycachetwo_type_value ON querycachetwo (qcc_type, qcc_value); +CREATE INDEX querycachetwo_title ON querycachetwo (qcc_type,qcc_namespace,qcc_title); +CREATE INDEX querycachetwo_titletwo ON querycachetwo (qcc_type,qcc_namespacetwo,qcc_titletwo); + diff --git a/www/wiki/maintenance/postgres/archives/patch-rc_cur_id-not-null.sql b/www/wiki/maintenance/postgres/archives/patch-rc_cur_id-not-null.sql new file mode 100644 index 00000000..2ca7edbf --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-rc_cur_id-not-null.sql @@ -0,0 +1 @@ +ALTER TABLE recentchanges ALTER rc_cur_id DROP NOT NULL; diff --git a/www/wiki/maintenance/postgres/archives/patch-redirect.sql b/www/wiki/maintenance/postgres/archives/patch-redirect.sql new file mode 100644 index 00000000..d2922d3b --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-redirect.sql @@ -0,0 +1,7 @@ +CREATE TABLE redirect ( + rd_from INTEGER NOT NULL REFERENCES page(page_id) ON DELETE CASCADE, + rd_namespace SMALLINT NOT NULL, + rd_title TEXT NOT NULL +); +CREATE INDEX redirect_ns_title ON redirect (rd_namespace,rd_title,rd_from); + diff --git a/www/wiki/maintenance/postgres/archives/patch-remove-archive2.sql b/www/wiki/maintenance/postgres/archives/patch-remove-archive2.sql new file mode 100644 index 00000000..20bac385 --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-remove-archive2.sql @@ -0,0 +1,3 @@ +DROP VIEW archive; +ALTER TABLE archive2 RENAME TO archive; +ALTER TABLE archive ADD ar_len INTEGER; diff --git a/www/wiki/maintenance/postgres/archives/patch-rename-iwl_prefix.sql b/www/wiki/maintenance/postgres/archives/patch-rename-iwl_prefix.sql new file mode 100644 index 00000000..0eb792ea --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-rename-iwl_prefix.sql @@ -0,0 +1,2 @@ +DROP INDEX iwl_prefix; +CREATE UNIQUE INDEX iwl_prefix_title_from ON iwlinks (iwl_prefix, iwl_title, iwl_from); diff --git a/www/wiki/maintenance/postgres/archives/patch-revision_rev_user_fkey.sql b/www/wiki/maintenance/postgres/archives/patch-revision_rev_user_fkey.sql new file mode 100644 index 00000000..721aadd5 --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-revision_rev_user_fkey.sql @@ -0,0 +1,4 @@ +ALTER TABLE revision DROP CONSTRAINT revision_rev_user_fkey; +ALTER TABLE revision ADD CONSTRAINT revision_rev_user_fkey + FOREIGN KEY (rev_user) REFERENCES mwuser(user_id) ON DELETE RESTRICT; + diff --git a/www/wiki/maintenance/postgres/archives/patch-site_stats-pk.sql b/www/wiki/maintenance/postgres/archives/patch-site_stats-pk.sql new file mode 100644 index 00000000..faa5e9f8 --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-site_stats-pk.sql @@ -0,0 +1,3 @@ +ALTER TABLE site_stats DROP CONSTRAINT site_stats_ss_row_id_key; +ALTER TABLE site_stats ADD PRIMARY KEY (ss_row_id); +ALTER TABLE site_stats ALTER ss_row_id SET DEFAULT 0; diff --git a/www/wiki/maintenance/postgres/archives/patch-sites.sql b/www/wiki/maintenance/postgres/archives/patch-sites.sql new file mode 100644 index 00000000..a4f9ed9e --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-sites.sql @@ -0,0 +1,31 @@ +CREATE SEQUENCE sites_site_id_seq; +CREATE TABLE sites ( + site_id INTEGER NOT NULL PRIMARY KEY DEFAULT nextval('sites_site_id_seq'), + site_global_key TEXT NOT NULL, + site_type TEXT NOT NULL, + site_group TEXT NOT NULL, + site_source TEXT NOT NULL, + site_language TEXT NOT NULL, + site_protocol TEXT NOT NULL, + site_domain TEXT NOT NULL, + site_data TEXT NOT NULL, + site_forward SMALLINT NOT NULL, + site_config TEXT NOT NULL +); +CREATE UNIQUE INDEX site_global_key ON sites (site_global_key); +CREATE INDEX site_type ON sites (site_type); +CREATE INDEX site_group ON sites (site_group); +CREATE INDEX site_source ON sites (site_source); +CREATE INDEX site_language ON sites (site_language); +CREATE INDEX site_protocol ON sites (site_protocol); +CREATE INDEX site_domain ON sites (site_domain); +CREATE INDEX site_forward ON sites (site_forward); + +CREATE TABLE site_identifiers ( + si_site INTEGER NOT NULL, + si_type TEXT NOT NULL, + si_key TEXT NOT NULL +); +CREATE UNIQUE INDEX si_type_key ON site_identifiers (si_type, si_key); +CREATE INDEX si_site ON site_identifiers (si_site); +CREATE INDEX si_key ON site_identifiers (si_key); diff --git a/www/wiki/maintenance/postgres/archives/patch-tag_summary.sql b/www/wiki/maintenance/postgres/archives/patch-tag_summary.sql new file mode 100644 index 00000000..49e05e77 --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-tag_summary.sql @@ -0,0 +1,9 @@ +CREATE TABLE tag_summary ( + ts_rc_id INTEGER NULL, + ts_log_id INTEGER NULL, + ts_rev_id INTEGER NULL, + ts_tags TEXT NOT NULL +); +CREATE UNIQUE INDEX tag_summary_rc_id ON tag_summary(ts_rc_id); +CREATE UNIQUE INDEX tag_summary_log_id ON tag_summary(ts_log_id); +CREATE UNIQUE INDEX tag_summary_rev_id ON tag_summary(ts_rev_id); diff --git a/www/wiki/maintenance/postgres/archives/patch-testrun.sql b/www/wiki/maintenance/postgres/archives/patch-testrun.sql new file mode 100644 index 00000000..a131b5da --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-testrun.sql @@ -0,0 +1,30 @@ +-- +-- Optional tables for parserTests recording mode +-- With --record option, success data will be saved to these tables, +-- and comparisons of what's changed from the previous run will be +-- displayed at the end of each run. +-- +-- This file is for the Postgres version of the tables +-- + +-- Note: "if exists" will not work on older versions of Postgres +DROP TABLE IF EXISTS testitem; +DROP TABLE IF EXISTS testrun; +DROP SEQUENCE IF EXISTS testrun_id_seq; + +CREATE SEQUENCE testrun_id_seq; +CREATE TABLE testrun ( + tr_id INTEGER PRIMARY KEY NOT NULL DEFAULT nextval('testrun_id_seq'), + tr_date TIMESTAMPTZ, + tr_mw_version TEXT, + tr_php_version TEXT, + tr_db_version TEXT, + tr_uname TEXT +); + +CREATE TABLE testitem ( + ti_run INTEGER NOT NULL REFERENCES testrun(tr_id) ON DELETE CASCADE, + ti_name TEXT NOT NULL, + ti_success SMALLINT NOT NULL +); +CREATE UNIQUE INDEX testitem_uniq ON testitem(ti_run, ti_name); diff --git a/www/wiki/maintenance/postgres/archives/patch-textsearch_bug66650.sql b/www/wiki/maintenance/postgres/archives/patch-textsearch_bug66650.sql new file mode 100644 index 00000000..e4f5681c --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-textsearch_bug66650.sql @@ -0,0 +1,5 @@ +UPDATE /*_*/pagecontent SET textvector=to_tsvector(old_text) +WHERE textvector IS NULL AND old_id IN +(SELECT max(rev_text_id) FROM revision GROUP BY rev_page); + +INSERT INTO /*_*/updatelog(ul_key) VALUES ('patch-textsearch_bug66650.sql'); diff --git a/www/wiki/maintenance/postgres/archives/patch-ts2pagetitle.sql b/www/wiki/maintenance/postgres/archives/patch-ts2pagetitle.sql new file mode 100644 index 00000000..4ac985e3 --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-ts2pagetitle.sql @@ -0,0 +1,13 @@ +CREATE OR REPLACE FUNCTION ts2_page_title() +RETURNS TRIGGER +LANGUAGE plpgsql AS +$mw$ +BEGIN +IF TG_OP = 'INSERT' THEN + NEW.titlevector = to_tsvector('default',REPLACE(NEW.page_title,'/',' ')); +ELSIF NEW.page_title != OLD.page_title THEN + NEW.titlevector := to_tsvector('default',REPLACE(NEW.page_title,'/',' ')); +END IF; +RETURN NEW; +END; +$mw$; diff --git a/www/wiki/maintenance/postgres/archives/patch-tsearch2funcs.sql b/www/wiki/maintenance/postgres/archives/patch-tsearch2funcs.sql new file mode 100644 index 00000000..c24efef3 --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-tsearch2funcs.sql @@ -0,0 +1,29 @@ +-- Should be run on Postgres 8.3 or newer to remove the 'default' + +CREATE OR REPLACE FUNCTION ts2_page_title() +RETURNS TRIGGER +LANGUAGE plpgsql AS +$mw$ +BEGIN +IF TG_OP = 'INSERT' THEN + NEW.titlevector = to_tsvector(REPLACE(NEW.page_title,'/',' ')); +ELSIF NEW.page_title != OLD.page_title THEN + NEW.titlevector := to_tsvector(REPLACE(NEW.page_title,'/',' ')); +END IF; +RETURN NEW; +END; +$mw$; + +CREATE OR REPLACE FUNCTION ts2_page_text() +RETURNS TRIGGER +LANGUAGE plpgsql AS +$mw$ +BEGIN +IF TG_OP = 'INSERT' THEN + NEW.textvector = to_tsvector(NEW.old_text); +ELSIF NEW.old_text != OLD.old_text THEN + NEW.textvector := to_tsvector(NEW.old_text); +END IF; +RETURN NEW; +END; +$mw$; diff --git a/www/wiki/maintenance/postgres/archives/patch-update_sequences.sql b/www/wiki/maintenance/postgres/archives/patch-update_sequences.sql new file mode 100644 index 00000000..94f7be4f --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-update_sequences.sql @@ -0,0 +1,20 @@ +ALTER TABLE revision RENAME rev_rev_id_val TO revision_rev_id_seq; +ALTER TABLE revision ALTER COLUMN rev_id SET DEFAULT NEXTVAL('revision_rev_id_seq'); + +ALTER TABLE pagecontent RENAME text_old_id_val TO text_old_id_seq; +ALTER TABLE pagecontent ALTER COLUMN old_id SET DEFAULT nextval('text_old_id_seq'); + +ALTER TABLE category RENAME category_id_seq TO category_cat_id_seq; +ALTER TABLE category ALTER COLUMN cat_id SET DEFAULT nextval('category_cat_id_seq'); + +ALTER TABLE ipblocks RENAME ipblocks_ipb_id_val TO ipblocks_ipb_id_seq; +ALTER TABLE ipblocks ALTER COLUMN ipb_id SET DEFAULT nextval('ipblocks_ipb_id_seq'); + +ALTER TABLE recentchanges RENAME rc_rc_id_seq TO recentchanges_rc_id_seq; +ALTER TABLE recentchanges ALTER COLUMN rc_id SET DEFAULT nextval('recentchanges_rc_id_seq'); + +ALTER TABLE logging RENAME log_log_id_seq TO logging_log_id_seq; +ALTER TABLE logging ALTER COLUMN log_id SET DEFAULT nextval('logging_log_id_seq'); + +ALTER TABLE page_restrictions RENAME pr_id_val TO page_restrictions_pr_id_seq; +ALTER TABLE page_restrictions ALTER COLUMN pr_id SET DEFAULT nextval('page_restrictions_pr_id_seq'); diff --git a/www/wiki/maintenance/postgres/archives/patch-updatelog.sql b/www/wiki/maintenance/postgres/archives/patch-updatelog.sql new file mode 100644 index 00000000..dda80aa4 --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-updatelog.sql @@ -0,0 +1,4 @@ + +CREATE TABLE updatelog ( + ul_key TEXT NOT NULL PRIMARY KEY +); diff --git a/www/wiki/maintenance/postgres/archives/patch-uploadstash.sql b/www/wiki/maintenance/postgres/archives/patch-uploadstash.sql new file mode 100644 index 00000000..8fd9fb99 --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-uploadstash.sql @@ -0,0 +1,24 @@ +CREATE SEQUENCE uploadstash_us_id_seq; +CREATE TYPE media_type AS ENUM ('UNKNOWN','BITMAP','DRAWING','AUDIO','VIDEO','MULTIMEDIA','OFFICE','TEXT','EXECUTABLE','ARCHIVE'); + +CREATE TABLE uploadstash ( + us_id INTEGER PRIMARY KEY NOT NULL DEFAULT nextval('uploadstash_us_id_seq'), + us_user INTEGER, + us_key TEXT, + us_orig_path TEXT, + us_path TEXT, + us_source_type TEXT, + us_timestamp TIMESTAMPTZ, + us_status TEXT, + us_size INTEGER, + us_sha1 TEXT, + us_mime TEXT, + us_media_type media_type DEFAULT NULL, + us_image_width INTEGER, + us_image_height INTEGER, + us_image_bits INTEGER +); + +CREATE INDEX us_user_idx ON uploadstash (us_user); +CREATE UNIQUE INDEX us_key_idx ON uploadstash (us_key); +CREATE INDEX us_timestamp_idx ON uploadstash (us_timestamp); diff --git a/www/wiki/maintenance/postgres/archives/patch-uploadstash_sequence.sql b/www/wiki/maintenance/postgres/archives/patch-uploadstash_sequence.sql new file mode 100644 index 00000000..550b794e --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-uploadstash_sequence.sql @@ -0,0 +1,2 @@ +ALTER TABLE uploadstash RENAME us_id_seq TO uploadstash_us_id_seq; +ALTER TABLE uploadstash ALTER COLUMN us_id SET DEFAULT NEXTVAL('uploadstash_us_id_seq'); diff --git a/www/wiki/maintenance/postgres/archives/patch-user_former_groups.sql b/www/wiki/maintenance/postgres/archives/patch-user_former_groups.sql new file mode 100644 index 00000000..1ba011e3 --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-user_former_groups.sql @@ -0,0 +1,5 @@ +CREATE TABLE user_former_groups ( + ufg_user INTEGER NULL REFERENCES mwuser(user_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + ufg_group TEXT NOT NULL +); +CREATE UNIQUE INDEX ufg_user_group ON user_former_groups (ufg_user, ufg_group); diff --git a/www/wiki/maintenance/postgres/archives/patch-user_properties.sql b/www/wiki/maintenance/postgres/archives/patch-user_properties.sql new file mode 100644 index 00000000..b40fa85f --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-user_properties.sql @@ -0,0 +1,8 @@ +CREATE TABLE user_properties( + up_user INTEGER NULL REFERENCES mwuser(user_id) ON DELETE CASCADE, + up_property TEXT NOT NULL, + up_value TEXT +); + +CREATE UNIQUE INDEX user_properties_user_property on user_properties (up_user,up_property); +CREATE INDEX user_properties_property on user_properties (up_property); diff --git a/www/wiki/maintenance/postgres/archives/patch-valid_tag.sql b/www/wiki/maintenance/postgres/archives/patch-valid_tag.sql new file mode 100644 index 00000000..98575c6e --- /dev/null +++ b/www/wiki/maintenance/postgres/archives/patch-valid_tag.sql @@ -0,0 +1,3 @@ +CREATE TABLE valid_tag ( + vt_tag TEXT NOT NULL PRIMARY KEY +); diff --git a/www/wiki/maintenance/postgres/compare_schemas.pl b/www/wiki/maintenance/postgres/compare_schemas.pl new file mode 100755 index 00000000..bb08237b --- /dev/null +++ b/www/wiki/maintenance/postgres/compare_schemas.pl @@ -0,0 +1,567 @@ +#!/usr/bin/perl + +## Rough check that the base and postgres "tables.sql" are in sync +## Should be run from maintenance/postgres +## Checks a few other things as well... + +use strict; +use warnings; +use Data::Dumper; +use Cwd; + +#check_valid_sql(); + +my @old = ('../tables.sql'); +my $new = 'tables.sql'; +my @xfile; + +## Read in exceptions and other metadata +my %ok; +while (<DATA>) { + next unless /^(\w+)\s*:\s*([^#]+)/; + my ($name,$val) = ($1,$2); + chomp $val; + if ($name eq 'RENAME') { + die "Invalid rename\n" unless $val =~ /(\w+)\s+(\w+)/; + $ok{OLD}{$1} = $2; + $ok{NEW}{$2} = $1; + next; + } + if ($name eq 'XFILE') { + push @xfile, $val; + next; + } + for (split /\s+/ => $val) { + $ok{$name}{$_} = 0; + } +} + +my $datatype = join '|' => qw( +bool +tinyint smallint int bigint real float +tinytext mediumtext text char varchar varbinary binary +timestamp datetime +tinyblob mediumblob blob +); +$datatype .= q{|ENUM\([\"\w\', ]+\)}; +$datatype = qr{($datatype)}; + +my $typeval = qr{(\(\d+\))?}; + +my $typeval2 = qr{ signed| unsigned| binary| NOT NULL| NULL| PRIMARY KEY| AUTO_INCREMENT| default ['\-\d\w"]+| REFERENCES .+CASCADE}; + +my $indextype = join '|' => qw(INDEX KEY FULLTEXT), 'PRIMARY KEY', 'UNIQUE INDEX', 'UNIQUE KEY'; +$indextype = qr{$indextype}; + +my $engine = qr{TYPE|ENGINE}; + +my $tabletype = qr{InnoDB|MyISAM|HEAP|HEAP MAX_ROWS=\d+|InnoDB MAX_ROWS=\d+ AVG_ROW_LENGTH=\d+}; + +my $charset = qr{utf8|binary}; + +open my $newfh, '<', $new or die qq{Could not open $new: $!\n}; + + +my ($table,%old); + +## Read in the xfiles +my %xinfo; +for my $xfile (@xfile) { + print "Loading $xfile\n"; + my $info = parse_sql($xfile); + for (keys %$info) { + $xinfo{$_} = $info->{$_}; + } +} + +for my $oldfile (@old) { + print "Loading $oldfile\n"; + my $info = parse_sql($oldfile); + for (keys %xinfo) { + $info->{$_} = $xinfo{$_}; + } + $old{$oldfile} = $info; +} + +sub parse_sql { + + my $oldfile = shift; + + open my $oldfh, '<', $oldfile or die qq{Could not open $oldfile: $!\n}; + + my %info; + while (<$oldfh>) { + next if /^\s*\-\-/ or /^\s+$/; + s/\s*\-\- [\w ]+$//; + chomp; + + if (/CREATE\s*TABLE/i) { + if (m{^CREATE TABLE /\*_\*/(\w+) \($}) { + $table = $1; + } + elsif (m{^CREATE TABLE /\*\$wgDBprefix\*/(\w+) \($}) { + $table = $1; + } + else { + die qq{Invalid CREATE TABLE at line $. of $oldfile\n}; + } + $info{$table}{name}=$table; + } + elsif (m{^\) /\*\$wgDBTableOptions\*/}) { + $info{$table}{engine} = 'ENGINE'; + $info{$table}{type} = 'variable'; + } + elsif (/^\) ($engine)=($tabletype);$/) { + $info{$table}{engine}=$1; + $info{$table}{type}=$2; + } + elsif (/^\) ($engine)=($tabletype), DEFAULT CHARSET=($charset);$/) { + $info{$table}{engine}=$1; + $info{$table}{type}=$2; + $info{$table}{charset}=$3; + } + elsif (/^ (\w+) $datatype$typeval$typeval2{0,4},?$/) { + $info{$table}{column}{$1} = $2; + my $extra = $3 || ''; + $info{$table}{columnfull}{$1} = "$2$extra"; + } + elsif (m{^ UNIQUE KEY (\w+) \((.+?)\)}) { + } + elsif (m{^CREATE (?:UNIQUE )?(?:FULLTEXT )?INDEX /\*i\*/(\w+) ON /\*_\*/(\w+) \((.+?)\);}) { + } + elsif (m{^\s*PRIMARY KEY \([\w,]+\)}) { + } + else { + die "Cannot parse line $. of $oldfile:\n$_\n"; + } + + } + close $oldfh or die qq{Could not close "$oldfile": $!\n}; + + return \%info; + +} ## end of parse_sql + +for my $oldfile (@old) { + +## Begin non-standard indent + +## MySQL sanity checks +for my $table (sort keys %{$old{$oldfile}}) { + my $t = $old{$oldfile}{$table}; + if ($t->{engine} eq 'TYPE') { + die "Invalid engine for $oldfile: $t->{engine}\n" unless $t->{name} eq 'profiling'; + } + my $charset = $t->{charset} || ''; + if ($oldfile !~ /binary/ and $charset eq 'binary') { + die "Invalid charset for $oldfile: $charset\n"; + } +} + +my $dtypelist = join '|' => qw( +SMALLINT INTEGER BIGINT NUMERIC SERIAL +TEXT CHAR VARCHAR +BYTEA +TIMESTAMPTZ +CIDR +); +my $dtype = qr{($dtypelist)}; +my %new; +my ($infunction,$inview,$inrule,$lastcomma) = (0,0,0,0); +my %custom_type; +seek $newfh, 0, 0; +while (<$newfh>) { + next if /^\s*\-\-/ or /^\s*$/; + s/\s*\-\- [\w ']+$//; + next if /^BEGIN;/ or /^SET / or /^COMMIT;/; + next if /^CREATE SEQUENCE/; + next if /^CREATE(?: UNIQUE)? INDEX/; + next if /^CREATE FUNCTION/; + next if /^CREATE TRIGGER/ or /^ FOR EACH ROW/; + next if /^INSERT INTO/ or /^ VALUES \(/; + next if /^ALTER TABLE/; + next if /^DROP SEQUENCE/; + next if /^DROP FUNCTION/; + + if (/^CREATE TYPE (\w+)/) { + die "Type $1 declared more than once!\n" if $custom_type{$1}++; + $dtype = qr{($dtypelist|$1)}; + next; + } + + chomp; + + if (/^\$mw\$;?$/) { + $infunction = $infunction ? 0 : 1; + next; + } + next if $infunction; + + next if /^CREATE VIEW/ and $inview = 1; + if ($inview) { + /;$/ and $inview = 0; + next; + } + + next if /^CREATE RULE/ and $inrule = 1; + if ($inrule) { + /;$/ and $inrule = 0; + next; + } + + if (/^CREATE TABLE "?(\w+)"? \($/) { + $table = $1; + $new{$table}{name}=$table; + $lastcomma = 1; + } + elsif (/^\);$/) { + if ($lastcomma) { + warn "Stray comma before line $.\n"; + } + } + elsif (/^ (\w+) +$dtype.*?(,?)(?: --.*)?$/) { + $new{$table}{column}{$1} = $2; + if (!$lastcomma) { + print "Missing comma before line $. of $new\n"; + } + $lastcomma = $3 ? 1 : 0; + } + elsif (m{^\s*PRIMARY KEY \([\w,]+\)}) { + $lastcomma = 0; + } + else { + die "Cannot parse line $. of $new:\n$_\n"; + } +} + +## Which column types are okay to map from mysql to postgres? +my $COLMAP = q{ +## INTS: +tinyint SMALLINT +int INTEGER SERIAL +smallint SMALLINT +bigint BIGINT +real NUMERIC +float NUMERIC + +## TEXT: +varchar(15) TEXT +varchar(32) TEXT +varchar(70) TEXT +varchar(255) TEXT +varchar TEXT +text TEXT +tinytext TEXT +ENUM TEXT + +## TIMESTAMPS: +varbinary(14) TIMESTAMPTZ +binary(14) TIMESTAMPTZ +datetime TIMESTAMPTZ +timestamp TIMESTAMPTZ + +## BYTEA: +mediumblob BYTEA + +## OTHER: +bool SMALLINT # Sigh + +}; +## Allow specific exceptions to the above +my $COLMAPOK = q{ +## User inputted text strings: +ar_comment tinyblob TEXT +fa_description tinyblob TEXT +img_description tinyblob TEXT +ipb_reason tinyblob TEXT +log_action varbinary(32) TEXT +log_type varbinary(32) TEXT +oi_description tinyblob TEXT +rev_comment tinyblob TEXT +rc_log_action varbinary(255) TEXT +rc_log_type varbinary(255) TEXT + +## Simple text-only strings: +ar_flags tinyblob TEXT +cf_name varbinary(255) TEXT +cf_value blob TEXT +ar_sha1 varbinary(32) TEXT +cl_collation varbinary(32) TEXT +cl_sortkey varbinary(230) TEXT +ct_params blob TEXT +fa_minor_mime varbinary(100) TEXT +fa_storage_group varbinary(16) TEXT # Just 'deleted' for now, should stay plain text +fa_storage_key varbinary(64) TEXT # sha1 plus text extension +ipb_address tinyblob TEXT # IP address or username +ipb_range_end tinyblob TEXT # hexadecimal +ipb_range_start tinyblob TEXT # hexadecimal +img_minor_mime varbinary(100) TEXT +lc_lang varbinary(32) TEXT +lc_value varbinary(32) TEXT +img_sha1 varbinary(32) TEXT +iw_wikiid varchar(64) TEXT +job_cmd varbinary(60) TEXT # Should we limit to 60 as well? +keyname varbinary(255) TEXT # No tablename prefix (objectcache) +ll_lang varbinary(20) TEXT # Language code +lc_value mediumblob TEXT +log_params blob TEXT # LF separated list of args +log_type varbinary(10) TEXT +ls_field varbinary(32) TEXT +md_deps mediumblob TEXT # JSON +md_module varbinary(255) TEXT +md_skin varbinary(32) TEXT +mr_blob mediumblob TEXT # JSON +mr_lang varbinary(32) TEXT +mr_resource varbinary(255) TEXT +mrl_message varbinary(255) TEXT +mrl_resource varbinary(255) TEXT +oi_minor_mime varbinary(100) TEXT +oi_sha1 varbinary(32) TEXT +old_flags tinyblob TEXT +old_text mediumblob TEXT +pp_propname varbinary(60) TEXT +pp_value blob TEXT +page_restrictions tinyblob TEXT # CSV string +pf_server varchar(30) TEXT +pr_level varbinary(60) TEXT +pr_type varbinary(60) TEXT +pt_create_perm varbinary(60) TEXT +pt_reason tinyblob TEXT +qc_type varbinary(32) TEXT +qcc_type varbinary(32) TEXT +qci_type varbinary(32) TEXT +rc_params blob TEXT +rev_sha1 varbinary(32) TEXT +rlc_to_blob blob TEXT +ts_tags blob TEXT +ufg_group varbinary(32) TEXT +ug_group varbinary(32) TEXT +ul_value blob TEXT +up_property varbinary(255) TEXT +up_value blob TEXT +us_sha1 varchar(31) TEXT +us_source_type varchar(50) TEXT +us_status varchar(50) TEXT +user_email_token binary(32) TEXT +user_ip varbinary(40) TEXT +user_newpassword tinyblob TEXT +user_options blob TEXT +user_password tinyblob TEXT +user_token binary(32) TEXT +iwl_prefix varbinary(20) TEXT + +## Text URLs: +el_index blob TEXT +el_to blob TEXT +iw_api blob TEXT +iw_url blob TEXT +tb_url blob TEXT +tc_url varbinary(255) TEXT + +## Deprecated or not yet used: +ar_text mediumblob TEXT +job_params blob TEXT +log_deleted tinyint INTEGER # Not used yet, but keep it INTEGER for safety +rc_type tinyint CHAR + +## Number tweaking: +fa_bits int SMALLINT # bits per pixel +fa_height int SMALLINT +fa_width int SMALLINT # Hope we don't see an image this wide... +hc_id int BIGINT # Odd that site_stats is all bigint... +img_bits int SMALLINT # bits per image should stay sane +oi_bits int SMALLINT + +## True binary fields, usually due to gzdeflate and/or serialize: +math_inputhash varbinary(16) BYTEA +math_outputhash varbinary(16) BYTEA + +## Namespaces: not need for such a high range +ar_namespace int SMALLINT +job_namespace int SMALLINT +log_namespace int SMALLINT +page_namespace int SMALLINT +pl_namespace int SMALLINT +pt_namespace int SMALLINT +qc_namespace int SMALLINT +rc_namespace int SMALLINT +rd_namespace int SMALLINT +rlc_to_namespace int SMALLINT +tl_namespace int SMALLINT +wl_namespace int SMALLINT + +## Easy enough to change if a wiki ever does grow this big: +ss_active_users bigint INTEGER +ss_good_articles bigint INTEGER +ss_total_edits bigint INTEGER +ss_total_pages bigint INTEGER +ss_users bigint INTEGER + +## True IP - keep an eye on these, coders tend to make textual assumptions +rc_ip varbinary(40) CIDR # Want to keep an eye on this + +## Others: +tc_time int TIMESTAMPTZ + + +}; + +my %colmap; +for (split /\n/ => $COLMAP) { + next unless /^\w/; + s/(.*?)#.*/$1/; + my ($col,@maps) = split / +/, $_; + for (@maps) { + $colmap{$col}{$_} = 1; + } +} + +my %colmapok; +for (split /\n/ => $COLMAPOK) { + next unless /^\w/; + my ($col,$old,$new) = split / +/, $_; + $colmapok{$col}{$old}{$new} = 1; +} + +## Old but not new +for my $t (sort keys %{$old{$oldfile}}) { + if (!exists $new{$t} and !exists $ok{OLD}{$t}) { + print "Table not in $new: $t\n"; + next; + } + next if exists $ok{OLD}{$t} and !$ok{OLD}{$t}; + my $newt = exists $ok{OLD}{$t} ? $ok{OLD}{$t} : $t; + my $oldcol = $old{$oldfile}{$t}{column}; + my $oldcolfull = $old{$oldfile}{$t}{columnfull}; + my $newcol = $new{$newt}{column}; + for my $c (keys %$oldcol) { + if (!exists $newcol->{$c}) { + print "Column $t.$c not in $new\n"; + next; + } + } + for my $c (sort keys %$newcol) { + if (!exists $oldcol->{$c}) { + print "Column $t.$c not in $oldfile\n"; + next; + } + ## Column types (roughly) match up? + my $new = $newcol->{$c}; + my $old = $oldcolfull->{$c}; + + ## Known exceptions: + next if exists $colmapok{$c}{$old}{$new}; + + $old =~ s/ENUM.*/ENUM/; + + next if $old eq 'ENUM' and $new eq 'media_type'; + + if (! exists $colmap{$old}{$new}) { + print "Column types for $t.$c do not match: $old does not map to $new\n"; + } + } +} +## New but not old: +for (sort keys %new) { + if (!exists $old{$oldfile}{$_} and !exists $ok{NEW}{$_}) { + print "Not in $oldfile: $_\n"; + next; + } +} + + +} ## end each file to be parsed + + +sub check_valid_sql { + + ## Check for a few common problems in most php files + + my $olddir = getcwd(); + chdir("../.."); + for my $basedir (qw/includes extensions/) { + scan_dir($basedir); + } + chdir $olddir; + + return; + +} ## end of check_valid_sql + + +sub scan_dir { + + my $dir = shift; + + opendir my $dh, $dir or die qq{Could not opendir $dir: $!\n}; + #print "Scanning $dir...\n"; + for my $file (grep { -f "$dir/$_" and /\.php$/ } readdir $dh) { + find_problems("$dir/$file"); + } + rewinddir $dh; + for my $subdir (grep { -d "$dir/$_" and ! /\./ } readdir $dh) { + scan_dir("$dir/$subdir"); + } + closedir $dh or die qq{Closedir failed: $!\n}; + return; + +} ## end of scan_dir + +sub find_problems { + + my $file = shift; + open my $fh, '<', $file or die qq{Could not open "$file": $!\n}; + my $lastline = ''; + my $inarray = 0; + while (<$fh>) { + if (/FORCE INDEX/ and $file !~ /Database\w*\.php/) { + warn "Found FORCE INDEX string at line $. of $file\n"; + } + if (/REPLACE INTO/ and $file !~ /Database\w*\.php/) { + warn "Found REPLACE INTO string at line $. of $file\n"; + } + if (/\bIF\s*\(/ and $file !~ /DatabaseMySQL\.php/) { + warn "Found IF string at line $. of $file\n"; + } + if (/\bCONCAT\b/ and $file !~ /Database\w*\.php/) { + warn "Found CONCAT string at line $. of $file\n"; + } + if (/\bGROUP\s+BY\s*\d\b/i and $file !~ /Database\w*\.php/) { + warn "Found GROUP BY # at line $. of $file\n"; + } + if (/wfGetDB\s*\(\s+\)/io) { + warn "wfGETDB is missing parameters at line $. of $file\n"; + } + if (/=\s*array\s*\(\s*$/) { + $inarray = 1; + next; + } + if ($inarray) { + if (/\s*\);\s*$/) { + $inarray = 0; + next; + } + next if ! /\w/ or /array\(\s*$/ or /^\s*#/ or m{^\s*//}; + if (! /,/) { + my $nextline = <$fh>; + last if ! defined $nextline; + if ($nextline =~ /^\s*\)[;,]/) { + $inarray = 0; + next; + } + #warn "Array is missing a comma? Line $. of $file\n"; + } + } + } + close $fh or die qq{Could not close "$file": $!\n}; + return; + +} ## end of find_problems + + +__DATA__ +## Known exceptions +OLD: searchindex ## We use tsearch2 directly on the page table instead +RENAME: user mwuser ## Reserved word causing lots of problems +RENAME: text pagecontent ## Reserved word +XFILE: ../archives/patch-profiling.sql diff --git a/www/wiki/maintenance/postgres/mediawiki_mysql2postgres.pl b/www/wiki/maintenance/postgres/mediawiki_mysql2postgres.pl new file mode 100755 index 00000000..34837e1b --- /dev/null +++ b/www/wiki/maintenance/postgres/mediawiki_mysql2postgres.pl @@ -0,0 +1,441 @@ +#!/usr/bin/perl + +## Convert data from a MySQL mediawiki database into a Postgres mediawiki database + +## NOTE: It is probably easier to dump your wiki using maintenance/dumpBackup.php +## and then import it with maintenance/importDump.php + +## If having UTF-8 problems, there are reports that adding --compatible=postgresql +## may help. + +use strict; +use warnings; +use Data::Dumper; +use Getopt::Long; + +use vars qw(%table %tz %special @torder $COM); +my $VERSION = '1.2'; + +## The following options can be changed via command line arguments: +my $MYSQLDB = ''; +my $MYSQLUSER = ''; + +## If the following are zero-length, we omit their arguments entirely: +my $MYSQLHOST = ''; +my $MYSQLPASSWORD = ''; +my $MYSQLSOCKET = ''; + +## Name of the dump file created +my $MYSQLDUMPFILE = 'mediawiki_upgrade.pg'; + +## How verbose should this script be (0, 1, or 2) +my $verbose = 0; + +my $help = 0; + +my $USAGE = " +Usage: $0 --db=<dbname> --user=<user> [OPTION]... +Example: $0 --db=wikidb --user=wikiuser --pass=sushi + +Converts a MediaWiki schema from MySQL to Postgres +Options: + db Name of the MySQL database + user MySQL database username + pass MySQL database password + host MySQL database host + socket MySQL database socket + verbose Verbosity, increases with multiple uses +"; + +GetOptions + ( + 'db=s' => \$MYSQLDB, + 'user=s' => \$MYSQLUSER, + 'pass=s' => \$MYSQLPASSWORD, + 'host=s' => \$MYSQLHOST, + 'socket=s' => \$MYSQLSOCKET, + 'verbose+' => \$verbose, + 'help' => \$help, + ); + +die $USAGE + if ! length $MYSQLDB + or ! length $MYSQLUSER + or $help; + +## The Postgres schema file: should not be changed +my $PG_SCHEMA = 'tables.sql'; + +## What version we default to when we can't parse the old schema +my $MW_DEFAULT_VERSION = 110; + +## Try and find a working version of mysqldump +$verbose and warn "Locating the mysqldump executable\n"; +my @MYSQLDUMP = ('/usr/local/bin/mysqldump', '/usr/bin/mysqldump'); +my $MYSQLDUMP; +for my $mytry (@MYSQLDUMP) { + next if ! -e $mytry; + -x $mytry or die qq{Not an executable file: "$mytry"\n}; + my $version = qx{$mytry -V}; + $version =~ /^mysqldump\s+Ver\s+\d+/ or die qq{Program at "$mytry" does not act like mysqldump\n}; + $MYSQLDUMP = $mytry; +} +$MYSQLDUMP or die qq{Could not find the mysqldump program\n}; + +## Flags we use for mysqldump +my @MYSQLDUMPARGS = qw( +--skip-lock-tables +--complete-insert +--skip-extended-insert +--skip-add-drop-table +--skip-add-locks +--skip-disable-keys +--skip-set-charset +--skip-comments +--skip-quote-names +); + + +$verbose and warn "Checking that mysqldump can handle our flags\n"; +## Make sure this version can handle all the flags we want. +## Combine with user dump below +my $MYSQLDUMPARGS = join ' ' => @MYSQLDUMPARGS; +## Argh. Any way to make this work on Win32? +my $version = qx{$MYSQLDUMP $MYSQLDUMPARGS 2>&1}; +if ($version =~ /unknown option/) { + die qq{Sorry, you need to use a newer version of the mysqldump program than the one at "$MYSQLDUMP"\n}; +} + +push @MYSQLDUMPARGS, "--user=$MYSQLUSER"; +length $MYSQLPASSWORD and push @MYSQLDUMPARGS, "--password=$MYSQLPASSWORD"; +length $MYSQLHOST and push @MYSQLDUMPARGS, "--host=$MYSQLHOST"; + +## Open the dump file to hold the mysqldump output +open my $mdump, '+>', $MYSQLDUMPFILE or die qq{Could not open "$MYSQLDUMPFILE": $!\n}; +print qq{Writing file "$MYSQLDUMPFILE"\n}; + +open my $mfork2, '-|' or exec $MYSQLDUMP, @MYSQLDUMPARGS, '--no-data', $MYSQLDB; +my $oldselect = select $mdump; + +print while <$mfork2>; + +## Slurp in the current schema +my $current_schema; +seek $mdump, 0, 0; +{ + local $/; + $current_schema = <$mdump>; +} +seek $mdump, 0, 0; +truncate $mdump, 0; + +warn qq{Trying to determine database version...\n} if $verbose; + +my $current_version = 0; +if ($current_schema =~ /CREATE TABLE \S+cur /) { + $current_version = 103; +} +elsif ($current_schema =~ /CREATE TABLE \S+brokenlinks /) { + $current_version = 104; +} +elsif ($current_schema !~ /CREATE TABLE \S+templatelinks /) { + $current_version = 105; +} +elsif ($current_schema !~ /CREATE TABLE \S+validate /) { + $current_version = 106; +} +elsif ($current_schema !~ /ipb_auto tinyint/) { + $current_version = 107; +} +elsif ($current_schema !~ /CREATE TABLE \S+profiling /) { + $current_version = 108; +} +elsif ($current_schema !~ /CREATE TABLE \S+querycachetwo /) { + $current_version = 109; +} +else { + $current_version = $MW_DEFAULT_VERSION; +} + +if (!$current_version) { + warn qq{WARNING! Could not figure out the old version, assuming MediaWiki $MW_DEFAULT_VERSION\n}; + $current_version = $MW_DEFAULT_VERSION; +} + +## Check for a table prefix: +my $table_prefix = ''; +if ($current_schema =~ /CREATE TABLE (\S+)querycache /) { + $table_prefix = $1; +} + +warn qq{Old schema is from MediaWiki version $current_version\n} if $verbose; +warn qq{Table prefix is "$table_prefix"\n} if $verbose and length $table_prefix; + +$verbose and warn qq{Writing file "$MYSQLDUMPFILE"\n}; +my $now = scalar localtime; +my $conninfo = ''; +$MYSQLHOST and $conninfo .= "\n-- host $MYSQLHOST"; +$MYSQLSOCKET and $conninfo .= "\n-- socket $MYSQLSOCKET"; + +print qq{ +-- Dump of MySQL Mediawiki tables for import into a Postgres Mediawiki schema +-- Performed by the program: $0 +-- Version: $VERSION +-- Author: Greg Sabino Mullane <greg\@turnstep.com> Comments welcome +-- +-- This file was created: $now +-- Executable used: $MYSQLDUMP +-- Connection information: +-- database: $MYSQLDB +-- user: $MYSQLUSER$conninfo + +-- This file can be imported manually with psql like so: +-- psql -p port# -h hostname -U username -f $MYSQLDUMPFILE databasename +-- This will overwrite any existing MediaWiki information, so be careful + +}; + +## psql specific stuff +print q{ +\\set ON_ERROR_STOP +BEGIN; +SET client_min_messages = 'WARNING'; +SET timezone = 'GMT'; +SET DateStyle = 'ISO, YMD'; +}; + +warn qq{Reading in the Postgres schema information\n} if $verbose; +open my $schema, '<', $PG_SCHEMA + or die qq{Could not open "$PG_SCHEMA": make sure this script is run from maintenance/postgres/\n}; +my $t; +while (<$schema>) { + if (/CREATE TABLE\s+(\S+)/) { + $t = $1; + $table{$t}={}; + $verbose > 1 and warn qq{ Found table $t\n}; + } + elsif (/^ +(\w+)\s+TIMESTAMP/) { + $tz{$t}{$1}++; + $verbose > 1 and warn qq{ Got a timestamp for column $1\n}; + } + elsif (/REFERENCES\s*([^( ]+)/) { + my $ref = $1; + exists $table{$ref} or die qq{No parent table $ref found for $t\n}; + $table{$t}{$ref}++; + } +} +close $schema or die qq{Could not close "$PG_SCHEMA": $!\n}; + +## Read in special cases and table/version information +$verbose and warn qq{Reading in schema exception information\n}; +my %version_tables; +while (<DATA>) { + if (/^VERSION\s+(\d+\.\d+):\s+(.+)/) { + my $list = join '|' => split /\s+/ => $2; + $version_tables{$1} = qr{\b$list\b}; + next; + } + next unless /^(\w+)\s*(.*)/; + $special{$1} = $2||''; + $special{$2} = $1 if length $2; +} + +## Determine the order of tables based on foreign key constraints +$verbose and warn qq{Figuring out order of tables to dump\n}; +my %dumped; +my $bail = 0; +{ + my $found=0; + T: for my $t (sort keys %table) { + next if exists $dumped{$t} and $dumped{$t} >= 1; + $found=1; + for my $dep (sort keys %{$table{$t}}) { + next T if ! exists $dumped{$dep} or $dumped{$dep} < 0; + } + $dumped{$t} = -1 if ! exists $dumped{$t}; + ## Skip certain tables that are not imported + next if exists $special{$t} and !$special{$t}; + push @torder, $special{$t} || $t; + } + last if !$found; + push @torder, '---'; + for (values %dumped) { $_+=2; } + die "Too many loops!\n" if $bail++ > 1000; + redo; +} + +## Prepare the Postgres database for the move +$verbose and warn qq{Writing Postgres transformation information\n}; + +print "\n-- Empty out all existing tables\n"; +$verbose and warn qq{Writing truncates to empty existing tables\n}; + + +for my $t (@torder, 'objectcache', 'querycache') { + next if $t eq '---'; + my $tname = $special{$t}||$t; + printf qq{TRUNCATE TABLE %-20s CASCADE;\n}, qq{"$tname"}; +} +print "\n\n"; + +print qq{-- Temporarily rename pagecontent to "${table_prefix}text"\n}; +print qq{ALTER TABLE pagecontent RENAME TO "${table_prefix}text";\n\n}; + +print qq{-- Allow rc_ip to contain empty string, will convert at end\n}; +print qq{ALTER TABLE recentchanges ALTER rc_ip TYPE text USING host(rc_ip);\n\n}; + +print "-- Changing all timestamp fields to handle raw integers\n"; +for my $t (sort keys %tz) { + next if $t eq 'archive2'; + for my $c (sort keys %{$tz{$t}}) { + printf "ALTER TABLE %-18s ALTER %-25s TYPE TEXT;\n", $t, $c; + } +} +print "\n"; + +print q{ +INSERT INTO page VALUES (0,-1,'Dummy Page','',0,0,0,default,now(),0,10); +}; + +## If we have a table _prefix, we need to temporarily rename all of our Postgres +## tables temporarily for the import. Perhaps consider making this an auto-schema +## thing in the future. +if (length $table_prefix) { + print qq{\n\n-- Temporarily renaming tables to accomodate the table_prefix "$table_prefix"\n\n}; + for my $t (@torder) { + next if $t eq '---' or $t eq 'text' or $t eq 'user'; + my $tname = $special{$t}||$t; + printf qq{ALTER TABLE %-18s RENAME TO "${table_prefix}$tname";\n}, qq{"$tname"}; + } +} + + +## Try and dump the ill-named "user" table: +## We do this table alone because "user" is a reserved word. +print q{ + +SET escape_string_warning TO 'off'; +\\o /dev/null + +-- Postgres uses a table name of "mwuser" instead of "user" + +-- Create a dummy user to satisfy fk contraints especially with revisions +SELECT setval('user_user_id_seq',0,'false'); +INSERT INTO mwuser + VALUES (DEFAULT,'Anonymous','',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,now(),now()); + +}; + +push @MYSQLDUMPARGS, '--no-create-info'; + +$verbose and warn qq{Dumping "user" table\n}; +$verbose > 2 and warn Dumper \@MYSQLDUMPARGS; +my $usertable = "${table_prefix}user"; +open my $mfork, '-|' or exec $MYSQLDUMP, @MYSQLDUMPARGS, $MYSQLDB, $usertable; +## Unfortunately, there is no easy way to catch errors +my $numusers = 0; +while (<$mfork>) { + ++$numusers and print if s/INSERT INTO $usertable/INSERT INTO mwuser/; +} +close $mfork; +if ($numusers < 1) { + warn qq{No users found, probably a connection error.\n}; + print qq{ERROR: No users found, connection failed, or table "$usertable" does not exist. Dump aborted.\n}; + close $mdump or die qq{Could not close "$MYSQLDUMPFILE": $!\n}; + exit; +} +print "\n-- Users loaded: $numusers\n\n-- Loading rest of the mediawiki schema:\n"; + +warn qq{Dumping all other tables from the MySQL schema\n} if $verbose; + +## Dump the rest of the tables, in chunks based on constraints +## We do not need the user table: +my @dumplist = grep { $_ ne 'user'} @torder; +my @alist; +{ + undef @alist; + PICKATABLE: { + my $tname = shift @dumplist; + ## XXX Make this dynamic below + for my $ver (sort {$b <=> $a } keys %version_tables) { + redo PICKATABLE if $tname =~ $version_tables{$ver}; + } + $tname = "${table_prefix}$tname" if length $table_prefix; + next if $tname !~ /^\w/; + push @alist, $tname; + $verbose and warn " $tname...\n"; + pop @alist and last if index($alist[-1],'---') >= 0; + redo if @dumplist; + } + + ## Dump everything else + open my $mfork2, '-|' or exec $MYSQLDUMP, @MYSQLDUMPARGS, $MYSQLDB, @alist; + print while <$mfork2>; + close $mfork2; + warn qq{Finished dumping from MySQL\n} if $verbose; + + redo if @dumplist; +} + +warn qq{Writing information to return Postgres database to normal\n} if $verbose; +print qq{ALTER TABLE "${table_prefix}text" RENAME TO pagecontent;\n}; +print qq{ALTER TABLE ${table_prefix}recentchanges ALTER rc_ip TYPE cidr USING\n}; +print qq{ CASE WHEN rc_ip = '' THEN NULL ELSE rc_ip::cidr END;\n}; + +## Return tables to their original names if a table prefix was used. +if (length $table_prefix) { + print qq{\n\n-- Renaming tables by removing table prefix "$table_prefix"\n\n}; + my $maxsize = 18; + for (@torder) { + $maxsize = length "$_$table_prefix" if length "$_$table_prefix" > $maxsize; + } + for my $t (@torder) { + next if $t eq '---' or $t eq 'text' or $t eq 'user'; + my $tname = $special{$t}||$t; + printf qq{ALTER TABLE %*s RENAME TO "$tname";\n}, $maxsize+1, qq{"${table_prefix}$tname"}; + } +} + +print qq{\n\n--Returning timestamps to normal\n}; +for my $t (sort keys %tz) { + next if $t eq 'archive2'; + for my $c (sort keys %{$tz{$t}}) { + printf "ALTER TABLE %-18s ALTER %-25s TYPE timestamptz\n". + " USING TO_TIMESTAMP($c,'YYYYMMDDHHMISS');\n", $t, $c; + } +} + +## Reset sequences +print q{ +SELECT setval('filearchive_fa_id_seq', 1+coalesce(max(fa_id) ,0),false) FROM filearchive; +SELECT setval('ipblocks_ipb_id_seq', 1+coalesce(max(ipb_id) ,0),false) FROM ipblocks; +SELECT setval('job_job_id_seq', 1+coalesce(max(job_id) ,0),false) FROM job; +SELECT setval('logging_log_id_seq', 1+coalesce(max(log_id) ,0),false) FROM logging; +SELECT setval('page_page_id_seq', 1+coalesce(max(page_id),0),false) FROM page; +SELECT setval('page_restrictions_pr_id_seq', 1+coalesce(max(pr_id) ,0),false) FROM page_restrictions; +SELECT setval('recentchanges_rc_id_seq', 1+coalesce(max(rc_id) ,0),false) FROM recentchanges; +SELECT setval('revision_rev_id_seq', 1+coalesce(max(rev_id) ,0),false) FROM revision; +SELECT setval('text_old_id_seq', 1+coalesce(max(old_id) ,0),false) FROM pagecontent; +SELECT setval('user_user_id_seq', 1+coalesce(max(user_id),0),false) FROM mwuser; +}; + +print "COMMIT;\n\\o\n\n-- End of dump\n\n"; +select $oldselect; +close $mdump or die qq{Could not close "$MYSQLDUMPFILE": $!\n}; +exit; + + +__DATA__ +## Known remappings: either indicate the MySQL name, +## or leave blank if it should be skipped +pagecontent text +mwuser user +archive2 +profiling +objectcache + +## Which tables to ignore depending on the version +VERSION 1.6: externallinks job templatelinks transcache +VERSION 1.7: filearchive langlinks querycache_info +VERSION 1.9: querycachetwo page_restrictions redirect + diff --git a/www/wiki/maintenance/postgres/tables.sql b/www/wiki/maintenance/postgres/tables.sql new file mode 100644 index 00000000..da9c8648 --- /dev/null +++ b/www/wiki/maintenance/postgres/tables.sql @@ -0,0 +1,798 @@ +-- SQL to create the initial tables for the MediaWiki database. +-- This is read and executed by the install script; you should +-- not have to run it by itself unless doing a manual install. +-- This is the PostgreSQL version. +-- For information about each table, please see the notes in maintenance/tables.sql +-- Please make sure all dollar-quoting uses $mw$ at the start of the line +-- TODO: Change CHAR/SMALLINT to BOOL (still used in a non-bool fashion in PHP code) + +BEGIN; +SET client_min_messages = 'ERROR'; + +DROP SEQUENCE IF EXISTS user_user_id_seq CASCADE; +DROP SEQUENCE IF EXISTS page_page_id_seq CASCADE; +DROP SEQUENCE IF EXISTS revision_rev_id_seq CASCADE; +DROP SEQUENCE IF EXISTS comment_comment_id_seq CASCADE; +DROP SEQUENCE IF EXISTS text_old_id_seq CASCADE; +DROP SEQUENCE IF EXISTS page_restrictions_pr_id_seq CASCADE; +DROP SEQUENCE IF EXISTS ipblocks_ipb_id_seq CASCADE; +DROP SEQUENCE IF EXISTS filearchive_fa_id_seq CASCADE; +DROP SEQUENCE IF EXISTS uploadstash_us_id_seq CASCADE; +DROP SEQUENCE IF EXISTS recentchanges_rc_id_seq CASCADE; +DROP SEQUENCE IF EXISTS watchlist_wl_id_seq CASCADE; +DROP SEQUENCE IF EXISTS logging_log_id_seq CASCADE; +DROP SEQUENCE IF EXISTS job_job_id_seq CASCADE; +DROP SEQUENCE IF EXISTS category_cat_id_seq CASCADE; +DROP SEQUENCE IF EXISTS archive_ar_id_seq CASCADE; +DROP SEQUENCE IF EXISTS externallinks_el_id_seq CASCADE; +DROP SEQUENCE IF EXISTS sites_site_id_seq CASCADE; +DROP SEQUENCE IF EXISTS change_tag_ct_id_seq CASCADE; +DROP SEQUENCE IF EXISTS tag_summary_ts_id_seq CASCADE; +DROP FUNCTION IF EXISTS page_deleted() CASCADE; +DROP FUNCTION IF EXISTS ts2_page_title() CASCADE; +DROP FUNCTION IF EXISTS ts2_page_text() CASCADE; +DROP FUNCTION IF EXISTS add_interwiki(TEXT,INT,SMALLINT) CASCADE; +DROP TYPE IF EXISTS media_type CASCADE; + +CREATE SEQUENCE user_user_id_seq MINVALUE 0 START WITH 0; +CREATE TABLE mwuser ( -- replace reserved word 'user' + user_id INTEGER NOT NULL PRIMARY KEY DEFAULT nextval('user_user_id_seq'), + user_name TEXT NOT NULL UNIQUE, + user_real_name TEXT, + user_password TEXT, + user_newpassword TEXT, + user_newpass_time TIMESTAMPTZ, + user_token TEXT, + user_email TEXT, + user_email_token TEXT, + user_email_token_expires TIMESTAMPTZ, + user_email_authenticated TIMESTAMPTZ, + user_touched TIMESTAMPTZ, + user_registration TIMESTAMPTZ, + user_editcount INTEGER, + user_password_expires TIMESTAMPTZ NULL +); +CREATE INDEX user_email_token_idx ON mwuser (user_email_token); + +-- Create a dummy user to satisfy fk contraints especially with revisions +INSERT INTO mwuser + VALUES (DEFAULT,'Anonymous','',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,now(),now()); + +CREATE TABLE user_groups ( + ug_user INTEGER NULL REFERENCES mwuser(user_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + ug_group TEXT NOT NULL, + ug_expiry TIMESTAMPTZ NULL, + PRIMARY KEY(ug_user, ug_group) +); +CREATE INDEX user_groups_group ON user_groups (ug_group); +CREATE INDEX user_groups_expiry ON user_groups (ug_expiry); + +CREATE TABLE user_former_groups ( + ufg_user INTEGER NULL REFERENCES mwuser(user_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + ufg_group TEXT NOT NULL +); +CREATE UNIQUE INDEX ufg_user_group ON user_former_groups (ufg_user, ufg_group); + +CREATE TABLE user_newtalk ( + user_id INTEGER NOT NULL REFERENCES mwuser(user_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + user_ip TEXT NULL, + user_last_timestamp TIMESTAMPTZ +); +CREATE INDEX user_newtalk_id_idx ON user_newtalk (user_id); +CREATE INDEX user_newtalk_ip_idx ON user_newtalk (user_ip); + +CREATE TABLE bot_passwords ( + bp_user INTEGER NOT NULL, + bp_app_id TEXT NOT NULL, + bp_password TEXT NOT NULL, + bp_token TEXT NOT NULL, + bp_restrictions TEXT NOT NULL, + bp_grants TEXT NOT NULL, + PRIMARY KEY ( bp_user, bp_app_id ) +); + +CREATE SEQUENCE page_page_id_seq; +CREATE TABLE page ( + page_id INTEGER NOT NULL PRIMARY KEY DEFAULT nextval('page_page_id_seq'), + page_namespace SMALLINT NOT NULL, + page_title TEXT NOT NULL, + page_restrictions TEXT, + page_is_redirect SMALLINT NOT NULL DEFAULT 0, + page_is_new SMALLINT NOT NULL DEFAULT 0, + page_random NUMERIC(15,14) NOT NULL DEFAULT RANDOM(), + page_touched TIMESTAMPTZ, + page_links_updated TIMESTAMPTZ NULL, + page_latest INTEGER NOT NULL, -- FK? + page_len INTEGER NOT NULL, + page_content_model TEXT, + page_lang TEXT DEFAULT NULL +); +CREATE UNIQUE INDEX page_unique_name ON page (page_namespace, page_title); +CREATE INDEX page_main_title ON page (page_title text_pattern_ops) WHERE page_namespace = 0; +CREATE INDEX page_talk_title ON page (page_title text_pattern_ops) WHERE page_namespace = 1; +CREATE INDEX page_user_title ON page (page_title text_pattern_ops) WHERE page_namespace = 2; +CREATE INDEX page_utalk_title ON page (page_title text_pattern_ops) WHERE page_namespace = 3; +CREATE INDEX page_project_title ON page (page_title text_pattern_ops) WHERE page_namespace = 4; +CREATE INDEX page_mediawiki_title ON page (page_title text_pattern_ops) WHERE page_namespace = 8; +CREATE INDEX page_random_idx ON page (page_random); +CREATE INDEX page_len_idx ON page (page_len); + +CREATE FUNCTION page_deleted() RETURNS TRIGGER LANGUAGE plpgsql AS +$mw$ +BEGIN +DELETE FROM recentchanges WHERE rc_namespace = OLD.page_namespace AND rc_title = OLD.page_title; +RETURN NULL; +END; +$mw$; + +CREATE TRIGGER page_deleted AFTER DELETE ON page + FOR EACH ROW EXECUTE PROCEDURE page_deleted(); + +CREATE SEQUENCE revision_rev_id_seq; +CREATE TABLE revision ( + rev_id INTEGER NOT NULL UNIQUE DEFAULT nextval('revision_rev_id_seq'), + rev_page INTEGER NULL REFERENCES page (page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + rev_text_id INTEGER NULL, -- FK + rev_comment TEXT NOT NULL DEFAULT '', + rev_user INTEGER NOT NULL REFERENCES mwuser(user_id) ON DELETE RESTRICT DEFERRABLE INITIALLY DEFERRED, + rev_user_text TEXT NOT NULL, + rev_timestamp TIMESTAMPTZ NOT NULL, + rev_minor_edit SMALLINT NOT NULL DEFAULT 0, + rev_deleted SMALLINT NOT NULL DEFAULT 0, + rev_len INTEGER NULL, + rev_parent_id INTEGER NULL, + rev_sha1 TEXT NOT NULL DEFAULT '', + rev_content_model TEXT, + rev_content_format TEXT +); +CREATE UNIQUE INDEX revision_unique ON revision (rev_page, rev_id); +CREATE INDEX rev_text_id_idx ON revision (rev_text_id); +CREATE INDEX rev_timestamp_idx ON revision (rev_timestamp); +CREATE INDEX rev_user_idx ON revision (rev_user); +CREATE INDEX rev_user_text_idx ON revision (rev_user_text); + +CREATE TABLE revision_comment_temp ( + revcomment_rev INTEGER NOT NULL, + revcomment_comment_id INTEGER NOT NULL, + PRIMARY KEY (revcomment_rev, revcomment_comment_id) +); +CREATE UNIQUE INDEX revcomment_rev ON revision_comment_temp (revcomment_rev); + +CREATE SEQUENCE ip_changes_ipc_rev_id_seq; + +CREATE TABLE ip_changes ( + ipc_rev_id INTEGER PRIMARY KEY NOT NULL DEFAULT nextval('ip_changes_ipc_rev_id_seq'), + ipc_rev_timestamp TIMESTAMPTZ NOT NULL, + ipc_hex BYTEA NOT NULL DEFAULT '' +); + +CREATE INDEX ipc_rev_timestamp ON ip_changes (ipc_rev_timestamp); +CREATE INDEX ipc_hex_time ON ip_changes (ipc_hex,ipc_rev_timestamp); + +CREATE SEQUENCE text_old_id_seq; +CREATE TABLE pagecontent ( -- replaces reserved word 'text' + old_id INTEGER NOT NULL PRIMARY KEY DEFAULT nextval('text_old_id_seq'), + old_text TEXT, + old_flags TEXT +); + + +CREATE SEQUENCE comment_comment_id_seq; +CREATE TABLE comment ( + comment_id INTEGER NOT NULL PRIMARY KEY DEFAULT nextval('comment_comment_id_seq'), + comment_hash INTEGER NOT NULL, + comment_text TEXT NOT NULL, + comment_data TEXT +); +CREATE INDEX comment_hash ON comment (comment_hash); + + +CREATE SEQUENCE page_restrictions_pr_id_seq; +CREATE TABLE page_restrictions ( + pr_id INTEGER NOT NULL UNIQUE DEFAULT nextval('page_restrictions_pr_id_seq'), + pr_page INTEGER NULL REFERENCES page (page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + pr_type TEXT NOT NULL, + pr_level TEXT NOT NULL, + pr_cascade SMALLINT NOT NULL, + pr_user INTEGER NULL, + pr_expiry TIMESTAMPTZ NULL +); +ALTER TABLE page_restrictions ADD CONSTRAINT page_restrictions_pk PRIMARY KEY (pr_page,pr_type); + +CREATE TABLE page_props ( + pp_page INTEGER NOT NULL REFERENCES page (page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + pp_propname TEXT NOT NULL, + pp_value TEXT NOT NULL, + pp_sortkey FLOAT +); +ALTER TABLE page_props ADD CONSTRAINT page_props_pk PRIMARY KEY (pp_page,pp_propname); +CREATE INDEX page_props_propname ON page_props (pp_propname); +CREATE UNIQUE INDEX pp_propname_page ON page_props (pp_propname,pp_page); +CREATE INDEX pp_propname_sortkey_page ON page_props (pp_propname, pp_sortkey, pp_page) WHERE (pp_sortkey IS NOT NULL); + +CREATE SEQUENCE archive_ar_id_seq; +CREATE TABLE archive ( + ar_id INTEGER NOT NULL PRIMARY KEY DEFAULT nextval('archive_ar_id_seq'), + ar_namespace SMALLINT NOT NULL, + ar_title TEXT NOT NULL, + ar_text TEXT, -- technically should be bytea, but not used anymore + ar_page_id INTEGER NULL, + ar_parent_id INTEGER NULL, + ar_sha1 TEXT NOT NULL DEFAULT '', + ar_comment TEXT NOT NULL DEFAULT '', + ar_comment_id INTEGER NOT NULL DEFAULT 0, + ar_user INTEGER NULL REFERENCES mwuser(user_id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED, + ar_user_text TEXT NOT NULL, + ar_timestamp TIMESTAMPTZ NOT NULL, + ar_minor_edit SMALLINT NOT NULL DEFAULT 0, + ar_flags TEXT, + ar_rev_id INTEGER, + ar_text_id INTEGER, + ar_deleted SMALLINT NOT NULL DEFAULT 0, + ar_len INTEGER NULL, + ar_content_model TEXT, + ar_content_format TEXT +); +CREATE INDEX archive_name_title_timestamp ON archive (ar_namespace,ar_title,ar_timestamp); +CREATE INDEX archive_user_text ON archive (ar_user_text); + + +CREATE TABLE redirect ( + rd_from INTEGER NOT NULL REFERENCES page(page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + rd_namespace SMALLINT NOT NULL, + rd_title TEXT NOT NULL, + rd_interwiki TEXT NULL, + rd_fragment TEXT NULL +); +CREATE INDEX redirect_ns_title ON redirect (rd_namespace,rd_title,rd_from); + + +CREATE TABLE pagelinks ( + pl_from INTEGER NOT NULL REFERENCES page(page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + pl_from_namespace INTEGER NOT NULL DEFAULT 0, + pl_namespace SMALLINT NOT NULL, + pl_title TEXT NOT NULL +); +CREATE UNIQUE INDEX pagelink_unique ON pagelinks (pl_from,pl_namespace,pl_title); +CREATE INDEX pagelinks_title ON pagelinks (pl_title); + +CREATE TABLE templatelinks ( + tl_from INTEGER NOT NULL REFERENCES page(page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + tl_from_namespace INTEGER NOT NULL DEFAULT 0, + tl_namespace SMALLINT NOT NULL, + tl_title TEXT NOT NULL +); +CREATE UNIQUE INDEX templatelinks_unique ON templatelinks (tl_namespace,tl_title,tl_from); +CREATE INDEX templatelinks_from ON templatelinks (tl_from); + +CREATE TABLE imagelinks ( + il_from INTEGER NOT NULL REFERENCES page(page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + il_from_namespace INTEGER NOT NULL DEFAULT 0, + il_to TEXT NOT NULL +); +CREATE UNIQUE INDEX il_from ON imagelinks (il_to,il_from); + +CREATE TABLE categorylinks ( + cl_from INTEGER NOT NULL REFERENCES page(page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + cl_to TEXT NOT NULL, + cl_sortkey TEXT NULL, + cl_timestamp TIMESTAMPTZ NOT NULL, + cl_sortkey_prefix TEXT NOT NULL DEFAULT '', + cl_collation TEXT NOT NULL DEFAULT 0, + cl_type TEXT NOT NULL DEFAULT 'page' +); +CREATE UNIQUE INDEX cl_from ON categorylinks (cl_from, cl_to); +CREATE INDEX cl_sortkey ON categorylinks (cl_to, cl_sortkey, cl_from); + +CREATE SEQUENCE externallinks_el_id_seq; +CREATE TABLE externallinks ( + el_id INTEGER NOT NULL PRIMARY KEY DEFAULT nextval('externallinks_el_id_seq'), + el_from INTEGER NOT NULL REFERENCES page(page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + el_to TEXT NOT NULL, + el_index TEXT NOT NULL, + el_index_60 BYTEA NOT NULL DEFAULT '' +); +CREATE INDEX externallinks_from_to ON externallinks (el_from,el_to); +CREATE INDEX externallinks_index ON externallinks (el_index); +CREATE INDEX el_index_60 ON externallinks (el_index_60, el_id); +CREATE INDEX el_from_index_60 ON externallinks (el_from, el_index_60, el_id); + +CREATE TABLE langlinks ( + ll_from INTEGER NOT NULL REFERENCES page (page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + ll_lang TEXT, + ll_title TEXT +); +CREATE UNIQUE INDEX langlinks_unique ON langlinks (ll_from,ll_lang); +CREATE INDEX langlinks_lang_title ON langlinks (ll_lang,ll_title); + + +CREATE TABLE site_stats ( + ss_row_id INTEGER NOT NULL PRIMARY KEY DEFAULT 0, + ss_total_edits INTEGER DEFAULT 0, + ss_good_articles INTEGER DEFAULT 0, + ss_total_pages INTEGER DEFAULT -1, + ss_users INTEGER DEFAULT -1, + ss_active_users INTEGER DEFAULT -1, + ss_admins INTEGER DEFAULT -1, + ss_images INTEGER DEFAULT 0 +); + + +CREATE SEQUENCE ipblocks_ipb_id_seq; +CREATE TABLE ipblocks ( + ipb_id INTEGER NOT NULL PRIMARY KEY DEFAULT nextval('ipblocks_ipb_id_seq'), + ipb_address TEXT NULL, + ipb_user INTEGER NULL REFERENCES mwuser(user_id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED, + ipb_by INTEGER NOT NULL REFERENCES mwuser(user_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + ipb_by_text TEXT NOT NULL DEFAULT '', + ipb_reason TEXT NOT NULL DEFAULT '', + ipb_reason_id INTEGER NOT NULL DEFAULT 0, + ipb_timestamp TIMESTAMPTZ NOT NULL, + ipb_auto SMALLINT NOT NULL DEFAULT 0, + ipb_anon_only SMALLINT NOT NULL DEFAULT 0, + ipb_create_account SMALLINT NOT NULL DEFAULT 1, + ipb_enable_autoblock SMALLINT NOT NULL DEFAULT 1, + ipb_expiry TIMESTAMPTZ NOT NULL, + ipb_range_start TEXT, + ipb_range_end TEXT, + ipb_deleted SMALLINT NOT NULL DEFAULT 0, + ipb_block_email SMALLINT NOT NULL DEFAULT 0, + ipb_allow_usertalk SMALLINT NOT NULL DEFAULT 0, + ipb_parent_block_id INTEGER NULL REFERENCES ipblocks(ipb_id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED + +); +CREATE UNIQUE INDEX ipb_address_unique ON ipblocks (ipb_address,ipb_user,ipb_auto,ipb_anon_only); +CREATE INDEX ipb_user ON ipblocks (ipb_user); +CREATE INDEX ipb_range ON ipblocks (ipb_range_start,ipb_range_end); +CREATE INDEX ipb_parent_block_id ON ipblocks (ipb_parent_block_id); + + +CREATE TABLE image ( + img_name TEXT NOT NULL PRIMARY KEY, + img_size INTEGER NOT NULL, + img_width INTEGER NOT NULL, + img_height INTEGER NOT NULL, + img_metadata BYTEA NOT NULL DEFAULT '', + img_bits SMALLINT, + img_media_type TEXT, + img_major_mime TEXT DEFAULT 'unknown', + img_minor_mime TEXT DEFAULT 'unknown', + img_description TEXT NOT NULL DEFAULT '', + img_user INTEGER NULL REFERENCES mwuser(user_id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED, + img_user_text TEXT NOT NULL, + img_timestamp TIMESTAMPTZ, + img_sha1 TEXT NOT NULL DEFAULT '' +); +CREATE INDEX img_size_idx ON image (img_size); +CREATE INDEX img_timestamp_idx ON image (img_timestamp); +CREATE INDEX img_sha1 ON image (img_sha1); + +CREATE TABLE image_comment_temp ( + imgcomment_name TEXT NOT NULL, + imgcomment_description_id INTEGER NOT NULL, + PRIMARY KEY (imgcomment_name, imgcomment_description_id) +); +CREATE UNIQUE INDEX imgcomment_name ON image_comment_temp (imgcomment_name); + +CREATE TABLE oldimage ( + oi_name TEXT NOT NULL, + oi_archive_name TEXT NOT NULL, + oi_size INTEGER NOT NULL, + oi_width INTEGER NOT NULL, + oi_height INTEGER NOT NULL, + oi_bits SMALLINT NULL, + oi_description TEXT NOT NULL DEFAULT '', + oi_description_id INTEGER NOT NULL DEFAULT 0, + oi_user INTEGER NULL REFERENCES mwuser(user_id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED, + oi_user_text TEXT NOT NULL, + oi_timestamp TIMESTAMPTZ NULL, + oi_metadata BYTEA NOT NULL DEFAULT '', + oi_media_type TEXT NULL, + oi_major_mime TEXT NULL DEFAULT 'unknown', + oi_minor_mime TEXT NULL DEFAULT 'unknown', + oi_deleted SMALLINT NOT NULL DEFAULT 0, + oi_sha1 TEXT NOT NULL DEFAULT '' +); +ALTER TABLE oldimage ADD CONSTRAINT oldimage_oi_name_fkey_cascaded FOREIGN KEY (oi_name) REFERENCES image(img_name) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED; +CREATE INDEX oi_name_timestamp ON oldimage (oi_name,oi_timestamp); +CREATE INDEX oi_name_archive_name ON oldimage (oi_name,oi_archive_name); +CREATE INDEX oi_sha1 ON oldimage (oi_sha1); + + +CREATE SEQUENCE filearchive_fa_id_seq; +CREATE TABLE filearchive ( + fa_id INTEGER NOT NULL PRIMARY KEY DEFAULT nextval('filearchive_fa_id_seq'), + fa_name TEXT NOT NULL, + fa_archive_name TEXT, + fa_storage_group TEXT, + fa_storage_key TEXT, + fa_deleted_user INTEGER NULL REFERENCES mwuser(user_id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED, + fa_deleted_timestamp TIMESTAMPTZ NOT NULL, + fa_deleted_reason TEXT NOT NULL DEFAULT '', + fa_deleted_reason_id INTEGER NOT NULL DEFAULT 0, + fa_size INTEGER NOT NULL, + fa_width INTEGER NOT NULL, + fa_height INTEGER NOT NULL, + fa_metadata BYTEA NOT NULL DEFAULT '', + fa_bits SMALLINT, + fa_media_type TEXT, + fa_major_mime TEXT DEFAULT 'unknown', + fa_minor_mime TEXT DEFAULT 'unknown', + fa_description TEXT NOT NULL DEFAULT '', + fa_description_id INTEGER NOT NULL DEFAULT 0, + fa_user INTEGER NULL REFERENCES mwuser(user_id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED, + fa_user_text TEXT NOT NULL, + fa_timestamp TIMESTAMPTZ, + fa_deleted SMALLINT NOT NULL DEFAULT 0, + fa_sha1 TEXT NOT NULL DEFAULT '' +); +CREATE INDEX fa_name_time ON filearchive (fa_name, fa_timestamp); +CREATE INDEX fa_dupe ON filearchive (fa_storage_group, fa_storage_key); +CREATE INDEX fa_notime ON filearchive (fa_deleted_timestamp); +CREATE INDEX fa_nouser ON filearchive (fa_deleted_user); +CREATE INDEX fa_sha1 ON filearchive (fa_sha1); + +CREATE SEQUENCE uploadstash_us_id_seq; +CREATE TYPE media_type AS ENUM ('UNKNOWN','BITMAP','DRAWING','AUDIO','VIDEO','MULTIMEDIA','OFFICE','TEXT','EXECUTABLE','ARCHIVE','3D'); + +CREATE TABLE uploadstash ( + us_id INTEGER PRIMARY KEY NOT NULL DEFAULT nextval('uploadstash_us_id_seq'), + us_user INTEGER, + us_key TEXT, + us_orig_path TEXT, + us_path TEXT, + us_props BYTEA, + us_source_type TEXT, + us_timestamp TIMESTAMPTZ, + us_status TEXT, + us_chunk_inx INTEGER NULL, + us_size INTEGER, + us_sha1 TEXT, + us_mime TEXT, + us_media_type media_type DEFAULT NULL, + us_image_width INTEGER, + us_image_height INTEGER, + us_image_bits SMALLINT +); + +CREATE INDEX us_user_idx ON uploadstash (us_user); +CREATE UNIQUE INDEX us_key_idx ON uploadstash (us_key); +CREATE INDEX us_timestamp_idx ON uploadstash (us_timestamp); + + +CREATE SEQUENCE recentchanges_rc_id_seq; +CREATE TABLE recentchanges ( + rc_id INTEGER NOT NULL PRIMARY KEY DEFAULT nextval('recentchanges_rc_id_seq'), + rc_timestamp TIMESTAMPTZ NOT NULL, + rc_cur_time TIMESTAMPTZ NULL, + rc_user INTEGER NULL REFERENCES mwuser(user_id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED, + rc_user_text TEXT NOT NULL, + rc_namespace SMALLINT NOT NULL, + rc_title TEXT NOT NULL, + rc_comment TEXT NOT NULL DEFAULT '', + rc_comment_id INTEGER NOT NULL DEFAULT 0, + rc_minor SMALLINT NOT NULL DEFAULT 0, + rc_bot SMALLINT NOT NULL DEFAULT 0, + rc_new SMALLINT NOT NULL DEFAULT 0, + rc_cur_id INTEGER NULL, + rc_this_oldid INTEGER NOT NULL, + rc_last_oldid INTEGER NOT NULL, + rc_type SMALLINT NOT NULL DEFAULT 0, + rc_source TEXT NOT NULL, + rc_patrolled SMALLINT NOT NULL DEFAULT 0, + rc_ip CIDR, + rc_old_len INTEGER, + rc_new_len INTEGER, + rc_deleted SMALLINT NOT NULL DEFAULT 0, + rc_logid INTEGER NOT NULL DEFAULT 0, + rc_log_type TEXT, + rc_log_action TEXT, + rc_params TEXT +); +CREATE INDEX rc_timestamp ON recentchanges (rc_timestamp); +CREATE INDEX rc_timestamp_bot ON recentchanges (rc_timestamp) WHERE rc_bot = 0; +CREATE INDEX rc_namespace_title ON recentchanges (rc_namespace, rc_title); +CREATE INDEX rc_cur_id ON recentchanges (rc_cur_id); +CREATE INDEX new_name_timestamp ON recentchanges (rc_new, rc_namespace, rc_timestamp); +CREATE INDEX rc_ip ON recentchanges (rc_ip); +CREATE INDEX rc_name_type_patrolled_timestamp ON recentchanges (rc_namespace, rc_type, rc_patrolled, rc_timestamp); + + +CREATE SEQUENCE watchlist_wl_id_seq; +CREATE TABLE watchlist ( + wl_id INTEGER NOT NULL PRIMARY KEY DEFAULT nextval('watchlist_wl_id_seq'), + wl_user INTEGER NOT NULL REFERENCES mwuser(user_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + wl_namespace SMALLINT NOT NULL DEFAULT 0, + wl_title TEXT NOT NULL, + wl_notificationtimestamp TIMESTAMPTZ +); +CREATE UNIQUE INDEX wl_user_namespace_title ON watchlist (wl_namespace, wl_title, wl_user); +CREATE INDEX wl_user ON watchlist (wl_user); +CREATE INDEX wl_user_notificationtimestamp ON watchlist (wl_user, wl_notificationtimestamp); + + +CREATE TABLE interwiki ( + iw_prefix TEXT NOT NULL UNIQUE, + iw_url TEXT NOT NULL, + iw_local SMALLINT NOT NULL, + iw_trans SMALLINT NOT NULL DEFAULT 0, + iw_api TEXT NOT NULL DEFAULT '', + iw_wikiid TEXT NOT NULL DEFAULT '' +); + + +CREATE TABLE querycache ( + qc_type TEXT NOT NULL, + qc_value INTEGER NOT NULL, + qc_namespace SMALLINT NOT NULL, + qc_title TEXT NOT NULL +); +CREATE INDEX querycache_type_value ON querycache (qc_type, qc_value); + +CREATE TABLE querycache_info ( + qci_type TEXT UNIQUE, + qci_timestamp TIMESTAMPTZ NULL +); + +CREATE TABLE querycachetwo ( + qcc_type TEXT NOT NULL, + qcc_value INTEGER NOT NULL DEFAULT 0, + qcc_namespace INTEGER NOT NULL DEFAULT 0, + qcc_title TEXT NOT NULL DEFAULT '', + qcc_namespacetwo INTEGER NOT NULL DEFAULT 0, + qcc_titletwo TEXT NOT NULL DEFAULT '' +); +CREATE INDEX querycachetwo_type_value ON querycachetwo (qcc_type, qcc_value); +CREATE INDEX querycachetwo_title ON querycachetwo (qcc_type,qcc_namespace,qcc_title); +CREATE INDEX querycachetwo_titletwo ON querycachetwo (qcc_type,qcc_namespacetwo,qcc_titletwo); + +CREATE TABLE objectcache ( + keyname TEXT UNIQUE, + value BYTEA NOT NULL DEFAULT '', + exptime TIMESTAMPTZ NOT NULL +); +CREATE INDEX objectcacache_exptime ON objectcache (exptime); + +CREATE TABLE transcache ( + tc_url TEXT NOT NULL UNIQUE, + tc_contents TEXT NOT NULL, + tc_time TIMESTAMPTZ NOT NULL +); + + +CREATE SEQUENCE logging_log_id_seq; +CREATE TABLE logging ( + log_id INTEGER NOT NULL PRIMARY KEY DEFAULT nextval('logging_log_id_seq'), + log_type TEXT NOT NULL, + log_action TEXT NOT NULL, + log_timestamp TIMESTAMPTZ NOT NULL, + log_user INTEGER REFERENCES mwuser(user_id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED, + log_namespace SMALLINT NOT NULL, + log_title TEXT NOT NULL, + log_comment TEXT NOT NULL DEFAULT '', + log_comment_id INTEGER NOT NULL DEFAULT 0, + log_params TEXT, + log_deleted SMALLINT NOT NULL DEFAULT 0, + log_user_text TEXT NOT NULL DEFAULT '', + log_page INTEGER +); +CREATE INDEX logging_type_name ON logging (log_type, log_timestamp); +CREATE INDEX logging_user_time ON logging (log_timestamp, log_user); +CREATE INDEX logging_page_time ON logging (log_namespace, log_title, log_timestamp); +CREATE INDEX logging_times ON logging (log_timestamp); +CREATE INDEX logging_user_type_time ON logging (log_user, log_type, log_timestamp); +CREATE INDEX logging_page_id_time ON logging (log_page, log_timestamp); +CREATE INDEX logging_user_text_type_time ON logging (log_user_text, log_type, log_timestamp); +CREATE INDEX logging_user_text_time ON logging (log_user_text, log_timestamp); + +CREATE TABLE log_search ( + ls_field TEXT NOT NULL, + ls_value TEXT NOT NULL, + ls_log_id INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (ls_field,ls_value,ls_log_id) +); +CREATE INDEX ls_log_id ON log_search (ls_log_id); + + +CREATE SEQUENCE job_job_id_seq; +CREATE TABLE job ( + job_id INTEGER NOT NULL PRIMARY KEY DEFAULT nextval('job_job_id_seq'), + job_cmd TEXT NOT NULL, + job_namespace SMALLINT NOT NULL, + job_title TEXT NOT NULL, + job_timestamp TIMESTAMPTZ, + job_params TEXT NOT NULL, + job_random INTEGER NOT NULL DEFAULT 0, + job_attempts INTEGER NOT NULL DEFAULT 0, + job_token TEXT NOT NULL DEFAULT '', + job_token_timestamp TIMESTAMPTZ, + job_sha1 TEXT NOT NULL DEFAULT '' +); +CREATE INDEX job_sha1 ON job (job_sha1); +CREATE INDEX job_cmd_token ON job (job_cmd, job_token, job_random); +CREATE INDEX job_cmd_token_id ON job (job_cmd, job_token, job_id); +CREATE INDEX job_cmd_namespace_title ON job (job_cmd, job_namespace, job_title); +CREATE INDEX job_timestamp_idx ON job (job_timestamp); + +-- Tsearch2 2 stuff. Will fail if we don't have proper access to the tsearch2 tables +-- Version 8.3 or higher only. Previous versions would need another parmeter for to_tsvector. +-- Make sure you also change patch-tsearch2funcs.sql if the funcs below change. + +ALTER TABLE page ADD titlevector tsvector; +CREATE FUNCTION ts2_page_title() RETURNS TRIGGER LANGUAGE plpgsql AS +$mw$ +BEGIN +IF TG_OP = 'INSERT' THEN + NEW.titlevector = to_tsvector(REPLACE(NEW.page_title,'/',' ')); +ELSIF NEW.page_title != OLD.page_title THEN + NEW.titlevector := to_tsvector(REPLACE(NEW.page_title,'/',' ')); +END IF; +RETURN NEW; +END; +$mw$; + +CREATE TRIGGER ts2_page_title BEFORE INSERT OR UPDATE ON page + FOR EACH ROW EXECUTE PROCEDURE ts2_page_title(); + + +ALTER TABLE pagecontent ADD textvector tsvector; +CREATE FUNCTION ts2_page_text() RETURNS TRIGGER LANGUAGE plpgsql AS +$mw$ +BEGIN +IF TG_OP = 'INSERT' THEN + NEW.textvector = to_tsvector(NEW.old_text); +ELSIF NEW.old_text != OLD.old_text THEN + NEW.textvector := to_tsvector(NEW.old_text); +END IF; +RETURN NEW; +END; +$mw$; + +CREATE TRIGGER ts2_page_text BEFORE INSERT OR UPDATE ON pagecontent + FOR EACH ROW EXECUTE PROCEDURE ts2_page_text(); + +-- These are added by the setup script due to version compatibility issues +-- If using 8.1, we switch from "gin" to "gist" + +CREATE INDEX ts2_page_title ON page USING gin(titlevector); +CREATE INDEX ts2_page_text ON pagecontent USING gin(textvector); + +CREATE FUNCTION add_interwiki (TEXT,INT,SMALLINT) RETURNS INT LANGUAGE SQL AS +$mw$ + INSERT INTO interwiki (iw_prefix, iw_url, iw_local) VALUES ($1,$2,$3); + SELECT 1; +$mw$; + +-- This table is not used unless profiling is turned on +CREATE TABLE profiling ( + pf_count INTEGER NOT NULL DEFAULT 0, + pf_time FLOAT NOT NULL DEFAULT 0, + pf_memory FLOAT NOT NULL DEFAULT 0, + pf_name TEXT NOT NULL, + pf_server TEXT NULL +); +CREATE UNIQUE INDEX pf_name_server ON profiling (pf_name, pf_server); + +CREATE TABLE protected_titles ( + pt_namespace SMALLINT NOT NULL, + pt_title TEXT NOT NULL, + pt_user INTEGER NULL REFERENCES mwuser(user_id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED, + pt_reason TEXT NOT NULL DEFAULT '', + pt_reason_id INTEGER NOT NULL DEFAULT 0, + pt_timestamp TIMESTAMPTZ NOT NULL, + pt_expiry TIMESTAMPTZ NULL, + pt_create_perm TEXT NOT NULL DEFAULT '' +); +CREATE UNIQUE INDEX protected_titles_unique ON protected_titles(pt_namespace, pt_title); + + +CREATE TABLE updatelog ( + ul_key TEXT NOT NULL PRIMARY KEY, + ul_value TEXT +); + + +CREATE SEQUENCE category_cat_id_seq; +CREATE TABLE category ( + cat_id INTEGER NOT NULL PRIMARY KEY DEFAULT nextval('category_cat_id_seq'), + cat_title TEXT NOT NULL, + cat_pages INTEGER NOT NULL DEFAULT 0, + cat_subcats INTEGER NOT NULL DEFAULT 0, + cat_files INTEGER NOT NULL DEFAULT 0, + cat_hidden SMALLINT NOT NULL DEFAULT 0 +); +CREATE UNIQUE INDEX category_title ON category(cat_title); +CREATE INDEX category_pages ON category(cat_pages); + +CREATE SEQUENCE change_tag_ct_id_seq; +CREATE TABLE change_tag ( + ct_id INTEGER NOT NULL PRIMARY KEY DEFAULT nextval('change_tag_ct_id_seq'), + ct_rc_id INTEGER NULL, + ct_log_id INTEGER NULL, + ct_rev_id INTEGER NULL, + ct_tag TEXT NOT NULL, + ct_params TEXT NULL +); +CREATE UNIQUE INDEX change_tag_rc_tag ON change_tag(ct_rc_id,ct_tag); +CREATE UNIQUE INDEX change_tag_log_tag ON change_tag(ct_log_id,ct_tag); +CREATE UNIQUE INDEX change_tag_rev_tag ON change_tag(ct_rev_id,ct_tag); +CREATE INDEX change_tag_tag_id ON change_tag(ct_tag,ct_rc_id,ct_rev_id,ct_log_id); + +CREATE SEQUENCE tag_summary_ts_id_seq; +CREATE TABLE tag_summary ( + ts_id INTEGER NOT NULL PRIMARY KEY DEFAULT nextval('tag_summary_ts_id_seq'), + ts_rc_id INTEGER NULL, + ts_log_id INTEGER NULL, + ts_rev_id INTEGER NULL, + ts_tags TEXT NOT NULL +); +CREATE UNIQUE INDEX tag_summary_rc_id ON tag_summary(ts_rc_id); +CREATE UNIQUE INDEX tag_summary_log_id ON tag_summary(ts_log_id); +CREATE UNIQUE INDEX tag_summary_rev_id ON tag_summary(ts_rev_id); + +CREATE TABLE valid_tag ( + vt_tag TEXT NOT NULL PRIMARY KEY +); + +CREATE TABLE user_properties ( + up_user INTEGER NULL REFERENCES mwuser(user_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + up_property TEXT NOT NULL, + up_value TEXT +); +CREATE UNIQUE INDEX user_properties_user_property ON user_properties (up_user,up_property); +CREATE INDEX user_properties_property ON user_properties (up_property); + +CREATE TABLE l10n_cache ( + lc_lang TEXT NOT NULL, + lc_key TEXT NOT NULL, + lc_value BYTEA NOT NULL +); +CREATE INDEX l10n_cache_lc_lang_key ON l10n_cache (lc_lang, lc_key); + +CREATE TABLE iwlinks ( + iwl_from INTEGER NOT NULL DEFAULT 0, + iwl_prefix TEXT NOT NULL DEFAULT '', + iwl_title TEXT NOT NULL DEFAULT '' +); +CREATE UNIQUE INDEX iwl_from ON iwlinks (iwl_from, iwl_prefix, iwl_title); +CREATE UNIQUE INDEX iwl_prefix_title_from ON iwlinks (iwl_prefix, iwl_title, iwl_from); +CREATE UNIQUE INDEX iwl_prefix_from_title ON iwlinks (iwl_prefix, iwl_from, iwl_title); + +CREATE TABLE module_deps ( + md_module TEXT NOT NULL, + md_skin TEXT NOT NULL, + md_deps TEXT NOT NULL +); +CREATE UNIQUE INDEX md_module_skin ON module_deps (md_module, md_skin); + +CREATE SEQUENCE sites_site_id_seq; +CREATE TABLE sites ( + site_id INTEGER NOT NULL PRIMARY KEY DEFAULT nextval('sites_site_id_seq'), + site_global_key TEXT NOT NULL, + site_type TEXT NOT NULL, + site_group TEXT NOT NULL, + site_source TEXT NOT NULL, + site_language TEXT NOT NULL, + site_protocol TEXT NOT NULL, + site_domain TEXT NOT NULL, + site_data TEXT NOT NULL, + site_forward SMALLINT NOT NULL, + site_config TEXT NOT NULL +); +CREATE UNIQUE INDEX site_global_key ON sites (site_global_key); +CREATE INDEX site_type ON sites (site_type); +CREATE INDEX site_group ON sites (site_group); +CREATE INDEX site_source ON sites (site_source); +CREATE INDEX site_language ON sites (site_language); +CREATE INDEX site_protocol ON sites (site_protocol); +CREATE INDEX site_domain ON sites (site_domain); +CREATE INDEX site_forward ON sites (site_forward); + +CREATE TABLE site_identifiers ( + si_site INTEGER NOT NULL, + si_type TEXT NOT NULL, + si_key TEXT NOT NULL +); +CREATE UNIQUE INDEX si_type_key ON site_identifiers (si_type, si_key); +CREATE INDEX si_site ON site_identifiers (si_site); +CREATE INDEX si_key ON site_identifiers (si_key); diff --git a/www/wiki/maintenance/postgres/update-keys.sql b/www/wiki/maintenance/postgres/update-keys.sql new file mode 100644 index 00000000..b8585515 --- /dev/null +++ b/www/wiki/maintenance/postgres/update-keys.sql @@ -0,0 +1,34 @@ +-- SQL to insert update keys into the initial tables after a +-- fresh installation of MediaWiki's database. +-- This is read and executed by the install script; you should +-- not have to run it by itself unless doing a manual install. +-- Insert keys here if either the unnecessary would cause heavy +-- processing or could potentially cause trouble by lowering field +-- sizes, adding constraints, etc. +-- When adjusting field sizes, it is recommended removing old +-- patches but to play safe, update keys should also inserted here. + +-- The /*_*/ comments in this and other files are +-- replaced with the defined table prefix by the installer +-- and updater scripts. If you are installing or running +-- updates manually, you will need to manually insert the +-- table prefix if any when running these scripts. +-- + +INSERT INTO /*_*/updatelog (ul_key, ul_value) + VALUES( 'filearchive-fa_major_mime-patch-fa_major_mime-chemical.sql', null ); +INSERT INTO /*_*/updatelog (ul_key, ul_value) + VALUES( 'image-img_major_mime-patch-img_major_mime-chemical.sql', null ); +INSERT INTO /*_*/updatelog (ul_key, ul_value) + VALUES( 'oldimage-oi_major_mime-patch-oi_major_mime-chemical.sql', null ); +INSERT INTO /*_*/updatelog (ul_key, ul_value) + VALUES( 'user_groups-ug_group-patch-ug_group-length-increase-255.sql', null ); +INSERT INTO /*_*/updatelog (ul_key, ul_value) + VALUES( 'user_former_groups-ufg_group-patch-ufg_group-length-increase-255.sql', null ); +INSERT INTO /*_*/updatelog (ul_key, ul_value) + VALUES( 'user_properties-up_property-patch-up_property.sql', null ); + +-- PostgreSQL-specific patches. + +INSERT INTO /*_*/updatelog (ul_key, ul_value) + VALUES( 'patch-textsearch_bug66650.sql', null ); diff --git a/www/wiki/maintenance/preprocessDump.php b/www/wiki/maintenance/preprocessDump.php new file mode 100644 index 00000000..17d97b05 --- /dev/null +++ b/www/wiki/maintenance/preprocessDump.php @@ -0,0 +1,98 @@ +<?php +/** + * Take page text out of an XML dump file and preprocess it to obj. + * It may be useful for getting preprocessor statistics or filling the + * preprocessor cache. + * + * Copyright © 2011 Platonides - https://www.mediawiki.org/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/dumpIterator.php'; + +/** + * Maintenance script that takes page text out of an XML dump file and + * preprocesses it to obj. + * + * @ingroup Maintenance + */ +class PreprocessDump extends DumpIterator { + + /* Variables for dressing up as a parser */ + public $mTitle = 'PreprocessDump'; + public $mPPNodeCount = 0; + + public function getStripList() { + global $wgParser; + + return $wgParser->getStripList(); + } + + public function __construct() { + parent::__construct(); + $this->addOption( 'cache', 'Use and populate the preprocessor cache.', false, false ); + $this->addOption( 'preprocessor', 'Preprocessor to use.', false, false ); + } + + public function getDbType() { + return Maintenance::DB_NONE; + } + + public function checkOptions() { + global $wgParser, $wgParserConf, $wgPreprocessorCacheThreshold; + + if ( !$this->hasOption( 'cache' ) ) { + $wgPreprocessorCacheThreshold = false; + } + + if ( $this->hasOption( 'preprocessor' ) ) { + $name = $this->getOption( 'preprocessor' ); + } elseif ( isset( $wgParserConf['preprocessorClass'] ) ) { + $name = $wgParserConf['preprocessorClass']; + } else { + $name = 'Preprocessor_DOM'; + } + + $wgParser->firstCallInit(); + $this->mPreprocessor = new $name( $this ); + } + + /** + * Callback function for each revision, preprocessToObj() + * @param Revision $rev + */ + public function processRevision( $rev ) { + $content = $rev->getContent( Revision::RAW ); + + if ( $content->getModel() !== CONTENT_MODEL_WIKITEXT ) { + return; + } + + try { + $this->mPreprocessor->preprocessToObj( strval( $content->getNativeData() ), 0 ); + } catch ( Exception $e ) { + $this->error( "Caught exception " . $e->getMessage() . " in " + . $rev->getTitle()->getPrefixedText() ); + } + } +} + +$maintClass = "PreprocessDump"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/preprocessorFuzzTest.php b/www/wiki/maintenance/preprocessorFuzzTest.php new file mode 100644 index 00000000..2503ed25 --- /dev/null +++ b/www/wiki/maintenance/preprocessorFuzzTest.php @@ -0,0 +1,274 @@ +<?php +/** + * Performs fuzz-style testing of MediaWiki's preprocessor. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +$optionsWithoutArgs = [ 'verbose' ]; +require_once __DIR__ . '/commandLine.inc'; + +$wgHooks['BeforeParserFetchTemplateAndtitle'][] = 'PPFuzzTester::templateHook'; + +class PPFuzzTester { + public $hairs = [ + '[[', ']]', '{{', '{{', '}}', '}}', '{{{', '}}}', + '<', '>', '<nowiki', '<gallery', '</nowiki>', '</gallery>', '<nOwIkI>', '</NoWiKi>', + '<!--', '-->', + "\n==", "==\n", + '|', '=', "\n", ' ', "\t", "\x7f", + '~~', '~~~', '~~~~', 'subst:', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', + 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', + + // extensions + // '<ref>', '</ref>', '<references/>', + ]; + public $minLength = 0; + public $maxLength = 20; + public $maxTemplates = 5; + // public $outputTypes = [ 'OT_HTML', 'OT_WIKI', 'OT_PREPROCESS' ]; + public $entryPoints = [ 'testSrvus', 'testPst', 'testPreprocess' ]; + public $verbose = false; + + /** + * @var bool|PPFuzzTest + */ + private static $currentTest = false; + + function execute() { + if ( !file_exists( 'results' ) ) { + mkdir( 'results' ); + } + if ( !is_dir( 'results' ) ) { + echo "Unable to create 'results' directory\n"; + exit( 1 ); + } + $overallStart = microtime( true ); + $reportInterval = 1000; + for ( $i = 1; true; $i++ ) { + $t = -microtime( true ); + try { + self::$currentTest = new PPFuzzTest( $this ); + self::$currentTest->execute(); + $passed = 'passed'; + } catch ( Exception $e ) { + $testReport = self::$currentTest->getReport(); + $exceptionReport = $e->getText(); + $hash = md5( $testReport ); + file_put_contents( "results/ppft-$hash.in", serialize( self::$currentTest ) ); + file_put_contents( "results/ppft-$hash.fail", + "Input:\n$testReport\n\nException report:\n$exceptionReport\n" ); + print "Test $hash failed\n"; + $passed = 'failed'; + } + $t += microtime( true ); + + if ( $this->verbose ) { + printf( "Test $passed in %.3f seconds\n", $t ); + print self::$currentTest->getReport(); + } + + $reportMetric = ( microtime( true ) - $overallStart ) / $i * $reportInterval; + if ( $reportMetric > 25 ) { + if ( substr( $reportInterval, 0, 1 ) === '1' ) { + $reportInterval /= 2; + } else { + $reportInterval /= 5; + } + } elseif ( $reportMetric < 4 ) { + if ( substr( $reportInterval, 0, 1 ) === '1' ) { + $reportInterval *= 5; + } else { + $reportInterval *= 2; + } + } + if ( $i % $reportInterval == 0 ) { + print "$i tests done\n"; + /* + $testReport = self::$currentTest->getReport(); + $filename = 'results/ppft-' . md5( $testReport ) . '.pass'; + file_put_contents( $filename, "Input:\n$testReport\n" );*/ + } + } + } + + function makeInputText( $max = false ) { + if ( $max === false ) { + $max = $this->maxLength; + } + $length = mt_rand( $this->minLength, $max ); + $s = ''; + for ( $i = 0; $i < $length; $i++ ) { + $hairIndex = mt_rand( 0, count( $this->hairs ) - 1 ); + $s .= $this->hairs[$hairIndex]; + } + // Send through the UTF-8 normaliser + // This resolves a few differences between the old preprocessor and the + // XML-based one, which doesn't like illegals and converts line endings. + // It's done by the MW UI, so it's a reasonably legitimate thing to do. + global $wgContLang; + $s = $wgContLang->normalize( $s ); + + return $s; + } + + function makeTitle() { + return Title::newFromText( mt_rand( 0, 1000000 ), mt_rand( 0, 10 ) ); + } + + /* + function pickOutputType() { + $count = count( $this->outputTypes ); + return $this->outputTypes[ mt_rand( 0, $count - 1 ) ]; + }*/ + + function pickEntryPoint() { + $count = count( $this->entryPoints ); + + return $this->entryPoints[mt_rand( 0, $count - 1 )]; + } +} + +class PPFuzzTest { + public $templates, $mainText, $title, $entryPoint, $output; + + function __construct( $tester ) { + global $wgMaxSigChars; + $this->parent = $tester; + $this->mainText = $tester->makeInputText(); + $this->title = $tester->makeTitle(); + // $this->outputType = $tester->pickOutputType(); + $this->entryPoint = $tester->pickEntryPoint(); + $this->nickname = $tester->makeInputText( $wgMaxSigChars + 10 ); + $this->fancySig = (bool)mt_rand( 0, 1 ); + $this->templates = []; + } + + /** + * @param Title $title + * @return array + */ + function templateHook( $title ) { + $titleText = $title->getPrefixedDBkey(); + + if ( !isset( $this->templates[$titleText] ) ) { + $finalTitle = $title; + if ( count( $this->templates ) >= $this->parent->maxTemplates ) { + // Too many templates + $text = false; + } else { + if ( !mt_rand( 0, 1 ) ) { + // Redirect + $finalTitle = $this->parent->makeTitle(); + } + if ( !mt_rand( 0, 5 ) ) { + // Doesn't exist + $text = false; + } else { + $text = $this->parent->makeInputText(); + } + } + $this->templates[$titleText] = [ + 'text' => $text, + 'finalTitle' => $finalTitle ]; + } + + return $this->templates[$titleText]; + } + + function execute() { + global $wgParser, $wgUser; + + $wgUser = new PPFuzzUser; + $wgUser->mName = 'Fuzz'; + $wgUser->mFrom = 'name'; + $wgUser->ppfz_test = $this; + + $options = ParserOptions::newFromUser( $wgUser ); + $options->setTemplateCallback( [ $this, 'templateHook' ] ); + $options->setTimestamp( wfTimestampNow() ); + $this->output = call_user_func( + [ $wgParser, $this->entryPoint ], + $this->mainText, + $this->title, + $options + ); + + return $this->output; + } + + function getReport() { + $s = "Title: " . $this->title->getPrefixedDBkey() . "\n" . +// "Output type: {$this->outputType}\n" . + "Entry point: {$this->entryPoint}\n" . + "User: " . ( $this->fancySig ? 'fancy' : 'no-fancy' ) . + ' ' . var_export( $this->nickname, true ) . "\n" . + "Main text: " . var_export( $this->mainText, true ) . "\n"; + foreach ( $this->templates as $titleText => $template ) { + $finalTitle = $template['finalTitle']; + if ( $finalTitle != $titleText ) { + $s .= "[[$titleText]] -> [[$finalTitle]]: " . var_export( $template['text'], true ) . "\n"; + } else { + $s .= "[[$titleText]]: " . var_export( $template['text'], true ) . "\n"; + } + } + $s .= "Output: " . var_export( $this->output, true ) . "\n"; + + return $s; + } +} + +class PPFuzzUser extends User { + public $ppfz_test, $mDataLoaded; + + function load() { + if ( $this->mDataLoaded ) { + return; + } + $this->mDataLoaded = true; + $this->loadDefaults( $this->mName ); + } + + function getOption( $oname, $defaultOverride = null, $ignoreHidden = false ) { + if ( $oname === 'fancysig' ) { + return $this->ppfz_test->fancySig; + } elseif ( $oname === 'nickname' ) { + return $this->ppfz_test->nickname; + } else { + return parent::getOption( $oname, $defaultOverride, $ignoreHidden ); + } + } +} + +ini_set( 'memory_limit', '50M' ); +if ( isset( $args[0] ) ) { + $testText = file_get_contents( $args[0] ); + if ( !$testText ) { + print "File not found\n"; + exit( 1 ); + } + $test = unserialize( $testText ); + $result = $test->execute(); + print "Test passed.\n"; +} else { + $tester = new PPFuzzTester; + $tester->verbose = isset( $options['verbose'] ); + $tester->execute(); +} diff --git a/www/wiki/maintenance/protect.php b/www/wiki/maintenance/protect.php new file mode 100644 index 00000000..f6bb2532 --- /dev/null +++ b/www/wiki/maintenance/protect.php @@ -0,0 +1,93 @@ +<?php +/** + * Protect or unprotect a page. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that protects or unprotects a page. + * + * @ingroup Maintenance + */ +class Protect extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Protect or unprotect a page from the command line.' ); + $this->addOption( 'unprotect', 'Removes protection' ); + $this->addOption( 'semiprotect', 'Adds semi-protection' ); + $this->addOption( 'cascade', 'Add cascading protection' ); + $this->addOption( 'user', 'Username to protect with', false, true, 'u' ); + $this->addOption( 'reason', 'Reason for un/protection', false, true, 'r' ); + $this->addArg( 'title', 'Title to protect', true ); + } + + public function execute() { + $userName = $this->getOption( 'user', false ); + $reason = $this->getOption( 'reason', '' ); + + $cascade = $this->hasOption( 'cascade' ); + + $protection = "sysop"; + if ( $this->hasOption( 'semiprotect' ) ) { + $protection = "autoconfirmed"; + } elseif ( $this->hasOption( 'unprotect' ) ) { + $protection = ""; + } + + if ( $userName === false ) { + $user = User::newSystemUser( 'Maintenance script', [ 'steal' => true ] ); + } else { + $user = User::newFromName( $userName ); + } + if ( !$user ) { + $this->error( "Invalid username", true ); + } + + // @todo FIXME: This is reset 7 lines down. + $restrictions = [ 'edit' => $protection, 'move' => $protection ]; + + $t = Title::newFromText( $this->getArg() ); + if ( !$t ) { + $this->error( "Invalid title", true ); + } + + $restrictions = []; + foreach ( $t->getRestrictionTypes() as $type ) { + $restrictions[$type] = $protection; + } + + # un/protect the article + $this->output( "Updating protection status... " ); + + $page = WikiPage::factory( $t ); + $status = $page->doUpdateRestrictions( $restrictions, [], $cascade, $reason, $user ); + + if ( $status->isOK() ) { + $this->output( "done\n" ); + } else { + $this->output( "failed\n" ); + } + } +} + +$maintClass = "Protect"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/pruneFileCache.php b/www/wiki/maintenance/pruneFileCache.php new file mode 100644 index 00000000..8e6978d5 --- /dev/null +++ b/www/wiki/maintenance/pruneFileCache.php @@ -0,0 +1,111 @@ +<?php +/** + * Prune file cache for pages, objects, resources, etc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that prunes file cache for pages, objects, resources, etc. + * + * @ingroup Maintenance + */ +class PruneFileCache extends Maintenance { + + protected $minSurviveTimestamp; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Build file cache for content pages' ); + $this->addOption( 'agedays', 'How many days old files must be in order to delete', true, true ); + $this->addOption( 'subdir', 'Prune one $wgFileCacheDirectory subdirectory name', false, true ); + } + + public function execute() { + global $wgUseFileCache, $wgFileCacheDirectory; + + if ( !$wgUseFileCache ) { + $this->error( "Nothing to do -- \$wgUseFileCache is disabled.", true ); + } + + $age = $this->getOption( 'agedays' ); + if ( !ctype_digit( $age ) ) { + $this->error( "Non-integer 'age' parameter given.", true ); + } + // Delete items with a TS older than this + $this->minSurviveTimestamp = time() - ( 86400 * $age ); + + $dir = $wgFileCacheDirectory; + if ( !is_dir( $dir ) ) { + $this->error( "Nothing to do -- \$wgFileCacheDirectory directory not found.", true ); + } + + $subDir = $this->getOption( 'subdir' ); + if ( $subDir !== null ) { + if ( !is_dir( "$dir/$subDir" ) ) { + $this->error( "The specified subdirectory `$subDir` does not exist.", true ); + } + $this->output( "Pruning `$dir/$subDir` directory...\n" ); + $this->prune_directory( "$dir/$subDir", 'report' ); + $this->output( "Done pruning `$dir/$subDir` directory\n" ); + } else { + $this->output( "Pruning `$dir` directory...\n" ); + // Note: don't prune things like .cdb files on the top level! + $this->prune_directory( $dir, 'report' ); + $this->output( "Done pruning `$dir` directory\n" ); + } + } + + /** + * @param string $dir + * @param string|bool $report Use 'report' to report the directories being scanned + */ + protected function prune_directory( $dir, $report = false ) { + $tsNow = time(); + $dirHandle = opendir( $dir ); + while ( false !== ( $file = readdir( $dirHandle ) ) ) { + // Skip ".", "..", and also any dirs or files like ".svn" or ".htaccess" + if ( $file[0] != "." ) { + $path = $dir . '/' . $file; // absolute + if ( is_dir( $path ) ) { + if ( $report === 'report' ) { + $this->output( "Scanning `$path`...\n" ); + } + $this->prune_directory( $path ); + } else { + $mts = filemtime( $path ); + // Sanity check the file extension against known cache types + if ( $mts < $this->minSurviveTimestamp + && preg_match( '/\.(?:html|cache)(?:\.gz)?$/', $file ) + && unlink( $path ) + ) { + $daysOld = round( ( $tsNow - $mts ) / 86400, 2 ); + $this->output( "Deleted `$path` [days=$daysOld]\n" ); + } + } + } + } + closedir( $dirHandle ); + } +} + +$maintClass = "PruneFileCache"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/purgeChangedFiles.php b/www/wiki/maintenance/purgeChangedFiles.php new file mode 100644 index 00000000..3c0fc7e5 --- /dev/null +++ b/www/wiki/maintenance/purgeChangedFiles.php @@ -0,0 +1,262 @@ +<?php +/** + * Scan the logging table and purge affected files within a timeframe. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that scans the deletion log and purges affected files + * within a timeframe. + * + * @ingroup Maintenance + */ +class PurgeChangedFiles extends Maintenance { + /** + * Mapping from type option to log type and actions. + * @var array + */ + private static $typeMappings = [ + 'created' => [ + 'upload' => [ 'upload' ], + 'import' => [ 'upload', 'interwiki' ], + ], + 'deleted' => [ + 'delete' => [ 'delete', 'revision' ], + 'suppress' => [ 'delete', 'revision' ], + ], + 'modified' => [ + 'upload' => [ 'overwrite', 'revert' ], + 'move' => [ 'move', 'move_redir' ], + ], + ]; + + /** + * @var string + */ + private $startTimestamp; + + /** + * @var string + */ + private $endTimestamp; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Scan the logging table and purge files and thumbnails.' ); + $this->addOption( 'starttime', 'Starting timestamp', true, true ); + $this->addOption( 'endtime', 'Ending timestamp', true, true ); + $this->addOption( 'type', 'Comma-separated list of types of changes to send purges for (' . + implode( ',', array_keys( self::$typeMappings ) ) . ',all)', false, true ); + $this->addOption( 'htcp-dest', 'HTCP announcement destination (IP:port)', false, true ); + $this->addOption( 'dry-run', 'Do not send purge requests' ); + $this->addOption( 'sleep-per-batch', 'Milliseconds to sleep between batches', false, true ); + $this->addOption( 'verbose', 'Show more output', false, false, 'v' ); + $this->setBatchSize( 100 ); + } + + public function execute() { + global $wgHTCPRouting; + + if ( $this->hasOption( 'htcp-dest' ) ) { + $parts = explode( ':', $this->getOption( 'htcp-dest' ) ); + if ( count( $parts ) < 2 ) { + // Add default htcp port + $parts[] = '4827'; + } + + // Route all HTCP messages to provided host:port + $wgHTCPRouting = [ + '' => [ 'host' => $parts[0], 'port' => $parts[1] ], + ]; + $this->verbose( "HTCP broadcasts to {$parts[0]}:{$parts[1]}\n" ); + } + + // Find out which actions we should be concerned with + $typeOpt = $this->getOption( 'type', 'all' ); + $validTypes = array_keys( self::$typeMappings ); + if ( $typeOpt === 'all' ) { + // Convert 'all' to all registered types + $typeOpt = implode( ',', $validTypes ); + } + $typeList = explode( ',', $typeOpt ); + foreach ( $typeList as $type ) { + if ( !in_array( $type, $validTypes ) ) { + $this->error( "\nERROR: Unknown type: {$type}\n" ); + $this->maybeHelp( true ); + } + } + + // Validate the timestamps + $dbr = $this->getDB( DB_REPLICA ); + $this->startTimestamp = $dbr->timestamp( $this->getOption( 'starttime' ) ); + $this->endTimestamp = $dbr->timestamp( $this->getOption( 'endtime' ) ); + + if ( $this->startTimestamp > $this->endTimestamp ) { + $this->error( "\nERROR: starttime after endtime\n" ); + $this->maybeHelp( true ); + } + + // Turn on verbose when dry-run is enabled + if ( $this->hasOption( 'dry-run' ) ) { + $this->mOptions['verbose'] = 1; + } + + $this->verbose( 'Purging files that were: ' . implode( ', ', $typeList ) . "\n" ); + foreach ( $typeList as $type ) { + $this->verbose( "Checking for {$type} files...\n" ); + $this->purgeFromLogType( $type ); + if ( !$this->hasOption( 'dry-run' ) ) { + $this->verbose( "...{$type} files purged.\n\n" ); + } + } + } + + /** + * Purge cache and thumbnails for changes of the given type. + * + * @param string $type Type of change to find + */ + protected function purgeFromLogType( $type ) { + $repo = RepoGroup::singleton()->getLocalRepo(); + $dbr = $this->getDB( DB_REPLICA ); + + foreach ( self::$typeMappings[$type] as $logType => $logActions ) { + $this->verbose( "Scanning for {$logType}/" . implode( ',', $logActions ) . "\n" ); + + $res = $dbr->select( + 'logging', + [ 'log_title', 'log_timestamp', 'log_params' ], + [ + 'log_namespace' => NS_FILE, + 'log_type' => $logType, + 'log_action' => $logActions, + 'log_timestamp >= ' . $dbr->addQuotes( $this->startTimestamp ), + 'log_timestamp <= ' . $dbr->addQuotes( $this->endTimestamp ), + ], + __METHOD__ + ); + + $bSize = 0; + foreach ( $res as $row ) { + $file = $repo->newFile( Title::makeTitle( NS_FILE, $row->log_title ) ); + + if ( $this->hasOption( 'dry-run' ) ) { + $this->verbose( "{$type}[{$row->log_timestamp}]: {$row->log_title}\n" ); + continue; + } + + // Purge current version and its thumbnails + $file->purgeCache(); + // Purge the old versions and their thumbnails + foreach ( $file->getHistory() as $oldFile ) { + $oldFile->purgeCache(); + } + + if ( $logType === 'delete' ) { + // If there is an orphaned storage file... delete it + if ( !$file->exists() && $repo->fileExists( $file->getPath() ) ) { + $dpath = $this->getDeletedPath( $repo, $file ); + if ( $repo->fileExists( $dpath ) ) { + // Sanity check to avoid data loss + $repo->getBackend()->delete( [ 'src' => $file->getPath() ] ); + $this->verbose( "Deleted orphan file: {$file->getPath()}.\n" ); + } else { + $this->error( "File was not deleted: {$file->getPath()}.\n" ); + } + } + + // Purge items from fileachive table (rows are likely here) + $this->purgeFromArchiveTable( $repo, $file ); + } elseif ( $logType === 'move' ) { + // Purge the target file as well + + $params = unserialize( $row->log_params ); + if ( isset( $params['4::target'] ) ) { + $target = $params['4::target']; + $targetFile = $repo->newFile( Title::makeTitle( NS_FILE, $target ) ); + $targetFile->purgeCache(); + $this->verbose( "Purged file {$target}; move target @{$row->log_timestamp}.\n" ); + } + } + + $this->verbose( "Purged file {$row->log_title}; {$type} @{$row->log_timestamp}.\n" ); + + if ( $this->hasOption( 'sleep-per-batch' ) && ++$bSize > $this->mBatchSize ) { + $bSize = 0; + // sleep-per-batch is milliseconds, usleep wants micro seconds. + usleep( 1000 * (int)$this->getOption( 'sleep-per-batch' ) ); + } + } + } + } + + protected function purgeFromArchiveTable( LocalRepo $repo, LocalFile $file ) { + $dbr = $repo->getReplicaDB(); + $res = $dbr->select( + 'filearchive', + [ 'fa_archive_name' ], + [ 'fa_name' => $file->getName() ], + __METHOD__ + ); + + foreach ( $res as $row ) { + if ( $row->fa_archive_name === null ) { + // Was not an old version (current version names checked already) + continue; + } + $ofile = $repo->newFromArchiveName( $file->getTitle(), $row->fa_archive_name ); + // If there is an orphaned storage file still there...delete it + if ( !$file->exists() && $repo->fileExists( $ofile->getPath() ) ) { + $dpath = $this->getDeletedPath( $repo, $ofile ); + if ( $repo->fileExists( $dpath ) ) { + // Sanity check to avoid data loss + $repo->getBackend()->delete( [ 'src' => $ofile->getPath() ] ); + $this->output( "Deleted orphan file: {$ofile->getPath()}.\n" ); + } else { + $this->error( "File was not deleted: {$ofile->getPath()}.\n" ); + } + } + $file->purgeOldThumbnails( $row->fa_archive_name ); + } + } + + protected function getDeletedPath( LocalRepo $repo, LocalFile $file ) { + $hash = $repo->getFileSha1( $file->getPath() ); + $key = "{$hash}.{$file->getExtension()}"; + + return $repo->getDeletedHashPath( $key ) . $key; + } + + /** + * Send an output message iff the 'verbose' option has been provided. + * + * @param string $msg Message to output + */ + protected function verbose( $msg ) { + if ( $this->hasOption( 'verbose' ) ) { + $this->output( $msg ); + } + } +} + +$maintClass = "PurgeChangedFiles"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/purgeChangedPages.php b/www/wiki/maintenance/purgeChangedPages.php new file mode 100644 index 00000000..cf65c693 --- /dev/null +++ b/www/wiki/maintenance/purgeChangedPages.php @@ -0,0 +1,194 @@ +<?php +/** + * Send purge requests for pages edited in date range to squid/varnish. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +use Wikimedia\Rdbms\ResultWrapper; + +/** + * Maintenance script that sends purge requests for pages edited in a date + * range to squid/varnish. + * + * Can be used to recover from an HTCP message partition or other major cache + * layer interruption. + * + * @ingroup Maintenance + */ +class PurgeChangedPages extends Maintenance { + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Send purge requests for edits in date range to squid/varnish' ); + $this->addOption( 'starttime', 'Starting timestamp', true, true ); + $this->addOption( 'endtime', 'Ending timestamp', true, true ); + $this->addOption( 'htcp-dest', 'HTCP announcement destination (IP:port)', false, true ); + $this->addOption( 'sleep-per-batch', 'Milliseconds to sleep between batches', false, true ); + $this->addOption( 'dry-run', 'Do not send purge requests' ); + $this->addOption( 'verbose', 'Show more output', false, false, 'v' ); + $this->setBatchSize( 100 ); + } + + public function execute() { + global $wgHTCPRouting; + + if ( $this->hasOption( 'htcp-dest' ) ) { + $parts = explode( ':', $this->getOption( 'htcp-dest' ) ); + if ( count( $parts ) < 2 ) { + // Add default htcp port + $parts[] = '4827'; + } + + // Route all HTCP messages to provided host:port + $wgHTCPRouting = [ + '' => [ 'host' => $parts[0], 'port' => $parts[1] ], + ]; + if ( $this->hasOption( 'verbose' ) ) { + $this->output( "HTCP broadcasts to {$parts[0]}:{$parts[1]}\n" ); + } + } + + $dbr = $this->getDB( DB_REPLICA ); + $minTime = $dbr->timestamp( $this->getOption( 'starttime' ) ); + $maxTime = $dbr->timestamp( $this->getOption( 'endtime' ) ); + + if ( $maxTime < $minTime ) { + $this->error( "\nERROR: starttime after endtime\n" ); + $this->maybeHelp( true ); + } + + $stuckCount = 0; // loop breaker + while ( true ) { + // Adjust bach size if we are stuck in a second that had many changes + $bSize = $this->mBatchSize + ( $stuckCount * $this->mBatchSize ); + + $res = $dbr->select( + [ 'page', 'revision' ], + [ + 'rev_timestamp', + 'page_namespace', + 'page_title', + ], + [ + "rev_timestamp > " . $dbr->addQuotes( $minTime ), + "rev_timestamp <= " . $dbr->addQuotes( $maxTime ), + // Only get rows where the revision is the latest for the page. + // Other revisions would be duplicate and we don't need to purge if + // there has been an edit after the interesting time window. + "page_latest = rev_id", + ], + __METHOD__, + [ 'ORDER BY' => 'rev_timestamp', 'LIMIT' => $bSize ], + [ + 'page' => [ 'INNER JOIN', 'rev_page=page_id' ], + ] + ); + + if ( !$res->numRows() ) { + // nothing more found so we are done + break; + } + + // Kludge to not get stuck in loops for batches with the same timestamp + list( $rows, $lastTime ) = $this->pageableSortedRows( $res, 'rev_timestamp', $bSize ); + if ( !count( $rows ) ) { + ++$stuckCount; + continue; + } + // Reset suck counter + $stuckCount = 0; + + $this->output( "Processing changes from {$minTime} to {$lastTime}.\n" ); + + // Advance past the last row next time + $minTime = $lastTime; + + // Create list of URLs from page_namespace + page_title + $urls = []; + foreach ( $rows as $row ) { + $title = Title::makeTitle( $row->page_namespace, $row->page_title ); + $urls[] = $title->getInternalURL(); + } + + if ( $this->hasOption( 'dry-run' ) || $this->hasOption( 'verbose' ) ) { + $this->output( implode( "\n", $urls ) . "\n" ); + if ( $this->hasOption( 'dry-run' ) ) { + continue; + } + } + + // Send batch of purge requests out to squids + $squid = new CdnCacheUpdate( $urls, count( $urls ) ); + $squid->doUpdate(); + + if ( $this->hasOption( 'sleep-per-batch' ) ) { + // sleep-per-batch is milliseconds, usleep wants micro seconds. + usleep( 1000 * (int)$this->getOption( 'sleep-per-batch' ) ); + } + } + + $this->output( "Done!\n" ); + } + + /** + * Remove all the rows in a result set with the highest value for column + * $column unless the number of rows is less $limit. This returns the new + * array of rows and the highest value of column $column for the rows left. + * The ordering of rows is maintained. + * + * This is useful for paging on mostly-unique values that may sometimes + * have large clumps of identical values. It should be safe to do the next + * query on items with a value higher than the highest of the rows returned here. + * If this returns an empty array for a non-empty query result, then all the rows + * had the same column value and the query should be repeated with a higher LIMIT. + * + * @todo move this elsewhere + * + * @param ResultWrapper $res Query result sorted by $column (ascending) + * @param string $column + * @param int $limit + * @return array (array of rows, string column value) + */ + protected function pageableSortedRows( ResultWrapper $res, $column, $limit ) { + $rows = iterator_to_array( $res, false ); + $count = count( $rows ); + if ( !$count ) { + return [ [], null ]; // nothing to do + } elseif ( $count < $limit ) { + return [ $rows, $rows[$count - 1]->$column ]; // no more rows left + } + $lastValue = $rows[$count - 1]->$column; // should be the highest + for ( $i = $count - 1; $i >= 0; --$i ) { + if ( $rows[$i]->$column === $lastValue ) { + unset( $rows[$i] ); + } else { + break; + } + } + $lastValueLeft = count( $rows ) ? $rows[count( $rows ) - 1]->$column : null; + + return [ $rows, $lastValueLeft ]; + } +} + +$maintClass = "PurgeChangedPages"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/purgeList.php b/www/wiki/maintenance/purgeList.php new file mode 100644 index 00000000..5ca7918e --- /dev/null +++ b/www/wiki/maintenance/purgeList.php @@ -0,0 +1,147 @@ +<?php +/** + * Send purge requests for listed pages to squid + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that sends purge requests for listed pages to squid. + * + * @ingroup Maintenance + */ +class PurgeList extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Send purge requests for listed pages to squid' ); + $this->addOption( 'purge', 'Whether to update page_touched.', false, false ); + $this->addOption( 'namespace', 'Namespace number', false, true ); + $this->addOption( 'all', 'Purge all pages', false, false ); + $this->addOption( 'delay', 'Number of seconds to delay between each purge', false, true ); + $this->addOption( 'verbose', 'Show more output', false, false, 'v' ); + $this->setBatchSize( 100 ); + } + + public function execute() { + if ( $this->hasOption( 'all' ) ) { + $this->purgeNamespace( false ); + } elseif ( $this->hasOption( 'namespace' ) ) { + $this->purgeNamespace( intval( $this->getOption( 'namespace' ) ) ); + } else { + $this->doPurge(); + } + $this->output( "Done!\n" ); + } + + /** + * Purge URL coming from stdin + */ + private function doPurge() { + $stdin = $this->getStdin(); + $urls = []; + + while ( !feof( $stdin ) ) { + $page = trim( fgets( $stdin ) ); + if ( preg_match( '%^https?://%', $page ) ) { + $urls[] = $page; + } elseif ( $page !== '' ) { + $title = Title::newFromText( $page ); + if ( $title ) { + $url = $title->getInternalURL(); + $this->output( "$url\n" ); + $urls[] = $url; + if ( $this->getOption( 'purge' ) ) { + $title->invalidateCache(); + } + } else { + $this->output( "(Invalid title '$page')\n" ); + } + } + } + $this->output( "Purging " . count( $urls ) . " urls\n" ); + $this->sendPurgeRequest( $urls ); + } + + /** + * Purge a namespace or all pages + * + * @param int|bool $namespace + */ + private function purgeNamespace( $namespace = false ) { + $dbr = $this->getDB( DB_REPLICA ); + $startId = 0; + if ( $namespace === false ) { + $conds = []; + } else { + $conds = [ 'page_namespace' => $namespace ]; + } + while ( true ) { + $res = $dbr->select( 'page', + [ 'page_id', 'page_namespace', 'page_title' ], + $conds + [ 'page_id > ' . $dbr->addQuotes( $startId ) ], + __METHOD__, + [ + 'LIMIT' => $this->mBatchSize, + 'ORDER BY' => 'page_id' + + ] + ); + if ( !$res->numRows() ) { + break; + } + $urls = []; + foreach ( $res as $row ) { + $title = Title::makeTitle( $row->page_namespace, $row->page_title ); + $url = $title->getInternalURL(); + $urls[] = $url; + $startId = $row->page_id; + } + $this->sendPurgeRequest( $urls ); + } + } + + /** + * Helper to purge an array of $urls + * @param array $urls List of URLS to purge from squids + */ + private function sendPurgeRequest( $urls ) { + if ( $this->hasOption( 'delay' ) ) { + $delay = floatval( $this->getOption( 'delay' ) ); + foreach ( $urls as $url ) { + if ( $this->hasOption( 'verbose' ) ) { + $this->output( $url . "\n" ); + } + $u = new CdnCacheUpdate( [ $url ] ); + $u->doUpdate(); + usleep( $delay * 1e6 ); + } + } else { + if ( $this->hasOption( 'verbose' ) ) { + $this->output( implode( "\n", $urls ) . "\n" ); + } + $u = new CdnCacheUpdate( $urls ); + $u->doUpdate(); + } + } +} + +$maintClass = "PurgeList"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/purgeModuleDeps.php b/www/wiki/maintenance/purgeModuleDeps.php new file mode 100644 index 00000000..feeeb65b --- /dev/null +++ b/www/wiki/maintenance/purgeModuleDeps.php @@ -0,0 +1,72 @@ +<?php +/** + * Remove all cache entries for ResourceLoader modules from the database. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @author Timo Tijhof + */ + +use Wikimedia\Rdbms\IDatabase; + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to purge the module_deps database cache table. + * + * @ingroup Maintenance + */ +class PurgeModuleDeps extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( + 'Remove all cache entries for ResourceLoader modules from the database' ); + $this->setBatchSize( 500 ); + } + + public function execute() { + $this->output( "Cleaning up module_deps table...\n" ); + + $dbw = $this->getDB( DB_MASTER ); + $res = $dbw->select( 'module_deps', [ 'md_module', 'md_skin' ], [], __METHOD__ ); + $rows = iterator_to_array( $res, false ); + + $modDeps = $dbw->tableName( 'module_deps' ); + $i = 1; + foreach ( array_chunk( $rows, $this->mBatchSize ) as $chunk ) { + // WHERE ( mod=A AND skin=A ) OR ( mod=A AND skin=B) .. + $conds = array_map( function ( stdClass $row ) use ( $dbw ) { + return $dbw->makeList( (array)$row, IDatabase::LIST_AND ); + }, $chunk ); + $conds = $dbw->makeList( $conds, IDatabase::LIST_OR ); + + $this->beginTransaction( $dbw, __METHOD__ ); + $dbw->query( "DELETE FROM $modDeps WHERE $conds", __METHOD__ ); + $numRows = $dbw->affectedRows(); + $this->output( "Batch $i: $numRows rows\n" ); + $this->commitTransaction( $dbw, __METHOD__ ); + + $i++; + } + + $this->output( "Done\n" ); + } +} + +$maintClass = 'PurgeModuleDeps'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/purgeOldText.php b/www/wiki/maintenance/purgeOldText.php new file mode 100644 index 00000000..1b78c7d9 --- /dev/null +++ b/www/wiki/maintenance/purgeOldText.php @@ -0,0 +1,45 @@ +<?php +/** + * Purge old text records from the database + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @author Rob Church <robchur@gmail.com> + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that purges old text records from the database. + * + * @ingroup Maintenance + */ +class PurgeOldText extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Purge old text records from the database' ); + $this->addOption( 'purge', 'Performs the deletion' ); + } + + public function execute() { + $this->purgeRedundantText( $this->hasOption( 'purge' ) ); + } +} + +$maintClass = "PurgeOldText"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/purgePage.php b/www/wiki/maintenance/purgePage.php new file mode 100644 index 00000000..44d390ea --- /dev/null +++ b/www/wiki/maintenance/purgePage.php @@ -0,0 +1,78 @@ +<?php +/** + * Purges a specific page. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that purges a list of pages passed through stdin + * + * @ingroup Maintenance + */ +class PurgePage extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Purge page.' ); + $this->addOption( 'skip-exists-check', 'Skip page existence check', false, false ); + } + + public function execute() { + $stdin = $this->getStdin(); + + while ( !feof( $stdin ) ) { + $title = trim( fgets( $stdin ) ); + if ( $title != '' ) { + $this->purge( $title ); + } + } + } + + private function purge( $title ) { + $title = Title::newFromText( $title ); + + if ( is_null( $title ) ) { + $this->error( 'Invalid page title' ); + return; + } + + $page = WikiPage::factory( $title ); + + if ( is_null( $page ) ) { + $this->error( "Could not instantiate page object" ); + return; + } + + if ( !$this->getOption( 'skip-exists-check' ) && !$page->exists() ) { + $this->error( "Page doesn't exist" ); + return; + } + + if ( $page->doPurge() ) { + $this->output( "Purged\n" ); + } else { + $this->error( "Purge failed" ); + } + } +} + +$maintClass = "PurgePage"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/purgeParserCache.php b/www/wiki/maintenance/purgeParserCache.php new file mode 100644 index 00000000..da2d850e --- /dev/null +++ b/www/wiki/maintenance/purgeParserCache.php @@ -0,0 +1,97 @@ +<?php +/** + * Remove old objects from the parser cache. + * This only works when the parser cache is in an SQL database. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require __DIR__ . '/Maintenance.php'; + +use MediaWiki\MediaWikiServices; + +/** + * Maintenance script to remove old objects from the parser cache. + * + * @ingroup Maintenance + */ +class PurgeParserCache extends Maintenance { + public $lastProgress; + + private $usleep = 0; + + function __construct() { + parent::__construct(); + $this->addDescription( "Remove old objects from the parser cache. " . + "This only works when the parser cache is in an SQL database." ); + $this->addOption( 'expiredate', 'Delete objects expiring before this date.', false, true ); + $this->addOption( + 'age', + 'Delete objects created more than this many seconds ago, assuming ' . + '$wgParserCacheExpireTime has remained consistent.', + false, + true ); + $this->addOption( 'msleep', 'Milliseconds to sleep between purge chunks', false, true ); + } + + function execute() { + global $wgParserCacheExpireTime; + + $inputDate = $this->getOption( 'expiredate' ); + $inputAge = $this->getOption( 'age' ); + if ( $inputDate !== null ) { + $date = wfTimestamp( TS_MW, strtotime( $inputDate ) ); + } elseif ( $inputAge !== null ) { + $date = wfTimestamp( TS_MW, time() + $wgParserCacheExpireTime - intval( $inputAge ) ); + } else { + $this->error( "Must specify either --expiredate or --age", 1 ); + return; + } + $this->usleep = 1e3 * $this->getOption( 'msleep', 0 ); + + $english = Language::factory( 'en' ); + $this->output( "Deleting objects expiring before " . + $english->timeanddate( $date ) . "\n" ); + + $pc = MediaWikiServices::getInstance()->getParserCache()->getCacheStorage(); + $success = $pc->deleteObjectsExpiringBefore( $date, [ $this, 'showProgressAndWait' ] ); + if ( !$success ) { + $this->error( "\nCannot purge this kind of parser cache.", 1 ); + } + $this->showProgressAndWait( 100 ); + $this->output( "\nDone\n" ); + } + + public function showProgressAndWait( $percent ) { + usleep( $this->usleep ); // avoid lag; T150124 + + $percentString = sprintf( "%.2f", $percent ); + if ( $percentString === $this->lastProgress ) { + return; + } + $this->lastProgress = $percentString; + + $stars = floor( $percent / 2 ); + $this->output( '[' . str_repeat( '*', $stars ) . str_repeat( '.', 50 - $stars ) . '] ' . + "$percentString%\r" ); + } +} + +$maintClass = 'PurgeParserCache'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/reassignEdits.php b/www/wiki/maintenance/reassignEdits.php new file mode 100644 index 00000000..7a0e4fc0 --- /dev/null +++ b/www/wiki/maintenance/reassignEdits.php @@ -0,0 +1,199 @@ +<?php +/** + * Reassign edits from a user or IP address to another user + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @author Rob Church <robchur@gmail.com> + * @licence GNU General Public Licence 2.0 or later + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that reassigns edits from a user or IP address + * to another user. + * + * @ingroup Maintenance + */ +class ReassignEdits extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Reassign edits from one user to another' ); + $this->addOption( "force", "Reassign even if the target user doesn't exist" ); + $this->addOption( "norc", "Don't update the recent changes table" ); + $this->addOption( "report", "Print out details of what would be changed, but don't update it" ); + $this->addArg( 'from', 'Old user to take edits from' ); + $this->addArg( 'to', 'New user to give edits to' ); + } + + public function execute() { + if ( $this->hasArg( 0 ) && $this->hasArg( 1 ) ) { + # Set up the users involved + $from = $this->initialiseUser( $this->getArg( 0 ) ); + $to = $this->initialiseUser( $this->getArg( 1 ) ); + + # If the target doesn't exist, and --force is not set, stop here + if ( $to->getId() || $this->hasOption( 'force' ) ) { + # Reassign the edits + $report = $this->hasOption( 'report' ); + $this->doReassignEdits( $from, $to, !$this->hasOption( 'norc' ), $report ); + # If reporting, and there were items, advise the user to run without --report + if ( $report ) { + $this->output( "Run the script again without --report to update.\n" ); + } + } else { + $ton = $to->getName(); + $this->error( "User '{$ton}' not found." ); + } + } + } + + /** + * Reassign edits from one user to another + * + * @param User $from User to take edits from + * @param User $to User to assign edits to + * @param bool $rc Update the recent changes table + * @param bool $report Don't change things; just echo numbers + * @return int Number of entries changed, or that would be changed + */ + private function doReassignEdits( &$from, &$to, $rc = false, $report = false ) { + $dbw = $this->getDB( DB_MASTER ); + $this->beginTransaction( $dbw, __METHOD__ ); + + # Count things + $this->output( "Checking current edits..." ); + $res = $dbw->select( + 'revision', + 'COUNT(*) AS count', + $this->userConditions( $from, 'rev_user', 'rev_user_text' ), + __METHOD__ + ); + $row = $dbw->fetchObject( $res ); + $cur = $row->count; + $this->output( "found {$cur}.\n" ); + + $this->output( "Checking deleted edits..." ); + $res = $dbw->select( + 'archive', + 'COUNT(*) AS count', + $this->userConditions( $from, 'ar_user', 'ar_user_text' ), + __METHOD__ + ); + $row = $dbw->fetchObject( $res ); + $del = $row->count; + $this->output( "found {$del}.\n" ); + + # Don't count recent changes if we're not supposed to + if ( $rc ) { + $this->output( "Checking recent changes..." ); + $res = $dbw->select( + 'recentchanges', + 'COUNT(*) AS count', + $this->userConditions( $from, 'rc_user', 'rc_user_text' ), + __METHOD__ + ); + $row = $dbw->fetchObject( $res ); + $rec = $row->count; + $this->output( "found {$rec}.\n" ); + } else { + $rec = 0; + } + + $total = $cur + $del + $rec; + $this->output( "\nTotal entries to change: {$total}\n" ); + + if ( !$report ) { + if ( $total ) { + # Reassign edits + $this->output( "\nReassigning current edits..." ); + $dbw->update( 'revision', $this->userSpecification( $to, 'rev_user', 'rev_user_text' ), + $this->userConditions( $from, 'rev_user', 'rev_user_text' ), __METHOD__ ); + $this->output( "done.\nReassigning deleted edits..." ); + $dbw->update( 'archive', $this->userSpecification( $to, 'ar_user', 'ar_user_text' ), + $this->userConditions( $from, 'ar_user', 'ar_user_text' ), __METHOD__ ); + $this->output( "done.\n" ); + # Update recent changes if required + if ( $rc ) { + $this->output( "Updating recent changes..." ); + $dbw->update( 'recentchanges', $this->userSpecification( $to, 'rc_user', 'rc_user_text' ), + $this->userConditions( $from, 'rc_user', 'rc_user_text' ), __METHOD__ ); + $this->output( "done.\n" ); + } + } + } + + $this->commitTransaction( $dbw, __METHOD__ ); + + return (int)$total; + } + + /** + * Return the most efficient set of user conditions + * i.e. a user => id mapping, or a user_text => text mapping + * + * @param User $user User for the condition + * @param string $idfield Field name containing the identifier + * @param string $utfield Field name containing the user text + * @return array + */ + private function userConditions( &$user, $idfield, $utfield ) { + return $user->getId() + ? [ $idfield => $user->getId() ] + : [ $utfield => $user->getName() ]; + } + + /** + * Return user specifications + * i.e. user => id, user_text => text + * + * @param User $user User for the spec + * @param string $idfield Field name containing the identifier + * @param string $utfield Field name containing the user text + * @return array + */ + private function userSpecification( &$user, $idfield, $utfield ) { + return [ $idfield => $user->getId(), $utfield => $user->getName() ]; + } + + /** + * Initialise the user object + * + * @param string $username Username or IP address + * @return User + */ + private function initialiseUser( $username ) { + if ( User::isIP( $username ) ) { + $user = new User(); + $user->setId( 0 ); + $user->setName( $username ); + } else { + $user = User::newFromName( $username ); + if ( !$user ) { + $this->error( "Invalid username", true ); + } + } + $user->load(); + + return $user; + } +} + +$maintClass = "ReassignEdits"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/rebuildFileCache.php b/www/wiki/maintenance/rebuildFileCache.php new file mode 100644 index 00000000..fe3944c8 --- /dev/null +++ b/www/wiki/maintenance/rebuildFileCache.php @@ -0,0 +1,182 @@ +<?php +/** + * Build file cache for content pages + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that builds file cache for content pages. + * + * @ingroup Maintenance + */ +class RebuildFileCache extends Maintenance { + private $enabled = true; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Build file cache for content pages' ); + $this->addOption( 'start', 'Page_id to start from', false, true ); + $this->addOption( 'end', 'Page_id to end on', false, true ); + $this->addOption( 'overwrite', 'Refresh page cache' ); + $this->setBatchSize( 100 ); + } + + public function finalSetup() { + global $wgDebugToolbar, $wgUseFileCache; + + $this->enabled = $wgUseFileCache; + // Script will handle capturing output and saving it itself + $wgUseFileCache = false; + // Debug toolbar makes content uncacheable so we disable it. + // Has to be done before Setup.php initialize MWDebug + $wgDebugToolbar = false; + // Avoid DB writes (like enotif/counters) + MediaWiki\MediaWikiServices::getInstance()->getReadOnlyMode() + ->setReason( 'Building cache' ); + + parent::finalSetup(); + } + + public function execute() { + global $wgRequestTime; + + if ( !$this->enabled ) { + $this->error( "Nothing to do -- \$wgUseFileCache is disabled.", true ); + } + + $start = $this->getOption( 'start', "0" ); + if ( !ctype_digit( $start ) ) { + $this->error( "Invalid value for start parameter.", true ); + } + $start = intval( $start ); + + $end = $this->getOption( 'end', "0" ); + if ( !ctype_digit( $end ) ) { + $this->error( "Invalid value for end parameter.", true ); + } + $end = intval( $end ); + + $this->output( "Building content page file cache from page {$start}!\n" ); + + $dbr = $this->getDB( DB_REPLICA ); + $overwrite = $this->hasOption( 'overwrite' ); + $start = ( $start > 0 ) + ? $start + : $dbr->selectField( 'page', 'MIN(page_id)', false, __METHOD__ ); + $end = ( $end > 0 ) + ? $end + : $dbr->selectField( 'page', 'MAX(page_id)', false, __METHOD__ ); + if ( !$start ) { + $this->error( "Nothing to do.", true ); + } + + $_SERVER['HTTP_ACCEPT_ENCODING'] = 'bgzip'; // hack, no real client + + # Do remaining chunk + $end += $this->mBatchSize - 1; + $blockStart = $start; + $blockEnd = $start + $this->mBatchSize - 1; + + $dbw = $this->getDB( DB_MASTER ); + // Go through each page and save the output + while ( $blockEnd <= $end ) { + // Get the pages + $res = $dbr->select( 'page', + [ 'page_namespace', 'page_title', 'page_id' ], + [ 'page_namespace' => MWNamespace::getContentNamespaces(), + "page_id BETWEEN $blockStart AND $blockEnd" ], + __METHOD__, + [ 'ORDER BY' => 'page_id ASC', 'USE INDEX' => 'PRIMARY' ] + ); + + $this->beginTransaction( $dbw, __METHOD__ ); // for any changes + foreach ( $res as $row ) { + $rebuilt = false; + + $title = Title::makeTitleSafe( $row->page_namespace, $row->page_title ); + if ( null == $title ) { + $this->output( "Page {$row->page_id} has bad title\n" ); + continue; // broken title? + } + + $context = new RequestContext(); + $context->setTitle( $title ); + $article = Article::newFromTitle( $title, $context ); + $context->setWikiPage( $article->getPage() ); + + // Some extensions like FlaggedRevs while error out if this is unset + RequestContext::getMain()->setTitle( $title ); + + // If the article is cacheable, then load it + if ( $article->isFileCacheable( HTMLFileCache::MODE_REBUILD ) ) { + $viewCache = new HTMLFileCache( $title, 'view' ); + $historyCache = new HTMLFileCache( $title, 'history' ); + if ( $viewCache->isCacheGood() && $historyCache->isCacheGood() ) { + if ( $overwrite ) { + $rebuilt = true; + } else { + $this->output( "Page '$title' (id {$row->page_id}) already cached\n" ); + continue; // done already! + } + } + + MediaWiki\suppressWarnings(); // header notices + // Cache ?action=view + $wgRequestTime = microtime( true ); # T24852 + ob_start(); + $article->view(); + $context->getOutput()->output(); + $context->getOutput()->clearHTML(); + $viewHtml = ob_get_clean(); + $viewCache->saveToFileCache( $viewHtml ); + // Cache ?action=history + $wgRequestTime = microtime( true ); # T24852 + ob_start(); + Action::factory( 'history', $article, $context )->show(); + $context->getOutput()->output(); + $context->getOutput()->clearHTML(); + $historyHtml = ob_get_clean(); + $historyCache->saveToFileCache( $historyHtml ); + MediaWiki\restoreWarnings(); + + if ( $rebuilt ) { + $this->output( "Re-cached page '$title' (id {$row->page_id})..." ); + } else { + $this->output( "Cached page '$title' (id {$row->page_id})..." ); + } + $this->output( "[view: " . strlen( $viewHtml ) . " bytes; " . + "history: " . strlen( $historyHtml ) . " bytes]\n" ); + } else { + $this->output( "Page '$title' (id {$row->page_id}) not cacheable\n" ); + } + } + $this->commitTransaction( $dbw, __METHOD__ ); // commit any changes (just for sanity) + + $blockStart += $this->mBatchSize; + $blockEnd += $this->mBatchSize; + } + $this->output( "Done!\n" ); + } +} + +$maintClass = "RebuildFileCache"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/rebuildImages.php b/www/wiki/maintenance/rebuildImages.php new file mode 100644 index 00000000..109350cd --- /dev/null +++ b/www/wiki/maintenance/rebuildImages.php @@ -0,0 +1,234 @@ +<?php +/** + * Update image metadata records. + * + * Usage: php rebuildImages.php [--missing] [--dry-run] + * Options: + * --missing Crawl the uploads dir for images without records, and + * add them only. + * + * Copyright © 2005 Brion Vibber <brion@pobox.com> + * https://www.mediawiki.org/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @author Brion Vibber <brion at pobox.com> + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +use Wikimedia\Rdbms\IMaintainableDatabase; + +/** + * Maintenance script to update image metadata records. + * + * @ingroup Maintenance + */ +class ImageBuilder extends Maintenance { + + /** + * @var IMaintainableDatabase + */ + protected $dbw; + + function __construct() { + parent::__construct(); + + global $wgUpdateCompatibleMetadata; + // make sure to update old, but compatible img_metadata fields. + $wgUpdateCompatibleMetadata = true; + + $this->addDescription( 'Script to update image metadata records' ); + + $this->addOption( 'missing', 'Check for files without associated database record' ); + $this->addOption( 'dry-run', 'Only report, don\'t update the database' ); + } + + public function execute() { + $this->dbw = $this->getDB( DB_MASTER ); + $this->dryrun = $this->hasOption( 'dry-run' ); + if ( $this->dryrun ) { + MediaWiki\MediaWikiServices::getInstance()->getReadOnlyMode() + ->setReason( 'Dry run mode, image upgrades are suppressed' ); + } + + if ( $this->hasOption( 'missing' ) ) { + $this->crawlMissing(); + } else { + $this->build(); + } + } + + /** + * @return FileRepo + */ + function getRepo() { + if ( !isset( $this->repo ) ) { + $this->repo = RepoGroup::singleton()->getLocalRepo(); + } + + return $this->repo; + } + + function build() { + $this->buildImage(); + $this->buildOldImage(); + } + + function init( $count, $table ) { + $this->processed = 0; + $this->updated = 0; + $this->count = $count; + $this->startTime = microtime( true ); + $this->table = $table; + } + + function progress( $updated ) { + $this->updated += $updated; + $this->processed++; + if ( $this->processed % 100 != 0 ) { + return; + } + $portion = $this->processed / $this->count; + $updateRate = $this->updated / $this->processed; + + $now = microtime( true ); + $delta = $now - $this->startTime; + $estimatedTotalTime = $delta / $portion; + $eta = $this->startTime + $estimatedTotalTime; + $rate = $this->processed / $delta; + + $this->output( sprintf( "%s: %6.2f%% done on %s; ETA %s [%d/%d] %.2f/sec <%.2f%% updated>\n", + wfTimestamp( TS_DB, intval( $now ) ), + $portion * 100.0, + $this->table, + wfTimestamp( TS_DB, intval( $eta ) ), + $this->processed, + $this->count, + $rate, + $updateRate * 100.0 ) ); + flush(); + } + + function buildTable( $table, $key, $callback ) { + $count = $this->dbw->selectField( $table, 'count(*)', '', __METHOD__ ); + $this->init( $count, $table ); + $this->output( "Processing $table...\n" ); + + $result = $this->getDB( DB_REPLICA )->select( $table, '*', [], __METHOD__ ); + + foreach ( $result as $row ) { + $update = call_user_func( $callback, $row, null ); + if ( $update ) { + $this->progress( 1 ); + } else { + $this->progress( 0 ); + } + } + $this->output( "Finished $table... $this->updated of $this->processed rows updated\n" ); + } + + function buildImage() { + $callback = [ $this, 'imageCallback' ]; + $this->buildTable( 'image', 'img_name', $callback ); + } + + function imageCallback( $row, $copy ) { + // Create a File object from the row + // This will also upgrade it + $file = $this->getRepo()->newFileFromRow( $row ); + + return $file->getUpgraded(); + } + + function buildOldImage() { + $this->buildTable( 'oldimage', 'oi_archive_name', [ $this, 'oldimageCallback' ] ); + } + + function oldimageCallback( $row, $copy ) { + // Create a File object from the row + // This will also upgrade it + if ( $row->oi_archive_name == '' ) { + $this->output( "Empty oi_archive_name for oi_name={$row->oi_name}\n" ); + + return false; + } + $file = $this->getRepo()->newFileFromRow( $row ); + + return $file->getUpgraded(); + } + + function crawlMissing() { + $this->getRepo()->enumFiles( [ $this, 'checkMissingImage' ] ); + } + + function checkMissingImage( $fullpath ) { + $filename = wfBaseName( $fullpath ); + $row = $this->dbw->selectRow( 'image', + [ 'img_name' ], + [ 'img_name' => $filename ], + __METHOD__ ); + + if ( !$row ) { // file not registered + $this->addMissingImage( $filename, $fullpath ); + } + } + + function addMissingImage( $filename, $fullpath ) { + global $wgContLang; + + $timestamp = $this->dbw->timestamp( $this->getRepo()->getFileTimestamp( $fullpath ) ); + + $altname = $wgContLang->checkTitleEncoding( $filename ); + if ( $altname != $filename ) { + if ( $this->dryrun ) { + $filename = $altname; + $this->output( "Estimating transcoding... $altname\n" ); + } else { + # @todo FIXME: create renameFile() + $filename = $this->renameFile( $filename ); + } + } + + if ( $filename == '' ) { + $this->output( "Empty filename for $fullpath\n" ); + + return; + } + if ( !$this->dryrun ) { + $file = wfLocalFile( $filename ); + if ( !$file->recordUpload( + '', + '(recovered file, missing upload log entry)', + '', + '', + '', + false, + $timestamp + ) ) { + $this->output( "Error uploading file $fullpath\n" ); + + return; + } + } + $this->output( $fullpath . "\n" ); + } +} + +$maintClass = 'ImageBuilder'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/rebuildLocalisationCache.php b/www/wiki/maintenance/rebuildLocalisationCache.php new file mode 100644 index 00000000..48602de0 --- /dev/null +++ b/www/wiki/maintenance/rebuildLocalisationCache.php @@ -0,0 +1,181 @@ +<?php + +/** + * Rebuild the localisation cache. Useful if you disabled automatic updates + * using $wgLocalisationCacheConf['manualRecache'] = true; + * + * Usage: + * php rebuildLocalisationCache.php [--force] [--threads=N] + * + * Use --force to rebuild all files, even the ones that are not out of date. + * Use --threads=N to fork more threads. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to rebuild the localisation cache. + * + * @ingroup Maintenance + */ +class RebuildLocalisationCache extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Rebuild the localisation cache' ); + $this->addOption( 'force', 'Rebuild all files, even ones not out of date' ); + $this->addOption( 'threads', 'Fork more than one thread', false, true ); + $this->addOption( 'outdir', 'Override the output directory (normally $wgCacheDirectory)', + false, true ); + $this->addOption( 'lang', 'Only rebuild these languages, comma separated.', + false, true ); + } + + public function finalSetup() { + # This script needs to be run to build the inital l10n cache. But if + # $wgLanguageCode is not 'en', it won't be able to run because there is + # no l10n cache. Break the cycle by forcing $wgLanguageCode = 'en'. + global $wgLanguageCode; + $wgLanguageCode = 'en'; + parent::finalSetup(); + } + + public function execute() { + global $wgLocalisationCacheConf; + + $force = $this->hasOption( 'force' ); + $threads = $this->getOption( 'threads', 1 ); + if ( $threads < 1 || $threads != intval( $threads ) ) { + $this->output( "Invalid thread count specified; running single-threaded.\n" ); + $threads = 1; + } + if ( $threads > 1 && wfIsWindows() ) { + $this->output( "Threaded rebuild is not supported on Windows; running single-threaded.\n" ); + $threads = 1; + } + if ( $threads > 1 && !function_exists( 'pcntl_fork' ) ) { + $this->output( "PHP pcntl extension is not present; running single-threaded.\n" ); + $threads = 1; + } + + $conf = $wgLocalisationCacheConf; + $conf['manualRecache'] = false; // Allow fallbacks to create CDB files + if ( $force ) { + $conf['forceRecache'] = true; + } + if ( $this->hasOption( 'outdir' ) ) { + $conf['storeDirectory'] = $this->getOption( 'outdir' ); + } + $lc = new LocalisationCacheBulkLoad( $conf ); + + $allCodes = array_keys( Language::fetchLanguageNames( null, 'mwfile' ) ); + if ( $this->hasOption( 'lang' ) ) { + # Validate requested languages + $codes = array_intersect( $allCodes, + explode( ',', $this->getOption( 'lang' ) ) ); + # Bailed out if nothing is left + if ( count( $codes ) == 0 ) { + $this->error( 'None of the languages specified exists.', 1 ); + } + } else { + # By default get all languages + $codes = $allCodes; + } + sort( $codes ); + + // Initialise and split into chunks + $numRebuilt = 0; + $total = count( $codes ); + $chunks = array_chunk( $codes, ceil( count( $codes ) / $threads ) ); + $pids = []; + $parentStatus = 0; + foreach ( $chunks as $codes ) { + // Do not fork for only one thread + $pid = ( $threads > 1 ) ? pcntl_fork() : -1; + + if ( $pid === 0 ) { + // Child, reseed because there is no bug in PHP: + // https://bugs.php.net/bug.php?id=42465 + mt_srand( getmypid() ); + + $this->doRebuild( $codes, $lc, $force ); + exit( 0 ); + } elseif ( $pid === -1 ) { + // Fork failed or one thread, do it serialized + $numRebuilt += $this->doRebuild( $codes, $lc, $force ); + } else { + // Main thread + $pids[] = $pid; + } + } + // Wait for all children + foreach ( $pids as $pid ) { + $status = 0; + pcntl_waitpid( $pid, $status ); + if ( pcntl_wexitstatus( $status ) ) { + // Pass a fatal error code through to the caller + $parentStatus = pcntl_wexitstatus( $status ); + } + } + + if ( !$pids ) { + $this->output( "$numRebuilt languages rebuilt out of $total\n" ); + if ( $numRebuilt === 0 ) { + $this->output( "Use --force to rebuild the caches which are still fresh.\n" ); + } + } + if ( $parentStatus ) { + exit( $parentStatus ); + } + } + + /** + * Helper function to rebuild list of languages codes. Prints the code + * for each language which is rebuilt. + * @param array $codes List of language codes to rebuild. + * @param LocalisationCache $lc Instance of LocalisationCacheBulkLoad (?) + * @param bool $force Rebuild up-to-date languages + * @return int Number of rebuilt languages + */ + private function doRebuild( $codes, $lc, $force ) { + $numRebuilt = 0; + foreach ( $codes as $code ) { + if ( $force || $lc->isExpired( $code ) ) { + $this->output( "Rebuilding $code...\n" ); + $lc->recache( $code ); + $numRebuilt++; + } + } + + return $numRebuilt; + } + + /** + * Sets whether a run of this maintenance script has the force parameter set + * + * @param bool $forced + */ + public function setForce( $forced = true ) { + $this->mOptions['force'] = $forced; + } +} + +$maintClass = "RebuildLocalisationCache"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/rebuildSitesCache.php b/www/wiki/maintenance/rebuildSitesCache.php new file mode 100644 index 00000000..230e86d4 --- /dev/null +++ b/www/wiki/maintenance/rebuildSitesCache.php @@ -0,0 +1,68 @@ +<?php + +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to dump a SiteStore as a static json file. + * + * @ingroup Maintenance + */ +class RebuildSitesCache extends Maintenance { + + public function __construct() { + parent::__construct(); + + $this->addDescription( 'Cache sites as json for file-based lookup.' ); + $this->addOption( 'file', 'File to output the json to', false, true ); + } + + public function execute() { + $sitesCacheFileBuilder = new SitesCacheFileBuilder( + \MediaWiki\MediaWikiServices::getInstance()->getSiteLookup(), + $this->getCacheFile() + ); + + $sitesCacheFileBuilder->build(); + } + + /** + * @return string + */ + private function getCacheFile() { + if ( $this->hasOption( 'file' ) ) { + $jsonFile = $this->getOption( 'file' ); + } else { + $jsonFile = $this->getConfig()->get( 'SitesCacheFile' ); + + if ( $jsonFile === false ) { + $this->error( 'Error: No file set in configuration for SitesCacheFile.', 1 ); + } + } + + return $jsonFile; + } + +} + +$maintClass = "RebuildSitesCache"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/rebuildall.php b/www/wiki/maintenance/rebuildall.php new file mode 100644 index 00000000..95822ca2 --- /dev/null +++ b/www/wiki/maintenance/rebuildall.php @@ -0,0 +1,67 @@ +<?php +/** + * Rebuild link tracking tables from scratch. This takes several + * hours, depending on the database size and server configuration. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that rebuilds link tracking tables from scratch. + * + * @ingroup Maintenance + */ +class RebuildAll extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Rebuild links, text index and recent changes' ); + } + + public function getDbType() { + return Maintenance::DB_ADMIN; + } + + public function execute() { + // Rebuild the text index + if ( $this->getDB( DB_REPLICA )->getType() != 'postgres' ) { + $this->output( "** Rebuilding fulltext search index (if you abort " + . "this will break searching; run this script again to fix):\n" ); + $rebuildText = $this->runChild( 'RebuildTextIndex', 'rebuildtextindex.php' ); + $rebuildText->execute(); + } + + // Rebuild RC + $this->output( "\n\n** Rebuilding recentchanges table:\n" ); + $rebuildRC = $this->runChild( 'RebuildRecentchanges', 'rebuildrecentchanges.php' ); + $rebuildRC->execute(); + + // Rebuild link tables + $this->output( "\n\n** Rebuilding links tables -- this can take a long time. " + . "It should be safe to abort via ctrl+C if you get bored.\n" ); + $rebuildLinks = $this->runChild( 'RefreshLinks', 'refreshLinks.php' ); + $rebuildLinks->execute(); + + $this->output( "Done.\n" ); + } +} + +$maintClass = "RebuildAll"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/rebuildmessages.php b/www/wiki/maintenance/rebuildmessages.php new file mode 100644 index 00000000..a47e50d9 --- /dev/null +++ b/www/wiki/maintenance/rebuildmessages.php @@ -0,0 +1,57 @@ +<?php +/** + * Purge all languages from the message cache. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that purges all languages from the message cache. + * + * @ingroup Maintenance + */ +class RebuildMessages extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Purge all language messages from the cache' ); + } + + public function execute() { + global $wgLocalDatabases, $wgDBname, $wgEnableSidebarCache, $messageMemc; + if ( $wgLocalDatabases ) { + $databases = $wgLocalDatabases; + } else { + $databases = [ $wgDBname ]; + } + + foreach ( $databases as $db ) { + $this->output( "Deleting message cache for {$db}... " ); + $messageMemc->delete( "{$db}:messages" ); + if ( $wgEnableSidebarCache ) { + $messageMemc->delete( "{$db}:sidebar" ); + } + $this->output( "Deleted\n" ); + } + } +} + +$maintClass = "RebuildMessages"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/rebuildrecentchanges.php b/www/wiki/maintenance/rebuildrecentchanges.php new file mode 100644 index 00000000..a2cf3c5b --- /dev/null +++ b/www/wiki/maintenance/rebuildrecentchanges.php @@ -0,0 +1,499 @@ +<?php +/** + * Rebuild recent changes from scratch. This takes several hours, + * depending on the database size and server configuration. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @todo Document + */ + +require_once __DIR__ . '/Maintenance.php'; +use MediaWiki\MediaWikiServices; + +/** + * Maintenance script that rebuilds recent changes from scratch. + * + * @ingroup Maintenance + */ +class RebuildRecentchanges extends Maintenance { + /** @var int UNIX timestamp */ + private $cutoffFrom; + /** @var int UNIX timestamp */ + private $cutoffTo; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Rebuild recent changes' ); + + $this->addOption( + 'from', + "Only rebuild rows in requested time range (in YYYYMMDDHHMMSS format)", + false, + true + ); + $this->addOption( + 'to', + "Only rebuild rows in requested time range (in YYYYMMDDHHMMSS format)", + false, + true + ); + $this->setBatchSize( 200 ); + } + + public function execute() { + if ( + ( $this->hasOption( 'from' ) && !$this->hasOption( 'to' ) ) || + ( !$this->hasOption( 'from' ) && $this->hasOption( 'to' ) ) + ) { + $this->error( "Both 'from' and 'to' must be given, or neither", 1 ); + } + + $this->rebuildRecentChangesTablePass1(); + $this->rebuildRecentChangesTablePass2(); + $this->rebuildRecentChangesTablePass3(); + $this->rebuildRecentChangesTablePass4(); + $this->rebuildRecentChangesTablePass5(); + if ( !( $this->hasOption( 'from' ) && $this->hasOption( 'to' ) ) ) { + $this->purgeFeeds(); + } + $this->output( "Done.\n" ); + } + + /** + * Rebuild pass 1: Insert `recentchanges` entries for page revisions. + */ + private function rebuildRecentChangesTablePass1() { + $dbw = $this->getDB( DB_MASTER ); + $revCommentStore = new CommentStore( 'rev_comment' ); + $rcCommentStore = new CommentStore( 'rc_comment' ); + + if ( $this->hasOption( 'from' ) && $this->hasOption( 'to' ) ) { + $this->cutoffFrom = wfTimestamp( TS_UNIX, $this->getOption( 'from' ) ); + $this->cutoffTo = wfTimestamp( TS_UNIX, $this->getOption( 'to' ) ); + + $sec = $this->cutoffTo - $this->cutoffFrom; + $days = $sec / 24 / 3600; + $this->output( "Rebuilding range of $sec seconds ($days days)\n" ); + } else { + global $wgRCMaxAge; + + $days = $wgRCMaxAge / 24 / 3600; + $this->output( "Rebuilding \$wgRCMaxAge=$wgRCMaxAge seconds ($days days)\n" ); + + $this->cutoffFrom = time() - $wgRCMaxAge; + $this->cutoffTo = time(); + } + + $this->output( "Clearing recentchanges table for time range...\n" ); + $rcids = $dbw->selectFieldValues( + 'recentchanges', + 'rc_id', + [ + 'rc_timestamp > ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffFrom ) ), + 'rc_timestamp < ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffTo ) ) + ] + ); + foreach ( array_chunk( $rcids, $this->mBatchSize ) as $rcidBatch ) { + $dbw->delete( 'recentchanges', [ 'rc_id' => $rcidBatch ], __METHOD__ ); + wfGetLBFactory()->waitForReplication(); + } + + $this->output( "Loading from page and revision tables...\n" ); + + $commentQuery = $revCommentStore->getJoin(); + $res = $dbw->select( + [ 'revision', 'page' ] + $commentQuery['tables'], + [ + 'rev_timestamp', + 'rev_user', + 'rev_user_text', + 'rev_minor_edit', + 'rev_id', + 'rev_deleted', + 'page_namespace', + 'page_title', + 'page_is_new', + 'page_id' + ] + $commentQuery['fields'], + [ + 'rev_timestamp > ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffFrom ) ), + 'rev_timestamp < ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffTo ) ) + ], + __METHOD__, + [ 'ORDER BY' => 'rev_timestamp DESC' ], + [ + 'page' => [ 'JOIN', 'rev_page=page_id' ], + ] + $commentQuery['joins'] + ); + + $this->output( "Inserting from page and revision tables...\n" ); + $inserted = 0; + foreach ( $res as $row ) { + $comment = $revCommentStore->getComment( $row ); + $dbw->insert( + 'recentchanges', + [ + 'rc_timestamp' => $row->rev_timestamp, + 'rc_user' => $row->rev_user, + 'rc_user_text' => $row->rev_user_text, + 'rc_namespace' => $row->page_namespace, + 'rc_title' => $row->page_title, + 'rc_minor' => $row->rev_minor_edit, + 'rc_bot' => 0, + 'rc_new' => $row->page_is_new, + 'rc_cur_id' => $row->page_id, + 'rc_this_oldid' => $row->rev_id, + 'rc_last_oldid' => 0, // is this ok? + 'rc_type' => $row->page_is_new ? RC_NEW : RC_EDIT, + 'rc_source' => $row->page_is_new ? RecentChange::SRC_NEW : RecentChange::SRC_EDIT, + 'rc_deleted' => $row->rev_deleted + ] + $rcCommentStore->insert( $dbw, $comment ), + __METHOD__ + ); + if ( ( ++$inserted % $this->mBatchSize ) == 0 ) { + wfGetLBFactory()->waitForReplication(); + } + } + } + + /** + * Rebuild pass 2: Enhance entries for page revisions with references to the previous revision + * (rc_last_oldid, rc_new etc.) and size differences (rc_old_len, rc_new_len). + */ + private function rebuildRecentChangesTablePass2() { + $dbw = $this->getDB( DB_MASTER ); + + $this->output( "Updating links and size differences...\n" ); + + # Fill in the rc_last_oldid field, which points to the previous edit + $res = $dbw->select( + 'recentchanges', + [ 'rc_cur_id', 'rc_this_oldid', 'rc_timestamp' ], + [ + "rc_timestamp > " . $dbw->addQuotes( $dbw->timestamp( $this->cutoffFrom ) ), + "rc_timestamp < " . $dbw->addQuotes( $dbw->timestamp( $this->cutoffTo ) ) + ], + __METHOD__, + [ 'ORDER BY' => 'rc_cur_id,rc_timestamp' ] + ); + + $lastCurId = 0; + $lastOldId = 0; + $lastSize = null; + $updated = 0; + foreach ( $res as $obj ) { + $new = 0; + + if ( $obj->rc_cur_id != $lastCurId ) { + # Switch! Look up the previous last edit, if any + $lastCurId = intval( $obj->rc_cur_id ); + $emit = $obj->rc_timestamp; + + $row = $dbw->selectRow( + 'revision', + [ 'rev_id', 'rev_len' ], + [ 'rev_page' => $lastCurId, "rev_timestamp < " . $dbw->addQuotes( $emit ) ], + __METHOD__, + [ 'ORDER BY' => 'rev_timestamp DESC' ] + ); + if ( $row ) { + $lastOldId = intval( $row->rev_id ); + # Grab the last text size if available + $lastSize = !is_null( $row->rev_len ) ? intval( $row->rev_len ) : null; + } else { + # No previous edit + $lastOldId = 0; + $lastSize = null; + $new = 1; // probably true + } + } + + if ( $lastCurId == 0 ) { + $this->output( "Uhhh, something wrong? No curid\n" ); + } else { + # Grab the entry's text size + $size = (int)$dbw->selectField( + 'revision', + 'rev_len', + [ 'rev_id' => $obj->rc_this_oldid ], + __METHOD__ + ); + + $dbw->update( + 'recentchanges', + [ + 'rc_last_oldid' => $lastOldId, + 'rc_new' => $new, + 'rc_type' => $new ? RC_NEW : RC_EDIT, + 'rc_source' => $new === 1 ? RecentChange::SRC_NEW : RecentChange::SRC_EDIT, + 'rc_old_len' => $lastSize, + 'rc_new_len' => $size, + ], + [ + 'rc_cur_id' => $lastCurId, + 'rc_this_oldid' => $obj->rc_this_oldid, + 'rc_timestamp' => $obj->rc_timestamp // index usage + ], + __METHOD__ + ); + + $lastOldId = intval( $obj->rc_this_oldid ); + $lastSize = $size; + + if ( ( ++$updated % $this->mBatchSize ) == 0 ) { + wfGetLBFactory()->waitForReplication(); + } + } + } + } + + /** + * Rebuild pass 3: Insert `recentchanges` entries for action logs. + */ + private function rebuildRecentChangesTablePass3() { + global $wgLogTypes, $wgLogRestrictions; + + $dbw = $this->getDB( DB_MASTER ); + $logCommentStore = new CommentStore( 'log_comment' ); + $rcCommentStore = new CommentStore( 'rc_comment' ); + + $this->output( "Loading from user, page, and logging tables...\n" ); + + $commentQuery = $logCommentStore->getJoin(); + $res = $dbw->select( + [ 'user', 'logging', 'page' ] + $commentQuery['tables'], + [ + 'log_timestamp', + 'log_user', + 'user_name', + 'log_namespace', + 'log_title', + 'page_id', + 'log_type', + 'log_action', + 'log_id', + 'log_params', + 'log_deleted' + ] + $commentQuery['fields'], + [ + 'log_timestamp > ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffFrom ) ), + 'log_timestamp < ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffTo ) ), + 'log_user=user_id', + // Some logs don't go in RC since they are private. + // @FIXME: core/extensions also have spammy logs that don't go in RC. + 'log_type' => array_diff( $wgLogTypes, array_keys( $wgLogRestrictions ) ), + ], + __METHOD__, + [ 'ORDER BY' => 'log_timestamp DESC' ], + [ + 'page' => + [ 'LEFT JOIN', [ 'log_namespace=page_namespace', 'log_title=page_title' ] ] + ] + $commentQuery['joins'] + ); + + $field = $dbw->fieldInfo( 'recentchanges', 'rc_cur_id' ); + + $inserted = 0; + foreach ( $res as $row ) { + $comment = $logCommentStore->getComment( $row ); + $dbw->insert( + 'recentchanges', + [ + 'rc_timestamp' => $row->log_timestamp, + 'rc_user' => $row->log_user, + 'rc_user_text' => $row->user_name, + 'rc_namespace' => $row->log_namespace, + 'rc_title' => $row->log_title, + 'rc_minor' => 0, + 'rc_bot' => 0, + 'rc_patrolled' => 1, + 'rc_new' => 0, + 'rc_this_oldid' => 0, + 'rc_last_oldid' => 0, + 'rc_type' => RC_LOG, + 'rc_source' => RecentChange::SRC_LOG, + 'rc_cur_id' => $field->isNullable() + ? $row->page_id + : (int)$row->page_id, // NULL => 0, + 'rc_log_type' => $row->log_type, + 'rc_log_action' => $row->log_action, + 'rc_logid' => $row->log_id, + 'rc_params' => $row->log_params, + 'rc_deleted' => $row->log_deleted + ] + $rcCommentStore->insert( $dbw, $comment ), + __METHOD__ + ); + + if ( ( ++$inserted % $this->mBatchSize ) == 0 ) { + wfGetLBFactory()->waitForReplication(); + } + } + } + + /** + * Rebuild pass 4: Mark bot and autopatrolled entries. + */ + private function rebuildRecentChangesTablePass4() { + global $wgUseRCPatrol, $wgMiserMode; + + $dbw = $this->getDB( DB_MASTER ); + + list( $recentchanges, $usergroups, $user ) = + $dbw->tableNamesN( 'recentchanges', 'user_groups', 'user' ); + + # @FIXME: recognize other bot account groups (not the same as users with 'bot' rights) + # @NOTE: users with 'bot' rights choose when edits are bot edits or not. That information + # may be lost at this point (aside from joining on the patrol log table entries). + $botgroups = [ 'bot' ]; + $autopatrolgroups = $wgUseRCPatrol ? User::getGroupsWithPermission( 'autopatrol' ) : []; + + # Flag our recent bot edits + if ( $botgroups ) { + $botwhere = $dbw->makeList( $botgroups ); + + $this->output( "Flagging bot account edits...\n" ); + + # Find all users that are bots + $sql = "SELECT DISTINCT user_name FROM $usergroups, $user " . + "WHERE ug_group IN($botwhere) AND user_id = ug_user"; + $res = $dbw->query( $sql, __METHOD__ ); + + $botusers = []; + foreach ( $res as $obj ) { + $botusers[] = $obj->user_name; + } + + # Fill in the rc_bot field + if ( $botusers ) { + $rcids = $dbw->selectFieldValues( + 'recentchanges', + 'rc_id', + [ + 'rc_user_text' => $botusers, + "rc_timestamp > " . $dbw->addQuotes( $dbw->timestamp( $this->cutoffFrom ) ), + "rc_timestamp < " . $dbw->addQuotes( $dbw->timestamp( $this->cutoffTo ) ) + ], + __METHOD__ + ); + + foreach ( array_chunk( $rcids, $this->mBatchSize ) as $rcidBatch ) { + $dbw->update( + 'recentchanges', + [ 'rc_bot' => 1 ], + [ 'rc_id' => $rcidBatch ], + __METHOD__ + ); + wfGetLBFactory()->waitForReplication(); + } + } + } + + # Flag our recent autopatrolled edits + if ( !$wgMiserMode && $autopatrolgroups ) { + $patrolwhere = $dbw->makeList( $autopatrolgroups ); + $patrolusers = []; + + $this->output( "Flagging auto-patrolled edits...\n" ); + + # Find all users in RC with autopatrol rights + $sql = "SELECT DISTINCT user_name FROM $usergroups, $user " . + "WHERE ug_group IN($patrolwhere) AND user_id = ug_user"; + $res = $dbw->query( $sql, __METHOD__ ); + + foreach ( $res as $obj ) { + $patrolusers[] = $dbw->addQuotes( $obj->user_name ); + } + + # Fill in the rc_patrolled field + if ( $patrolusers ) { + $patrolwhere = implode( ',', $patrolusers ); + $sql2 = "UPDATE $recentchanges SET rc_patrolled=1 " . + "WHERE rc_user_text IN($patrolwhere) " . + "AND rc_timestamp > " . $dbw->addQuotes( $dbw->timestamp( $this->cutoffFrom ) ) . ' ' . + "AND rc_timestamp < " . $dbw->addQuotes( $dbw->timestamp( $this->cutoffTo ) ); + $dbw->query( $sql2 ); + } + } + } + + /** + * Rebuild pass 5: Delete duplicate entries where we generate both a page revision and a log entry + * for a single action (upload only, at the moment, but potentially also move, protect, ...). + */ + private function rebuildRecentChangesTablePass5() { + $dbw = wfGetDB( DB_MASTER ); + + $this->output( "Removing duplicate revision and logging entries...\n" ); + + $res = $dbw->select( + [ 'logging', 'log_search' ], + [ 'ls_value', 'ls_log_id' ], + [ + 'ls_log_id = log_id', + 'ls_field' => 'associated_rev_id', + 'log_type' => 'upload', + 'log_timestamp > ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffFrom ) ), + 'log_timestamp < ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffTo ) ), + ], + __METHOD__ + ); + + $updates = 0; + foreach ( $res as $obj ) { + $rev_id = $obj->ls_value; + $log_id = $obj->ls_log_id; + + // Mark the logging row as having an associated rev id + $dbw->update( + 'recentchanges', + /*SET*/ [ 'rc_this_oldid' => $rev_id ], + /*WHERE*/ [ 'rc_logid' => $log_id ], + __METHOD__ + ); + + // Delete the revision row + $dbw->delete( + 'recentchanges', + /*WHERE*/ [ 'rc_this_oldid' => $rev_id, 'rc_logid' => 0 ], + __METHOD__ + ); + + if ( ( ++$updates % $this->mBatchSize ) == 0 ) { + wfGetLBFactory()->waitForReplication(); + } + } + } + + /** + * Purge cached feeds in $wanCache + */ + private function purgeFeeds() { + global $wgFeedClasses; + + $this->output( "Deleting feed timestamps.\n" ); + + $wanCache = MediaWikiServices::getInstance()->getMainWANObjectCache(); + foreach ( $wgFeedClasses as $feed => $className ) { + $wanCache->delete( $wanCache->makeKey( 'rcfeed', $feed, 'timestamp' ) ); # Good enough for now. + } + } +} + +$maintClass = "RebuildRecentchanges"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/rebuildtextindex.php b/www/wiki/maintenance/rebuildtextindex.php new file mode 100644 index 00000000..faa4d962 --- /dev/null +++ b/www/wiki/maintenance/rebuildtextindex.php @@ -0,0 +1,164 @@ +<?php +/** + * Rebuild search index table from scratch. This may take several + * hours, depending on the database size and server configuration. + * + * Postgres is trigger-based and should never need rebuilding. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @todo document + */ + +require_once __DIR__ . '/Maintenance.php'; + +use Wikimedia\Rdbms\IMaintainableDatabase; +use Wikimedia\Rdbms\DatabaseSqlite; + +/** + * Maintenance script that rebuilds search index table from scratch. + * + * @ingroup Maintenance + */ +class RebuildTextIndex extends Maintenance { + const RTI_CHUNK_SIZE = 500; + + /** + * @var IMaintainableDatabase + */ + private $db; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Rebuild search index table from scratch' ); + } + + public function getDbType() { + return Maintenance::DB_ADMIN; + } + + public function execute() { + // Shouldn't be needed for Postgres + $this->db = $this->getDB( DB_MASTER ); + if ( $this->db->getType() == 'postgres' ) { + $this->error( "This script is not needed when using Postgres.\n", true ); + } + + if ( $this->db->getType() == 'sqlite' ) { + if ( !DatabaseSqlite::getFulltextSearchModule() ) { + $this->error( "Your version of SQLite module for PHP doesn't " + . "support full-text search (FTS3).\n", true ); + } + if ( !$this->db->checkForEnabledSearch() ) { + $this->error( "Your database schema is not configured for " + . "full-text search support. Run update.php.\n", true ); + } + } + + if ( $this->db->getType() == 'mysql' ) { + $this->dropMysqlTextIndex(); + $this->clearSearchIndex(); + $this->populateSearchIndex(); + $this->createMysqlTextIndex(); + } else { + $this->clearSearchIndex(); + $this->populateSearchIndex(); + } + + $this->output( "Done.\n" ); + } + + /** + * Populates the search index with content from all pages + */ + protected function populateSearchIndex() { + $res = $this->db->select( 'page', 'MAX(page_id) AS count' ); + $s = $this->db->fetchObject( $res ); + $count = $s->count; + $this->output( "Rebuilding index fields for {$count} pages...\n" ); + $n = 0; + + $fields = array_merge( + Revision::selectPageFields(), + Revision::selectFields(), + Revision::selectTextFields() + ); + + while ( $n < $count ) { + if ( $n ) { + $this->output( $n . "\n" ); + } + $end = $n + self::RTI_CHUNK_SIZE - 1; + + $res = $this->db->select( [ 'page', 'revision', 'text' ], $fields, + [ "page_id BETWEEN $n AND $end", 'page_latest = rev_id', 'rev_text_id = old_id' ], + __METHOD__ + ); + + foreach ( $res as $s ) { + $title = Title::makeTitle( $s->page_namespace, $s->page_title ); + try { + $rev = new Revision( $s ); + $content = $rev->getContent(); + + $u = new SearchUpdate( $s->page_id, $title, $content ); + $u->doUpdate(); + } catch ( MWContentSerializationException $ex ) { + $this->output( "Failed to deserialize content of revision {$s->rev_id} of page " + . "`" . $title->getPrefixedDBkey() . "`!\n" ); + } + } + $n += self::RTI_CHUNK_SIZE; + } + } + + /** + * (MySQL only) Drops fulltext index before populating the table. + */ + private function dropMysqlTextIndex() { + $searchindex = $this->db->tableName( 'searchindex' ); + if ( $this->db->indexExists( 'searchindex', 'si_title', __METHOD__ ) ) { + $this->output( "Dropping index...\n" ); + $sql = "ALTER TABLE $searchindex DROP INDEX si_title, DROP INDEX si_text"; + $this->db->query( $sql, __METHOD__ ); + } + } + + /** + * (MySQL only) Adds back fulltext index after populating the table. + */ + private function createMysqlTextIndex() { + $searchindex = $this->db->tableName( 'searchindex' ); + $this->output( "\nRebuild the index...\n" ); + $sql = "ALTER TABLE $searchindex ADD FULLTEXT si_title (si_title), " . + "ADD FULLTEXT si_text (si_text)"; + $this->db->query( $sql, __METHOD__ ); + } + + /** + * Deletes everything from search index. + */ + private function clearSearchIndex() { + $this->output( 'Clearing searchindex table...' ); + $this->db->delete( 'searchindex', '*', __METHOD__ ); + $this->output( "Done\n" ); + } +} + +$maintClass = "RebuildTextIndex"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/recountCategories.php b/www/wiki/maintenance/recountCategories.php new file mode 100644 index 00000000..a4bfa989 --- /dev/null +++ b/www/wiki/maintenance/recountCategories.php @@ -0,0 +1,172 @@ +<?php +/** + * Refreshes category counts. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +use MediaWiki\MediaWikiServices; + +/** + * Maintenance script that refreshes category membership counts in the category + * table. + * + * (The populateCategory.php script will also recalculate counts, but + * recountCategories only updates rows that need to be updated, making it more + * efficient.) + * + * @ingroup Maintenance + */ +class RecountCategories extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( <<<'TEXT' +This script refreshes the category membership counts stored in the category +table. As time passes, these counts often drift from the actual number of +category members. The script identifies rows where the value in the category +table does not match the number of categorylinks rows for that category, and +updates the category table accordingly. + +To fully refresh the data in the category table, you need to run this script +three times: once in each mode. Alternatively, just one mode can be run if +required. +TEXT + ); + $this->addOption( + 'mode', + '(REQUIRED) Which category count column to recompute: "pages", "subcats" or "files".', + true, + true + ); + $this->addOption( + 'begin', + 'Only recount categories with cat_id greater than the given value', + false, + true + ); + $this->addOption( + 'throttle', + 'Wait this many milliseconds after each batch. Default: 0', + false, + true + ); + + $this->setBatchSize( 500 ); + } + + public function execute() { + $this->mode = $this->getOption( 'mode' ); + if ( !in_array( $this->mode, [ 'pages', 'subcats', 'files' ] ) ) { + $this->error( 'Please specify a valid mode: one of "pages", "subcats" or "files".', 1 ); + } + + $this->minimumId = intval( $this->getOption( 'begin', 0 ) ); + + // do the work, batch by batch + $affectedRows = 0; + while ( ( $result = $this->doWork() ) !== false ) { + $affectedRows += $result; + usleep( $this->getOption( 'throttle', 0 ) * 1000 ); + } + + $this->output( "Done! Updated the {$this->mode} counts of $affectedRows categories.\n" . + "Now run the script using the other --mode options if you haven't already.\n" ); + if ( $this->mode === 'pages' ) { + $this->output( + "Also run 'php cleanupEmptyCategories.php --mode remove' to remove empty,\n" . + "nonexistent categories from the category table.\n\n" ); + } + } + + protected function doWork() { + $this->output( "Finding up to {$this->mBatchSize} drifted rows " . + "starting at cat_id {$this->minimumId}...\n" ); + + $countingConds = [ 'cl_to = cat_title' ]; + if ( $this->mode === 'subcats' ) { + $countingConds['cl_type'] = 'subcat'; + } elseif ( $this->mode === 'files' ) { + $countingConds['cl_type'] = 'file'; + } + + $dbr = $this->getDB( DB_REPLICA, 'vslow' ); + $countingSubquery = $dbr->selectSQLText( 'categorylinks', + 'COUNT(*)', + $countingConds, + __METHOD__ ); + + // First, let's find out which categories have drifted and need to be updated. + // The query counts the categorylinks for each category on the replica DB, + // but this data can't be used for updating the master, so we don't include it + // in the results. + $idsToUpdate = $dbr->selectFieldValues( 'category', + 'cat_id', + [ + 'cat_id > ' . $this->minimumId, + "cat_{$this->mode} != ($countingSubquery)" + ], + __METHOD__, + [ 'LIMIT' => $this->mBatchSize ] + ); + if ( !$idsToUpdate ) { + return false; + } + $this->output( "Updating cat_{$this->mode} field on " . + count( $idsToUpdate ) . " rows...\n" ); + + // In the next batch, start where this query left off. The rows selected + // in this iteration shouldn't be selected again after being updated, but + // we still keep track of where we are up to, as extra protection against + // infinite loops. + $this->minimumId = end( $idsToUpdate ); + + // Now, on master, find the correct counts for these categories. + $dbw = $this->getDB( DB_MASTER ); + $res = $dbw->select( 'category', + [ 'cat_id', 'count' => "($countingSubquery)" ], + [ 'cat_id' => $idsToUpdate ], + __METHOD__ ); + + // Update the category counts on the rows we just identified. + // This logic is equivalent to Category::refreshCounts, except here, we + // don't remove rows when cat_pages is zero and the category description page + // doesn't exist - instead we print a suggestion to run + // cleanupEmptyCategories.php. + $affectedRows = 0; + foreach ( $res as $row ) { + $dbw->update( 'category', + [ "cat_{$this->mode}" => $row->count ], + [ + 'cat_id' => $row->cat_id, + "cat_{$this->mode} != {$row->count}", + ], + __METHOD__ ); + $affectedRows += $dbw->affectedRows(); + } + + MediaWikiServices::getInstance()->getDBLoadBalancerFactory()->waitForReplication(); + + return $affectedRows; + } +} + +$maintClass = 'RecountCategories'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/refreshFileHeaders.php b/www/wiki/maintenance/refreshFileHeaders.php new file mode 100644 index 00000000..bca1c964 --- /dev/null +++ b/www/wiki/maintenance/refreshFileHeaders.php @@ -0,0 +1,116 @@ +<?php +/** + * Refresh file headers from metadata. + * + * Usage: php refreshFileHeaders.php + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to refresh file headers from metadata + * + * @ingroup Maintenance + */ +class RefreshFileHeaders extends Maintenance { + function __construct() { + parent::__construct(); + $this->addDescription( 'Script to update file HTTP headers' ); + $this->addOption( 'verbose', 'Output information about each file.', false, false, 'v' ); + $this->addOption( 'start', 'Name of file to start with', false, true ); + $this->addOption( 'end', 'Name of file to end with', false, true ); + $this->setBatchSize( 200 ); + } + + public function execute() { + $repo = RepoGroup::singleton()->getLocalRepo(); + $start = str_replace( ' ', '_', $this->getOption( 'start', '' ) ); // page on img_name + $end = str_replace( ' ', '_', $this->getOption( 'end', '' ) ); // page on img_name + + $count = 0; + $dbr = $this->getDB( DB_REPLICA ); + + do { + $conds = [ "img_name > {$dbr->addQuotes( $start )}" ]; + + if ( strlen( $end ) ) { + $conds[] = "img_name <= {$dbr->addQuotes( $end )}"; + } + + $res = $dbr->select( 'image', '*', $conds, + __METHOD__, [ 'LIMIT' => $this->mBatchSize, 'ORDER BY' => 'img_name ASC' ] ); + + if ( $res->numRows() > 0 ) { + $row1 = $res->current(); + $this->output( "Processing next {$res->numRows()} row(s) starting with {$row1->img_name}.\n" ); + $res->rewind(); + } + + $backendOperations = []; + + foreach ( $res as $row ) { + $file = $repo->newFileFromRow( $row ); + $headers = $file->getContentHeaders(); + + if ( count( $headers ) ) { + $backendOperations[] = [ + 'op' => 'describe', 'src' => $file->getPath(), 'headers' => $headers + ]; + } + + // Do all of the older file versions... + foreach ( $file->getHistory() as $oldFile ) { + $headers = $oldFile->getContentHeaders(); + if ( count( $headers ) ) { + $backendOperations[] = [ + 'op' => 'describe', 'src' => $oldFile->getPath(), 'headers' => $headers + ]; + } + } + + if ( $this->hasOption( 'verbose' ) ) { + $this->output( "Queued headers update for file '{$row->img_name}'.\n" ); + } + + $start = $row->img_name; // advance + } + + $backendOperationsCount = count( $backendOperations ); + $count += $backendOperationsCount; + + $this->output( "Updating headers for {$backendOperationsCount} file(s).\n" ); + $this->updateFileHeaders( $repo, $backendOperations ); + } while ( $res->numRows() === $this->mBatchSize ); + + $this->output( "Done. Updated headers for $count file(s).\n" ); + } + + protected function updateFileHeaders( $repo, $backendOperations ) { + $status = $repo->getBackend()->doQuickOperations( $backendOperations ); + + if ( !$status->isGood() ) { + $this->error( "Encountered error: " . print_r( $status, true ) ); + } + } +} + +$maintClass = 'RefreshFileHeaders'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/refreshImageMetadata.php b/www/wiki/maintenance/refreshImageMetadata.php new file mode 100644 index 00000000..f6e9e9c3 --- /dev/null +++ b/www/wiki/maintenance/refreshImageMetadata.php @@ -0,0 +1,260 @@ +<?php +/** + * Refresh image metadata fields. See also rebuildImages.php + * + * Usage: php refreshImageMetadata.php + * + * Copyright © 2011 Brian Wolff + * https://www.mediawiki.org/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @author Brian Wolff + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +use Wikimedia\Rdbms\IDatabase; +use Wikimedia\Rdbms\IMaintainableDatabase; + +/** + * Maintenance script to refresh image metadata fields. + * + * @ingroup Maintenance + */ +class RefreshImageMetadata extends Maintenance { + + /** + * @var IMaintainableDatabase + */ + protected $dbw; + + function __construct() { + parent::__construct(); + + $this->addDescription( 'Script to update image metadata records' ); + $this->setBatchSize( 200 ); + + $this->addOption( + 'force', + 'Reload metadata from file even if the metadata looks ok', + false, + false, + 'f' + ); + $this->addOption( + 'broken-only', + 'Only fix really broken records, leave old but still compatible records alone.' + ); + $this->addOption( + 'verbose', + 'Output extra information about each upgraded/non-upgraded file.', + false, + false, + 'v' + ); + $this->addOption( 'start', 'Name of file to start with', false, true ); + $this->addOption( 'end', 'Name of file to end with', false, true ); + + $this->addOption( + 'mediatype', + 'Only refresh files with this media type, e.g. BITMAP, UNKNOWN etc.', + false, + true + ); + $this->addOption( + 'mime', + "Only refresh files with this MIME type. Can accept wild-card 'image/*'. " + . "Potentially inefficient unless 'mediatype' is also specified", + false, + true + ); + $this->addOption( + 'metadata-contains', + '(Inefficient!) Only refresh files where the img_metadata field ' + . 'contains this string. Can be used if its known a specific ' + . 'property was being extracted incorrectly.', + false, + true + ); + } + + public function execute() { + $force = $this->hasOption( 'force' ); + $brokenOnly = $this->hasOption( 'broken-only' ); + $verbose = $this->hasOption( 'verbose' ); + $start = $this->getOption( 'start', false ); + $this->setupParameters( $force, $brokenOnly ); + + $upgraded = 0; + $leftAlone = 0; + $error = 0; + + $dbw = $this->getDB( DB_MASTER ); + if ( $this->mBatchSize <= 0 ) { + $this->error( "Batch size is too low...", 12 ); + } + + $repo = RepoGroup::singleton()->getLocalRepo(); + $conds = $this->getConditions( $dbw ); + + // For the WHERE img_name > 'foo' condition that comes after doing a batch + $conds2 = []; + if ( $start !== false ) { + $conds2[] = 'img_name >= ' . $dbw->addQuotes( $start ); + } + + $options = [ + 'LIMIT' => $this->mBatchSize, + 'ORDER BY' => 'img_name ASC', + ]; + + do { + $res = $dbw->select( + 'image', + '*', + array_merge( $conds, $conds2 ), + __METHOD__, + $options + ); + + if ( $res->numRows() > 0 ) { + $row1 = $res->current(); + $this->output( "Processing next {$res->numRows()} row(s) starting with {$row1->img_name}.\n" ); + $res->rewind(); + } + + foreach ( $res as $row ) { + try { + // LocalFile will upgrade immediately here if obsolete + $file = $repo->newFileFromRow( $row ); + if ( $file->getUpgraded() ) { + // File was upgraded. + $upgraded++; + $newLength = strlen( $file->getMetadata() ); + $oldLength = strlen( $row->img_metadata ); + if ( $newLength < $oldLength - 5 ) { + // If after updating, the metadata is smaller then + // what it was before, that's probably not a good thing + // because we extract more data with time, not less. + // Thus this probably indicates an error of some sort, + // or at the very least is suspicious. Have the - 5 just + // to weed out any inconsequential changes. + $error++; + $this->output( + "Warning: File:{$row->img_name} used to have " . + "$oldLength bytes of metadata but now has $newLength bytes.\n" + ); + } elseif ( $verbose ) { + $this->output( "Refreshed File:{$row->img_name}.\n" ); + } + } else { + $leftAlone++; + if ( $force ) { + $file->upgradeRow(); + $newLength = strlen( $file->getMetadata() ); + $oldLength = strlen( $row->img_metadata ); + if ( $newLength < $oldLength - 5 ) { + $error++; + $this->output( + "Warning: File:{$row->img_name} used to have " . + "$oldLength bytes of metadata but now has $newLength bytes. (forced)\n" + ); + } + if ( $verbose ) { + $this->output( "Forcibly refreshed File:{$row->img_name}.\n" ); + } + } else { + if ( $verbose ) { + $this->output( "Skipping File:{$row->img_name}.\n" ); + } + } + } + } catch ( Exception $e ) { + $this->output( "{$row->img_name} failed. {$e->getMessage()}\n" ); + } + } + $conds2 = [ 'img_name > ' . $dbw->addQuotes( $row->img_name ) ]; + wfWaitForSlaves(); + } while ( $res->numRows() === $this->mBatchSize ); + + $total = $upgraded + $leftAlone; + if ( $force ) { + $this->output( "\nFinished refreshing file metadata for $total files. " + . "$upgraded needed to be refreshed, $leftAlone did not need to " + . "be but were refreshed anyways, and $error refreshes were suspicious.\n" ); + } else { + $this->output( "\nFinished refreshing file metadata for $total files. " + . "$upgraded were refreshed, $leftAlone were already up to date, " + . "and $error refreshes were suspicious.\n" ); + } + } + + /** + * @param IDatabase $dbw + * @return array + */ + function getConditions( $dbw ) { + $conds = []; + + $end = $this->getOption( 'end', false ); + $mime = $this->getOption( 'mime', false ); + $mediatype = $this->getOption( 'mediatype', false ); + $like = $this->getOption( 'metadata-contains', false ); + + if ( $end !== false ) { + $conds[] = 'img_name <= ' . $dbw->addQuotes( $end ); + } + if ( $mime !== false ) { + list( $major, $minor ) = File::splitMime( $mime ); + $conds['img_major_mime'] = $major; + if ( $minor !== '*' ) { + $conds['img_minor_mime'] = $minor; + } + } + if ( $mediatype !== false ) { + $conds['img_media_type'] = $mediatype; + } + if ( $like ) { + $conds[] = 'img_metadata ' . $dbw->buildLike( $dbw->anyString(), $like, $dbw->anyString() ); + } + + return $conds; + } + + /** + * @param bool $force + * @param bool $brokenOnly + */ + function setupParameters( $force, $brokenOnly ) { + global $wgUpdateCompatibleMetadata; + + if ( $brokenOnly ) { + $wgUpdateCompatibleMetadata = false; + } else { + $wgUpdateCompatibleMetadata = true; + } + + if ( $brokenOnly && $force ) { + $this->error( 'Cannot use --broken-only and --force together. ', 2 ); + } + } +} + +$maintClass = 'RefreshImageMetadata'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/refreshLinks.php b/www/wiki/maintenance/refreshLinks.php new file mode 100644 index 00000000..b099aff4 --- /dev/null +++ b/www/wiki/maintenance/refreshLinks.php @@ -0,0 +1,493 @@ +<?php +/** + * Refresh link tables. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +use Wikimedia\Rdbms\IDatabase; + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to refresh link tables. + * + * @ingroup Maintenance + */ +class RefreshLinks extends Maintenance { + const REPORTING_INTERVAL = 100; + + /** @var int|bool */ + protected $namespace = false; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Refresh link tables' ); + $this->addOption( 'dfn-only', 'Delete links from nonexistent articles only' ); + $this->addOption( 'new-only', 'Only affect articles with just a single edit' ); + $this->addOption( 'redirects-only', 'Only fix redirects, not all links' ); + $this->addOption( 'old-redirects-only', 'Only fix redirects with no redirect table entry' ); + $this->addOption( 'e', 'Last page id to refresh', false, true ); + $this->addOption( 'dfn-chunk-size', 'Maximum number of existent IDs to check per ' . + 'query, default 100000', false, true ); + $this->addOption( 'namespace', 'Only fix pages in this namespace', false, true ); + $this->addOption( 'category', 'Only fix pages in this category', false, true ); + $this->addOption( 'tracking-category', 'Only fix pages in this tracking category', false, true ); + $this->addArg( 'start', 'Page_id to start from, default 1', false ); + $this->setBatchSize( 100 ); + } + + public function execute() { + // Note that there is a difference between not specifying the start + // and end IDs and using the minimum and maximum values from the page + // table. In the latter case, deleteLinksFromNonexistent() will not + // delete entries for nonexistent IDs that fall outside the range. + $start = (int)$this->getArg( 0 ) ?: null; + $end = (int)$this->getOption( 'e' ) ?: null; + $dfnChunkSize = (int)$this->getOption( 'dfn-chunk-size', 100000 ); + $ns = $this->getOption( 'namespace' ); + if ( $ns === null ) { + $this->namespace = false; + } else { + $this->namespace = (int)$ns; + } + if ( ( $category = $this->getOption( 'category', false ) ) !== false ) { + $title = Title::makeTitleSafe( NS_CATEGORY, $category ); + if ( !$title ) { + $this->error( "'$category' is an invalid category name!\n", true ); + } + $this->refreshCategory( $title ); + } elseif ( ( $category = $this->getOption( 'tracking-category', false ) ) !== false ) { + $this->refreshTrackingCategory( $category ); + } elseif ( !$this->hasOption( 'dfn-only' ) ) { + $new = $this->hasOption( 'new-only' ); + $redir = $this->hasOption( 'redirects-only' ); + $oldRedir = $this->hasOption( 'old-redirects-only' ); + $this->doRefreshLinks( $start, $new, $end, $redir, $oldRedir ); + $this->deleteLinksFromNonexistent( null, null, $this->mBatchSize, $dfnChunkSize ); + } else { + $this->deleteLinksFromNonexistent( $start, $end, $this->mBatchSize, $dfnChunkSize ); + } + } + + private function namespaceCond() { + return $this->namespace !== false + ? [ 'page_namespace' => $this->namespace ] + : []; + } + + /** + * Do the actual link refreshing. + * @param int|null $start Page_id to start from + * @param bool $newOnly Only do pages with 1 edit + * @param int|null $end Page_id to stop at + * @param bool $redirectsOnly Only fix redirects + * @param bool $oldRedirectsOnly Only fix redirects without redirect entries + */ + private function doRefreshLinks( $start, $newOnly = false, + $end = null, $redirectsOnly = false, $oldRedirectsOnly = false + ) { + $dbr = $this->getDB( DB_REPLICA, [ 'vslow' ] ); + + if ( $start === null ) { + $start = 1; + } + + // Give extensions a chance to optimize settings + Hooks::run( 'MaintenanceRefreshLinksInit', [ $this ] ); + + $what = $redirectsOnly ? "redirects" : "links"; + + if ( $oldRedirectsOnly ) { + # This entire code path is cut-and-pasted from below. Hurrah. + + $conds = [ + "page_is_redirect=1", + "rd_from IS NULL", + self::intervalCond( $dbr, 'page_id', $start, $end ), + ] + $this->namespaceCond(); + + $res = $dbr->select( + [ 'page', 'redirect' ], + 'page_id', + $conds, + __METHOD__, + [], + [ 'redirect' => [ "LEFT JOIN", "page_id=rd_from" ] ] + ); + $num = $res->numRows(); + $this->output( "Refreshing $num old redirects from $start...\n" ); + + $i = 0; + + foreach ( $res as $row ) { + if ( !( ++$i % self::REPORTING_INTERVAL ) ) { + $this->output( "$i\n" ); + wfWaitForSlaves(); + } + $this->fixRedirect( $row->page_id ); + } + } elseif ( $newOnly ) { + $this->output( "Refreshing $what from " ); + $res = $dbr->select( 'page', + [ 'page_id' ], + [ + 'page_is_new' => 1, + self::intervalCond( $dbr, 'page_id', $start, $end ), + ] + $this->namespaceCond(), + __METHOD__ + ); + $num = $res->numRows(); + $this->output( "$num new articles...\n" ); + + $i = 0; + foreach ( $res as $row ) { + if ( !( ++$i % self::REPORTING_INTERVAL ) ) { + $this->output( "$i\n" ); + wfWaitForSlaves(); + } + if ( $redirectsOnly ) { + $this->fixRedirect( $row->page_id ); + } else { + self::fixLinksFromArticle( $row->page_id, $this->namespace ); + } + } + } else { + if ( !$end ) { + $maxPage = $dbr->selectField( 'page', 'max(page_id)', false ); + $maxRD = $dbr->selectField( 'redirect', 'max(rd_from)', false ); + $end = max( $maxPage, $maxRD ); + } + $this->output( "Refreshing redirects table.\n" ); + $this->output( "Starting from page_id $start of $end.\n" ); + + for ( $id = $start; $id <= $end; $id++ ) { + if ( !( $id % self::REPORTING_INTERVAL ) ) { + $this->output( "$id\n" ); + wfWaitForSlaves(); + } + $this->fixRedirect( $id ); + } + + if ( !$redirectsOnly ) { + $this->output( "Refreshing links tables.\n" ); + $this->output( "Starting from page_id $start of $end.\n" ); + + for ( $id = $start; $id <= $end; $id++ ) { + if ( !( $id % self::REPORTING_INTERVAL ) ) { + $this->output( "$id\n" ); + wfWaitForSlaves(); + } + self::fixLinksFromArticle( $id, $this->namespace ); + } + } + } + } + + /** + * Update the redirect entry for a given page. + * + * This methods bypasses the "redirect" table to get the redirect target, + * and parses the page's content to fetch it. This allows to be sure that + * the redirect target is up to date and valid. + * This is particularly useful when modifying namespaces to be sure the + * entry in the "redirect" table points to the correct page and not to an + * invalid one. + * + * @param int $id The page ID to check + */ + private function fixRedirect( $id ) { + $page = WikiPage::newFromID( $id ); + $dbw = $this->getDB( DB_MASTER ); + + if ( $page === null ) { + // This page doesn't exist (any more) + // Delete any redirect table entry for it + $dbw->delete( 'redirect', [ 'rd_from' => $id ], + __METHOD__ ); + + return; + } elseif ( $this->namespace !== false + && !$page->getTitle()->inNamespace( $this->namespace ) + ) { + return; + } + + $rt = null; + $content = $page->getContent( Revision::RAW ); + if ( $content !== null ) { + $rt = $content->getUltimateRedirectTarget(); + } + + if ( $rt === null ) { + // The page is not a redirect + // Delete any redirect table entry for it + $dbw->delete( 'redirect', [ 'rd_from' => $id ], __METHOD__ ); + $fieldValue = 0; + } else { + $page->insertRedirectEntry( $rt ); + $fieldValue = 1; + } + + // Update the page table to be sure it is an a consistent state + $dbw->update( 'page', [ 'page_is_redirect' => $fieldValue ], + [ 'page_id' => $id ], __METHOD__ ); + } + + /** + * Run LinksUpdate for all links on a given page_id + * @param int $id The page_id + * @param int|bool $ns Only fix links if it is in this namespace + */ + public static function fixLinksFromArticle( $id, $ns = false ) { + $page = WikiPage::newFromID( $id ); + + LinkCache::singleton()->clear(); + + if ( $page === null ) { + return; + } elseif ( $ns !== false + && !$page->getTitle()->inNamespace( $ns ) ) { + return; + } + + $content = $page->getContent( Revision::RAW ); + if ( $content === null ) { + return; + } + + $updates = $content->getSecondaryDataUpdates( + $page->getTitle(), /* $old = */ null, /* $recursive = */ false ); + foreach ( $updates as $update ) { + DeferredUpdates::addUpdate( $update ); + DeferredUpdates::doUpdates(); + } + } + + /** + * Removes non-existing links from pages from pagelinks, imagelinks, + * categorylinks, templatelinks, externallinks, interwikilinks, langlinks and redirect tables. + * + * @param int|null $start Page_id to start from + * @param int|null $end Page_id to stop at + * @param int $batchSize The size of deletion batches + * @param int $chunkSize Maximum number of existent IDs to check per query + * + * @author Merlijn van Deen <valhallasw@arctus.nl> + */ + private function deleteLinksFromNonexistent( $start = null, $end = null, $batchSize = 100, + $chunkSize = 100000 + ) { + wfWaitForSlaves(); + $this->output( "Deleting illegal entries from the links tables...\n" ); + $dbr = $this->getDB( DB_REPLICA, [ 'vslow' ] ); + do { + // Find the start of the next chunk. This is based only + // on existent page_ids. + $nextStart = $dbr->selectField( + 'page', + 'page_id', + [ self::intervalCond( $dbr, 'page_id', $start, $end ) ] + + $this->namespaceCond(), + __METHOD__, + [ 'ORDER BY' => 'page_id', 'OFFSET' => $chunkSize ] + ); + + if ( $nextStart !== false ) { + // To find the end of the current chunk, subtract one. + // This will serve to limit the number of rows scanned in + // dfnCheckInterval(), per query, to at most the sum of + // the chunk size and deletion batch size. + $chunkEnd = $nextStart - 1; + } else { + // This is the last chunk. Check all page_ids up to $end. + $chunkEnd = $end; + } + + $fmtStart = $start !== null ? "[$start" : '(-INF'; + $fmtChunkEnd = $chunkEnd !== null ? "$chunkEnd]" : 'INF)'; + $this->output( " Checking interval $fmtStart, $fmtChunkEnd\n" ); + $this->dfnCheckInterval( $start, $chunkEnd, $batchSize ); + + $start = $nextStart; + + } while ( $nextStart !== false ); + } + + /** + * @see RefreshLinks::deleteLinksFromNonexistent() + * @param int|null $start Page_id to start from + * @param int|null $end Page_id to stop at + * @param int $batchSize The size of deletion batches + */ + private function dfnCheckInterval( $start = null, $end = null, $batchSize = 100 ) { + $dbw = $this->getDB( DB_MASTER ); + $dbr = $this->getDB( DB_REPLICA, [ 'vslow' ] ); + + $linksTables = [ // table name => page_id field + 'pagelinks' => 'pl_from', + 'imagelinks' => 'il_from', + 'categorylinks' => 'cl_from', + 'templatelinks' => 'tl_from', + 'externallinks' => 'el_from', + 'iwlinks' => 'iwl_from', + 'langlinks' => 'll_from', + 'redirect' => 'rd_from', + 'page_props' => 'pp_page', + ]; + + foreach ( $linksTables as $table => $field ) { + $this->output( " $table: 0" ); + $tableStart = $start; + $counter = 0; + do { + $ids = $dbr->selectFieldValues( + $table, + $field, + [ + self::intervalCond( $dbr, $field, $tableStart, $end ), + "$field NOT IN ({$dbr->selectSQLText( 'page', 'page_id' )})", + ], + __METHOD__, + [ 'DISTINCT', 'ORDER BY' => $field, 'LIMIT' => $batchSize ] + ); + + $numIds = count( $ids ); + if ( $numIds ) { + $counter += $numIds; + $dbw->delete( $table, [ $field => $ids ], __METHOD__ ); + $this->output( ", $counter" ); + $tableStart = $ids[$numIds - 1] + 1; + wfWaitForSlaves(); + } + + } while ( $numIds >= $batchSize && ( $end === null || $tableStart <= $end ) ); + + $this->output( " deleted.\n" ); + } + } + + /** + * Build a SQL expression for a closed interval (i.e. BETWEEN). + * + * By specifying a null $start or $end, it is also possible to create + * half-bounded or unbounded intervals using this function. + * + * @param IDatabase $db + * @param string $var Field name + * @param mixed $start First value to include or null + * @param mixed $end Last value to include or null + * @return string + */ + private static function intervalCond( IDatabase $db, $var, $start, $end ) { + if ( $start === null && $end === null ) { + return "$var IS NOT NULL"; + } elseif ( $end === null ) { + return "$var >= {$db->addQuotes( $start )}"; + } elseif ( $start === null ) { + return "$var <= {$db->addQuotes( $end )}"; + } else { + return "$var BETWEEN {$db->addQuotes( $start )} AND {$db->addQuotes( $end )}"; + } + } + + /** + * Refershes links for pages in a tracking category + * + * @param string $category Category key + */ + private function refreshTrackingCategory( $category ) { + $cats = $this->getPossibleCategories( $category ); + + if ( !$cats ) { + $this->error( "Tracking category '$category' is disabled\n" ); + // Output to stderr but don't bail out, + } + + foreach ( $cats as $cat ) { + $this->refreshCategory( $cat ); + } + } + + /** + * Refreshes links to a category + * + * @param Title $category + */ + private function refreshCategory( Title $category ) { + $this->output( "Refreshing pages in category '{$category->getText()}'...\n" ); + + $dbr = $this->getDB( DB_REPLICA ); + $conds = [ + 'page_id=cl_from', + 'cl_to' => $category->getDBkey(), + ]; + if ( $this->namespace !== false ) { + $conds['page_namespace'] = $this->namespace; + } + + $i = 0; + $timestamp = ''; + $lastId = 0; + do { + $finalConds = $conds; + $timestamp = $dbr->addQuotes( $timestamp ); + $finalConds [] = + "(cl_timestamp > $timestamp OR (cl_timestamp = $timestamp AND cl_from > $lastId))"; + $res = $dbr->select( [ 'page', 'categorylinks' ], + [ 'page_id', 'cl_timestamp' ], + $finalConds, + __METHOD__, + [ + 'ORDER BY' => [ 'cl_timestamp', 'cl_from' ], + 'LIMIT' => $this->mBatchSize, + ] + ); + + foreach ( $res as $row ) { + if ( !( ++$i % self::REPORTING_INTERVAL ) ) { + $this->output( "$i\n" ); + wfWaitForSlaves(); + } + $lastId = $row->page_id; + $timestamp = $row->cl_timestamp; + self::fixLinksFromArticle( $row->page_id ); + } + + } while ( $res->numRows() == $this->mBatchSize ); + } + + /** + * Returns a list of possible categories for a given tracking category key + * + * @param string $categoryKey + * @return Title[] + */ + private function getPossibleCategories( $categoryKey ) { + $trackingCategories = new TrackingCategories( $this->getConfig() ); + $cats = $trackingCategories->getTrackingCategories(); + if ( isset( $cats[$categoryKey] ) ) { + return $cats[$categoryKey]['cats']; + } + $this->error( "Unknown tracking category {$categoryKey}\n", true ); + } +} + +$maintClass = 'RefreshLinks'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/removeInvalidEmails.php b/www/wiki/maintenance/removeInvalidEmails.php new file mode 100644 index 00000000..1034005a --- /dev/null +++ b/www/wiki/maintenance/removeInvalidEmails.php @@ -0,0 +1,78 @@ +<?php + +require_once __DIR__ . '/Maintenance.php'; + +/** + * A script to remove emails that are invalid from + * the user_email column of the user table. Emails + * are validated before users can add them, but + * this was not always the case so older users may + * have invalid ones. + * + * By default it does a dry-run, pass --commit + * to actually update the database. + */ +class RemoveInvalidEmails extends Maintenance { + + private $commit = false; + + public function __construct() { + parent::__construct(); + $this->addOption( 'commit', 'Whether to actually update the database', false, false ); + $this->setBatchSize( 500 ); + } + public function execute() { + $this->commit = $this->hasOption( 'commit' ); + $dbr = $this->getDB( DB_REPLICA ); + $dbw = $this->getDB( DB_MASTER ); + $lastId = 0; + do { + $rows = $dbr->select( + 'user', + [ 'user_id', 'user_email' ], + [ + 'user_id > ' . $dbr->addQuotes( $lastId ), + 'user_email != ""', + 'user_email_authenticated IS NULL' + ], + __METHOD__, + [ 'LIMIT' => $this->mBatchSize ] + ); + $count = $rows->numRows(); + $badIds = []; + foreach ( $rows as $row ) { + if ( !Sanitizer::validateEmail( trim( $row->user_email ) ) ) { + $this->output( "Found bad email: {$row->user_email} for user #{$row->user_id}\n" ); + $badIds[] = $row->user_id; + } + if ( $row->user_id > $lastId ) { + $lastId = $row->user_id; + } + } + + if ( $badIds ) { + $badCount = count( $badIds ); + if ( $this->commit ) { + $this->output( "Removing $badCount emails from the database.\n" ); + $dbw->update( + 'user', + [ 'user_email' => '' ], + [ 'user_id' => $badIds ], + __METHOD__ + ); + foreach ( $badIds as $badId ) { + User::newFromId( $badId )->invalidateCache(); + } + wfWaitForSlaves(); + } else { + $this->output( "Would have removed $badCount emails from the database.\n" ); + + } + } + } while ( $count !== 0 ); + $this->output( "Done.\n" ); + } +} + +$maintClass = 'RemoveInvalidEmails'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/removeUnusedAccounts.php b/www/wiki/maintenance/removeUnusedAccounts.php new file mode 100644 index 00000000..c750784e --- /dev/null +++ b/www/wiki/maintenance/removeUnusedAccounts.php @@ -0,0 +1,135 @@ +<?php +/** + * Remove unused user accounts from the database + * An unused account is one which has made no edits + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @author Rob Church <robchur@gmail.com> + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that removes unused user accounts from the database. + * + * @ingroup Maintenance + */ +class RemoveUnusedAccounts extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addOption( 'delete', 'Actually delete the account' ); + $this->addOption( 'ignore-groups', 'List of comma-separated groups to exclude', false, true ); + $this->addOption( 'ignore-touched', 'Skip accounts touched in last N days', false, true ); + } + + public function execute() { + $this->output( "Remove unused accounts\n\n" ); + + # Do an initial scan for inactive accounts and report the result + $this->output( "Checking for unused user accounts...\n" ); + $del = []; + $dbr = $this->getDB( DB_REPLICA ); + $res = $dbr->select( 'user', [ 'user_id', 'user_name', 'user_touched' ], '', __METHOD__ ); + if ( $this->hasOption( 'ignore-groups' ) ) { + $excludedGroups = explode( ',', $this->getOption( 'ignore-groups' ) ); + } else { + $excludedGroups = []; + } + $touched = $this->getOption( 'ignore-touched', "1" ); + if ( !ctype_digit( $touched ) ) { + $this->error( "Please put a valid positive integer on the --ignore-touched parameter.", true ); + } + $touchedSeconds = 86400 * $touched; + foreach ( $res as $row ) { + # Check the account, but ignore it if it's within a $excludedGroups + # group or if it's touched within the $touchedSeconds seconds. + $instance = User::newFromId( $row->user_id ); + if ( count( array_intersect( $instance->getEffectiveGroups(), $excludedGroups ) ) == 0 + && $this->isInactiveAccount( $row->user_id, true ) + && wfTimestamp( TS_UNIX, $row->user_touched ) < wfTimestamp( TS_UNIX, time() - $touchedSeconds ) + ) { + # Inactive; print out the name and flag it + $del[] = $row->user_id; + $this->output( $row->user_name . "\n" ); + } + } + $count = count( $del ); + $this->output( "...found {$count}.\n" ); + + # If required, go back and delete each marked account + if ( $count > 0 && $this->hasOption( 'delete' ) ) { + $this->output( "\nDeleting unused accounts..." ); + $dbw = $this->getDB( DB_MASTER ); + $dbw->delete( 'user', [ 'user_id' => $del ], __METHOD__ ); + $dbw->delete( 'user_groups', [ 'ug_user' => $del ], __METHOD__ ); + $dbw->delete( 'user_former_groups', [ 'ufg_user' => $del ], __METHOD__ ); + $dbw->delete( 'user_properties', [ 'up_user' => $del ], __METHOD__ ); + $dbw->delete( 'logging', [ 'log_user' => $del ], __METHOD__ ); + $dbw->delete( 'recentchanges', [ 'rc_user' => $del ], __METHOD__ ); + $this->output( "done.\n" ); + # Update the site_stats.ss_users field + $users = $dbw->selectField( 'user', 'COUNT(*)', [], __METHOD__ ); + $dbw->update( + 'site_stats', + [ 'ss_users' => $users ], + [ 'ss_row_id' => 1 ], + __METHOD__ + ); + } elseif ( $count > 0 ) { + $this->output( "\nRun the script again with --delete to remove them from the database.\n" ); + } + $this->output( "\n" ); + } + + /** + * Could the specified user account be deemed inactive? + * (No edits, no deleted edits, no log entries, no current/old uploads) + * + * @param int $id User's ID + * @param bool $master Perform checking on the master + * @return bool + */ + private function isInactiveAccount( $id, $master = false ) { + $dbo = $this->getDB( $master ? DB_MASTER : DB_REPLICA ); + $checks = [ + 'revision' => 'rev', + 'archive' => 'ar', + 'image' => 'img', + 'oldimage' => 'oi', + 'filearchive' => 'fa' + ]; + $count = 0; + + $this->beginTransaction( $dbo, __METHOD__ ); + foreach ( $checks as $table => $fprefix ) { + $conds = [ $fprefix . '_user' => $id ]; + $count += (int)$dbo->selectField( $table, 'COUNT(*)', $conds, __METHOD__ ); + } + + $conds = [ 'log_user' => $id, 'log_type != ' . $dbo->addQuotes( 'newusers' ) ]; + $count += (int)$dbo->selectField( 'logging', 'COUNT(*)', $conds, __METHOD__ ); + + $this->commitTransaction( $dbo, __METHOD__ ); + + return $count == 0; + } +} + +$maintClass = "RemoveUnusedAccounts"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/renameDbPrefix.php b/www/wiki/maintenance/renameDbPrefix.php new file mode 100644 index 00000000..2772f04b --- /dev/null +++ b/www/wiki/maintenance/renameDbPrefix.php @@ -0,0 +1,94 @@ +<?php +/** + * Change the prefix of database tables. + * Run this script to after changing $wgDBprefix on a wiki. + * The wiki will have to get downtime to do this correctly. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that changes the prefix of database tables. + * + * @ingroup Maintenance + */ +class RenameDbPrefix extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addOption( "old", "Old db prefix [0 for none]", true, true ); + $this->addOption( "new", "New db prefix [0 for none]", true, true ); + } + + public function getDbType() { + return Maintenance::DB_ADMIN; + } + + public function execute() { + global $wgDBname; + + // Allow for no old prefix + if ( $this->getOption( 'old', 0 ) === '0' ) { + $old = ''; + } else { + // Use nice safe, sane, prefixes + preg_match( '/^[a-zA-Z]+_$/', $this->getOption( 'old' ), $m ); + $old = isset( $m[0] ) ? $m[0] : false; + } + // Allow for no new prefix + if ( $this->getOption( 'new', 0 ) === '0' ) { + $new = ''; + } else { + // Use nice safe, sane, prefixes + preg_match( '/^[a-zA-Z]+_$/', $this->getOption( 'new' ), $m ); + $new = isset( $m[0] ) ? $m[0] : false; + } + + if ( $old === false || $new === false ) { + $this->error( "Invalid prefix!", true ); + } + if ( $old === $new ) { + $this->output( "Same prefix. Nothing to rename!\n", true ); + } + + $this->output( "Renaming DB prefix for tables of $wgDBname from '$old' to '$new'\n" ); + $count = 0; + + $dbw = $this->getDB( DB_MASTER ); + $res = $dbw->query( "SHOW TABLES " . $dbw->buildLike( $old, $dbw->anyString() ) ); + foreach ( $res as $row ) { + // XXX: odd syntax. MySQL outputs an oddly cased "Tables of X" + // sort of message. Best not to try $row->x stuff... + $fields = get_object_vars( $row ); + // Silly for loop over one field... + foreach ( $fields as $table ) { + // $old should be regexp safe ([a-zA-Z_]) + $newTable = preg_replace( '/^' . $old . '/', $new, $table ); + $this->output( "Renaming table $table to $newTable\n" ); + $dbw->query( "RENAME TABLE $table TO $newTable" ); + } + $count++; + } + $this->output( "Done! [$count tables]\n" ); + } +} + +$maintClass = "RenameDbPrefix"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/renderDump.php b/www/wiki/maintenance/renderDump.php new file mode 100644 index 00000000..68a371c3 --- /dev/null +++ b/www/wiki/maintenance/renderDump.php @@ -0,0 +1,124 @@ +<?php +/** + * Take page text out of an XML dump file and render basic HTML out to files. + * This is *NOT* suitable for publishing or offline use; it's intended for + * running comparative tests of parsing behavior using real-world data. + * + * Templates etc are pulled from the local wiki database, not from the dump. + * + * Copyright (C) 2006 Brion Vibber <brion@pobox.com> + * https://www.mediawiki.org/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that takes page text out of an XML dump file + * and render basic HTML out to files. + * + * @ingroup Maintenance + */ +class DumpRenderer extends Maintenance { + + private $count = 0; + private $outputDirectory, $startTime; + + public function __construct() { + parent::__construct(); + $this->addDescription( + 'Take page text out of an XML dump file and render basic HTML out to files' ); + $this->addOption( 'output-dir', 'The directory to output the HTML files to', true, true ); + $this->addOption( 'prefix', 'Prefix for the rendered files (defaults to wiki)', false, true ); + $this->addOption( 'parser', 'Use an alternative parser class', false, true ); + } + + public function execute() { + $this->outputDirectory = $this->getOption( 'output-dir' ); + $this->prefix = $this->getOption( 'prefix', 'wiki' ); + $this->startTime = microtime( true ); + + if ( $this->hasOption( 'parser' ) ) { + global $wgParserConf; + $wgParserConf['class'] = $this->getOption( 'parser' ); + $this->prefix .= "-{$wgParserConf['class']}"; + } + + $source = new ImportStreamSource( $this->getStdin() ); + $importer = new WikiImporter( $source, $this->getConfig() ); + + $importer->setRevisionCallback( + [ $this, 'handleRevision' ] ); + + $importer->doImport(); + + $delta = microtime( true ) - $this->startTime; + $this->error( "Rendered {$this->count} pages in " . round( $delta, 2 ) . " seconds " ); + if ( $delta > 0 ) { + $this->error( round( $this->count / $delta, 2 ) . " pages/sec" ); + } + $this->error( "\n" ); + } + + /** + * Callback function for each revision, turn into HTML and save + * @param Revision $rev + */ + public function handleRevision( $rev ) { + $title = $rev->getTitle(); + if ( !$title ) { + $this->error( "Got bogus revision with null title!" ); + + return; + } + $display = $title->getPrefixedText(); + + $this->count++; + + $sanitized = rawurlencode( $display ); + $filename = sprintf( "%s/%s-%07d-%s.html", + $this->outputDirectory, + $this->prefix, + $this->count, + $sanitized ); + $this->output( sprintf( "%s\n", $filename, $display ) ); + + $user = new User(); + $options = ParserOptions::newFromUser( $user ); + + $content = $rev->getContent(); + $output = $content->getParserOutput( $title, null, $options ); + + file_put_contents( $filename, + "<!DOCTYPE html>\n" . + "<html lang=\"en\" dir=\"ltr\">\n" . + "<head>\n" . + "<meta charset=\"UTF-8\" />\n" . + "<title>" . htmlspecialchars( $display ) . "</title>\n" . + "</head>\n" . + "<body>\n" . + $output->getText() . + "</body>\n" . + "</html>" ); + } +} + +$maintClass = "DumpRenderer"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/resetUserEmail.php b/www/wiki/maintenance/resetUserEmail.php new file mode 100644 index 00000000..8d0873f1 --- /dev/null +++ b/www/wiki/maintenance/resetUserEmail.php @@ -0,0 +1,72 @@ +<?php +/** + * Reset user email. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that resets user email. + * + * @since 1.27 + * @ingroup Maintenance + */ +class ResetUserEmail extends Maintenance { + public function __construct() { + $this->addDescription( "Resets a user's email" ); + $this->addArg( 'user', 'Username or user ID, if starts with #', true ); + $this->addArg( 'email', 'Email to assign' ); + + $this->addOption( 'no-reset-password', 'Don\'t reset the user\'s password', false, false ); + + parent::__construct(); + } + + public function execute() { + $userName = $this->getArg( 0 ); + if ( preg_match( '/^#\d+$/', $userName ) ) { + $user = User::newFromId( substr( $userName, 1 ) ); + } else { + $user = User::newFromName( $userName ); + } + if ( !$user || !$user->getId() || !$user->loadFromId() ) { + $this->error( "Error: user '$userName' does not exist\n", 1 ); + } + + $email = $this->getArg( 1 ); + if ( !Sanitizer::validateEmail( $email ) ) { + $this->error( "Error: email '$email' is not valid\n", 1 ); + } + + // Code from https://wikitech.wikimedia.org/wiki/Password_reset + $user->setEmail( $email ); + $user->setEmailAuthenticationTimestamp( wfTimestampNow() ); + $user->saveSettings(); + + if ( !$this->hasOption( 'no-reset-password' ) ) { + // Kick whomever is currently controlling the account off + $user->setPassword( PasswordFactory::generateRandomPasswordString( 128 ) ); + } + } +} + +$maintClass = 'ResetUserEmail'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/resetUserTokens.php b/www/wiki/maintenance/resetUserTokens.php new file mode 100644 index 00000000..481da980 --- /dev/null +++ b/www/wiki/maintenance/resetUserTokens.php @@ -0,0 +1,120 @@ +<?php +/** + * Reset the user_token for all users on the wiki. Useful if you believe + * that your user table was acidentally leaked to an external source. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @author Daniel Friesen <mediawiki@danielfriesen.name> + * @author Chris Steipp <csteipp@wikimedia.org> + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to reset the user_token for all users on the wiki. + * + * @ingroup Maintenance + * @deprecated since 1.27, use $wgAuthenticationTokenVersion instead. + */ +class ResetUserTokens extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( + "Reset the user_token of all users on the wiki. Note that this may log some of them out.\n" + . "Deprecated, use \$wgAuthenticationTokenVersion instead." + ); + $this->addOption( 'nowarn', "Hides the 5 seconds warning", false, false ); + $this->addOption( + 'nulls', + 'Only reset tokens that are currently null (string of \x00\'s)', + false, + false + ); + $this->setBatchSize( 1000 ); + } + + public function execute() { + $this->nullsOnly = $this->getOption( 'nulls' ); + + if ( !$this->getOption( 'nowarn' ) ) { + if ( $this->nullsOnly ) { + $this->output( "The script is about to reset the user_token " + . "for USERS WITH NULL TOKENS in the database.\n" ); + } else { + $this->output( "The script is about to reset the user_token for ALL USERS in the database.\n" ); + $this->output( "This may log some of them out and is not necessary unless you believe your\n" ); + $this->output( "user table has been compromised.\n" ); + } + $this->output( "\n" ); + $this->output( "Abort with control-c in the next five seconds " + . "(skip this countdown with --nowarn) ... " ); + wfCountDown( 5 ); + } + + // We list user by user_id from one of the replica DBs + // We list user by user_id from one of the slave database + $dbr = $this->getDB( DB_REPLICA ); + + $where = []; + if ( $this->nullsOnly ) { + // Have to build this by hand, because \ is escaped in helper functions + $where = [ 'user_token = \'' . str_repeat( '\0', 32 ) . '\'' ]; + } + + $maxid = $dbr->selectField( 'user', 'MAX(user_id)', [], __METHOD__ ); + + $min = 0; + $max = $this->mBatchSize; + + do { + $result = $dbr->select( 'user', + [ 'user_id' ], + array_merge( + $where, + [ 'user_id > ' . $dbr->addQuotes( $min ), + 'user_id <= ' . $dbr->addQuotes( $max ) + ] + ), + __METHOD__ + ); + + foreach ( $result as $user ) { + $this->updateUser( $user->user_id ); + } + + $min = $max; + $max = $min + $this->mBatchSize; + + wfWaitForSlaves(); + } while ( $min <= $maxid ); + } + + private function updateUser( $userid ) { + $user = User::newFromId( $userid ); + $username = $user->getName(); + $this->output( 'Resetting user_token for "' . $username . '": ' ); + // Change value + $user->setToken(); + $user->saveSettings(); + $this->output( " OK\n" ); + } +} + +$maintClass = "ResetUserTokens"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/resources/update-oojs-ui.sh b/www/wiki/maintenance/resources/update-oojs-ui.sh new file mode 100755 index 00000000..799af4ca --- /dev/null +++ b/www/wiki/maintenance/resources/update-oojs-ui.sh @@ -0,0 +1,85 @@ +#!/bin/bash -eu + +# This script generates a commit that updates our copy of OOjs UI + +if [ -n "${2:-}" ] +then + # Too many parameters + echo >&2 "Usage: $0 [<version>]" + exit 1 +fi + +REPO_DIR=$(cd "$(dirname $0)/../.."; pwd) # Root dir of the git repo working tree +TARGET_DIR="resources/lib/oojs-ui" # Destination relative to the root of the repo +NPM_DIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'update-oojs-ui') # e.g. /tmp/update-oojs-ui.rI0I5Vir + +# Prepare working tree +cd "$REPO_DIR" +git reset composer.json +git checkout composer.json +git reset -- $TARGET_DIR +git checkout -- $TARGET_DIR +git fetch origin +git checkout -B upstream-oojs-ui origin/master + +# Fetch upstream version +cd $NPM_DIR +if [ -n "${1:-}" ] +then + npm install "oojs-ui@$1" +else + npm install oojs-ui +fi + +OOJSUI_VERSION=$(node -e 'console.log(require("./node_modules/oojs-ui/package.json").version);') +if [ "$OOJSUI_VERSION" == "" ] +then + echo 'Could not find OOjs UI version' + exit 1 +fi + +# Copy files, picking the necessary ones from source and distribution +rm -r "$REPO_DIR/$TARGET_DIR" +mkdir -p "$REPO_DIR/$TARGET_DIR/i18n" +mkdir -p "$REPO_DIR/$TARGET_DIR/images" +mkdir -p "$REPO_DIR/$TARGET_DIR/themes/wikimediaui/images" +mkdir -p "$REPO_DIR/$TARGET_DIR/themes/apex/images" +cp ./node_modules/oojs-ui/dist/oojs-ui-core.js{,.map} "$REPO_DIR/$TARGET_DIR" +cp ./node_modules/oojs-ui/dist/oojs-ui-core-{wikimediaui,apex}.css "$REPO_DIR/$TARGET_DIR" +cp ./node_modules/oojs-ui/dist/oojs-ui-widgets.js{,.map} "$REPO_DIR/$TARGET_DIR" +cp ./node_modules/oojs-ui/dist/oojs-ui-widgets-{wikimediaui,apex}.css "$REPO_DIR/$TARGET_DIR" +cp ./node_modules/oojs-ui/dist/oojs-ui-toolbars.js{,.map} "$REPO_DIR/$TARGET_DIR" +cp ./node_modules/oojs-ui/dist/oojs-ui-toolbars-{wikimediaui,apex}.css "$REPO_DIR/$TARGET_DIR" +cp ./node_modules/oojs-ui/dist/oojs-ui-windows.js{,.map} "$REPO_DIR/$TARGET_DIR" +cp ./node_modules/oojs-ui/dist/oojs-ui-windows-{wikimediaui,apex}.css "$REPO_DIR/$TARGET_DIR" +cp ./node_modules/oojs-ui/dist/oojs-ui-{wikimediaui,apex}.js{,.map} "$REPO_DIR/$TARGET_DIR" +cp -R ./node_modules/oojs-ui/dist/i18n "$REPO_DIR/$TARGET_DIR" +cp -R ./node_modules/oojs-ui/dist/images "$REPO_DIR/$TARGET_DIR" +cp -R ./node_modules/oojs-ui/dist/themes/wikimediaui/images "$REPO_DIR/$TARGET_DIR/themes/wikimediaui" +cp ./node_modules/oojs-ui/src/themes/wikimediaui/*.json "$REPO_DIR/$TARGET_DIR/themes/wikimediaui" +cp -R ./node_modules/oojs-ui/dist/themes/apex/images "$REPO_DIR/$TARGET_DIR/themes/apex" +cp ./node_modules/oojs-ui/src/themes/apex/*.json "$REPO_DIR/$TARGET_DIR/themes/apex" +cp ./node_modules/oojs-ui/dist/wikimedia-ui-base.less "$REPO_DIR/$TARGET_DIR" + +# Clean up temporary area +rm -rf "$NPM_DIR" + +# Generate commit +cd $REPO_DIR + +COMMITMSG=$(cat <<END +Update OOjs UI to v$OOJSUI_VERSION + +Release notes: + https://phabricator.wikimedia.org/diffusion/GOJU/browse/master/History.md;v$OOJSUI_VERSION +END +) + +# Update composer.json as well +composer require oojs/oojs-ui $OOJSUI_VERSION --no-update + +# Stage deletion, modification and creation of files. Then commit. +git add --update $TARGET_DIR +git add $TARGET_DIR +git add composer.json +git commit -m "$COMMITMSG" diff --git a/www/wiki/maintenance/resources/update-oojs.sh b/www/wiki/maintenance/resources/update-oojs.sh new file mode 100755 index 00000000..267bd961 --- /dev/null +++ b/www/wiki/maintenance/resources/update-oojs.sh @@ -0,0 +1,62 @@ +#!/bin/bash -eu + +# This script generates a commit that updates our copy of OOjs + +if [ -n "${2:-}" ] +then + # Too many parameters + echo >&2 "Usage: $0 [<version>]" + exit 1 +fi + +REPO_DIR=$(cd "$(dirname $0)/../.."; pwd) # Root dir of the git repo working tree +TARGET_DIR="resources/lib/oojs" # Destination relative to the root of the repo +NPM_DIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'update-oojs') # e.g. /tmp/update-oojs.rI0I5Vir + +# Prepare working tree +cd "$REPO_DIR" +git reset -- $TARGET_DIR +git checkout -- $TARGET_DIR +git fetch origin +git checkout -B upstream-oojs origin/master + +# Fetch upstream version +cd $NPM_DIR +if [ -n "${1:-}" ] +then + npm install "oojs@$1" +else + npm install oojs +fi + +OOJS_VERSION=$(node -e 'console.log(require("./node_modules/oojs/package.json").version);') +if [ "$OOJS_VERSION" == "" ] +then + echo 'Could not find OOjs version' + exit 1 +fi + +# Copy file(s) +rsync --force ./node_modules/oojs/dist/oojs.jquery.js "$REPO_DIR/$TARGET_DIR" +rsync --force ./node_modules/oojs/dist/AUTHORS.txt "$REPO_DIR/$TARGET_DIR" +rsync --force ./node_modules/oojs/dist/LICENSE-MIT "$REPO_DIR/$TARGET_DIR" +rsync --force ./node_modules/oojs/dist/README.md "$REPO_DIR/$TARGET_DIR" + +# Clean up temporary area +rm -rf "$NPM_DIR" + +# Generate commit +cd $REPO_DIR + +COMMITMSG=$(cat <<END +Update OOjs to v$OOJS_VERSION + +Release notes: + https://phabricator.wikimedia.org/diffusion/GOJS/browse/master/History.md;v$OOJS_VERSION +END +) + +# Stage deletion, modification and creation of files. Then commit. +git add --update $TARGET_DIR +git add $TARGET_DIR +git commit -m "$COMMITMSG" diff --git a/www/wiki/maintenance/rollbackEdits.php b/www/wiki/maintenance/rollbackEdits.php new file mode 100644 index 00000000..5ad7d4e1 --- /dev/null +++ b/www/wiki/maintenance/rollbackEdits.php @@ -0,0 +1,115 @@ +<?php +/** + * Rollback all edits by a given user or IP provided they're the most + * recent edit (just like real rollback) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to rollback all edits by a given user or IP provided + * they're the most recent edit. + * + * @ingroup Maintenance + */ +class RollbackEdits extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( + "Rollback all edits by a given user or IP provided they're the most recent edit" ); + $this->addOption( + 'titles', + 'A list of titles, none means all titles where the given user is the most recent', + false, + true + ); + $this->addOption( 'user', 'A user or IP to rollback all edits for', true, true ); + $this->addOption( 'summary', 'Edit summary to use', false, true ); + $this->addOption( 'bot', 'Mark the edits as bot' ); + } + + public function execute() { + $user = $this->getOption( 'user' ); + $username = User::isIP( $user ) ? $user : User::getCanonicalName( $user ); + if ( !$username ) { + $this->error( 'Invalid username', true ); + } + + $bot = $this->hasOption( 'bot' ); + $summary = $this->getOption( 'summary', $this->mSelf . ' mass rollback' ); + $titles = []; + $results = []; + if ( $this->hasOption( 'titles' ) ) { + foreach ( explode( '|', $this->getOption( 'titles' ) ) as $title ) { + $t = Title::newFromText( $title ); + if ( !$t ) { + $this->error( 'Invalid title, ' . $title ); + } else { + $titles[] = $t; + } + } + } else { + $titles = $this->getRollbackTitles( $user ); + } + + if ( !$titles ) { + $this->output( 'No suitable titles to be rolled back' ); + + return; + } + + $doer = User::newSystemUser( 'Maintenance script', [ 'steal' => true ] ); + + foreach ( $titles as $t ) { + $page = WikiPage::factory( $t ); + $this->output( 'Processing ' . $t->getPrefixedText() . '... ' ); + if ( !$page->commitRollback( $user, $summary, $bot, $results, $doer ) ) { + $this->output( "done\n" ); + } else { + $this->output( "failed\n" ); + } + } + } + + /** + * Get all pages that should be rolled back for a given user + * @param string $user A name to check against rev_user_text + * @return array + */ + private function getRollbackTitles( $user ) { + $dbr = $this->getDB( DB_REPLICA ); + $titles = []; + $results = $dbr->select( + [ 'page', 'revision' ], + [ 'page_namespace', 'page_title' ], + [ 'page_latest = rev_id', 'rev_user_text' => $user ], + __METHOD__ + ); + foreach ( $results as $row ) { + $titles[] = Title::makeTitle( $row->page_namespace, $row->page_title ); + } + + return $titles; + } +} + +$maintClass = 'RollbackEdits'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/runBatchedQuery.php b/www/wiki/maintenance/runBatchedQuery.php new file mode 100644 index 00000000..b0a2b924 --- /dev/null +++ b/www/wiki/maintenance/runBatchedQuery.php @@ -0,0 +1,115 @@ +<?php +/** + * Run a database query in batches and wait for replica DBs. This is used on large + * wikis to prevent replication lag from going through the roof when executing + * large write queries. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +use Wikimedia\Rdbms\IDatabase; + +/** + * Maintenance script to run a database query in batches and wait for replica DBs. + * + * @ingroup Maintenance + */ +class BatchedQueryRunner extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( + "Run an update query on all rows of a table. " . + "Waits for replicas at appropriate intervals." ); + $this->addOption( 'table', 'The table name', true, true ); + $this->addOption( 'set', 'The SET clause', true, true ); + $this->addOption( 'where', 'The WHERE clause', false, true ); + $this->addOption( 'key', 'A column name, the values of which are unique', true, true ); + $this->addOption( 'batch-size', 'The batch size (default 1000)', false, true ); + $this->addOption( 'db', 'The database name, or omit to use the current wiki.', false, true ); + } + + public function execute() { + $table = $this->getOption( 'table' ); + $key = $this->getOption( 'key' ); + $set = $this->getOption( 'set' ); + $where = $this->getOption( 'where', null ); + $where = $where === null ? [] : [ $where ]; + $batchSize = $this->getOption( 'batch-size', 1000 ); + + $dbName = $this->getOption( 'db', null ); + if ( $dbName === null ) { + $dbw = $this->getDB( DB_MASTER ); + } else { + $lbf = MediaWiki\MediaWikiServices::getInstance()->getDBLoadBalancerFactory(); + $lb = $lbf->getMainLB( $dbName ); + $dbw = $lb->getConnection( DB_MASTER, [], $dbName ); + } + + $selectConds = $where; + $prevEnd = false; + + $n = 1; + do { + $this->output( "Batch $n: " ); + $n++; + + // Note that the update conditions do not rely on atomicity of the + // SELECT query in order to guarantee that all rows are updated. The + // results of the SELECT are merely a partitioning hint. Simultaneous + // updates merely result in the wrong number of rows being updated + // in a batch. + + $res = $dbw->select( $table, $key, $selectConds, __METHOD__, + [ 'ORDER BY' => $key, 'LIMIT' => $batchSize ] ); + if ( $res->numRows() ) { + $res->seek( $res->numRows() - 1 ); + $row = $res->fetchObject(); + $end = $dbw->addQuotes( $row->$key ); + $selectConds = array_merge( $where, [ "$key > $end" ] ); + $updateConds = array_merge( $where, [ "$key <= $end" ] ); + } else { + $updateConds = $where; + } + if ( $prevEnd !== false ) { + $updateConds = array_merge( [ "$key > $prevEnd" ], $updateConds ); + } + + $query = "UPDATE " . $dbw->tableName( $table ) . + " SET " . $set . + " WHERE " . $dbw->makeList( $updateConds, IDatabase::LIST_AND ); + + $dbw->query( $query, __METHOD__ ); + + $prevEnd = $end; + + $affected = $dbw->affectedRows(); + $this->output( "$affected rows affected\n" ); + wfWaitForSlaves(); + } while ( $res->numRows() ); + } + + public function getDbType() { + return Maintenance::DB_ADMIN; + } +} + +$maintClass = "BatchedQueryRunner"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/runJobs.php b/www/wiki/maintenance/runJobs.php new file mode 100644 index 00000000..2e011fec --- /dev/null +++ b/www/wiki/maintenance/runJobs.php @@ -0,0 +1,119 @@ +<?php +/** + * Run pending jobs. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +use MediaWiki\Logger\LoggerFactory; + +/** + * Maintenance script that runs pending jobs. + * + * @ingroup Maintenance + */ +class RunJobs extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Run pending jobs' ); + $this->addOption( 'maxjobs', 'Maximum number of jobs to run', false, true ); + $this->addOption( 'maxtime', 'Maximum amount of wall-clock time', false, true ); + $this->addOption( 'type', 'Type of job to run', false, true ); + $this->addOption( 'procs', 'Number of processes to use', false, true ); + $this->addOption( 'nothrottle', 'Ignore job throttling configuration', false, false ); + $this->addOption( 'result', 'Set to JSON to print only a JSON response', false, true ); + $this->addOption( 'wait', 'Wait for new jobs instead of exiting', false, false ); + } + + public function memoryLimit() { + if ( $this->hasOption( 'memory-limit' ) ) { + return parent::memoryLimit(); + } + + // Don't eat all memory on the machine if we get a bad job. + return "150M"; + } + + public function execute() { + if ( $this->hasOption( 'procs' ) ) { + $procs = intval( $this->getOption( 'procs' ) ); + if ( $procs < 1 || $procs > 1000 ) { + $this->error( "Invalid argument to --procs", true ); + } elseif ( $procs != 1 ) { + $fc = new ForkController( $procs ); + if ( $fc->start() != 'child' ) { + exit( 0 ); + } + } + } + + $outputJSON = ( $this->getOption( 'result' ) === 'json' ); + $wait = $this->hasOption( 'wait' ); + + $runner = new JobRunner( LoggerFactory::getInstance( 'runJobs' ) ); + if ( !$outputJSON ) { + $runner->setDebugHandler( [ $this, 'debugInternal' ] ); + } + + $type = $this->getOption( 'type', false ); + $maxJobs = $this->getOption( 'maxjobs', false ); + $maxTime = $this->getOption( 'maxtime', false ); + $throttle = !$this->hasOption( 'nothrottle' ); + + while ( true ) { + $response = $runner->run( [ + 'type' => $type, + 'maxJobs' => $maxJobs, + 'maxTime' => $maxTime, + 'throttle' => $throttle, + ] ); + + if ( $outputJSON ) { + $this->output( FormatJson::encode( $response, true ) ); + } + + if ( + !$wait || + $response['reached'] === 'time-limit' || + $response['reached'] === 'job-limit' || + $response['reached'] === 'memory-limit' + ) { + break; + } + + if ( $maxJobs !== false ) { + $maxJobs -= count( $response['jobs'] ); + } + + sleep( 1 ); + } + } + + /** + * @param string $s + */ + public function debugInternal( $s ) { + $this->output( $s ); + } +} + +$maintClass = "RunJobs"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/runScript.php b/www/wiki/maintenance/runScript.php new file mode 100644 index 00000000..385db157 --- /dev/null +++ b/www/wiki/maintenance/runScript.php @@ -0,0 +1,64 @@ +<?php +/** + * Convenience maintenance script wrapper, useful for scripts + * or extensions located outside of standard locations. + * + * To use, give the maintenance script as a relative or full path. + * + * Example usage: + * + * If your pwd is mediawiki base folder: + * php maintenance/runScript.php extensions/Wikibase/lib/maintenance/dispatchChanges.php + * + * If your pwd is maintenance folder: + * php runScript.php ../extensions/Wikibase/lib/maintenance/dispatchChanges.php + * + * Or full path: + * php /var/www/mediawiki/maintenance/runScript.php maintenance/runJobs.php + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @author Katie Filbert < aude.wiki@gmail.com > + * @file + * @ingroup Maintenance + */ +$IP = getenv( 'MW_INSTALL_PATH' ); + +if ( $IP === false ) { + $IP = dirname( __DIR__ ); + + putenv( "MW_INSTALL_PATH=$IP" ); +} + +require_once "$IP/maintenance/Maintenance.php"; + +if ( !isset( $argv[1] ) ) { + fwrite( STDERR, "This script requires a maintainance script as an argument.\n" + . "Usage: runScript.php extensions/Wikibase/lib/maintenance/dispatchChanges\n" ); + exit( 1 ); +} + +$scriptFilename = $argv[1]; +array_shift( $argv ); + +$scriptFile = realpath( $scriptFilename ); + +if ( !$scriptFile ) { + fwrite( STDERR, "The MediaWiki script file \"{$scriptFilename}\" does not exist.\n" ); + exit( 1 ); +} + +require_once $scriptFile; diff --git a/www/wiki/maintenance/shell.php b/www/wiki/maintenance/shell.php new file mode 100644 index 00000000..65c353a2 --- /dev/null +++ b/www/wiki/maintenance/shell.php @@ -0,0 +1,100 @@ +<?php +/** + * Modern interactive shell within the MediaWiki engine. + * + * Merely wraps around http://psysh.org/ and drop an interactive PHP shell in + * the global scope. + * + * Copyright © 2017 Antoine Musso <hashar@free.fr> + * Copyright © 2017 Gergő Tisza <tgr.huwiki@gmail.com> + * Copyright © 2017 Justin Hileman <justin@justinhileman.info> + * Copyright © 2017 Wikimedia Foundation Inc. + * https://www.mediawiki.org/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * + * @author Antoine Musso <hashar@free.fr> + * @author Justin Hileman <justin@justinhileman.info> + * @author Gergő Tisza <tgr.huwiki@gmail.com> + */ + +use MediaWiki\Logger\LoggerFactory; +use MediaWiki\Logger\ConsoleSpi; +use MediaWiki\MediaWikiServices; + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Interactive shell with completion and global scope. + * + */ +class MediaWikiShell extends Maintenance { + + public function __construct() { + parent::__construct(); + $this->addOption( 'd', + 'For back compatibility with eval.php. ' . + '1 send debug to stderr. ' . + 'With 2 additionally initialize database with debugging ', + false, true + ); + } + + public function execute() { + if ( !class_exists( \Psy\Shell::class ) ) { + $this->error( 'PsySH not found. Please run composer with the --dev option.', 1 ); + } + + $traverser = new \PhpParser\NodeTraverser(); + $codeCleaner = new \Psy\CodeCleaner( null, null, $traverser ); + + // add this after initializing the code cleaner so all the default passes get added first + $traverser->addVisitor( new CodeCleanerGlobalsPass() ); + + $config = new \Psy\Configuration( [ 'codeCleaner' => $codeCleaner ] ); + $config->setUpdateCheck( \Psy\VersionUpdater\Checker::NEVER ); + $shell = new \Psy\Shell( $config ); + if ( $this->hasOption( 'd' ) ) { + $this->setupLegacy(); + } + + $shell->run(); + } + + /** + * For back compatibility with eval.php + */ + protected function setupLegacy() { + $d = intval( $this->getOption( 'd' ) ); + if ( $d > 0 ) { + LoggerFactory::registerProvider( new ConsoleSpi ); + // Some services hold Logger instances in object properties + MediaWikiServices::resetGlobalInstance(); + } + if ( $d > 1 ) { + # Set DBO_DEBUG (equivalent of $wgDebugDumpSql) + wfGetDB( DB_MASTER )->setFlag( DBO_DEBUG ); + wfGetDB( DB_REPLICA )->setFlag( DBO_DEBUG ); + } + } + +} + +$maintClass = 'MediaWikiShell'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/showJobs.php b/www/wiki/maintenance/showJobs.php new file mode 100644 index 00000000..0c68032e --- /dev/null +++ b/www/wiki/maintenance/showJobs.php @@ -0,0 +1,109 @@ +<?php +/** + * Report number of jobs currently waiting in master database. + * + * Based on runJobs.php + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @author Tim Starling + * @author Antoine Musso + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that reports the number of jobs currently waiting + * in master database. + * + * @ingroup Maintenance + */ +class ShowJobs extends Maintenance { + protected static $stateMethods = [ + 'unclaimed' => 'getAllQueuedJobs', + 'delayed' => 'getAllDelayedJobs', + 'claimed' => 'getAllAcquiredJobs', + 'abandoned' => 'getAllAbandonedJobs', + ]; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Show number of jobs waiting in master database' ); + $this->addOption( 'group', 'Show number of jobs per job type' ); + $this->addOption( 'list', 'Show a list of all jobs instead of counts' ); + $this->addOption( 'type', 'Only show/count jobs of a given type', false, true ); + $this->addOption( 'status', 'Filter list by state (unclaimed,delayed,claimed,abandoned)' ); + $this->addOption( 'limit', 'Limit of jobs listed' ); + } + + public function execute() { + $typeFilter = $this->getOption( 'type', '' ); + $stateFilter = $this->getOption( 'status', '' ); + $stateLimit = (float)$this->getOption( 'limit', INF ); + + $group = JobQueueGroup::singleton(); + + $filteredTypes = $typeFilter + ? [ $typeFilter ] + : $group->getQueueTypes(); + $filteredStates = $stateFilter + ? array_intersect_key( self::$stateMethods, [ $stateFilter => 1 ] ) + : self::$stateMethods; + + if ( $this->hasOption( 'list' ) ) { + $count = 0; + foreach ( $filteredTypes as $type ) { + $queue = $group->get( $type ); + foreach ( $filteredStates as $state => $method ) { + foreach ( $queue->$method() as $job ) { + /** @var Job $job */ + $this->output( $job->toString() . " status=$state\n" ); + if ( ++$count >= $stateLimit ) { + return; + } + } + } + } + } elseif ( $this->hasOption( 'group' ) ) { + foreach ( $filteredTypes as $type ) { + $queue = $group->get( $type ); + $delayed = $queue->getDelayedCount(); + $pending = $queue->getSize(); + $claimed = $queue->getAcquiredCount(); + $abandoned = $queue->getAbandonedCount(); + $active = max( 0, $claimed - $abandoned ); + if ( ( $pending + $claimed + $delayed + $abandoned ) > 0 ) { + $this->output( + "{$type}: $pending queued; " . + "$claimed claimed ($active active, $abandoned abandoned); " . + "$delayed delayed\n" + ); + } + } + } else { + $count = 0; + foreach ( $filteredTypes as $type ) { + $count += $group->get( $type )->getSize(); + } + $this->output( "$count\n" ); + } + } +} + +$maintClass = "ShowJobs"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/showSiteStats.php b/www/wiki/maintenance/showSiteStats.php new file mode 100644 index 00000000..5a151651 --- /dev/null +++ b/www/wiki/maintenance/showSiteStats.php @@ -0,0 +1,78 @@ +<?php + +/** + * Show the cached statistics. + * Give out the same output as [[Special:Statistics]] + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @author Antoine Musso <hashar at free dot fr> + * Based on initSiteStats.php by: + * @author Brion Vibber + * @author Rob Church <robchur@gmail.com> + * + * @license GNU General Public License 2.0 or later + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to show the cached statistics. + * + * @ingroup Maintenance + */ +class ShowSiteStats extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Show the cached statistics' ); + } + + public function execute() { + $fields = [ + 'ss_total_edits' => 'Total edits', + 'ss_good_articles' => 'Number of articles', + 'ss_total_pages' => 'Total pages', + 'ss_users' => 'Number of users', + 'ss_active_users' => 'Active users', + 'ss_images' => 'Number of images', + ]; + + // Get cached stats from a replica DB + $dbr = $this->getDB( DB_REPLICA ); + $stats = $dbr->selectRow( 'site_stats', '*', '', __METHOD__ ); + + // Get maximum size for each column + $max_length_value = $max_length_desc = 0; + foreach ( $fields as $field => $desc ) { + $max_length_value = max( $max_length_value, strlen( $stats->$field ) ); + $max_length_desc = max( $max_length_desc, strlen( $desc ) ); + } + + // Show them + foreach ( $fields as $field => $desc ) { + $this->output( sprintf( + "%-{$max_length_desc}s: %{$max_length_value}d\n", + $desc, + $stats->$field + ) ); + } + } +} + +$maintClass = "ShowSiteStats"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/sql.php b/www/wiki/maintenance/sql.php new file mode 100644 index 00000000..36e55f3e --- /dev/null +++ b/www/wiki/maintenance/sql.php @@ -0,0 +1,192 @@ +<?php +/** + * Send SQL queries from the specified file to the database, performing + * variable replacement along the way. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +use Wikimedia\Rdbms\ResultWrapper; +use Wikimedia\Rdbms\IDatabase; +use Wikimedia\Rdbms\DBQueryError; + +/** + * Maintenance script that sends SQL queries from the specified file to the database. + * + * @ingroup Maintenance + */ +class MwSql extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Send SQL queries to a MediaWiki database. ' . + 'Takes a file name containing SQL as argument or runs interactively.' ); + $this->addOption( 'query', + 'Run a single query instead of running interactively', false, true ); + $this->addOption( 'cluster', 'Use an external cluster by name', false, true ); + $this->addOption( 'wikidb', + 'The database wiki ID to use if not the current one', false, true ); + $this->addOption( 'replicadb', + 'Replica DB server to use instead of the master DB (can be "any")', false, true ); + } + + public function execute() { + global $IP; + + // We wan't to allow "" for the wikidb, meaning don't call select_db() + $wiki = $this->hasOption( 'wikidb' ) ? $this->getOption( 'wikidb' ) : false; + // Get the appropriate load balancer (for this wiki) + if ( $this->hasOption( 'cluster' ) ) { + $lb = wfGetLBFactory()->getExternalLB( $this->getOption( 'cluster' ) ); + } else { + $lb = wfGetLB( $wiki ); + } + // Figure out which server to use + $replicaDB = $this->getOption( 'replicadb', $this->getOption( 'slave', '' ) ); + if ( $replicaDB === 'any' ) { + $index = DB_REPLICA; + } elseif ( $replicaDB != '' ) { + $index = null; + $serverCount = $lb->getServerCount(); + for ( $i = 0; $i < $serverCount; ++$i ) { + if ( $lb->getServerName( $i ) === $replicaDB ) { + $index = $i; + break; + } + } + if ( $index === null ) { + $this->error( "No replica DB server configured with the name '$replicaDB'.", 1 ); + } + } else { + $index = DB_MASTER; + } + + /** @var IDatabase $db DB handle for the appropriate cluster/wiki */ + $db = $lb->getConnection( $index, [], $wiki ); + if ( $replicaDB != '' && $db->getLBInfo( 'master' ) !== null ) { + $this->error( "The server selected ({$db->getServer()}) is not a replica DB.", 1 ); + } + + if ( $index === DB_MASTER ) { + $updater = DatabaseUpdater::newForDB( $db, true, $this ); + $db->setSchemaVars( $updater->getSchemaVars() ); + } + + if ( $this->hasArg( 0 ) ) { + $file = fopen( $this->getArg( 0 ), 'r' ); + if ( !$file ) { + $this->error( "Unable to open input file", true ); + } + + $error = $db->sourceStream( $file, null, [ $this, 'sqlPrintResult' ] ); + if ( $error !== true ) { + $this->error( $error, true ); + } else { + exit( 0 ); + } + } + + if ( $this->hasOption( 'query' ) ) { + $query = $this->getOption( 'query' ); + $this->sqlDoQuery( $db, $query, /* dieOnError */ true ); + wfWaitForSlaves(); + return; + } + + if ( + function_exists( 'readline_add_history' ) && + Maintenance::posix_isatty( 0 /*STDIN*/ ) + ) { + $historyFile = isset( $_ENV['HOME'] ) ? + "{$_ENV['HOME']}/.mwsql_history" : "$IP/maintenance/.mwsql_history"; + readline_read_history( $historyFile ); + } else { + $historyFile = null; + } + + $wholeLine = ''; + $newPrompt = '> '; + $prompt = $newPrompt; + $doDie = !Maintenance::posix_isatty( 0 ); + while ( ( $line = Maintenance::readconsole( $prompt ) ) !== false ) { + if ( !$line ) { + # User simply pressed return key + continue; + } + $done = $db->streamStatementEnd( $wholeLine, $line ); + + $wholeLine .= $line; + + if ( !$done ) { + $wholeLine .= ' '; + $prompt = ' -> '; + continue; + } + if ( $historyFile ) { + # Delimiter is eated by streamStatementEnd, we add it + # up in the history (T39020) + readline_add_history( $wholeLine . ';' ); + readline_write_history( $historyFile ); + } + $this->sqlDoQuery( $db, $wholeLine, $doDie ); + $prompt = $newPrompt; + $wholeLine = ''; + } + wfWaitForSlaves(); + } + + protected function sqlDoQuery( IDatabase $db, $line, $dieOnError ) { + try { + $res = $db->query( $line ); + $this->sqlPrintResult( $res, $db ); + } catch ( DBQueryError $e ) { + $this->error( $e, $dieOnError ); + } + } + + /** + * Print the results, callback for $db->sourceStream() + * @param ResultWrapper|bool $res The results object + * @param IDatabase $db + */ + public function sqlPrintResult( $res, $db ) { + if ( !$res ) { + // Do nothing + return; + } elseif ( is_object( $res ) && $res->numRows() ) { + foreach ( $res as $row ) { + $this->output( print_r( $row, true ) ); + } + } else { + $affected = $db->affectedRows(); + $this->output( "Query OK, $affected row(s) affected\n" ); + } + } + + /** + * @return int DB_TYPE constant + */ + public function getDbType() { + return Maintenance::DB_ADMIN; + } +} + +$maintClass = "MwSql"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/sqlite.inc b/www/wiki/maintenance/sqlite.inc new file mode 100644 index 00000000..f14856a5 --- /dev/null +++ b/www/wiki/maintenance/sqlite.inc @@ -0,0 +1,96 @@ +<?php +/** + * Helper class for sqlite-specific scripts + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +use Wikimedia\Rdbms\DatabaseSqlite; +use Wikimedia\Rdbms\DBError; + +/** + * This class contains code common to different SQLite-related maintenance scripts + * + * @ingroup Maintenance + */ +class Sqlite { + + /** + * Checks whether PHP has SQLite support + * @return bool + */ + public static function isPresent() { + return extension_loaded( 'pdo_sqlite' ); + } + + /** + * Checks given files for correctness of SQL syntax. MySQL DDL will be converted to + * SQLite-compatible during processing. + * Will throw exceptions on SQL errors + * @param array|string $files + * @throws MWException + * @return bool True if no error or error string in case of errors + */ + public static function checkSqlSyntax( $files ) { + if ( !self::isPresent() ) { + throw new MWException( "Can't check SQL syntax: SQLite not found" ); + } + if ( !is_array( $files ) ) { + $files = [ $files ]; + } + + $allowedTypes = array_flip( [ + 'integer', + 'real', + 'text', + 'blob', // NULL type is omitted intentionally + ] ); + + $db = DatabaseSqlite::newStandaloneInstance( ':memory:' ); + try { + foreach ( $files as $file ) { + $err = $db->sourceFile( $file ); + if ( $err != true ) { + return $err; + } + } + + $tables = $db->query( "SELECT name FROM sqlite_master WHERE type='table'", __METHOD__ ); + foreach ( $tables as $table ) { + if ( strpos( $table->name, 'sqlite_' ) === 0 ) { + continue; + } + + $columns = $db->query( "PRAGMA table_info({$table->name})", __METHOD__ ); + foreach ( $columns as $col ) { + if ( !isset( $allowedTypes[strtolower( $col->type )] ) ) { + $db->close(); + + return "Table {$table->name} has column {$col->name} with non-native type '{$col->type}'"; + } + } + } + } catch ( DBError $e ) { + return $e->getMessage(); + } + $db->close(); + + return true; + } +} diff --git a/www/wiki/maintenance/sqlite.php b/www/wiki/maintenance/sqlite.php new file mode 100644 index 00000000..e74a86cf --- /dev/null +++ b/www/wiki/maintenance/sqlite.php @@ -0,0 +1,146 @@ +<?php +/** + * Performs some operations specific to SQLite database backend. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that performs some operations specific to SQLite database backend. + * + * @ingroup Maintenance + */ +class SqliteMaintenance extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Performs some operations specific to SQLite database backend' ); + $this->addOption( + 'vacuum', + 'Clean up database by removing deleted pages. Decreases database file size' + ); + $this->addOption( 'integrity', 'Check database for integrity' ); + $this->addOption( 'backup-to', 'Backup database to the given file', false, true ); + $this->addOption( 'check-syntax', 'Check SQL file(s) for syntax errors', false, true ); + } + + /** + * While we use database connection, this simple lie prevents useless --dbpass and + * --dbuser options from appearing in help message for this script. + * + * @return int DB constant + */ + public function getDbType() { + return Maintenance::DB_NONE; + } + + public function execute() { + // Should work even if we use a non-SQLite database + if ( $this->hasOption( 'check-syntax' ) ) { + $this->checkSyntax(); + + return; + } + + $this->db = $this->getDB( DB_MASTER ); + + if ( $this->db->getType() != 'sqlite' ) { + $this->error( "This maintenance script requires a SQLite database.\n" ); + + return; + } + + if ( $this->hasOption( 'vacuum' ) ) { + $this->vacuum(); + } + + if ( $this->hasOption( 'integrity' ) ) { + $this->integrityCheck(); + } + + if ( $this->hasOption( 'backup-to' ) ) { + $this->backup( $this->getOption( 'backup-to' ) ); + } + } + + private function vacuum() { + $prevSize = filesize( $this->db->getDbFilePath() ); + if ( $prevSize == 0 ) { + $this->error( "Can't vacuum an empty database.\n", true ); + } + + $this->output( 'VACUUM: ' ); + if ( $this->db->query( 'VACUUM' ) ) { + clearstatcache(); + $newSize = filesize( $this->db->getDbFilePath() ); + $this->output( sprintf( "Database size was %d, now %d (%.1f%% reduction).\n", + $prevSize, $newSize, ( $prevSize - $newSize ) * 100.0 / $prevSize ) ); + } else { + $this->output( 'Error\n' ); + } + } + + private function integrityCheck() { + $this->output( "Performing database integrity checks:\n" ); + $res = $this->db->query( 'PRAGMA integrity_check' ); + + if ( !$res || $res->numRows() == 0 ) { + $this->error( "Error: integrity check query returned nothing.\n" ); + + return; + } + + foreach ( $res as $row ) { + $this->output( $row->integrity_check ); + } + } + + private function backup( $fileName ) { + $this->output( "Backing up database:\n Locking..." ); + $this->db->query( 'BEGIN IMMEDIATE TRANSACTION', __METHOD__ ); + $ourFile = $this->db->getDbFilePath(); + $this->output( " Copying database file $ourFile to $fileName... " ); + MediaWiki\suppressWarnings( false ); + if ( !copy( $ourFile, $fileName ) ) { + $err = error_get_last(); + $this->error( " {$err['message']}" ); + } + MediaWiki\suppressWarnings( true ); + $this->output( " Releasing lock...\n" ); + $this->db->query( 'COMMIT TRANSACTION', __METHOD__ ); + } + + private function checkSyntax() { + if ( !Sqlite::isPresent() ) { + $this->error( "Error: SQLite support not found\n" ); + } + $files = [ $this->getOption( 'check-syntax' ) ]; + $files = array_merge( $files, $this->mArgs ); + $result = Sqlite::checkSqlSyntax( $files ); + if ( $result === true ) { + $this->output( "SQL syntax check: no errors detected.\n" ); + } else { + $this->error( "Error: $result\n" ); + } + } +} + +$maintClass = "SqliteMaintenance"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/sqlite/archives/initial-indexes.sql b/www/wiki/maintenance/sqlite/archives/initial-indexes.sql new file mode 100644 index 00000000..2d0c9eea --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/initial-indexes.sql @@ -0,0 +1,462 @@ +-- Correct for the total lack of indexes in the MW 1.13 SQLite schema +-- +-- Unique indexes need to be handled with INSERT SELECT since just running +-- the CREATE INDEX statement will fail if there are duplicate values. +-- +-- Ignore duplicates, several tables will have them (e.g. T18966) but in +-- most cases it's harmless to discard them. + +-------------------------------------------------------------------------------- +-- Drop temporary tables from aborted runs +-------------------------------------------------------------------------------- + +DROP TABLE IF EXISTS /*_*/user_tmp; +DROP TABLE IF EXISTS /*_*/user_groups_tmp; +DROP TABLE IF EXISTS /*_*/page_tmp; +DROP TABLE IF EXISTS /*_*/revision_tmp; +DROP TABLE IF EXISTS /*_*/pagelinks_tmp; +DROP TABLE IF EXISTS /*_*/templatelinks_tmp; +DROP TABLE IF EXISTS /*_*/imagelinks_tmp; +DROP TABLE IF EXISTS /*_*/categorylinks_tmp; +DROP TABLE IF EXISTS /*_*/category_tmp; +DROP TABLE IF EXISTS /*_*/langlinks_tmp; +DROP TABLE IF EXISTS /*_*/site_stats_tmp; +DROP TABLE IF EXISTS /*_*/ipblocks_tmp; +DROP TABLE IF EXISTS /*_*/watchlist_tmp; +DROP TABLE IF EXISTS /*_*/math_tmp; +DROP TABLE IF EXISTS /*_*/interwiki_tmp; +DROP TABLE IF EXISTS /*_*/page_restrictions_tmp; +DROP TABLE IF EXISTS /*_*/protected_titles_tmp; +DROP TABLE IF EXISTS /*_*/page_props_tmp; +DROP TABLE IF EXISTS /*_*/archive_tmp; +DROP TABLE IF EXISTS /*_*/externallinks_tmp; + +-------------------------------------------------------------------------------- +-- Create new tables +-------------------------------------------------------------------------------- + +CREATE TABLE /*_*/user_tmp ( + user_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + user_name varchar(255) binary NOT NULL default '', + user_real_name varchar(255) binary NOT NULL default '', + user_password tinyblob NOT NULL, + user_newpassword tinyblob NOT NULL, + user_newpass_time binary(14), + user_email tinytext NOT NULL, + user_options blob NOT NULL, + user_touched binary(14) NOT NULL default '', + user_token binary(32) NOT NULL default '', + user_email_authenticated binary(14), + user_email_token binary(32), + user_email_token_expires binary(14), + user_registration binary(14), + user_editcount int +); +CREATE UNIQUE INDEX /*i*/user_name ON /*_*/user_tmp (user_name); +CREATE INDEX /*i*/user_email_token ON /*_*/user_tmp (user_email_token); + + +CREATE TABLE /*_*/user_groups_tmp ( + ug_user int unsigned NOT NULL default 0, + ug_group varbinary(16) NOT NULL default '' +); + +CREATE UNIQUE INDEX /*i*/ug_user_group ON /*_*/user_groups_tmp (ug_user,ug_group); +CREATE INDEX /*i*/ug_group ON /*_*/user_groups_tmp (ug_group); + +CREATE TABLE /*_*/page_tmp ( + page_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + page_namespace int NOT NULL, + page_title varchar(255) binary NOT NULL, + page_restrictions tinyblob NOT NULL, + page_is_redirect tinyint unsigned NOT NULL default 0, + page_is_new tinyint unsigned NOT NULL default 0, + page_random real unsigned NOT NULL, + page_touched binary(14) NOT NULL default '', + page_latest int unsigned NOT NULL, + page_len int unsigned NOT NULL +); + +CREATE UNIQUE INDEX /*i*/name_title ON /*_*/page_tmp (page_namespace,page_title); +CREATE INDEX /*i*/page_random ON /*_*/page_tmp (page_random); +CREATE INDEX /*i*/page_len ON /*_*/page_tmp (page_len); + + +CREATE TABLE /*_*/revision_tmp ( + rev_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + rev_page int unsigned NOT NULL, + rev_text_id int unsigned NOT NULL, + rev_comment tinyblob NOT NULL, + rev_user int unsigned NOT NULL default 0, + rev_user_text varchar(255) binary NOT NULL default '', + rev_timestamp binary(14) NOT NULL default '', + rev_minor_edit tinyint unsigned NOT NULL default 0, + rev_deleted tinyint unsigned NOT NULL default 0, + rev_len int unsigned, + rev_parent_id int unsigned default NULL +); +CREATE UNIQUE INDEX /*i*/rev_page_id ON /*_*/revision_tmp (rev_page, rev_id); +CREATE INDEX /*i*/rev_timestamp ON /*_*/revision_tmp (rev_timestamp); +CREATE INDEX /*i*/page_timestamp ON /*_*/revision_tmp (rev_page,rev_timestamp); +CREATE INDEX /*i*/user_timestamp ON /*_*/revision_tmp (rev_user,rev_timestamp); +CREATE INDEX /*i*/usertext_timestamp ON /*_*/revision_tmp (rev_user_text,rev_timestamp); + +CREATE TABLE /*_*/pagelinks_tmp ( + pl_from int unsigned NOT NULL default 0, + pl_namespace int NOT NULL default 0, + pl_title varchar(255) binary NOT NULL default '' +); + +CREATE UNIQUE INDEX /*i*/pl_from ON /*_*/pagelinks_tmp (pl_from,pl_namespace,pl_title); +CREATE INDEX /*i*/pl_namespace_title ON /*_*/pagelinks_tmp (pl_namespace,pl_title,pl_from); + + +CREATE TABLE /*_*/templatelinks_tmp ( + tl_from int unsigned NOT NULL default 0, + tl_namespace int NOT NULL default 0, + tl_title varchar(255) binary NOT NULL default '' +); + +CREATE UNIQUE INDEX /*i*/tl_from ON /*_*/templatelinks_tmp (tl_from,tl_namespace,tl_title); +CREATE INDEX /*i*/tl_namespace_title ON /*_*/templatelinks_tmp (tl_namespace,tl_title,tl_from); + + +CREATE TABLE /*_*/imagelinks_tmp ( + il_from int unsigned NOT NULL default 0, + il_to varchar(255) binary NOT NULL default '' +) /*$wgDBTableOptions*/; +CREATE UNIQUE INDEX /*i*/il_from ON /*_*/imagelinks_tmp (il_from,il_to); +CREATE INDEX /*i*/il_to ON /*_*/imagelinks_tmp (il_to,il_from); + + +CREATE TABLE /*_*/categorylinks_tmp ( + cl_from int unsigned NOT NULL default 0, + cl_to varchar(255) binary NOT NULL default '', + cl_sortkey varchar(70) binary NOT NULL default '', + cl_timestamp timestamp NOT NULL +); +CREATE UNIQUE INDEX /*i*/cl_from ON /*_*/categorylinks_tmp (cl_from,cl_to); +CREATE INDEX /*i*/cl_sortkey ON /*_*/categorylinks_tmp (cl_to,cl_sortkey,cl_from); +CREATE INDEX /*i*/cl_timestamp ON /*_*/categorylinks_tmp (cl_to,cl_timestamp); + + +CREATE TABLE /*_*/category_tmp ( + cat_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + cat_title varchar(255) binary NOT NULL, + cat_pages int signed NOT NULL default 0, + cat_subcats int signed NOT NULL default 0, + cat_files int signed NOT NULL default 0, + cat_hidden tinyint unsigned NOT NULL default 0 +); +CREATE UNIQUE INDEX /*i*/cat_title ON /*_*/category_tmp (cat_title); +CREATE INDEX /*i*/cat_pages ON /*_*/category_tmp (cat_pages); + +CREATE TABLE /*_*/langlinks_tmp ( + ll_from int unsigned NOT NULL default 0, + ll_lang varbinary(20) NOT NULL default '', + ll_title varchar(255) binary NOT NULL default '' +); + +CREATE UNIQUE INDEX /*i*/ll_from ON /*_*/langlinks_tmp (ll_from, ll_lang); +CREATE INDEX /*i*/ll_lang_title ON /*_*/langlinks_tmp (ll_lang, ll_title); + + +CREATE TABLE /*_*/site_stats_tmp ( + ss_row_id int unsigned NOT NULL, + ss_total_edits bigint unsigned default 0, + ss_good_articles bigint unsigned default 0, + ss_total_pages bigint default '-1', + ss_users bigint default '-1', + ss_active_users bigint default '-1', + ss_admins int default '-1', + ss_images int default 0 +); +CREATE UNIQUE INDEX /*i*/ss_row_id ON /*_*/site_stats_tmp (ss_row_id); + + +CREATE TABLE /*_*/ipblocks_tmp ( + ipb_id int NOT NULL PRIMARY KEY AUTO_INCREMENT, + ipb_address tinyblob NOT NULL, + ipb_user int unsigned NOT NULL default 0, + ipb_by int unsigned NOT NULL default 0, + ipb_by_text varchar(255) binary NOT NULL default '', + ipb_reason tinyblob NOT NULL, + ipb_timestamp binary(14) NOT NULL default '', + ipb_auto bool NOT NULL default 0, + + -- If set to 1, block applies only to logged-out users + ipb_anon_only bool NOT NULL default 0, + ipb_create_account bool NOT NULL default 1, + ipb_enable_autoblock bool NOT NULL default '1', + ipb_expiry varbinary(14) NOT NULL default '', + ipb_range_start tinyblob NOT NULL, + ipb_range_end tinyblob NOT NULL, + ipb_deleted bool NOT NULL default 0, + ipb_block_email bool NOT NULL default 0, + ipb_allow_usertalk bool NOT NULL default 0 +); +CREATE UNIQUE INDEX /*i*/ipb_address ON /*_*/ipblocks_tmp (ipb_address(255), ipb_user, ipb_auto, ipb_anon_only); +CREATE INDEX /*i*/ipb_user ON /*_*/ipblocks_tmp (ipb_user); +CREATE INDEX /*i*/ipb_range ON /*_*/ipblocks_tmp (ipb_range_start(8), ipb_range_end(8)); +CREATE INDEX /*i*/ipb_timestamp ON /*_*/ipblocks_tmp (ipb_timestamp); +CREATE INDEX /*i*/ipb_expiry ON /*_*/ipblocks_tmp (ipb_expiry); + + +CREATE TABLE /*_*/watchlist_tmp ( + wl_user int unsigned NOT NULL, + wl_namespace int NOT NULL default 0, + wl_title varchar(255) binary NOT NULL default '', + wl_notificationtimestamp varbinary(14) +); + +CREATE UNIQUE INDEX /*i*/wl_user_namespace_title ON /*_*/watchlist_tmp (wl_user, wl_namespace, wl_title); +CREATE INDEX /*i*/namespace_title ON /*_*/watchlist_tmp (wl_namespace, wl_title); + + +CREATE TABLE /*_*/math_tmp ( + math_inputhash varbinary(16) NOT NULL, + math_outputhash varbinary(16) NOT NULL, + math_html_conservativeness tinyint NOT NULL, + math_html text, + math_mathml text +); + +CREATE UNIQUE INDEX /*i*/math_inputhash ON /*_*/math_tmp (math_inputhash); + + +CREATE TABLE /*_*/interwiki_tmp ( + iw_prefix varchar(32) NOT NULL, + iw_url blob NOT NULL, + iw_local bool NOT NULL, + iw_trans tinyint NOT NULL default 0 +); + +CREATE UNIQUE INDEX /*i*/iw_prefix ON /*_*/interwiki_tmp (iw_prefix); + + +CREATE TABLE /*_*/page_restrictions_tmp ( + pr_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + pr_page int NOT NULL, + pr_type varbinary(60) NOT NULL, + pr_level varbinary(60) NOT NULL, + pr_cascade tinyint NOT NULL, + pr_user int NULL, + pr_expiry varbinary(14) NULL +); + +CREATE UNIQUE INDEX /*i*/pr_pagetype ON /*_*/page_restrictions_tmp (pr_page,pr_type); +CREATE UNIQUE INDEX /*i*/pr_typelevel ON /*_*/page_restrictions_tmp (pr_type,pr_level); +CREATE UNIQUE INDEX /*i*/pr_level ON /*_*/page_restrictions_tmp (pr_level); +CREATE UNIQUE INDEX /*i*/pr_cascade ON /*_*/page_restrictions_tmp (pr_cascade); + +CREATE TABLE /*_*/protected_titles_tmp ( + pt_namespace int NOT NULL, + pt_title varchar(255) binary NOT NULL, + pt_user int unsigned NOT NULL, + pt_reason tinyblob, + pt_timestamp binary(14) NOT NULL, + pt_expiry varbinary(14) NOT NULL default '', + pt_create_perm varbinary(60) NOT NULL +); +CREATE UNIQUE INDEX /*i*/pt_namespace_title ON /*_*/protected_titles_tmp (pt_namespace,pt_title); +CREATE INDEX /*i*/pt_timestamp ON /*_*/protected_titles_tmp (pt_timestamp); + +CREATE TABLE /*_*/page_props_tmp ( + pp_page int NOT NULL, + pp_propname varbinary(60) NOT NULL, + pp_value blob NOT NULL +); +CREATE UNIQUE INDEX /*i*/pp_page_propname ON /*_*/page_props_tmp (pp_page,pp_propname); + +-- +-- Holding area for deleted articles, which may be viewed +-- or restored by admins through the Special:Undelete interface. +-- The fields generally correspond to the page, revision, and text +-- fields, with several caveats. +-- Cannot reasonably create views on this table, due to the presence of TEXT +-- columns. +CREATE TABLE /*$wgDBprefix*/archive_tmp ( + ar_id NOT NULL PRIMARY KEY clustered IDENTITY, + ar_namespace SMALLINT NOT NULL DEFAULT 0, + ar_title NVARCHAR(255) NOT NULL DEFAULT '', + ar_text NVARCHAR(MAX) NOT NULL, + ar_comment NVARCHAR(255) NOT NULL, + ar_user INT NULL REFERENCES /*$wgDBprefix*/[user](user_id) ON DELETE SET NULL, + ar_user_text NVARCHAR(255) NOT NULL, + ar_timestamp DATETIME NOT NULL DEFAULT GETDATE(), + ar_minor_edit BIT NOT NULL DEFAULT 0, + ar_flags NVARCHAR(255) NOT NULL, + ar_rev_id INT, + ar_text_id INT, + ar_deleted BIT NOT NULL DEFAULT 0, + ar_len INT DEFAULT NULL, + ar_page_id INT NULL, + ar_parent_id INT NULL +); +CREATE INDEX /*$wgDBprefix*/ar_name_title_timestamp ON /*$wgDBprefix*/archive_tmp(ar_namespace,ar_title,ar_timestamp); +CREATE INDEX /*$wgDBprefix*/ar_usertext_timestamp ON /*$wgDBprefix*/archive_tmp(ar_user_text,ar_timestamp); +CREATE INDEX /*$wgDBprefix*/ar_user_text ON /*$wgDBprefix*/archive_tmp(ar_user_text); + +-- +-- Track links to external URLs +-- IE >= 4 supports no more than 2083 characters in a URL +CREATE TABLE /*$wgDBprefix*/externallinks_tmp ( + el_id INT NOT NULL PRIMARY KEY clustered IDENTITY, + el_from INT NOT NULL DEFAULT '0', + el_to VARCHAR(2083) NOT NULL, + el_index VARCHAR(896) NOT NULL, +); +-- Maximum key length ON SQL Server is 900 bytes +CREATE INDEX /*$wgDBprefix*/externallinks_index ON /*$wgDBprefix*/externallinks_tmp(el_index); + +-------------------------------------------------------------------------------- +-- Populate the new tables using INSERT SELECT +-------------------------------------------------------------------------------- + +INSERT OR IGNORE INTO /*_*/user_tmp SELECT * FROM /*_*/user; +INSERT OR IGNORE INTO /*_*/user_groups_tmp SELECT * FROM /*_*/user_groups; +INSERT OR IGNORE INTO /*_*/page_tmp SELECT * FROM /*_*/page; +INSERT OR IGNORE INTO /*_*/revision_tmp SELECT * FROM /*_*/revision; +INSERT OR IGNORE INTO /*_*/pagelinks_tmp SELECT * FROM /*_*/pagelinks; +INSERT OR IGNORE INTO /*_*/templatelinks_tmp SELECT * FROM /*_*/templatelinks; +INSERT OR IGNORE INTO /*_*/imagelinks_tmp SELECT * FROM /*_*/imagelinks; +INSERT OR IGNORE INTO /*_*/categorylinks_tmp SELECT * FROM /*_*/categorylinks; +INSERT OR IGNORE INTO /*_*/category_tmp SELECT * FROM /*_*/category; +INSERT OR IGNORE INTO /*_*/langlinks_tmp SELECT * FROM /*_*/langlinks; +INSERT OR IGNORE INTO /*_*/site_stats_tmp SELECT * FROM /*_*/site_stats; +INSERT OR IGNORE INTO /*_*/ipblocks_tmp SELECT * FROM /*_*/ipblocks; +INSERT OR IGNORE INTO /*_*/watchlist_tmp SELECT * FROM /*_*/watchlist; +INSERT OR IGNORE INTO /*_*/math_tmp SELECT * FROM /*_*/math; +INSERT OR IGNORE INTO /*_*/interwiki_tmp SELECT * FROM /*_*/interwiki; +INSERT OR IGNORE INTO /*_*/page_restrictions_tmp SELECT * FROM /*_*/page_restrictions; +INSERT OR IGNORE INTO /*_*/protected_titles_tmp SELECT * FROM /*_*/protected_titles; +INSERT OR IGNORE INTO /*_*/page_props_tmp SELECT * FROM /*_*/page_props; +INSERT OR IGNORE INTO /*_*/archive_tmp SELECT * FROM /*_*/archive; +INSERT OR IGNORE INTO /*_*/externallinks_tmp SELECT * FROM /*_*/externallinks; + +-------------------------------------------------------------------------------- +-- Do the table renames +-------------------------------------------------------------------------------- + +DROP TABLE /*_*/user; +ALTER TABLE /*_*/user_tmp RENAME TO /*_*/user; +DROP TABLE /*_*/user_groups; +ALTER TABLE /*_*/user_groups_tmp RENAME TO /*_*/user_groups; +DROP TABLE /*_*/page; +ALTER TABLE /*_*/page_tmp RENAME TO /*_*/page; +DROP TABLE /*_*/revision; +ALTER TABLE /*_*/revision_tmp RENAME TO /*_*/revision; +DROP TABLE /*_*/pagelinks; +ALTER TABLE /*_*/pagelinks_tmp RENAME TO /*_*/pagelinks; +DROP TABLE /*_*/templatelinks; +ALTER TABLE /*_*/templatelinks_tmp RENAME TO /*_*/templatelinks; +DROP TABLE /*_*/imagelinks; +ALTER TABLE /*_*/imagelinks_tmp RENAME TO /*_*/imagelinks; +DROP TABLE /*_*/categorylinks; +ALTER TABLE /*_*/categorylinks_tmp RENAME TO /*_*/categorylinks; +DROP TABLE /*_*/category; +ALTER TABLE /*_*/category_tmp RENAME TO /*_*/category; +DROP TABLE /*_*/langlinks; +ALTER TABLE /*_*/langlinks_tmp RENAME TO /*_*/langlinks; +DROP TABLE /*_*/site_stats; +ALTER TABLE /*_*/site_stats_tmp RENAME TO /*_*/site_stats; +DROP TABLE /*_*/ipblocks; +ALTER TABLE /*_*/ipblocks_tmp RENAME TO /*_*/ipblocks; +DROP TABLE /*_*/watchlist; +ALTER TABLE /*_*/watchlist_tmp RENAME TO /*_*/watchlist; +DROP TABLE /*_*/math; +ALTER TABLE /*_*/math_tmp RENAME TO /*_*/math; +DROP TABLE /*_*/interwiki; +ALTER TABLE /*_*/interwiki_tmp RENAME TO /*_*/interwiki; +DROP TABLE /*_*/page_restrictions; +ALTER TABLE /*_*/page_restrictions_tmp RENAME TO /*_*/page_restrictions; +DROP TABLE /*_*/protected_titles; +ALTER TABLE /*_*/protected_titles_tmp RENAME TO /*_*/protected_titles; +DROP TABLE /*_*/page_props; +ALTER TABLE /*_*/page_props_tmp RENAME TO /*_*/page_props; +DROP TABLE /*_*/archive; +ALTER TABLE /*_*/archive_tmp RENAME TO /*_*/archive; +DROP TABLE /*_*/externalllinks; +ALTER TABLE /*_*/externallinks_tmp RENAME TO /*_*/externallinks; + +-------------------------------------------------------------------------------- +-- Drop and create tables with unique indexes but no valuable data +-------------------------------------------------------------------------------- + + +DROP TABLE IF EXISTS /*_*/searchindex; +CREATE TABLE /*_*/searchindex ( + si_page int unsigned NOT NULL, + si_title varchar(255) NOT NULL default '', + si_text mediumtext NOT NULL +); +CREATE UNIQUE INDEX /*i*/si_page ON /*_*/searchindex (si_page); +CREATE INDEX /*i*/si_title ON /*_*/searchindex (si_title); +CREATE INDEX /*i*/si_text ON /*_*/searchindex (si_text); + +DROP TABLE IF EXISTS /*_*/transcache; +CREATE TABLE /*_*/transcache ( + tc_url varbinary(255) NOT NULL, + tc_contents text, + tc_time int NOT NULL +) /*$wgDBTableOptions*/; +CREATE UNIQUE INDEX /*i*/tc_url_idx ON /*_*/transcache (tc_url); + +DROP TABLE IF EXISTS /*_*/querycache_info; +CREATE TABLE /*_*/querycache_info ( + qci_type varbinary(32) NOT NULL default '', + qci_timestamp binary(14) NOT NULL default '19700101000000' +) /*$wgDBTableOptions*/; +CREATE UNIQUE INDEX /*i*/qci_type ON /*_*/querycache_info (qci_type); + +-------------------------------------------------------------------------------- +-- Empty some cache tables to make the update faster +-------------------------------------------------------------------------------- + +DELETE FROM /*_*/querycache; +DELETE FROM /*_*/objectcache; +DELETE FROM /*_*/querycachetwo; + +-------------------------------------------------------------------------------- +-- Add indexes to tables with no unique indexes +-------------------------------------------------------------------------------- + +CREATE INDEX /*i*/un_user_id ON /*_*/user_newtalk (user_id); +CREATE INDEX /*i*/un_user_ip ON /*_*/user_newtalk (user_ip); +CREATE INDEX /*i*/name_title_timestamp ON /*_*/archive (ar_namespace,ar_title,ar_timestamp); +CREATE INDEX /*i*/ar_usertext_timestamp ON /*_*/archive (ar_user_text,ar_timestamp); +CREATE INDEX /*i*/el_from ON /*_*/externallinks (el_from, el_to(40)); +CREATE INDEX /*i*/el_to ON /*_*/externallinks (el_to(60), el_from); +CREATE INDEX /*i*/el_index ON /*_*/externallinks (el_index(60)); +CREATE INDEX /*i*/img_usertext_timestamp ON /*_*/image (img_user_text,img_timestamp); +CREATE INDEX /*i*/img_size ON /*_*/image (img_size); +CREATE INDEX /*i*/img_timestamp ON /*_*/image (img_timestamp); +CREATE INDEX /*i*/img_sha1 ON /*_*/image (img_sha1); +CREATE INDEX /*i*/oi_usertext_timestamp ON /*_*/oldimage (oi_user_text,oi_timestamp); +CREATE INDEX /*i*/oi_name_timestamp ON /*_*/oldimage (oi_name,oi_timestamp); +CREATE INDEX /*i*/oi_name_archive_name ON /*_*/oldimage (oi_name,oi_archive_name(14)); +CREATE INDEX /*i*/oi_sha1 ON /*_*/oldimage (oi_sha1); +CREATE INDEX /*i*/fa_name ON /*_*/filearchive (fa_name, fa_timestamp); +CREATE INDEX /*i*/fa_group_key ON /*_*/filearchive (fa_storage_group, fa_storage_key); +CREATE INDEX /*i*/fa_deleted_timestamp ON /*_*/filearchive (fa_deleted_timestamp); +CREATE INDEX /*i*/fa_user_timestamp ON /*_*/filearchive (fa_user_text,fa_timestamp); +CREATE INDEX /*i*/rc_timestamp ON /*_*/recentchanges (rc_timestamp); +CREATE INDEX /*i*/rc_namespace_title ON /*_*/recentchanges (rc_namespace, rc_title); +CREATE INDEX /*i*/rc_cur_id ON /*_*/recentchanges (rc_cur_id); +CREATE INDEX /*i*/new_name_timestamp ON /*_*/recentchanges (rc_new,rc_namespace,rc_timestamp); +CREATE INDEX /*i*/rc_ip ON /*_*/recentchanges (rc_ip); +CREATE INDEX /*i*/rc_ns_usertext ON /*_*/recentchanges (rc_namespace, rc_user_text); +CREATE INDEX /*i*/rc_user_text ON /*_*/recentchanges (rc_user_text, rc_timestamp); +CREATE INDEX /*i*/qc_type_value ON /*_*/querycache (qc_type,qc_value); +CREATE INDEX /*i*/oc_exptime ON /*_*/objectcache (exptime); +CREATE INDEX /*i*/type_time ON /*_*/logging (log_type, log_timestamp); +CREATE INDEX /*i*/user_time ON /*_*/logging (log_user, log_timestamp); +CREATE INDEX /*i*/page_time ON /*_*/logging (log_namespace, log_title, log_timestamp); +CREATE INDEX /*i*/times ON /*_*/logging (log_timestamp); +CREATE INDEX /*i*/job_cmd_namespace_title ON /*_*/job (job_cmd, job_namespace, job_title); +CREATE INDEX /*i*/rd_ns_title ON /*_*/redirect (rd_namespace,rd_title,rd_from); +CREATE INDEX /*i*/qcc_type ON /*_*/querycachetwo (qcc_type,qcc_value); +CREATE INDEX /*i*/qcc_title ON /*_*/querycachetwo (qcc_type,qcc_namespace,qcc_title); +CREATE INDEX /*i*/qcc_titletwo ON /*_*/querycachetwo (qcc_type,qcc_namespacetwo,qcc_titletwo); + +INSERT INTO /*_*/updatelog (ul_key) VALUES ('initial_indexes'); diff --git a/www/wiki/maintenance/sqlite/archives/patch-add-3d.sql b/www/wiki/maintenance/sqlite/archives/patch-add-3d.sql new file mode 100644 index 00000000..10d74fb9 --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-add-3d.sql @@ -0,0 +1,249 @@ +-- image + +CREATE TABLE /*_*/image_tmp ( + -- Filename. + -- This is also the title of the associated description page, + -- which will be in namespace 6 (NS_FILE). + img_name varchar(255) binary NOT NULL default '' PRIMARY KEY, + + -- File size in bytes. + img_size int unsigned NOT NULL default 0, + + -- For images, size in pixels. + img_width int NOT NULL default 0, + img_height int NOT NULL default 0, + + -- Extracted Exif metadata stored as a serialized PHP array. + img_metadata mediumblob NOT NULL, + + -- For images, bits per pixel if known. + img_bits int NOT NULL default 0, + + -- Media type as defined by the MEDIATYPE_xxx constants + img_media_type ENUM("UNKNOWN", "BITMAP", "DRAWING", "AUDIO", "VIDEO", "MULTIMEDIA", "OFFICE", "TEXT", "EXECUTABLE", "ARCHIVE", "3D") default NULL, + + -- major part of a MIME media type as defined by IANA + -- see https://www.iana.org/assignments/media-types/ + -- for "chemical" cf. http://dx.doi.org/10.1021/ci9803233 by the ACS + img_major_mime ENUM("unknown", "application", "audio", "image", "text", "video", "message", "model", "multipart", "chemical") NOT NULL default "unknown", + + -- minor part of a MIME media type as defined by IANA + -- the minor parts are not required to adher to any standard + -- but should be consistent throughout the database + -- see https://www.iana.org/assignments/media-types/ + img_minor_mime varbinary(100) NOT NULL default "unknown", + + -- Description field as entered by the uploader. + -- This is displayed in image upload history and logs. + img_description varbinary(767) NOT NULL, + + -- user_id and user_name of uploader. + img_user int unsigned NOT NULL default 0, + img_user_text varchar(255) binary NOT NULL, + + -- Time of the upload. + img_timestamp varbinary(14) NOT NULL default '', + + -- SHA-1 content hash in base-36 + img_sha1 varbinary(32) NOT NULL default '' +) /*$wgDBTableOptions*/; + +INSERT INTO /*_*/image_tmp + SELECT img_name, img_size, img_width, img_height, img_metadata, img_bits, + img_media_type, img_major_mime, img_minor_mime, img_description, + img_user, img_user_text, img_timestamp, img_sha1 + FROM /*_*/image; + +DROP TABLE /*_*/image; + +ALTER TABLE /*_*/image_tmp RENAME TO /*_*/image; + +-- Used by Special:Newimages and ApiQueryAllImages +CREATE INDEX /*i*/img_user_timestamp ON /*_*/image (img_user,img_timestamp); +CREATE INDEX /*i*/img_usertext_timestamp ON /*_*/image (img_user_text,img_timestamp); +-- Used by Special:ListFiles for sort-by-size +CREATE INDEX /*i*/img_size ON /*_*/image (img_size); +-- Used by Special:Newimages and Special:ListFiles +CREATE INDEX /*i*/img_timestamp ON /*_*/image (img_timestamp); +-- Used in API and duplicate search +CREATE INDEX /*i*/img_sha1 ON /*_*/image (img_sha1(10)); +-- Used to get media of one type +CREATE INDEX /*i*/img_media_mime ON /*_*/image (img_media_type,img_major_mime,img_minor_mime); + +-- oldimage + +CREATE TABLE /*_*/oldimage_tmp ( + -- Base filename: key to image.img_name + oi_name varchar(255) binary NOT NULL default '', + + -- Filename of the archived file. + -- This is generally a timestamp and '!' prepended to the base name. + oi_archive_name varchar(255) binary NOT NULL default '', + + -- Other fields as in image... + oi_size int unsigned NOT NULL default 0, + oi_width int NOT NULL default 0, + oi_height int NOT NULL default 0, + oi_bits int NOT NULL default 0, + oi_description varbinary(767) NOT NULL, + oi_user int unsigned NOT NULL default 0, + oi_user_text varchar(255) binary NOT NULL, + oi_timestamp binary(14) NOT NULL default '', + + oi_metadata mediumblob NOT NULL, + oi_media_type ENUM("UNKNOWN", "BITMAP", "DRAWING", "AUDIO", "VIDEO", "MULTIMEDIA", "OFFICE", "TEXT", "EXECUTABLE", "ARCHIVE", "3D") default NULL, + oi_major_mime ENUM("unknown", "application", "audio", "image", "text", "video", "message", "model", "multipart", "chemical") NOT NULL default "unknown", + oi_minor_mime varbinary(100) NOT NULL default "unknown", + oi_deleted tinyint unsigned NOT NULL default 0, + oi_sha1 varbinary(32) NOT NULL default '' +) /*$wgDBTableOptions*/; + +INSERT INTO /*_*/oldimage_tmp + SELECT oi_name, oi_archive_name, oi_size, oi_width, oi_height, oi_bits, + oi_description, oi_user, oi_user_text, oi_timestamp, oi_metadata, + oi_media_type, oi_major_mime, oi_minor_mime, oi_deleted, oi_sha1 + FROM /*_*/oldimage; + +DROP TABLE /*_*/oldimage; + +ALTER TABLE oldimage_tmp RENAME TO /*_*/oldimage; + +CREATE INDEX /*i*/oi_usertext_timestamp ON /*_*/oldimage (oi_user_text,oi_timestamp); +CREATE INDEX /*i*/oi_name_timestamp ON /*_*/oldimage (oi_name,oi_timestamp); +-- oi_archive_name truncated to 14 to avoid key length overflow +CREATE INDEX /*i*/oi_name_archive_name ON /*_*/oldimage (oi_name,oi_archive_name(14)); +CREATE INDEX /*i*/oi_sha1 ON /*_*/oldimage (oi_sha1(10)); + +-- filearchive + +CREATE TABLE /*_*/filearchive_tmp ( + -- Unique row id + fa_id int NOT NULL PRIMARY KEY AUTO_INCREMENT, + + -- Original base filename; key to image.img_name, page.page_title, etc + fa_name varchar(255) binary NOT NULL default '', + + -- Filename of archived file, if an old revision + fa_archive_name varchar(255) binary default '', + + -- Which storage bin (directory tree or object store) the file data + -- is stored in. Should be 'deleted' for files that have been deleted; + -- any other bin is not yet in use. + fa_storage_group varbinary(16), + + -- SHA-1 of the file contents plus extension, used as a key for storage. + -- eg 8f8a562add37052a1848ff7771a2c515db94baa9.jpg + -- + -- If NULL, the file was missing at deletion time or has been purged + -- from the archival storage. + fa_storage_key varbinary(64) default '', + + -- Deletion information, if this file is deleted. + fa_deleted_user int, + fa_deleted_timestamp binary(14) default '', + fa_deleted_reason varbinary(767) default '', + + -- Duped fields from image + fa_size int unsigned default 0, + fa_width int default 0, + fa_height int default 0, + fa_metadata mediumblob, + fa_bits int default 0, + fa_media_type ENUM("UNKNOWN", "BITMAP", "DRAWING", "AUDIO", "VIDEO", "MULTIMEDIA", "OFFICE", "TEXT", "EXECUTABLE", "ARCHIVE", "3D") default NULL, + fa_major_mime ENUM("unknown", "application", "audio", "image", "text", "video", "message", "model", "multipart", "chemical") default "unknown", + fa_minor_mime varbinary(100) default "unknown", + fa_description varbinary(767), + fa_user int unsigned default 0, + fa_user_text varchar(255) binary, + fa_timestamp binary(14) default '', + + -- Visibility of deleted revisions, bitfield + fa_deleted tinyint unsigned NOT NULL default 0, + + -- sha1 hash of file content + fa_sha1 varbinary(32) NOT NULL default '' +) /*$wgDBTableOptions*/; + +INSERT INTO /*_*/filearchive_tmp + SELECT fa_id, fa_name, fa_archive_name, fa_storage_group, fa_storage_key, fa_deleted_user, fa_deleted_timestamp, + fa_deleted_reason, fa_size, fa_width, fa_height, fa_metadata, fa_bits, fa_media_type, fa_major_mime, + fa_minor_mime, fa_description, fa_user, fa_user_text, fa_timestamp, fa_deleted, fa_sha1 + FROM /*_*/filearchive; + +DROP TABLE /*_*/filearchive; + +ALTER TABLE /*_*/filearchive_tmp RENAME TO /*_*/filearchive; + +-- pick out by image name +CREATE INDEX /*i*/fa_name ON /*_*/filearchive (fa_name, fa_timestamp); +-- pick out dupe files +CREATE INDEX /*i*/fa_storage_group ON /*_*/filearchive (fa_storage_group, fa_storage_key); +-- sort by deletion time +CREATE INDEX /*i*/fa_deleted_timestamp ON /*_*/filearchive (fa_deleted_timestamp); +-- sort by uploader +CREATE INDEX /*i*/fa_user_timestamp ON /*_*/filearchive (fa_user_text,fa_timestamp); +-- find file by sha1, 10 bytes will be enough for hashes to be indexed +CREATE INDEX /*i*/fa_sha1 ON /*_*/filearchive (fa_sha1(10)); + +-- uploadstash + +CREATE TABLE /*_*/uploadstash_tmp ( + us_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + + -- the user who uploaded the file. + us_user int unsigned NOT NULL, + + -- file key. this is how applications actually search for the file. + -- this might go away, or become the primary key. + us_key varchar(255) NOT NULL, + + -- the original path + us_orig_path varchar(255) NOT NULL, + + -- the temporary path at which the file is actually stored + us_path varchar(255) NOT NULL, + + -- which type of upload the file came from (sometimes) + us_source_type varchar(50), + + -- the date/time on which the file was added + us_timestamp varbinary(14) NOT NULL, + + us_status varchar(50) NOT NULL, + + -- chunk counter starts at 0, current offset is stored in us_size + us_chunk_inx int unsigned NULL, + + -- Serialized file properties from FSFile::getProps() + us_props blob, + + -- file size in bytes + us_size int unsigned NOT NULL, + -- this hash comes from FSFile::getSha1Base36(), and is 31 characters + us_sha1 varchar(31) NOT NULL, + us_mime varchar(255), + -- Media type as defined by the MEDIATYPE_xxx constants, should duplicate definition in the image table + us_media_type ENUM("UNKNOWN", "BITMAP", "DRAWING", "AUDIO", "VIDEO", "MULTIMEDIA", "OFFICE", "TEXT", "EXECUTABLE", "ARCHIVE", "3D") default NULL, + -- image-specific properties + us_image_width int unsigned, + us_image_height int unsigned, + us_image_bits smallint unsigned + +) /*$wgDBTableOptions*/; + +INSERT INTO /*_*/uploadstash_tmp + SELECT us_id, us_user, us_key, us_orig_path, us_path, us_source_type, + us_timestamp, us_status, us_chunk_inx, us_props, us_size, us_sha1, us_mime, + us_media_type, us_image_width, us_image_height, us_image_bits + FROM /*_*/uploadstash; + +DROP TABLE uploadstash; + +ALTER TABLE /*_*/uploadstash_tmp RENAME TO /*_*/uploadstash; + +-- sometimes there's a delete for all of a user's stuff. +CREATE INDEX /*i*/us_user ON /*_*/uploadstash (us_user); +-- pick out files by key, enforce key uniqueness +CREATE UNIQUE INDEX /*i*/us_key ON /*_*/uploadstash (us_key); +-- the abandoned upload cleanup script needs this +CREATE INDEX /*i*/us_timestamp ON /*_*/uploadstash (us_timestamp); diff --git a/www/wiki/maintenance/sqlite/archives/patch-archive-ar_id.sql b/www/wiki/maintenance/sqlite/archives/patch-archive-ar_id.sql new file mode 100644 index 00000000..00a9b071 --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-archive-ar_id.sql @@ -0,0 +1,39 @@ +DROP TABLE IF EXISTS /*_*/archive_tmp; + +CREATE TABLE /*$wgDBprefix*/archive_tmp ( + ar_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + ar_namespace int NOT NULL default 0, + ar_title varchar(255) binary NOT NULL default '', + ar_text mediumblob NOT NULL, + ar_comment tinyblob NOT NULL, + ar_user int unsigned NOT NULL default 0, + ar_user_text varchar(255) binary NOT NULL, + ar_timestamp binary(14) NOT NULL default '', + ar_minor_edit tinyint NOT NULL default 0, + ar_flags tinyblob NOT NULL, + ar_rev_id int unsigned, + ar_text_id int unsigned, + ar_deleted tinyint unsigned NOT NULL default 0, + ar_len int unsigned, + ar_page_id int unsigned, + ar_parent_id int unsigned default NULL, + ar_sha1 varbinary(32) NOT NULL default '', + ar_content_model varbinary(32) DEFAULT NULL, + ar_content_format varbinary(64) DEFAULT NULL +); + +INSERT OR IGNORE INTO /*_*/archive_tmp ( + ar_namespace, ar_title, ar_title, ar_text, ar_comment, ar_user, ar_user_text, ar_timestamp, + ar_minor_edit, ar_flags, ar_rev_id, ar_text_id, ar_deleted, ar_len, ar_page_id, ar_parent_id ) + SELECT + ar_namespace, ar_title, ar_title, ar_text, ar_comment, ar_user, ar_user_text, ar_timestamp, + ar_minor_edit, ar_flags, ar_rev_id, ar_text_id, ar_deleted, ar_len, ar_page_id, ar_parent_id + FROM /*_*/archive; + +DROP TABLE /*_*/archive; + +ALTER TABLE /*_*/archive_tmp RENAME TO /*_*/archive; + +CREATE INDEX /*i*/name_title_timestamp ON /*_*/archive (ar_namespace,ar_title,ar_timestamp); +CREATE INDEX /*i*/ar_usertext_timestamp ON /*_*/archive (ar_user_text,ar_timestamp); +CREATE INDEX /*i*/ar_revid ON /*_*/archive (ar_rev_id); diff --git a/www/wiki/maintenance/sqlite/archives/patch-archive_kill_ar_page_revid.sql b/www/wiki/maintenance/sqlite/archives/patch-archive_kill_ar_page_revid.sql new file mode 100644 index 00000000..0f3e9c7f --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-archive_kill_ar_page_revid.sql @@ -0,0 +1,3 @@ +-- Used for killing the wrong index added during SVN for 1.17 +-- Won't affect most people, but it doesn't need to exist +DROP INDEX IF EXISTS ar_page_revid;
\ No newline at end of file diff --git a/www/wiki/maintenance/sqlite/archives/patch-cat_hidden.sql b/www/wiki/maintenance/sqlite/archives/patch-cat_hidden.sql new file mode 100644 index 00000000..272b8ef3 --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-cat_hidden.sql @@ -0,0 +1,20 @@ +-- cat_hidden is no longer used, delete it + +CREATE TABLE /*_*/category_tmp ( + cat_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + cat_title varchar(255) binary NOT NULL, + cat_pages int signed NOT NULL default 0, + cat_subcats int signed NOT NULL default 0, + cat_files int signed NOT NULL default 0 +) /*$wgDBTableOptions*/; + +INSERT INTO /*_*/category_tmp + SELECT cat_id, cat_title, cat_pages, cat_subcats, cat_files + FROM /*_*/category; + +DROP TABLE /*_*/category; + +ALTER TABLE /*_*/category_tmp RENAME TO /*_*/category; + +CREATE UNIQUE INDEX /*i*/cat_title ON /*_*/category (cat_title); +CREATE INDEX /*i*/cat_pages ON /*_*/category (cat_pages); diff --git a/www/wiki/maintenance/sqlite/archives/patch-categorylinks-better-collation.sql b/www/wiki/maintenance/sqlite/archives/patch-categorylinks-better-collation.sql new file mode 100644 index 00000000..f32af134 --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-categorylinks-better-collation.sql @@ -0,0 +1,7 @@ +ALTER TABLE /*_*/categorylinks ADD COLUMN cl_sortkey_prefix TEXT NOT NULL default ''; +ALTER TABLE /*_*/categorylinks ADD COLUMN cl_collation BLOB NOT NULL default ''; +ALTER TABLE /*_*/categorylinks ADD COLUMN cl_type TEXT NOT NULL default 'page'; +CREATE INDEX cl_collation ON /*_*/categorylinks (cl_collation); +DROP INDEX cl_sortkey; +CREATE INDEX cl_sortkey ON /*_*/categorylinks (cl_to, cl_type, cl_sortkey, cl_from); +INSERT OR IGNORE INTO /*_*/updatelog (ul_key) VALUES ('cl_fields_update'); diff --git a/www/wiki/maintenance/sqlite/archives/patch-categorylinks-fix-pk.sql b/www/wiki/maintenance/sqlite/archives/patch-categorylinks-fix-pk.sql new file mode 100644 index 00000000..13a75a36 --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-categorylinks-fix-pk.sql @@ -0,0 +1,60 @@ +CREATE TABLE /*_*/categorylinks_tmp ( + -- Key to page_id of the page defined as a category member. + cl_from int unsigned NOT NULL default 0, + + -- Name of the category. + -- This is also the page_title of the category's description page; + -- all such pages are in namespace 14 (NS_CATEGORY). + cl_to varchar(255) binary NOT NULL default '', + + -- A binary string obtained by applying a sortkey generation algorithm + -- (Collation::getSortKey()) to page_title, or cl_sortkey_prefix . "\n" + -- . page_title if cl_sortkey_prefix is nonempty. + cl_sortkey varbinary(230) NOT NULL default '', + + -- A prefix for the raw sortkey manually specified by the user, either via + -- [[Category:Foo|prefix]] or {{defaultsort:prefix}}. If nonempty, it's + -- concatenated with a line break followed by the page title before the sortkey + -- conversion algorithm is run. We store this so that we can update + -- collations without reparsing all pages. + -- Note: If you change the length of this field, you also need to change + -- code in LinksUpdate.php. See T27254. + cl_sortkey_prefix varchar(255) binary NOT NULL default '', + + -- This isn't really used at present. Provided for an optional + -- sorting method by approximate addition time. + cl_timestamp timestamp NOT NULL, + + -- Stores $wgCategoryCollation at the time cl_sortkey was generated. This + -- can be used to install new collation versions, tracking which rows are not + -- yet updated. '' means no collation, this is a legacy row that needs to be + -- updated by updateCollation.php. In the future, it might be possible to + -- specify different collations per category. + cl_collation varbinary(32) NOT NULL default '', + + -- Stores whether cl_from is a category, file, or other page, so we can + -- paginate the three categories separately. This never has to be updated + -- after the page is created, since none of these page types can be moved to + -- any other. + cl_type ENUM('page', 'subcat', 'file') NOT NULL default 'page', + PRIMARY KEY (cl_from,cl_to) +) /*$wgDBTableOptions*/; + +INSERT INTO /*_*/categorylinks_tmp + SELECT * + FROM /*_*/categorylinks; + +DROP TABLE /*_*/categorylinks; + +ALTER TABLE /*_*/categorylinks_tmp RENAME TO /*_*/categorylinks; + +-- We always sort within a given category, and within a given type. FIXME: +-- Formerly this index didn't cover cl_type (since that didn't exist), so old +-- callers won't be using an index: fix this? +CREATE INDEX /*i*/cl_sortkey ON /*_*/categorylinks (cl_to,cl_type,cl_sortkey,cl_from); + +-- Used by the API (and some extensions) +CREATE INDEX /*i*/cl_timestamp ON /*_*/categorylinks (cl_to,cl_timestamp); + +-- Used when updating collation (e.g. updateCollation.php) +CREATE INDEX /*i*/cl_collation_ext ON /*_*/categorylinks (cl_collation, cl_to, cl_type, cl_from);
\ No newline at end of file diff --git a/www/wiki/maintenance/sqlite/archives/patch-change_tag-ct_id.sql b/www/wiki/maintenance/sqlite/archives/patch-change_tag-ct_id.sql new file mode 100644 index 00000000..1c010943 --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-change_tag-ct_id.sql @@ -0,0 +1,25 @@ +DROP TABLE IF EXISTS /*_*/change_tag_tmp; + +CREATE TABLE /*$wgDBprefix*/change_tag_tmp ( + ct_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + ct_rc_id int NULL, + ct_log_id int NULL, + ct_rev_id int NULL, + ct_tag varchar(255) NOT NULL, + ct_params blob NULL +); + +INSERT OR IGNORE INTO /*_*/change_tag_tmp ( + ct_rc_id, ct_log_id, ct_rev_id, ct_tag, ct_params ) + SELECT + ct_rc_id, ct_log_id, ct_rev_id, ct_tag, ct_params + FROM /*_*/change_tag; + +DROP TABLE /*_*/change_tag; + +ALTER TABLE /*_*/change_tag_tmp RENAME TO /*_*/change_tag; + +CREATE UNIQUE INDEX /*i*/change_tag_rc_tag ON /*_*/change_tag (ct_rc_id,ct_tag); +CREATE UNIQUE INDEX /*i*/change_tag_log_tag ON /*_*/change_tag (ct_log_id,ct_tag); +CREATE UNIQUE INDEX /*i*/change_tag_rev_tag ON /*_*/change_tag (ct_rev_id,ct_tag); +CREATE INDEX /*i*/change_tag_tag_id ON /*_*/change_tag (ct_tag,ct_rc_id,ct_rev_id,ct_log_id); diff --git a/www/wiki/maintenance/sqlite/archives/patch-comment-table.sql b/www/wiki/maintenance/sqlite/archives/patch-comment-table.sql new file mode 100644 index 00000000..f743b55c --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-comment-table.sql @@ -0,0 +1,332 @@ +-- +-- patch-comment-table.sql +-- +-- T166732. Add a `comment` table and various columns (and temporary tables) to reference it. +-- Sigh, sqlite, such trouble just to change the default value of a column. + +CREATE TABLE /*_*/comment ( + comment_id bigint unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + comment_hash INT NOT NULL, + comment_text BLOB NOT NULL, + comment_data BLOB +) /*$wgDBTableOptions*/; +CREATE INDEX /*i*/comment_hash ON /*_*/comment (comment_hash); + +CREATE TABLE /*_*/revision_comment_temp ( + revcomment_rev int unsigned NOT NULL, + revcomment_comment_id bigint unsigned NOT NULL, + PRIMARY KEY (revcomment_rev, revcomment_comment_id) +) /*$wgDBTableOptions*/; +CREATE UNIQUE INDEX /*i*/revcomment_rev ON /*_*/revision_comment_temp (revcomment_rev); + +CREATE TABLE /*_*/image_comment_temp ( + imgcomment_name varchar(255) binary NOT NULL, + imgcomment_description_id bigint unsigned NOT NULL, + PRIMARY KEY (imgcomment_name, imgcomment_description_id) +) /*$wgDBTableOptions*/; +CREATE UNIQUE INDEX /*i*/imgcomment_name ON /*_*/image_comment_temp (imgcomment_name); + +ALTER TABLE /*_*/recentchanges + ADD COLUMN rc_comment_id bigint unsigned NOT NULL DEFAULT 0; + +ALTER TABLE /*_*/logging + ADD COLUMN log_comment_id bigint unsigned NOT NULL DEFAULT 0; + +BEGIN; + +DROP TABLE IF EXISTS /*_*/revision_tmp; +CREATE TABLE /*_*/revision_tmp ( + rev_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + rev_page int unsigned NOT NULL, + rev_text_id int unsigned NOT NULL, + rev_comment varbinary(767) NOT NULL default '', + rev_user int unsigned NOT NULL default 0, + rev_user_text varchar(255) binary NOT NULL default '', + rev_timestamp binary(14) NOT NULL default '', + rev_minor_edit tinyint unsigned NOT NULL default 0, + rev_deleted tinyint unsigned NOT NULL default 0, + rev_len int unsigned, + rev_parent_id int unsigned default NULL, + rev_sha1 varbinary(32) NOT NULL default '', + rev_content_model varbinary(32) DEFAULT NULL, + rev_content_format varbinary(64) DEFAULT NULL +) /*$wgDBTableOptions*/ MAX_ROWS=10000000 AVG_ROW_LENGTH=1024; + +INSERT OR IGNORE INTO /*_*/revision_tmp ( + rev_id, rev_page, rev_text_id, rev_comment, rev_user, rev_user_text, + rev_timestamp, rev_minor_edit, rev_deleted, rev_len, rev_parent_id, + rev_sha1, rev_content_model, rev_content_format) + SELECT + rev_id, rev_page, rev_text_id, rev_comment, rev_user, rev_user_text, + rev_timestamp, rev_minor_edit, rev_deleted, rev_len, rev_parent_id, + rev_sha1, rev_content_model, rev_content_format + FROM /*_*/revision; + +DROP TABLE /*_*/revision; +ALTER TABLE /*_*/revision_tmp RENAME TO /*_*/revision; +CREATE INDEX /*i*/rev_page_id ON /*_*/revision (rev_page, rev_id); +CREATE INDEX /*i*/rev_timestamp ON /*_*/revision (rev_timestamp); +CREATE INDEX /*i*/page_timestamp ON /*_*/revision (rev_page,rev_timestamp); +CREATE INDEX /*i*/user_timestamp ON /*_*/revision (rev_user,rev_timestamp); +CREATE INDEX /*i*/usertext_timestamp ON /*_*/revision (rev_user_text,rev_timestamp); +CREATE INDEX /*i*/page_user_timestamp ON /*_*/revision (rev_page,rev_user,rev_timestamp); + +COMMIT; + +BEGIN; + +DROP TABLE IF EXISTS /*_*/archive_tmp; +CREATE TABLE /*_*/archive_tmp ( + ar_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + ar_namespace int NOT NULL default 0, + ar_title varchar(255) binary NOT NULL default '', + ar_text mediumblob NOT NULL, + ar_comment varbinary(767) NOT NULL default '', + ar_comment_id bigint unsigned NOT NULL DEFAULT 0, + ar_user int unsigned NOT NULL default 0, + ar_user_text varchar(255) binary NOT NULL, + ar_timestamp binary(14) NOT NULL default '', + ar_minor_edit tinyint NOT NULL default 0, + ar_flags tinyblob NOT NULL, + ar_rev_id int unsigned, + ar_text_id int unsigned, + ar_deleted tinyint unsigned NOT NULL default 0, + ar_len int unsigned, + ar_page_id int unsigned, + ar_parent_id int unsigned default NULL, + ar_sha1 varbinary(32) NOT NULL default '', + ar_content_model varbinary(32) DEFAULT NULL, + ar_content_format varbinary(64) DEFAULT NULL +) /*$wgDBTableOptions*/; + +INSERT OR IGNORE INTO /*_*/archive_tmp ( + ar_id, ar_namespace, ar_title, ar_text, ar_comment, ar_user, ar_user_text, + ar_timestamp, ar_minor_edit, ar_flags, ar_rev_id, ar_text_id, ar_deleted, + ar_len, ar_page_id, ar_parent_id, ar_sha1, ar_content_model, + ar_content_format) + SELECT + ar_id, ar_namespace, ar_title, ar_text, ar_comment, ar_user, ar_user_text, + ar_timestamp, ar_minor_edit, ar_flags, ar_rev_id, ar_text_id, ar_deleted, + ar_len, ar_page_id, ar_parent_id, ar_sha1, ar_content_model, + ar_content_format + FROM /*_*/archive; + +DROP TABLE /*_*/archive; +ALTER TABLE /*_*/archive_tmp RENAME TO /*_*/archive; +CREATE INDEX /*i*/name_title_timestamp ON /*_*/archive (ar_namespace,ar_title,ar_timestamp); +CREATE INDEX /*i*/ar_usertext_timestamp ON /*_*/archive (ar_user_text,ar_timestamp); +CREATE INDEX /*i*/ar_revid ON /*_*/archive (ar_rev_id); + +COMMIT; + +BEGIN; + +DROP TABLE IF EXISTS ipblocks_tmp; +CREATE TABLE /*_*/ipblocks_tmp ( + ipb_id int NOT NULL PRIMARY KEY AUTO_INCREMENT, + ipb_address tinyblob NOT NULL, + ipb_user int unsigned NOT NULL default 0, + ipb_by int unsigned NOT NULL default 0, + ipb_by_text varchar(255) binary NOT NULL default '', + ipb_reason varbinary(767) NOT NULL default '', + ipb_reason_id bigint unsigned NOT NULL DEFAULT 0, + ipb_timestamp binary(14) NOT NULL default '', + ipb_auto bool NOT NULL default 0, + ipb_anon_only bool NOT NULL default 0, + ipb_create_account bool NOT NULL default 1, + ipb_enable_autoblock bool NOT NULL default '1', + ipb_expiry varbinary(14) NOT NULL default '', + ipb_range_start tinyblob NOT NULL, + ipb_range_end tinyblob NOT NULL, + ipb_deleted bool NOT NULL default 0, + ipb_block_email bool NOT NULL default 0, + ipb_allow_usertalk bool NOT NULL default 0, + ipb_parent_block_id int default NULL +) /*$wgDBTableOptions*/; + +INSERT OR IGNORE INTO /*_*/ipblocks_tmp ( + ipb_id, ipb_address, ipb_user, ipb_by, ipb_by_text, ipb_reason, + ipb_timestamp, ipb_auto, ipb_anon_only, ipb_create_account, + ipb_enable_autoblock, ipb_expiry, ipb_range_start, ipb_range_end, + ipb_deleted, ipb_block_email, ipb_allow_usertalk, ipb_parent_block_id) + SELECT + ipb_id, ipb_address, ipb_user, ipb_by, ipb_by_text, ipb_reason, + ipb_timestamp, ipb_auto, ipb_anon_only, ipb_create_account, + ipb_enable_autoblock, ipb_expiry, ipb_range_start, ipb_range_end, + ipb_deleted, ipb_block_email, ipb_allow_usertalk, ipb_parent_block_id + FROM /*_*/ipblocks; + +DROP TABLE /*_*/ipblocks; +ALTER TABLE /*_*/ipblocks_tmp RENAME TO /*_*/ipblocks; +CREATE UNIQUE INDEX /*i*/ipb_address ON /*_*/ipblocks (ipb_address(255), ipb_user, ipb_auto, ipb_anon_only); +CREATE INDEX /*i*/ipb_user ON /*_*/ipblocks (ipb_user); +CREATE INDEX /*i*/ipb_range ON /*_*/ipblocks (ipb_range_start(8), ipb_range_end(8)); +CREATE INDEX /*i*/ipb_timestamp ON /*_*/ipblocks (ipb_timestamp); +CREATE INDEX /*i*/ipb_expiry ON /*_*/ipblocks (ipb_expiry); +CREATE INDEX /*i*/ipb_parent_block_id ON /*_*/ipblocks (ipb_parent_block_id); + +COMMIT; + +BEGIN; + +DROP TABLE IF EXISTS /*_*/image_tmp; +CREATE TABLE /*_*/image_tmp ( + img_name varchar(255) binary NOT NULL default '' PRIMARY KEY, + img_size int unsigned NOT NULL default 0, + img_width int NOT NULL default 0, + img_height int NOT NULL default 0, + img_metadata mediumblob NOT NULL, + img_bits int NOT NULL default 0, + img_media_type ENUM("UNKNOWN", "BITMAP", "DRAWING", "AUDIO", "VIDEO", "MULTIMEDIA", "OFFICE", "TEXT", "EXECUTABLE", "ARCHIVE") default NULL, + img_major_mime ENUM("unknown", "application", "audio", "image", "text", "video", "message", "model", "multipart", "chemical") NOT NULL default "unknown", + img_minor_mime varbinary(100) NOT NULL default "unknown", + img_description varbinary(767) NOT NULL default '', + img_user int unsigned NOT NULL default 0, + img_user_text varchar(255) binary NOT NULL, + img_timestamp varbinary(14) NOT NULL default '', + img_sha1 varbinary(32) NOT NULL default '' +) /*$wgDBTableOptions*/; + +INSERT OR IGNORE INTO /*_*/image_tmp ( + img_name, img_size, img_width, img_height, img_metadata, img_bits, + img_media_type, img_major_mime, img_minor_mime, img_description, img_user, + img_user_text, img_timestamp, img_sha1) + SELECT + img_name, img_size, img_width, img_height, img_metadata, img_bits, + img_media_type, img_major_mime, img_minor_mime, img_description, img_user, + img_user_text, img_timestamp, img_sha1 + FROM /*_*/image; + +DROP TABLE /*_*/image; +ALTER TABLE /*_*/image_tmp RENAME TO /*_*/image; +CREATE INDEX /*i*/img_user_timestamp ON /*_*/image (img_user,img_timestamp); +CREATE INDEX /*i*/img_usertext_timestamp ON /*_*/image (img_user_text,img_timestamp); +CREATE INDEX /*i*/img_size ON /*_*/image (img_size); +CREATE INDEX /*i*/img_timestamp ON /*_*/image (img_timestamp); +CREATE INDEX /*i*/img_sha1 ON /*_*/image (img_sha1(10)); +CREATE INDEX /*i*/img_media_mime ON /*_*/image (img_media_type,img_major_mime,img_minor_mime); + +COMMIT; + +BEGIN; + +DROP TABLE IF EXISTS /*_*/oldimage_tmp; +CREATE TABLE /*_*/oldimage_tmp ( + oi_name varchar(255) binary NOT NULL default '', + oi_archive_name varchar(255) binary NOT NULL default '', + oi_size int unsigned NOT NULL default 0, + oi_width int NOT NULL default 0, + oi_height int NOT NULL default 0, + oi_bits int NOT NULL default 0, + oi_description varbinary(767) NOT NULL default '', + oi_description_id bigint unsigned NOT NULL DEFAULT 0, + oi_user int unsigned NOT NULL default 0, + oi_user_text varchar(255) binary NOT NULL, + oi_timestamp binary(14) NOT NULL default '', + oi_metadata mediumblob NOT NULL, + oi_media_type ENUM("UNKNOWN", "BITMAP", "DRAWING", "AUDIO", "VIDEO", "MULTIMEDIA", "OFFICE", "TEXT", "EXECUTABLE", "ARCHIVE") default NULL, + oi_major_mime ENUM("unknown", "application", "audio", "image", "text", "video", "message", "model", "multipart", "chemical") NOT NULL default "unknown", + oi_minor_mime varbinary(100) NOT NULL default "unknown", + oi_deleted tinyint unsigned NOT NULL default 0, + oi_sha1 varbinary(32) NOT NULL default '' +) /*$wgDBTableOptions*/; + +INSERT OR IGNORE INTO /*_*/oldimage_tmp ( + oi_name, oi_archive_name, oi_size, oi_width, oi_height, oi_bits, + oi_description, oi_user, oi_user_text, oi_timestamp, oi_metadata, + oi_media_type, oi_major_mime, oi_minor_mime, oi_deleted, oi_sha1) + SELECT + oi_name, oi_archive_name, oi_size, oi_width, oi_height, oi_bits, + oi_description, oi_user, oi_user_text, oi_timestamp, oi_metadata, + oi_media_type, oi_major_mime, oi_minor_mime, oi_deleted, oi_sha1 + FROM /*_*/oldimage; + +DROP TABLE /*_*/oldimage; +ALTER TABLE /*_*/oldimage_tmp RENAME TO /*_*/oldimage; +CREATE INDEX /*i*/oi_usertext_timestamp ON /*_*/oldimage (oi_user_text,oi_timestamp); +CREATE INDEX /*i*/oi_name_timestamp ON /*_*/oldimage (oi_name,oi_timestamp); +CREATE INDEX /*i*/oi_name_archive_name ON /*_*/oldimage (oi_name,oi_archive_name(14)); +CREATE INDEX /*i*/oi_sha1 ON /*_*/oldimage (oi_sha1(10)); + +COMMIT; + +BEGIN; + +DROP TABLE IF EXISTS /*_*/filearchive_tmp; +CREATE TABLE /*_*/filearchive_tmp ( + fa_id int NOT NULL PRIMARY KEY AUTO_INCREMENT, + fa_name varchar(255) binary NOT NULL default '', + fa_archive_name varchar(255) binary default '', + fa_storage_group varbinary(16), + fa_storage_key varbinary(64) default '', + fa_deleted_user int, + fa_deleted_timestamp binary(14) default '', + fa_deleted_reason varbinary(767) default '', + fa_deleted_reason_id bigint unsigned NOT NULL DEFAULT 0, + fa_size int unsigned default 0, + fa_width int default 0, + fa_height int default 0, + fa_metadata mediumblob, + fa_bits int default 0, + fa_media_type ENUM("UNKNOWN", "BITMAP", "DRAWING", "AUDIO", "VIDEO", "MULTIMEDIA", "OFFICE", "TEXT", "EXECUTABLE", "ARCHIVE") default NULL, + fa_major_mime ENUM("unknown", "application", "audio", "image", "text", "video", "message", "model", "multipart", "chemical") default "unknown", + fa_minor_mime varbinary(100) default "unknown", + fa_description varbinary(767) default '', + fa_description_id bigint unsigned NOT NULL DEFAULT 0, + fa_user int unsigned default 0, + fa_user_text varchar(255) binary, + fa_timestamp binary(14) default '', + fa_deleted tinyint unsigned NOT NULL default 0, + fa_sha1 varbinary(32) NOT NULL default '' +) /*$wgDBTableOptions*/; + +INSERT OR IGNORE INTO /*_*/filearchive_tmp ( + fa_id, fa_name, fa_archive_name, fa_storage_group, fa_storage_key, + fa_deleted_user, fa_deleted_timestamp, fa_deleted_reason, fa_size, + fa_width, fa_height, fa_metadata, fa_bits, fa_media_type, fa_major_mime, + fa_minor_mime, fa_description, fa_user, fa_user_text, fa_timestamp, + fa_deleted, fa_sha1) + SELECT + fa_id, fa_name, fa_archive_name, fa_storage_group, fa_storage_key, + fa_deleted_user, fa_deleted_timestamp, fa_deleted_reason, fa_size, + fa_width, fa_height, fa_metadata, fa_bits, fa_media_type, fa_major_mime, + fa_minor_mime, fa_description, fa_user, fa_user_text, fa_timestamp, + fa_deleted, fa_sha1 + FROM /*_*/filearchive; + +DROP TABLE /*_*/filearchive; +ALTER TABLE /*_*/filearchive_tmp RENAME TO /*_*/filearchive; +CREATE INDEX /*i*/fa_name ON /*_*/filearchive (fa_name, fa_timestamp); +CREATE INDEX /*i*/fa_storage_group ON /*_*/filearchive (fa_storage_group, fa_storage_key); +CREATE INDEX /*i*/fa_deleted_timestamp ON /*_*/filearchive (fa_deleted_timestamp); +CREATE INDEX /*i*/fa_user_timestamp ON /*_*/filearchive (fa_user_text,fa_timestamp); +CREATE INDEX /*i*/fa_sha1 ON /*_*/filearchive (fa_sha1(10)); + +COMMIT; + +BEGIN; + +DROP TABLE IF EXISTS /*_*/protected_titles_tmp; +CREATE TABLE /*_*/protected_titles_tmp ( + pt_namespace int NOT NULL, + pt_title varchar(255) binary NOT NULL, + pt_user int unsigned NOT NULL, + pt_reason varbinary(767) default '', + pt_reason_id bigint unsigned NOT NULL DEFAULT 0, + pt_timestamp binary(14) NOT NULL, + pt_expiry varbinary(14) NOT NULL default '', + pt_create_perm varbinary(60) NOT NULL +) /*$wgDBTableOptions*/; + +INSERT OR IGNORE INTO /*_*/protected_titles_tmp ( + pt_namespace, pt_title, pt_user, pt_reason, pt_timestamp, pt_expiry, pt_create_perm) + SELECT + pt_namespace, pt_title, pt_user, pt_reason, pt_timestamp, pt_expiry, pt_create_perm + FROM /*_*/protected_titles; + +DROP TABLE /*_*/protected_titles; +ALTER TABLE /*_*/protected_titles_tmp RENAME TO /*_*/protected_titles; +CREATE UNIQUE INDEX /*i*/pt_namespace_title ON /*_*/protected_titles (pt_namespace,pt_title); +CREATE INDEX /*i*/pt_timestamp ON /*_*/protected_titles (pt_timestamp); + +COMMIT; diff --git a/www/wiki/maintenance/sqlite/archives/patch-drop-page_counter.sql b/www/wiki/maintenance/sqlite/archives/patch-drop-page_counter.sql new file mode 100644 index 00000000..ac8151da --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-drop-page_counter.sql @@ -0,0 +1,31 @@ +-- field is deprecated and no longer updated as of 1.25 +CREATE TABLE /*_*/page_tmp ( + page_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + page_namespace int NOT NULL, + page_title varchar(255) binary NOT NULL, + page_restrictions tinyblob NOT NULL, + page_is_redirect tinyint unsigned NOT NULL default 0, + page_is_new tinyint unsigned NOT NULL default 0, + page_random real unsigned NOT NULL, + page_touched binary(14) NOT NULL default '', + page_links_updated varbinary(14) NULL default NULL, + page_latest int unsigned NOT NULL, + page_len int unsigned NOT NULL, + page_content_model varbinary(32) DEFAULT NULL, + page_lang varbinary(35) DEFAULT NULL +) /*$wgDBTableOptions*/; + +INSERT INTO /*_*/page_tmp + SELECT page_id, page_namespace, page_title, page_restrictions, page_is_redirect, + page_is_new, page_random, page_touched, page_links_updated, page_latest, page_len, + page_content_model, page_lang + FROM /*_*/page; + +DROP TABLE /*_*/page; + +ALTER TABLE /*_*/page_tmp RENAME TO /*_*/page; + +CREATE UNIQUE INDEX /*i*/name_title ON /*_*/page (page_namespace,page_title); +CREATE INDEX /*i*/page_random ON /*_*/page (page_random); +CREATE INDEX /*i*/page_len ON /*_*/page (page_len); +CREATE INDEX /*i*/page_redirect_namespace_len ON /*_*/page (page_is_redirect, page_namespace, page_len); diff --git a/www/wiki/maintenance/sqlite/archives/patch-drop-rc_cur_time.sql b/www/wiki/maintenance/sqlite/archives/patch-drop-rc_cur_time.sql new file mode 100644 index 00000000..350479fb --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-drop-rc_cur_time.sql @@ -0,0 +1,45 @@ +-- rc_cur_time is no longer used, delete the field +CREATE TABLE /*_*/recentchanges_tmp ( + rc_id int NOT NULL PRIMARY KEY AUTO_INCREMENT, + rc_timestamp varbinary(14) NOT NULL default '', + rc_user int unsigned NOT NULL default 0, + rc_user_text varchar(255) binary NOT NULL, + rc_namespace int NOT NULL default 0, + rc_title varchar(255) binary NOT NULL default '', + rc_comment varchar(255) binary NOT NULL default '', + rc_minor tinyint unsigned NOT NULL default 0, + rc_bot tinyint unsigned NOT NULL default 0, + rc_new tinyint unsigned NOT NULL default 0, + rc_cur_id int unsigned NOT NULL default 0, + rc_this_oldid int unsigned NOT NULL default 0, + rc_last_oldid int unsigned NOT NULL default 0, + rc_type tinyint unsigned NOT NULL default 0, + rc_source varchar(16) binary not null default '', + rc_patrolled tinyint unsigned NOT NULL default 0, + rc_ip varbinary(40) NOT NULL default '', + rc_old_len int, + rc_new_len int, + rc_deleted tinyint unsigned NOT NULL default 0, + rc_logid int unsigned NOT NULL default 0, + rc_log_type varbinary(255) NULL default NULL, + rc_log_action varbinary(255) NULL default NULL, + rc_params blob NULL +) /*$wgDBTableOptions*/; + +INSERT INTO /*_*/recentchanges_tmp + SELECT rc_id, rc_timestamp, rc_user, rc_user_text, rc_namespace, rc_title, rc_comment, rc_minor, + rc_bot, rc_new, rc_cur_id, rc_this_oldid, rc_last_oldid, rc_type, rc_source, rc_patrolled, + rc_ip, rc_old_len, rc_new_len, rc_deleted, rc_logid, rc_log_type, rc_log_action, rc_params + FROM /*_*/recentchanges; + +DROP TABLE /*_*/recentchanges; + +ALTER TABLE /*_*/recentchanges_tmp RENAME TO /*_*/recentchanges; + +CREATE INDEX /*i*/rc_timestamp ON /*_*/recentchanges (rc_timestamp); +CREATE INDEX /*i*/rc_namespace_title ON /*_*/recentchanges (rc_namespace, rc_title); +CREATE INDEX /*i*/rc_cur_id ON /*_*/recentchanges (rc_cur_id); +CREATE INDEX /*i*/new_name_timestamp ON /*_*/recentchanges (rc_new,rc_namespace,rc_timestamp); +CREATE INDEX /*i*/rc_ip ON /*_*/recentchanges (rc_ip); +CREATE INDEX /*i*/rc_ns_usertext ON /*_*/recentchanges (rc_namespace, rc_user_text); +CREATE INDEX /*i*/rc_user_text ON /*_*/recentchanges (rc_user_text, rc_timestamp);
\ No newline at end of file diff --git a/www/wiki/maintenance/sqlite/archives/patch-drop-ss_admins.sql b/www/wiki/maintenance/sqlite/archives/patch-drop-ss_admins.sql new file mode 100644 index 00000000..39606630 --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-drop-ss_admins.sql @@ -0,0 +1,21 @@ +-- field is deprecated and no longer updated as of 1.5 +CREATE TABLE /*_*/site_stats_tmp ( + ss_row_id int unsigned NOT NULL, + ss_total_edits bigint unsigned default 0, + ss_good_articles bigint unsigned default 0, + ss_total_pages bigint default '-1', + ss_users bigint default '-1', + ss_active_users bigint default '-1', + ss_images int default 0 +) /*$wgDBTableOptions*/; + +INSERT INTO /*_*/site_stats_tmp + SELECT ss_row_id, ss_total_edits, ss_good_articles, + ss_total_pages, ss_users, ss_active_users, ss_images + FROM /*_*/site_stats; + +DROP TABLE /*_*/site_stats; + +ALTER TABLE /*_*/site_stats_tmp RENAME TO /*_*/site_stats; + +CREATE UNIQUE INDEX /*i*/ss_row_id ON /*_*/site_stats (ss_row_id);
\ No newline at end of file diff --git a/www/wiki/maintenance/sqlite/archives/patch-drop-ss_total_views.sql b/www/wiki/maintenance/sqlite/archives/patch-drop-ss_total_views.sql new file mode 100644 index 00000000..ad80988d --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-drop-ss_total_views.sql @@ -0,0 +1,21 @@ +-- field is deprecated and no longer updated as of 1.25 +CREATE TABLE /*_*/site_stats_tmp ( + ss_row_id int unsigned NOT NULL, + ss_total_edits bigint unsigned default 0, + ss_good_articles bigint unsigned default 0, + ss_total_pages bigint default '-1', + ss_users bigint default '-1', + ss_active_users bigint default '-1', + ss_images int default 0 +) /*$wgDBTableOptions*/; + +INSERT INTO /*_*/site_stats_tmp + SELECT ss_row_id, ss_total_edits, ss_good_articles, ss_total_pages, + ss_users, ss_active_users, ss_images + FROM /*_*/site_stats; + +DROP TABLE /*_*/site_stats; + +ALTER TABLE /*_*/site_stats_tmp RENAME TO /*_*/site_stats; + +CREATE UNIQUE INDEX /*i*/ss_row_id ON /*_*/site_stats (ss_row_id); diff --git a/www/wiki/maintenance/sqlite/archives/patch-drop-user_options.sql b/www/wiki/maintenance/sqlite/archives/patch-drop-user_options.sql new file mode 100644 index 00000000..5bc6a47c --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-drop-user_options.sql @@ -0,0 +1,31 @@ +-- Remove user_options field from user table + +CREATE TABLE /*_*/user_tmp ( + user_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + user_name varchar(255) binary NOT NULL default '', + user_real_name varchar(255) binary NOT NULL default '', + user_password tinyblob NOT NULL, + user_newpassword tinyblob NOT NULL, + user_newpass_time binary(14), + user_email tinytext NOT NULL, + user_touched binary(14) NOT NULL default '', + user_token binary(32) NOT NULL default '', + user_email_authenticated binary(14), + user_email_token binary(32), + user_email_token_expires binary(14), + user_registration binary(14), + user_editcount int +) /*$wgDBTableOptions*/; + +INSERT INTO /*_*/user_tmp + SELECT user_id, user_name, user_real_name, user_password, user_newpassword, user_newpass_time, user_email, user_touched, + user_token, user_email_authenticated, user_email_token, user_email_token_expires, user_registration, user_editcount + FROM /*_*/user; + +DROP TABLE /*_*/user; + +ALTER TABLE /*_*/user_tmp RENAME TO /*_*/user; + +CREATE UNIQUE INDEX /*i*/user_name ON /*_*/user (user_name); +CREATE INDEX /*i*/user_email_token ON /*_*/user (user_email_token); +CREATE INDEX /*i*/user_email ON /*_*/user (user_email(50)); diff --git a/www/wiki/maintenance/sqlite/archives/patch-editsummary-length.sql b/www/wiki/maintenance/sqlite/archives/patch-editsummary-length.sql new file mode 100644 index 00000000..f86b2ada --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-editsummary-length.sql @@ -0,0 +1,65 @@ +CREATE TABLE /*_*/filearchive_tmp ( + -- Unique row id + fa_id int NOT NULL PRIMARY KEY AUTO_INCREMENT, + + -- Original base filename; key to image.img_name, page.page_title, etc + fa_name varchar(255) binary NOT NULL default '', + + -- Filename of archived file, if an old revision + fa_archive_name varchar(255) binary default '', + + -- Which storage bin (directory tree or object store) the file data + -- is stored in. Should be 'deleted' for files that have been deleted; + -- any other bin is not yet in use. + fa_storage_group varbinary(16), + + -- SHA-1 of the file contents plus extension, used as a key for storage. + -- eg 8f8a562add37052a1848ff7771a2c515db94baa9.jpg + -- + -- If NULL, the file was missing at deletion time or has been purged + -- from the archival storage. + fa_storage_key varbinary(64) default '', + + -- Deletion information, if this file is deleted. + fa_deleted_user int, + fa_deleted_timestamp binary(14) default '', + fa_deleted_reason varbinary(767) default '', + -- Duped fields from image + fa_size int unsigned default 0, + fa_width int default 0, + fa_height int default 0, + fa_metadata mediumblob, + fa_bits int default 0, + fa_media_type ENUM("UNKNOWN", "BITMAP", "DRAWING", "AUDIO", "VIDEO", "MULTIMEDIA", "OFFICE", "TEXT", "EXECUTABLE", "ARCHIVE") default NULL, + fa_major_mime ENUM("unknown", "application", "audio", "image", "text", "video", "message", "model", "multipart", "chemical") default "unknown", + fa_minor_mime varbinary(100) default "unknown", + fa_description varbinary(767), + fa_user int unsigned default 0, + fa_user_text varchar(255) binary, + fa_timestamp binary(14) default '', + + -- Visibility of deleted revisions, bitfield + fa_deleted tinyint unsigned NOT NULL default 0, + + -- sha1 hash of file content + fa_sha1 varbinary(32) NOT NULL default '' +) /*$wgDBTableOptions*/; + + +INSERT INTO /*_*/filearchive_tmp + SELECT fa_id, fa_name, fa_archive_name, fa_storage_group, fa_storage_key, fa_deleted_user, fa_deleted_timestamp, + fa_deleted_reason, fa_size, fa_width, fa_height, fa_metadata, fa_bits, fa_media_type, fa_major_mime, + fa_minor_mime, fa_description, fa_user, fa_user_text, fa_timestamp, fa_deleted, fa_sha1 + FROM /*_*/filearchive; + +DROP TABLE /*_*/filearchive; + +ALTER TABLE /*_*/filearchive_tmp RENAME TO /*_*/filearchive; + + +CREATE INDEX /*i*/fa_name ON /*_*/filearchive (fa_name, fa_timestamp); +CREATE INDEX /*i*/fa_storage_group ON /*_*/filearchive (fa_storage_group, fa_storage_key); +CREATE INDEX /*i*/fa_deleted_timestamp ON /*_*/filearchive (fa_deleted_timestamp); +CREATE INDEX /*i*/fa_user_timestamp ON /*_*/filearchive (fa_user_text,fa_timestamp); +CREATE INDEX /*i*/fa_sha1 ON /*_*/filearchive (fa_sha1(10)); + diff --git a/www/wiki/maintenance/sqlite/archives/patch-externallinks-el_id.sql b/www/wiki/maintenance/sqlite/archives/patch-externallinks-el_id.sql new file mode 100644 index 00000000..0aad4071 --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-externallinks-el_id.sql @@ -0,0 +1,19 @@ +DROP TABLE IF EXISTS /*_*/externallinks_tmp; + +CREATE TABLE /*$wgDBprefix*/externallinks_tmp ( + el_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + el_from int unsigned NOT NULL default 0, + el_to blob NOT NULL, + el_index blob NOT NULL +); + +INSERT OR IGNORE INTO /*_*/externallinks_tmp (el_from, el_to, el_index) SELECT + el_from, el_to, el_index FROM /*_*/externallinks; + +DROP TABLE /*_*/externallinks; + +ALTER TABLE /*_*/externallinks_tmp RENAME TO /*_*/externallinks; + +CREATE INDEX /*i*/el_from ON /*_*/externallinks (el_from, el_to(40)); +CREATE INDEX /*i*/el_to ON /*_*/externallinks (el_to(60), el_from); +CREATE INDEX /*i*/el_index ON /*_*/externallinks (el_index(60));
\ No newline at end of file diff --git a/www/wiki/maintenance/sqlite/archives/patch-imagelinks-fix-pk.sql b/www/wiki/maintenance/sqlite/archives/patch-imagelinks-fix-pk.sql new file mode 100644 index 00000000..b48bea53 --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-imagelinks-fix-pk.sql @@ -0,0 +1,25 @@ +CREATE TABLE /*_*/imagelinks_tmp ( + -- Key to page_id of the page containing the image / media link. + il_from int unsigned NOT NULL default 0, + -- Namespace for this page + il_from_namespace int NOT NULL default 0, + + -- Filename of target image. + -- This is also the page_title of the file's description page; + -- all such pages are in namespace 6 (NS_FILE). + il_to varchar(255) binary NOT NULL default '', + PRIMARY KEY (il_from,il_to) +) /*$wgDBTableOptions*/; + +INSERT INTO /*_*/imagelinks_tmp + SELECT * FROM /*_*/imagelinks; + +DROP TABLE /*_*/imagelinks; + +ALTER TABLE /*_*/imagelinks_tmp RENAME TO /*_*/imagelinks; + +-- Reverse index, for Special:Whatlinkshere and file description page local usage +CREATE INDEX /*i*/il_to ON /*_*/imagelinks (il_to,il_from); + +-- Index for Special:Whatlinkshere with namespace filter +CREATE INDEX /*i*/il_backlinks_namespace ON /*_*/imagelinks (il_from_namespace,il_to,il_from);
\ No newline at end of file diff --git a/www/wiki/maintenance/sqlite/archives/patch-ip_changes.sql b/www/wiki/maintenance/sqlite/archives/patch-ip_changes.sql new file mode 100644 index 00000000..5f05672e --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-ip_changes.sql @@ -0,0 +1,23 @@ +-- +-- Every time an edit by a logged out user is saved, +-- a row is created in ip_changes. This stores +-- the IP as a hex representation so that we can more +-- easily find edits within an IP range. +-- +CREATE TABLE /*_*/ip_changes ( + -- Foreign key to the revision table, also serves as the unique primary key + ipc_rev_id int unsigned NOT NULL PRIMARY KEY DEFAULT '0', + + -- The timestamp of the revision + ipc_rev_timestamp binary(14) NOT NULL DEFAULT '', + + -- Hex representation of the IP address, as returned by IP::toHex() + -- For IPv4 it will resemble: ABCD1234 + -- For IPv6: v6-ABCD1234000000000000000000000000 + -- BETWEEN is then used to identify revisions within a given range + ipc_hex varbinary(35) NOT NULL DEFAULT '' + +) /*$wgDBTableOptions*/; + +CREATE INDEX /*i*/ipc_rev_timestamp ON /*_*/ip_changes (ipc_rev_timestamp); +CREATE INDEX /*i*/ipc_hex_time ON /*_*/ip_changes (ipc_hex,ipc_rev_timestamp); diff --git a/www/wiki/maintenance/sqlite/archives/patch-iw_api_and_wikiid.sql b/www/wiki/maintenance/sqlite/archives/patch-iw_api_and_wikiid.sql new file mode 100644 index 00000000..f9172b5e --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-iw_api_and_wikiid.sql @@ -0,0 +1,19 @@ +-- +-- Add iw_api and iw_wikiid to interwiki table +-- + + +CREATE TABLE /*_*/interwiki_tmp ( + iw_prefix TEXT NOT NULL, + iw_url BLOB NOT NULL, + iw_api BLOB NOT NULL, + iw_wikiid TEXT NOT NULL, + iw_local INTEGER NOT NULL, + iw_trans INTEGER NOT NULL default 0 +) /*$wgDBTableOptions*/; + +INSERT INTO /*_*/interwiki_tmp SELECT iw_prefix, iw_url, '', '', iw_local, iw_trans FROM /*_*/interwiki; +DROP TABLE /*_*/interwiki; +ALTER TABLE /*_*/interwiki_tmp RENAME TO /*_*/interwiki; + +CREATE UNIQUE INDEX /*i*/iw_prefix ON /*_*/interwiki (iw_prefix);
\ No newline at end of file diff --git a/www/wiki/maintenance/sqlite/archives/patch-iwlinks-fix-pk.sql b/www/wiki/maintenance/sqlite/archives/patch-iwlinks-fix-pk.sql new file mode 100644 index 00000000..91ce2519 --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-iwlinks-fix-pk.sql @@ -0,0 +1,24 @@ +CREATE TABLE /*_*/iwlinks_tmp ( + -- page_id of the referring page + iwl_from int unsigned NOT NULL default 0, + + -- Interwiki prefix code of the target + iwl_prefix varbinary(20) NOT NULL default '', + + -- Title of the target, including namespace + iwl_title varchar(255) binary NOT NULL default '', + PRIMARY KEY (iwl_from,iwl_prefix,iwl_title) +) /*$wgDBTableOptions*/; + +INSERT INTO /*_*/iwlinks_tmp + SELECT * FROM /*_*/iwlinks; + +DROP TABLE /*_*/iwlinks; + +ALTER TABLE /*_*/iwlinks_tmp RENAME TO /*_*/iwlinks; + +-- Index for ApiQueryIWBacklinks +CREATE INDEX /*i*/iwl_prefix_title_from ON /*_*/iwlinks (iwl_prefix, iwl_title, iwl_from); + +-- Index for ApiQueryIWLinks +CREATE INDEX /*i*/iwl_prefix_from_title ON /*_*/iwlinks (iwl_prefix, iwl_from, iwl_title);
\ No newline at end of file diff --git a/www/wiki/maintenance/sqlite/archives/patch-job_token.sql b/www/wiki/maintenance/sqlite/archives/patch-job_token.sql new file mode 100644 index 00000000..4e4d28fd --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-job_token.sql @@ -0,0 +1,8 @@ +ALTER TABLE /*_*/job ADD COLUMN job_random integer unsigned NOT NULL default 0; +ALTER TABLE /*_*/job ADD COLUMN job_token varbinary(32) NOT NULL default ''; +ALTER TABLE /*_*/job ADD COLUMN job_sha1 varbinary(32) NOT NULL default ''; +ALTER TABLE /*_*/job ADD COLUMN job_token_timestamp varbinary(14) NULL default NULL; + +CREATE INDEX /*i*/job_sha1 ON /*_*/job (job_sha1); +CREATE INDEX /*i*/job_cmd_token ON /*_*/job (job_cmd,job_token,job_random); + diff --git a/www/wiki/maintenance/sqlite/archives/patch-jobs-add-timestamp.sql b/www/wiki/maintenance/sqlite/archives/patch-jobs-add-timestamp.sql new file mode 100644 index 00000000..c5e6e711 --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-jobs-add-timestamp.sql @@ -0,0 +1,2 @@ +ALTER TABLE /*_*/job ADD COLUMN job_timestamp varbinary(14) NULL default NULL; +CREATE INDEX /*i*/job_timestamp ON /*_*/job(job_timestamp); diff --git a/www/wiki/maintenance/sqlite/archives/patch-kill-iwl_prefix.sql b/www/wiki/maintenance/sqlite/archives/patch-kill-iwl_prefix.sql new file mode 100644 index 00000000..78ed385e --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-kill-iwl_prefix.sql @@ -0,0 +1,7 @@ +-- +-- Kill the old iwl_prefix index, which may be present on some +-- installs if they ran update.php between it being added and being renamed +-- + +DROP INDEX IF EXISTS /*i*/iwl_prefix; + diff --git a/www/wiki/maintenance/sqlite/archives/patch-l10n_cache-primary-key.sql b/www/wiki/maintenance/sqlite/archives/patch-l10n_cache-primary-key.sql new file mode 100644 index 00000000..55df392c --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-l10n_cache-primary-key.sql @@ -0,0 +1,12 @@ +-- +-- patch-l10n_cache-primary-key.sql +-- +-- Bug T146591. Add l10n_cache primary key +DROP TABLE IF EXISTS /*_*/l10n_cache; + +CREATE TABLE /*$wgDBprefix*/l10n_cache ( + lc_lang varbinary(32) NOT NULL, + lc_key varchar(255) NOT NULL, + lc_value mediumblob NOT NULL, + PRIMARY KEY (lc_lang, lc_key) +) /*$wgDBTableOptions*/; diff --git a/www/wiki/maintenance/sqlite/archives/patch-langlinks-fix-pk.sql b/www/wiki/maintenance/sqlite/archives/patch-langlinks-fix-pk.sql new file mode 100644 index 00000000..da096ace --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-langlinks-fix-pk.sql @@ -0,0 +1,21 @@ +CREATE TABLE /*_*/langlinks_tmp ( + -- page_id of the referring page + ll_from int unsigned NOT NULL default 0, + + -- Language code of the target + ll_lang varbinary(20) NOT NULL default '', + + -- Title of the target, including namespace + ll_title varchar(255) binary NOT NULL default '', + PRIMARY KEY (ll_from,ll_lang) +) /*$wgDBTableOptions*/; + +INSERT INTO /*_*/langlinks_tmp + SELECT * FROM /*_*/langlinks; + +DROP TABLE /*_*/langlinks; + +ALTER TABLE /*_*/langlinks_tmp RENAME TO /*_*/langlinks; + +-- Index for ApiQueryLangbacklinks +CREATE INDEX /*i*/ll_lang ON /*_*/langlinks (ll_lang, ll_title);
\ No newline at end of file diff --git a/www/wiki/maintenance/sqlite/archives/patch-log_search-fix-pk.sql b/www/wiki/maintenance/sqlite/archives/patch-log_search-fix-pk.sql new file mode 100644 index 00000000..153e4150 --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-log_search-fix-pk.sql @@ -0,0 +1,18 @@ +CREATE TABLE /*_*/log_search_tmp ( + -- The type of ID (rev ID, log ID, rev timestamp, username) + ls_field varbinary(32) NOT NULL, + -- The value of the ID + ls_value varchar(255) NOT NULL, + -- Key to log_id + ls_log_id int unsigned NOT NULL default 0, + PRIMARY KEY (ls_field,ls_value,ls_log_id) +) /*$wgDBTableOptions*/; + +INSERT INTO /*_*/log_search_tmp + SELECT * FROM /*_*/log_search; + +DROP TABLE /*_*/log_search; + +ALTER TABLE /*_*/log_search_tmp RENAME TO /*_*/log_search; + +CREATE INDEX /*i*/ls_log_id ON /*_*/log_search (ls_log_id);
\ No newline at end of file diff --git a/www/wiki/maintenance/sqlite/archives/patch-log_search-rename-index.sql b/www/wiki/maintenance/sqlite/archives/patch-log_search-rename-index.sql new file mode 100644 index 00000000..4b98a0f2 --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-log_search-rename-index.sql @@ -0,0 +1 @@ +CREATE UNIQUE INDEX ls_field_val ON /*_*/log_search (ls_field,ls_value,ls_log_id); diff --git a/www/wiki/maintenance/sqlite/archives/patch-log_user_text.sql b/www/wiki/maintenance/sqlite/archives/patch-log_user_text.sql new file mode 100644 index 00000000..c7fcc75f --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-log_user_text.sql @@ -0,0 +1,5 @@ +ALTER TABLE /*$wgDBprefix*/logging ADD COLUMN log_user_text TEXT NOT NULL default ''; +ALTER TABLE /*$wgDBprefix*/logging ADD COLUMN log_page INTEGER NULL; + +CREATE INDEX /*i*/log_user_type_time ON /*_*/logging (log_user, log_type, log_timestamp); +CREATE INDEX /*i*/log_page_id_time ON /*_*/logging (log_page,log_timestamp); diff --git a/www/wiki/maintenance/sqlite/archives/patch-module_deps-fix-pk.sql b/www/wiki/maintenance/sqlite/archives/patch-module_deps-fix-pk.sql new file mode 100644 index 00000000..73bcbe23 --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-module_deps-fix-pk.sql @@ -0,0 +1,16 @@ +CREATE TABLE /*_*/module_deps_tmp ( + -- Module name + md_module varbinary(255) NOT NULL, + -- Module context vary (includes skin and language; called "md_skin" for legacy reasons) + md_skin varbinary(32) NOT NULL, + -- JSON blob with file dependencies + md_deps mediumblob NOT NULL, + PRIMARY KEY (md_module,md_skin) +) /*$wgDBTableOptions*/; + +INSERT INTO /*_*/module_deps_tmp + SELECT * FROM /*_*/module_deps; + +DROP TABLE /*_*/module_deps; + +ALTER TABLE /*_*/module_deps_tmp RENAME TO /*_*/module_deps;
\ No newline at end of file diff --git a/www/wiki/maintenance/sqlite/archives/patch-objectcache-fix-pk.sql b/www/wiki/maintenance/sqlite/archives/patch-objectcache-fix-pk.sql new file mode 100644 index 00000000..f2bef583 --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-objectcache-fix-pk.sql @@ -0,0 +1,14 @@ +CREATE TABLE /*_*/objectcache_tmp ( + keyname varbinary(255) NOT NULL default '' PRIMARY KEY, + value mediumblob, + exptime datetime +) /*$wgDBTableOptions*/; + +INSERT INTO /*_*/objectcache_tmp + SELECT * FROM /*_*/objectcache; + +DROP TABLE /*_*/objectcache; + +ALTER TABLE /*_*/objectcache_tmp RENAME TO /*_*/objectcache; + +CREATE INDEX /*i*/exptime ON /*_*/objectcache (exptime);
\ No newline at end of file diff --git a/www/wiki/maintenance/sqlite/archives/patch-page-page_lang.sql b/www/wiki/maintenance/sqlite/archives/patch-page-page_lang.sql new file mode 100644 index 00000000..8de2dc7b --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-page-page_lang.sql @@ -0,0 +1,3 @@ +-- Add page_lang column + +ALTER TABLE /*$wgDBprefix*/page ADD COLUMN page_lang TEXT default NULL; diff --git a/www/wiki/maintenance/sqlite/archives/patch-page_redirect_namespace_len.sql b/www/wiki/maintenance/sqlite/archives/patch-page_redirect_namespace_len.sql new file mode 100644 index 00000000..d9eedadd --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-page_redirect_namespace_len.sql @@ -0,0 +1,7 @@ +-- +-- Add the page_redirect_namespace_len index +-- + +CREATE INDEX /*i*/page_redirect_namespace_len ON /*_*/page (page_is_redirect, page_namespace, page_len); + + diff --git a/www/wiki/maintenance/sqlite/archives/patch-pagelinks-fix-pk.sql b/www/wiki/maintenance/sqlite/archives/patch-pagelinks-fix-pk.sql new file mode 100644 index 00000000..0e845865 --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-pagelinks-fix-pk.sql @@ -0,0 +1,27 @@ +CREATE TABLE /*_*/pagelinks_tmp ( + -- Key to the page_id of the page containing the link. + pl_from int unsigned NOT NULL default 0, + -- Namespace for this page + pl_from_namespace int NOT NULL default 0, + + -- Key to page_namespace/page_title of the target page. + -- The target page may or may not exist, and due to renames + -- and deletions may refer to different page records as time + -- goes by. + pl_namespace int NOT NULL default 0, + pl_title varchar(255) binary NOT NULL default '', + PRIMARY KEY (pl_from,pl_namespace,pl_title) +) /*$wgDBTableOptions*/; + +INSERT INTO /*_*/pagelinks_tmp + SELECT * FROM /*_*/pagelinks; + +DROP TABLE /*_*/pagelinks; + +ALTER TABLE /*_*/pagelinks_tmp RENAME TO /*_*/pagelinks; + +-- Reverse index, for Special:Whatlinkshere +CREATE INDEX /*i*/pl_namespace ON /*_*/pagelinks (pl_namespace,pl_title,pl_from); + +-- Index for Special:Whatlinkshere with namespace filter +CREATE INDEX /*i*/pl_backlinks_namespace ON /*_*/pagelinks (pl_from_namespace,pl_namespace,pl_title,pl_from); diff --git a/www/wiki/maintenance/sqlite/archives/patch-profiling.sql b/www/wiki/maintenance/sqlite/archives/patch-profiling.sql new file mode 100644 index 00000000..4a07283c --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-profiling.sql @@ -0,0 +1,12 @@ +-- profiling table +-- This is optional + +CREATE TABLE /*_*/profiling ( + pf_count int NOT NULL default 0, + pf_time float NOT NULL default 0, + pf_memory float NOT NULL default 0, + pf_name varchar(255) NOT NULL default '', + pf_server varchar(30) NOT NULL default '' +); + +CREATE UNIQUE INDEX /*i*/pf_name_server ON /*_*/profiling (pf_name, pf_server); diff --git a/www/wiki/maintenance/sqlite/archives/patch-querycache_info-fix-pk.sql b/www/wiki/maintenance/sqlite/archives/patch-querycache_info-fix-pk.sql new file mode 100644 index 00000000..d9483be4 --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-querycache_info-fix-pk.sql @@ -0,0 +1,15 @@ +CREATE TABLE /*_*/querycache_info_tmp ( + -- Special page name + -- Corresponds to a qc_type value + qci_type varbinary(32) NOT NULL default '' PRIMARY KEY, + + -- Timestamp of last update + qci_timestamp binary(14) NOT NULL default '19700101000000' +) /*$wgDBTableOptions*/; + +INSERT INTO /*_*/querycache_info_tmp + SELECT * FROM /*_*/querycache_info; + +DROP TABLE /*_*/querycache_info; + +ALTER TABLE /*_*/querycache_info_tmp RENAME TO /*_*/querycache_info;
\ No newline at end of file diff --git a/www/wiki/maintenance/sqlite/archives/patch-rc_moved.sql b/www/wiki/maintenance/sqlite/archives/patch-rc_moved.sql new file mode 100644 index 00000000..70248d54 --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-rc_moved.sql @@ -0,0 +1,46 @@ +-- rc_moved_to_ns and rc_moved_to_title is no longer used, delete the fields + +CREATE TABLE /*_*/recentchanges_tmp ( + rc_id int NOT NULL PRIMARY KEY AUTO_INCREMENT, + rc_timestamp varbinary(14) NOT NULL default '', + rc_cur_time varbinary(14) NOT NULL default '', + rc_user int unsigned NOT NULL default 0, + rc_user_text varchar(255) binary NOT NULL, + rc_namespace int NOT NULL default 0, + rc_title varchar(255) binary NOT NULL default '', + rc_comment varchar(255) binary NOT NULL default '', + rc_minor tinyint unsigned NOT NULL default 0, + rc_bot tinyint unsigned NOT NULL default 0, + rc_new tinyint unsigned NOT NULL default 0, + rc_cur_id int unsigned NOT NULL default 0, + rc_this_oldid int unsigned NOT NULL default 0, + rc_last_oldid int unsigned NOT NULL default 0, + rc_type tinyint unsigned NOT NULL default 0, + rc_patrolled tinyint unsigned NOT NULL default 0, + rc_ip varbinary(40) NOT NULL default '', + rc_old_len int, + rc_new_len int, + rc_deleted tinyint unsigned NOT NULL default 0, + rc_logid int unsigned NOT NULL default 0, + rc_log_type varbinary(255) NULL default NULL, + rc_log_action varbinary(255) NULL default NULL, + rc_params blob NULL +) /*$wgDBTableOptions*/; + +INSERT INTO /*_*/recentchanges_tmp + SELECT rc_id, rc_timestamp, rc_cur_time, rc_user, rc_user_text, rc_namespace, rc_title, rc_comment, + rc_minor, rc_bot, rc_new, rc_cur_id, rc_this_oldid, rc_last_oldid, rc_type, rc_patrolled, rc_ip, + rc_old_len, rc_new_len, rc_deleted, rc_logid, rc_log_type, rc_log_action, rc_params + FROM /*_*/recentchanges; + +DROP TABLE /*_*/recentchanges; + +ALTER TABLE /*_*/recentchanges_tmp RENAME TO /*_*/recentchanges; + +CREATE INDEX /*i*/rc_timestamp ON /*_*/recentchanges (rc_timestamp); +CREATE INDEX /*i*/rc_namespace_title ON /*_*/recentchanges (rc_namespace, rc_title); +CREATE INDEX /*i*/rc_cur_id ON /*_*/recentchanges (rc_cur_id); +CREATE INDEX /*i*/new_name_timestamp ON /*_*/recentchanges (rc_new,rc_namespace,rc_timestamp); +CREATE INDEX /*i*/rc_ip ON /*_*/recentchanges (rc_ip); +CREATE INDEX /*i*/rc_ns_usertext ON /*_*/recentchanges (rc_namespace, rc_user_text); +CREATE INDEX /*i*/rc_user_text ON /*_*/recentchanges (rc_user_text, rc_timestamp); diff --git a/www/wiki/maintenance/sqlite/archives/patch-rd_interwiki.sql b/www/wiki/maintenance/sqlite/archives/patch-rd_interwiki.sql new file mode 100644 index 00000000..ae4870a4 --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-rd_interwiki.sql @@ -0,0 +1,5 @@ +-- Add interwiki and fragment columns to redirect table + +ALTER TABLE /*$wgDBprefix*/redirect ADD COLUMN rd_interwiki TEXT default NULL; +ALTER TABLE /*$wgDBprefix*/redirect ADD COLUMN rd_fragment TEXT default NULL; + diff --git a/www/wiki/maintenance/sqlite/archives/patch-rename-iwl_prefix.sql b/www/wiki/maintenance/sqlite/archives/patch-rename-iwl_prefix.sql new file mode 100644 index 00000000..6d5b1bfa --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-rename-iwl_prefix.sql @@ -0,0 +1,5 @@ +-- +-- Recreates the iwl_prefix for the iwlinks table +-- +DROP INDEX IF EXISTS /*i*/iwl_prefix; +CREATE INDEX IF NOT EXISTS /*i*/iwl_prefix_title_from ON /*_*/iwlinks (iwl_prefix, iwl_title, iwl_from); diff --git a/www/wiki/maintenance/sqlite/archives/patch-revision-user-page-index.sql b/www/wiki/maintenance/sqlite/archives/patch-revision-user-page-index.sql new file mode 100644 index 00000000..a4554c8f --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-revision-user-page-index.sql @@ -0,0 +1,4 @@ +-- New index on revision table to allow searches for all edits by a given user +-- to a given page. Added 2007-08-28 + +CREATE INDEX /*i*/page_user_timestamp ON /*_*/revision (rev_page,rev_user,rev_timestamp); diff --git a/www/wiki/maintenance/sqlite/archives/patch-site_stats-fix-pk.sql b/www/wiki/maintenance/sqlite/archives/patch-site_stats-fix-pk.sql new file mode 100644 index 00000000..d785e984 --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-site_stats-fix-pk.sql @@ -0,0 +1,33 @@ +CREATE TABLE /*_*/site_stats_tmp ( + -- The single row should contain 1 here. + ss_row_id int unsigned NOT NULL PRIMARY KEY, + + -- Total number of edits performed. + ss_total_edits bigint unsigned default 0, + + -- An approximate count of pages matching the following criteria: + -- * in namespace 0 + -- * not a redirect + -- * contains the text '[[' + -- See Article::isCountable() in includes/Article.php + ss_good_articles bigint unsigned default 0, + + -- Total pages, theoretically equal to SELECT COUNT(*) FROM page; except faster + ss_total_pages bigint default '-1', + + -- Number of users, theoretically equal to SELECT COUNT(*) FROM user; + ss_users bigint default '-1', + + -- Number of users that still edit + ss_active_users bigint default '-1', + + -- Number of images, equivalent to SELECT COUNT(*) FROM image + ss_images int default 0 +) /*$wgDBTableOptions*/; + +INSERT INTO /*_*/site_stats_tmp + SELECT * FROM /*_*/site_stats; + +DROP TABLE /*_*/site_stats; + +ALTER TABLE /*_*/site_stats_tmp RENAME TO /*_*/site_stats;
\ No newline at end of file diff --git a/www/wiki/maintenance/sqlite/archives/patch-sites.sql b/www/wiki/maintenance/sqlite/archives/patch-sites.sql new file mode 100644 index 00000000..88392748 --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-sites.sql @@ -0,0 +1,71 @@ +-- Patch to add the sites and site_identifiers tables. +-- Licence: GNU GPL v2+ +-- Author: Jeroen De Dauw < jeroendedauw@gmail.com > + + +-- Holds all the sites known to the wiki. +CREATE TABLE IF NOT EXISTS /*_*/sites ( +-- Numeric id of the site + site_id INT UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT, + + -- Global identifier for the site, ie 'enwiktionary' + site_global_key varbinary(32) NOT NULL, + + -- Type of the site, ie 'mediawiki' + site_type varbinary(32) NOT NULL, + + -- Group of the site, ie 'wikipedia' + site_group varbinary(32) NOT NULL, + + -- Source of the site data, ie 'local', 'wikidata', 'my-magical-repo' + site_source varbinary(32) NOT NULL, + + -- Language code of the sites primary language. + site_language varbinary(32) NOT NULL, + + -- Protocol of the site, ie 'http://', 'irc://', '//' + -- This field is an index for lookups and is build from type specific data in site_data. + site_protocol varbinary(32) NOT NULL, + + -- Domain of the site in reverse order, ie 'org.mediawiki.www.' + -- This field is an index for lookups and is build from type specific data in site_data. + site_domain VARCHAR(255) NOT NULL, + + -- Type dependent site data. + site_data BLOB NOT NULL, + + -- If site.tld/path/key:pageTitle should forward users to the page on + -- the actual site, where "key" is the local identifier. + site_forward bool NOT NULL, + + -- Type dependent site config. + -- For instance if template transclusion should be allowed if it's a MediaWiki. + site_config BLOB NOT NULL +) /*$wgDBTableOptions*/; + +CREATE UNIQUE INDEX /*i*/sites_global_key ON /*_*/sites (site_global_key); +CREATE INDEX /*i*/sites_type ON /*_*/sites (site_type); +CREATE INDEX /*i*/sites_group ON /*_*/sites (site_group); +CREATE INDEX /*i*/sites_source ON /*_*/sites (site_source); +CREATE INDEX /*i*/sites_language ON /*_*/sites (site_language); +CREATE INDEX /*i*/sites_protocol ON /*_*/sites (site_protocol); +CREATE INDEX /*i*/sites_domain ON /*_*/sites (site_domain); +CREATE INDEX /*i*/sites_forward ON /*_*/sites (site_forward); + + + +-- Links local site identifiers to their corresponding site. +CREATE TABLE IF NOT EXISTS /*_*/site_identifiers ( + -- Key on site.site_id + si_site INT UNSIGNED NOT NULL, + + -- local key type, ie 'interwiki' or 'langlink' + si_type varbinary(32) NOT NULL, + + -- local key value, ie 'en' or 'wiktionary' + si_key varbinary(32) NOT NULL +) /*$wgDBTableOptions*/; + +CREATE UNIQUE INDEX /*i*/site_ids_type ON /*_*/site_identifiers (si_type, si_key); +CREATE INDEX /*i*/site_ids_site ON /*_*/site_identifiers (si_site); +CREATE INDEX /*i*/site_ids_key ON /*_*/site_identifiers (si_key);
\ No newline at end of file diff --git a/www/wiki/maintenance/sqlite/archives/patch-tag_summary-ts_id.sql b/www/wiki/maintenance/sqlite/archives/patch-tag_summary-ts_id.sql new file mode 100644 index 00000000..b6a12028 --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-tag_summary-ts_id.sql @@ -0,0 +1,23 @@ +DROP TABLE IF EXISTS /*_*/tag_summary_tmp; + +CREATE TABLE /*$wgDBprefix*/tag_summary_tmp ( + ts_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + ts_rc_id int NULL, + ts_log_id int NULL, + ts_rev_id int NULL, + ts_tags blob NOT NULL +); + +INSERT OR IGNORE INTO /*_*/tag_summary_tmp ( + ts_rc_id, ts_log_id, ts_rev_id, ts_tags ) + SELECT + ts_rc_id, ts_log_id, ts_rev_id, ts_tags + FROM /*_*/tag_summary; + +DROP TABLE /*_*/tag_summary; + +ALTER TABLE /*_*/tag_summary_tmp RENAME TO /*_*/tag_summary; + +CREATE UNIQUE INDEX /*i*/tag_summary_rc_id ON /*_*/tag_summary (ts_rc_id); +CREATE UNIQUE INDEX /*i*/tag_summary_log_id ON /*_*/tag_summary (ts_log_id); +CREATE UNIQUE INDEX /*i*/tag_summary_rev_id ON /*_*/tag_summary (ts_rev_id); diff --git a/www/wiki/maintenance/sqlite/archives/patch-tc-timestamp.sql b/www/wiki/maintenance/sqlite/archives/patch-tc-timestamp.sql new file mode 100644 index 00000000..5c09bf35 --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-tc-timestamp.sql @@ -0,0 +1,3 @@ +UPDATE /*_*/transcache SET tc_time = strftime('%Y%m%d%H%M%S', datetime(tc_time, 'unixepoch')); + +INSERT INTO /*_*/updatelog (ul_key) VALUES ('convert transcache field'); diff --git a/www/wiki/maintenance/sqlite/archives/patch-templatelinks-fix-pk.sql b/www/wiki/maintenance/sqlite/archives/patch-templatelinks-fix-pk.sql new file mode 100644 index 00000000..5f09f60d --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-templatelinks-fix-pk.sql @@ -0,0 +1,27 @@ +CREATE TABLE /*_*/templatelinks_tmp ( + -- Key to the page_id of the page containing the link. + tl_from int unsigned NOT NULL default 0, + -- Namespace for this page + tl_from_namespace int NOT NULL default 0, + + -- Key to page_namespace/page_title of the target page. + -- The target page may or may not exist, and due to renames + -- and deletions may refer to different page records as time + -- goes by. + tl_namespace int NOT NULL default 0, + tl_title varchar(255) binary NOT NULL default '', + PRIMARY KEY (tl_from,tl_namespace,tl_title) +) /*$wgDBTableOptions*/; + +INSERT INTO /*_*/templatelinks_tmp + SELECT * FROM /*_*/templatelinks; + +DROP TABLE /*_*/templatelinks; + +ALTER TABLE /*_*/templatelinks_tmp RENAME TO /*_*/templatelinks; + +-- Reverse index, for Special:Whatlinkshere +CREATE INDEX /*i*/tl_namespace ON /*_*/templatelinks (tl_namespace,tl_title,tl_from); + +-- Index for Special:Whatlinkshere with namespace filter +CREATE INDEX /*i*/tl_backlinks_namespace ON /*_*/templatelinks (tl_from_namespace,tl_namespace,tl_title,tl_from); diff --git a/www/wiki/maintenance/sqlite/archives/patch-text-fix-pk.sql b/www/wiki/maintenance/sqlite/archives/patch-text-fix-pk.sql new file mode 100644 index 00000000..380887b1 --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-text-fix-pk.sql @@ -0,0 +1,37 @@ +CREATE TABLE /*_*/text_tmp ( + -- Unique text storage key number. + -- Note that the 'oldid' parameter used in URLs does *not* + -- refer to this number anymore, but to rev_id. + -- + -- revision.rev_text_id is a key to this column + old_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + + -- Depending on the contents of the old_flags field, the text + -- may be convenient plain text, or it may be funkily encoded. + old_text mediumblob NOT NULL, + + -- Comma-separated list of flags: + -- gzip: text is compressed with PHP's gzdeflate() function. + -- utf-8: text was stored as UTF-8. + -- If $wgLegacyEncoding option is on, rows *without* this flag + -- will be converted to UTF-8 transparently at load time. Note + -- that due to a bug in a maintenance script, this flag may + -- have been stored as 'utf8' in some cases (T18841). + -- object: text field contained a serialized PHP object. + -- The object either contains multiple versions compressed + -- together to achieve a better compression ratio, or it refers + -- to another row where the text can be found. + -- external: text was stored in an external location specified by old_text. + -- Any additional flags apply to the data stored at that URL, not + -- the URL itself. The 'object' flag is *not* set for URLs of the + -- form 'DB://cluster/id/itemid', because the external storage + -- system itself decompresses these. + old_flags tinyblob NOT NULL +) /*$wgDBTableOptions*/ MAX_ROWS=10000000 AVG_ROW_LENGTH=10240; + +INSERT INTO /*_*/text_tmp + SELECT * FROM /*_*/text; + +DROP TABLE /*_*/text; + +ALTER TABLE /*_*/text_tmp RENAME TO /*_*/text;
\ No newline at end of file diff --git a/www/wiki/maintenance/sqlite/archives/patch-transcache-fix-pk.sql b/www/wiki/maintenance/sqlite/archives/patch-transcache-fix-pk.sql new file mode 100644 index 00000000..53f83e1f --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-transcache-fix-pk.sql @@ -0,0 +1,12 @@ +CREATE TABLE /*_*/transcache_tmp ( + tc_url varbinary(255) NOT NULL PRIMARY KEY, + tc_contents text, + tc_time binary(14) NOT NULL +) /*$wgDBTableOptions*/; + +INSERT INTO /*_*/transcache_tmp + SELECT * FROM /*_*/transcache; + +DROP TABLE /*_*/transcache; + +ALTER TABLE /*_*/transcache_tmp RENAME TO /*_*/transcache;
\ No newline at end of file diff --git a/www/wiki/maintenance/sqlite/archives/patch-ufg_group-length-increase-255.sql b/www/wiki/maintenance/sqlite/archives/patch-ufg_group-length-increase-255.sql new file mode 100644 index 00000000..edd0a3dc --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-ufg_group-length-increase-255.sql @@ -0,0 +1,15 @@ + CREATE TABLE /*_*/user_former_groups_tmp ( + ufg_user int unsigned NOT NULL default 0, + ufg_group varbinary(255) NOT NULL default '' +) /*$wgDBTableOptions*/; + +INSERT INTO /*_*/user_former_groups_tmp + SELECT ufg_user, ufg_group + FROM /*_*/user_former_groups; + +DROP TABLE /*_*/user_former_groups; + +ALTER TABLE /*_*/user_former_groups_tmp RENAME TO /*_*/user_former_groups; + +CREATE UNIQUE INDEX /*i*/ufg_user_group ON /*_*/user_former_groups (ufg_user,ufg_group); + diff --git a/www/wiki/maintenance/sqlite/archives/patch-ug_group-length-increase-255.sql b/www/wiki/maintenance/sqlite/archives/patch-ug_group-length-increase-255.sql new file mode 100644 index 00000000..3daeb7c6 --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-ug_group-length-increase-255.sql @@ -0,0 +1,15 @@ +CREATE TABLE /*_*/user_groups_tmp ( + ug_user int unsigned NOT NULL default 0, + ug_group varbinary(255) NOT NULL default '' +) /*$wgDBTableOptions*/; + +INSERT INTO /*_*/user_groups_tmp + SELECT ug_user, ug_group + FROM /*_*/user_groups; + +DROP TABLE /*_*/user_groups; + +ALTER TABLE /*_*/user_groups_tmp RENAME TO /*_*/user_groups; + +CREATE UNIQUE INDEX /*i*/ug_user_group ON /*_*/user_groups (ug_user,ug_group); +CREATE INDEX /*i*/ug_group ON /*_*/user_groups (ug_group); diff --git a/www/wiki/maintenance/sqlite/archives/patch-user_former_groups-fix-pk.sql b/www/wiki/maintenance/sqlite/archives/patch-user_former_groups-fix-pk.sql new file mode 100644 index 00000000..4f5d6225 --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-user_former_groups-fix-pk.sql @@ -0,0 +1,13 @@ +CREATE TABLE /*_*/user_former_groups_tmp ( + -- Key to user_id + ufg_user int unsigned NOT NULL default 0, + ufg_group varbinary(255) NOT NULL default '', + PRIMARY KEY (ufg_user,ufg_group) +) /*$wgDBTableOptions*/; + +INSERT INTO /*_*/user_former_groups_tmp + SELECT * FROM /*_*/user_former_groups; + +DROP TABLE /*_*/user_former_groups; + +ALTER TABLE /*_*/user_former_groups_tmp RENAME TO /*_*/user_former_groups;
\ No newline at end of file diff --git a/www/wiki/maintenance/sqlite/archives/patch-user_groups-ug_expiry.sql b/www/wiki/maintenance/sqlite/archives/patch-user_groups-ug_expiry.sql new file mode 100644 index 00000000..7fc89416 --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-user_groups-ug_expiry.sql @@ -0,0 +1,21 @@ +DROP TABLE IF EXISTS /*_*/user_groups_tmp; + +CREATE TABLE /*$wgDBprefix*/user_groups_tmp ( + ug_user int unsigned NOT NULL default 0, + ug_group varbinary(255) NOT NULL default '', + ug_expiry varbinary(14) NULL default NULL, + PRIMARY KEY (ug_user, ug_group) +); + +INSERT OR IGNORE INTO /*_*/user_groups_tmp ( + ug_user, ug_group ) + SELECT + ug_user, ug_group + FROM /*_*/user_groups; + +DROP TABLE /*_*/user_groups; + +ALTER TABLE /*_*/user_groups_tmp RENAME TO /*_*/user_groups; + +CREATE INDEX /*i*/ug_group ON /*_*/user_groups (ug_group); +CREATE INDEX /*i*/ug_expiry ON /*_*/user_groups (ug_expiry); diff --git a/www/wiki/maintenance/sqlite/archives/patch-user_properties-fix-pk.sql b/www/wiki/maintenance/sqlite/archives/patch-user_properties-fix-pk.sql new file mode 100644 index 00000000..8362d233 --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-user_properties-fix-pk.sql @@ -0,0 +1,20 @@ +CREATE TABLE /*_*/user_properties_tmp ( + -- Foreign key to user.user_id + up_user int NOT NULL, + + -- Name of the option being saved. This is indexed for bulk lookup. + up_property varbinary(255) NOT NULL, + + -- Property value as a string. + up_value blob, + PRIMARY KEY (up_user,up_property) +) /*$wgDBTableOptions*/; + +INSERT INTO /*_*/user_properties_tmp + SELECT * FROM /*_*/user_properties; + +DROP TABLE /*_*/user_properties; + +ALTER TABLE /*_*/user_properties_tmp RENAME TO /*_*/user_properties; + +CREATE INDEX /*i*/user_properties_property ON /*_*/user_properties (up_property);
\ No newline at end of file diff --git a/www/wiki/maintenance/sqlite/archives/patch-watchlist-wl_id.sql b/www/wiki/maintenance/sqlite/archives/patch-watchlist-wl_id.sql new file mode 100644 index 00000000..771f9b7e --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/patch-watchlist-wl_id.sql @@ -0,0 +1,23 @@ +DROP TABLE IF EXISTS /*_*/watchlist_tmp; + +CREATE TABLE /*$wgDBprefix*/watchlist_tmp ( + wl_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + wl_user INTEGER NOT NULL, + wl_namespace INTEGER NOT NULL default 0, + wl_title TEXT NOT NULL default '', + wl_notificationtimestamp BLOB +); + +INSERT OR IGNORE INTO /*_*/watchlist_tmp ( + wl_user, wl_namespace, wl_title, wl_notificationtimestamp ) + SELECT + wl_user, wl_namespace, wl_title, wl_notificationtimestamp + FROM /*_*/watchlist; + +DROP TABLE /*_*/watchlist; + +ALTER TABLE /*_*/watchlist_tmp RENAME TO /*_*/watchlist; + +CREATE UNIQUE INDEX /*i*/wl_user ON /*_*/watchlist (wl_user, wl_namespace, wl_title); +CREATE INDEX /*i*/namespace_title ON /*_*/watchlist (wl_namespace, wl_title); +CREATE INDEX /*i*/wl_user_notificationtimestamp ON /*_*/watchlist (wl_user, wl_notificationtimestamp); diff --git a/www/wiki/maintenance/sqlite/archives/searchindex-fts3.sql b/www/wiki/maintenance/sqlite/archives/searchindex-fts3.sql new file mode 100644 index 00000000..38cdfcfc --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/searchindex-fts3.sql @@ -0,0 +1,18 @@ +-- Patch that introduces fulltext search capabilities to SQLite schema +-- Requires that SQLite must be compiled with FTS3 module (comes with core amalgamation). +-- See https://sqlite.org/fts3.html for details of syntax. +-- Will fail if FTS3 is not present, +DROP TABLE IF EXISTS /*_*/searchindex; +CREATE VIRTUAL TABLE /*_*/searchindex USING FTS3( + -- Key to page_id + -- Disabled, instead we use the built-in rowid column + -- si_page INTEGER NOT NULL, + + -- Munged version of title + si_title, + + -- Munged version of body text + si_text +); + +INSERT INTO /*_*/updatelog (ul_key) VALUES ('fts3'); diff --git a/www/wiki/maintenance/sqlite/archives/searchindex-no-fts.sql b/www/wiki/maintenance/sqlite/archives/searchindex-no-fts.sql new file mode 100644 index 00000000..16247ffe --- /dev/null +++ b/www/wiki/maintenance/sqlite/archives/searchindex-no-fts.sql @@ -0,0 +1,25 @@ +-- Searchindex table definition for cases when no full-text search SQLite module is present +-- (currently, only FTS3 is supported). +-- Use it if you are moving your database from environment with FTS support +-- to environment without it. + +DROP TABLE IF EXISTS /*_*/searchindex; + +-- These are pieces of FTS3-enabled searchindex +DROP TABLE IF EXISTS /*_*/searchindex_content; +DROP TABLE IF EXISTS /*_*/searchindex_segdir; +DROP TABLE IF EXISTS /*_*/searchindex_segments; + +CREATE TABLE /*_*/searchindex ( + -- Key to page_id + -- Disabled, instead we use the built-in rowid column + -- si_page INTEGER NOT NULL, + + -- Munged version of title + si_title TEXT, + + -- Munged version of body text + si_text TEXT +); + +DELETE FROM /*_*/updatelog WHERE ul_key='fts3';
\ No newline at end of file diff --git a/www/wiki/maintenance/storage/blob_tracking.sql b/www/wiki/maintenance/storage/blob_tracking.sql new file mode 100644 index 00000000..fbc407c7 --- /dev/null +++ b/www/wiki/maintenance/storage/blob_tracking.sql @@ -0,0 +1,56 @@ + +-- Table for tracking blobs prior to recompression or similar maintenance operations + +CREATE TABLE /*$wgDBprefix*/blob_tracking ( + -- page.page_id + -- This may be zero for orphan or deleted text + -- Note that this is for compression grouping only -- it doesn't need to be + -- accurate at the time recompressTracked is run. Operations such as a + -- delete/undelete cycle may make it inaccurate. + bt_page integer not null, + + -- revision.rev_id + -- This may be zero for orphan or deleted text + -- Like bt_page, it does not need to be accurate when recompressTracked is run. + bt_rev_id integer not null, + + -- text.old_id + bt_text_id integer not null, + + -- The ES cluster + bt_cluster varbinary(255), + + -- The ES blob ID + bt_blob_id integer not null, + + -- The CGZ content hash, or null + bt_cgz_hash varbinary(255), + + -- The URL this blob is to be moved to + bt_new_url varbinary(255), + + -- True if the text table has been updated to point to bt_new_url + bt_moved bool not null default 0, + + -- Primary key + -- Note that text_id is not unique due to null edits (protection, move) + -- moveTextRow(), commit(), trackOrphanText() + PRIMARY KEY (bt_text_id, bt_rev_id), + + -- Sort by page for easy CGZ recompression + -- doAllPages(), doAllOrphans(), doPage(), finishIncompleteMoves() + KEY (bt_moved, bt_page, bt_text_id), + + -- Key for determining the revisions using a given blob + -- Not used by any scripts yet + KEY (bt_cluster, bt_blob_id, bt_cgz_hash) + +) /*$wgDBTableOptions*/; + +-- Tracking table for blob rows that aren't tracked by the text table +CREATE TABLE /*$wgDBprefix*/blob_orphans ( + bo_cluster varbinary(255), + bo_blob_id integer not null, + + PRIMARY KEY (bo_cluster, bo_blob_id) +) /*$wgDBTableOptions*/; diff --git a/www/wiki/maintenance/storage/blobs.sql b/www/wiki/maintenance/storage/blobs.sql new file mode 100644 index 00000000..979e68a9 --- /dev/null +++ b/www/wiki/maintenance/storage/blobs.sql @@ -0,0 +1,7 @@ +-- Blobs table for external storage + +CREATE TABLE /*$wgDBprefix*/blobs ( + blob_id integer UNSIGNED NOT NULL AUTO_INCREMENT, + blob_text longblob, + PRIMARY KEY (blob_id) +) ENGINE=InnoDB; diff --git a/www/wiki/maintenance/storage/checkStorage.php b/www/wiki/maintenance/storage/checkStorage.php new file mode 100644 index 00000000..9045870d --- /dev/null +++ b/www/wiki/maintenance/storage/checkStorage.php @@ -0,0 +1,535 @@ +<?php +/** + * Fsck for MediaWiki + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance ExternalStorage + */ + +use MediaWiki\MediaWikiServices; + +if ( !defined( 'MEDIAWIKI' ) ) { + $optionsWithoutArgs = [ 'fix' ]; + require_once __DIR__ . '/../commandLine.inc'; + + $cs = new CheckStorage; + $fix = isset( $options['fix'] ); + if ( isset( $args[0] ) ) { + $xml = $args[0]; + } else { + $xml = false; + } + $cs->check( $fix, $xml ); +} + +// ---------------------------------------------------------------------------------- + +/** + * Maintenance script to do various checks on external storage. + * + * @fixme this should extend the base Maintenance class + * @ingroup Maintenance ExternalStorage + */ +class CheckStorage { + const CONCAT_HEADER = 'O:27:"concatenatedgziphistoryblob"'; + public $oldIdMap, $errors; + public $dbStore = null; + + public $errorDescriptions = [ + 'restore text' => 'Damaged text, need to be restored from a backup', + 'restore revision' => 'Damaged revision row, need to be restored from a backup', + 'unfixable' => 'Unexpected errors with no automated fixing method', + 'fixed' => 'Errors already fixed', + 'fixable' => 'Errors which would already be fixed if --fix was specified', + ]; + + function check( $fix = false, $xml = '' ) { + $dbr = wfGetDB( DB_REPLICA ); + if ( $fix ) { + print "Checking, will fix errors if possible...\n"; + } else { + print "Checking...\n"; + } + $maxRevId = $dbr->selectField( 'revision', 'MAX(rev_id)', false, __METHOD__ ); + $chunkSize = 1000; + $flagStats = []; + $objectStats = []; + $knownFlags = [ 'external', 'gzip', 'object', 'utf-8' ]; + $this->errors = [ + 'restore text' => [], + 'restore revision' => [], + 'unfixable' => [], + 'fixed' => [], + 'fixable' => [], + ]; + + for ( $chunkStart = 1; $chunkStart < $maxRevId; $chunkStart += $chunkSize ) { + $chunkEnd = $chunkStart + $chunkSize - 1; + // print "$chunkStart of $maxRevId\n"; + + // Fetch revision rows + $this->oldIdMap = []; + $dbr->ping(); + $res = $dbr->select( 'revision', [ 'rev_id', 'rev_text_id' ], + [ "rev_id BETWEEN $chunkStart AND $chunkEnd" ], __METHOD__ ); + foreach ( $res as $row ) { + $this->oldIdMap[$row->rev_id] = $row->rev_text_id; + } + $dbr->freeResult( $res ); + + if ( !count( $this->oldIdMap ) ) { + continue; + } + + // Fetch old_flags + $missingTextRows = array_flip( $this->oldIdMap ); + $externalRevs = []; + $objectRevs = []; + $res = $dbr->select( 'text', [ 'old_id', 'old_flags' ], + 'old_id IN (' . implode( ',', $this->oldIdMap ) . ')', __METHOD__ ); + foreach ( $res as $row ) { + /** + * @var $flags int + */ + $flags = $row->old_flags; + $id = $row->old_id; + + // Create flagStats row if it doesn't exist + $flagStats = $flagStats + [ $flags => 0 ]; + // Increment counter + $flagStats[$flags]++; + + // Not missing + unset( $missingTextRows[$row->old_id] ); + + // Check for external or object + if ( $flags == '' ) { + $flagArray = []; + } else { + $flagArray = explode( ',', $flags ); + } + if ( in_array( 'external', $flagArray ) ) { + $externalRevs[] = $id; + } elseif ( in_array( 'object', $flagArray ) ) { + $objectRevs[] = $id; + } + + // Check for unrecognised flags + if ( $flags == '0' ) { + // This is a known bug from 2004 + // It's safe to just erase the old_flags field + if ( $fix ) { + $this->error( 'fixed', "Warning: old_flags set to 0", $id ); + $dbw = wfGetDB( DB_MASTER ); + $dbw->ping(); + $dbw->update( 'text', [ 'old_flags' => '' ], + [ 'old_id' => $id ], __METHOD__ ); + echo "Fixed\n"; + } else { + $this->error( 'fixable', "Warning: old_flags set to 0", $id ); + } + } elseif ( count( array_diff( $flagArray, $knownFlags ) ) ) { + $this->error( 'unfixable', "Error: invalid flags field \"$flags\"", $id ); + } + } + $dbr->freeResult( $res ); + + // Output errors for any missing text rows + foreach ( $missingTextRows as $oldId => $revId ) { + $this->error( 'restore revision', "Error: missing text row", $oldId ); + } + + // Verify external revisions + $externalConcatBlobs = []; + $externalNormalBlobs = []; + if ( count( $externalRevs ) ) { + $res = $dbr->select( 'text', [ 'old_id', 'old_flags', 'old_text' ], + [ 'old_id IN (' . implode( ',', $externalRevs ) . ')' ], __METHOD__ ); + foreach ( $res as $row ) { + $urlParts = explode( '://', $row->old_text, 2 ); + if ( count( $urlParts ) !== 2 || $urlParts[1] == '' ) { + $this->error( 'restore text', "Error: invalid URL \"{$row->old_text}\"", $row->old_id ); + continue; + } + list( $proto, ) = $urlParts; + if ( $proto != 'DB' ) { + $this->error( 'restore text', "Error: invalid external protocol \"$proto\"", $row->old_id ); + continue; + } + $path = explode( '/', $row->old_text ); + $cluster = $path[2]; + $id = $path[3]; + if ( isset( $path[4] ) ) { + $externalConcatBlobs[$cluster][$id][] = $row->old_id; + } else { + $externalNormalBlobs[$cluster][$id][] = $row->old_id; + } + } + $dbr->freeResult( $res ); + } + + // Check external concat blobs for the right header + $this->checkExternalConcatBlobs( $externalConcatBlobs ); + + // Check external normal blobs for existence + if ( count( $externalNormalBlobs ) ) { + if ( is_null( $this->dbStore ) ) { + $this->dbStore = new ExternalStoreDB; + } + foreach ( $externalConcatBlobs as $cluster => $xBlobIds ) { + $blobIds = array_keys( $xBlobIds ); + $extDb =& $this->dbStore->getSlave( $cluster ); + $blobsTable = $this->dbStore->getTable( $extDb ); + $res = $extDb->select( $blobsTable, + [ 'blob_id' ], + [ 'blob_id IN( ' . implode( ',', $blobIds ) . ')' ], __METHOD__ ); + foreach ( $res as $row ) { + unset( $xBlobIds[$row->blob_id] ); + } + $extDb->freeResult( $res ); + // Print errors for missing blobs rows + foreach ( $xBlobIds as $blobId => $oldId ) { + $this->error( 'restore text', "Error: missing target $blobId for one-part ES URL", $oldId ); + } + } + } + + // Check local objects + $dbr->ping(); + $concatBlobs = []; + $curIds = []; + if ( count( $objectRevs ) ) { + $headerLength = 300; + $res = $dbr->select( + 'text', + [ 'old_id', 'old_flags', "LEFT(old_text, $headerLength) AS header" ], + [ 'old_id IN (' . implode( ',', $objectRevs ) . ')' ], + __METHOD__ + ); + foreach ( $res as $row ) { + $oldId = $row->old_id; + $matches = []; + if ( !preg_match( '/^O:(\d+):"(\w+)"/', $row->header, $matches ) ) { + $this->error( 'restore text', "Error: invalid object header", $oldId ); + continue; + } + + $className = strtolower( $matches[2] ); + if ( strlen( $className ) != $matches[1] ) { + $this->error( + 'restore text', + "Error: invalid object header, wrong class name length", + $oldId + ); + continue; + } + + $objectStats = $objectStats + [ $className => 0 ]; + $objectStats[$className]++; + + switch ( $className ) { + case 'concatenatedgziphistoryblob': + // Good + break; + case 'historyblobstub': + case 'historyblobcurstub': + if ( strlen( $row->header ) == $headerLength ) { + $this->error( 'unfixable', "Error: overlong stub header", $oldId ); + continue; + } + $stubObj = unserialize( $row->header ); + if ( !is_object( $stubObj ) ) { + $this->error( 'restore text', "Error: unable to unserialize stub object", $oldId ); + continue; + } + if ( $className == 'historyblobstub' ) { + $concatBlobs[$stubObj->mOldId][] = $oldId; + } else { + $curIds[$stubObj->mCurId][] = $oldId; + } + break; + default: + $this->error( 'unfixable', "Error: unrecognised object class \"$className\"", $oldId ); + } + } + $dbr->freeResult( $res ); + } + + // Check local concat blob validity + $externalConcatBlobs = []; + if ( count( $concatBlobs ) ) { + $headerLength = 300; + $res = $dbr->select( + 'text', + [ 'old_id', 'old_flags', "LEFT(old_text, $headerLength) AS header" ], + [ 'old_id IN (' . implode( ',', array_keys( $concatBlobs ) ) . ')' ], + __METHOD__ + ); + foreach ( $res as $row ) { + $flags = explode( ',', $row->old_flags ); + if ( in_array( 'external', $flags ) ) { + // Concat blob is in external storage? + if ( in_array( 'object', $flags ) ) { + $urlParts = explode( '/', $row->header ); + if ( $urlParts[0] != 'DB:' ) { + $this->error( + 'unfixable', + "Error: unrecognised external storage type \"{$urlParts[0]}", + $row->old_id + ); + } else { + $cluster = $urlParts[2]; + $id = $urlParts[3]; + if ( !isset( $externalConcatBlobs[$cluster][$id] ) ) { + $externalConcatBlobs[$cluster][$id] = []; + } + $externalConcatBlobs[$cluster][$id] = array_merge( + $externalConcatBlobs[$cluster][$id], $concatBlobs[$row->old_id] + ); + } + } else { + $this->error( + 'unfixable', + "Error: invalid flags \"{$row->old_flags}\" on concat bulk row {$row->old_id}", + $concatBlobs[$row->old_id] ); + } + } elseif ( strcasecmp( + substr( $row->header, 0, strlen( self::CONCAT_HEADER ) ), + self::CONCAT_HEADER + ) ) { + $this->error( + 'restore text', + "Error: Incorrect object header for concat bulk row {$row->old_id}", + $concatBlobs[$row->old_id] + ); + } # else good + + unset( $concatBlobs[$row->old_id] ); + } + $dbr->freeResult( $res ); + } + + // Check targets of unresolved stubs + $this->checkExternalConcatBlobs( $externalConcatBlobs ); + // next chunk + } + + print "\n\nErrors:\n"; + foreach ( $this->errors as $name => $errors ) { + if ( count( $errors ) ) { + $description = $this->errorDescriptions[$name]; + echo "$description: " . implode( ',', array_keys( $errors ) ) . "\n"; + } + } + + if ( count( $this->errors['restore text'] ) && $fix ) { + if ( (string)$xml !== '' ) { + $this->restoreText( array_keys( $this->errors['restore text'] ), $xml ); + } else { + echo "Can't fix text, no XML backup specified\n"; + } + } + + print "\nFlag statistics:\n"; + $total = array_sum( $flagStats ); + foreach ( $flagStats as $flag => $count ) { + printf( "%-30s %10d %5.2f%%\n", $flag, $count, $count / $total * 100 ); + } + print "\nLocal object statistics:\n"; + $total = array_sum( $objectStats ); + foreach ( $objectStats as $className => $count ) { + printf( "%-30s %10d %5.2f%%\n", $className, $count, $count / $total * 100 ); + } + } + + function error( $type, $msg, $ids ) { + if ( is_array( $ids ) && count( $ids ) == 1 ) { + $ids = reset( $ids ); + } + if ( is_array( $ids ) ) { + $revIds = []; + foreach ( $ids as $id ) { + $revIds = array_merge( $revIds, array_keys( $this->oldIdMap, $id ) ); + } + print "$msg in text rows " . implode( ', ', $ids ) . + ", revisions " . implode( ', ', $revIds ) . "\n"; + } else { + $id = $ids; + $revIds = array_keys( $this->oldIdMap, $id ); + if ( count( $revIds ) == 1 ) { + print "$msg in old_id $id, rev_id {$revIds[0]}\n"; + } else { + print "$msg in old_id $id, revisions " . implode( ', ', $revIds ) . "\n"; + } + } + $this->errors[$type] = $this->errors[$type] + array_flip( $revIds ); + } + + function checkExternalConcatBlobs( $externalConcatBlobs ) { + if ( !count( $externalConcatBlobs ) ) { + return; + } + + if ( is_null( $this->dbStore ) ) { + $this->dbStore = new ExternalStoreDB; + } + + foreach ( $externalConcatBlobs as $cluster => $oldIds ) { + $blobIds = array_keys( $oldIds ); + $extDb =& $this->dbStore->getSlave( $cluster ); + $blobsTable = $this->dbStore->getTable( $extDb ); + $headerLength = strlen( self::CONCAT_HEADER ); + $res = $extDb->select( $blobsTable, + [ 'blob_id', "LEFT(blob_text, $headerLength) AS header" ], + [ 'blob_id IN( ' . implode( ',', $blobIds ) . ')' ], __METHOD__ ); + foreach ( $res as $row ) { + if ( strcasecmp( $row->header, self::CONCAT_HEADER ) ) { + $this->error( + 'restore text', + "Error: invalid header on target $cluster/{$row->blob_id} of two-part ES URL", + $oldIds[$row->blob_id] + ); + } + unset( $oldIds[$row->blob_id] ); + } + $extDb->freeResult( $res ); + + // Print errors for missing blobs rows + foreach ( $oldIds as $blobId => $oldIds2 ) { + $this->error( + 'restore text', + "Error: missing target $cluster/$blobId for two-part ES URL", + $oldIds2 + ); + } + } + } + + function restoreText( $revIds, $xml ) { + global $wgDBname; + $tmpDir = wfTempDir(); + + if ( !count( $revIds ) ) { + return; + } + + print "Restoring text from XML backup...\n"; + + $revFileName = "$tmpDir/broken-revlist-$wgDBname"; + $filteredXmlFileName = "$tmpDir/filtered-$wgDBname.xml"; + + // Write revision list + if ( !file_put_contents( $revFileName, implode( "\n", $revIds ) ) ) { + echo "Error writing revision list, can't restore text\n"; + + return; + } + + // Run mwdumper + echo "Filtering XML dump...\n"; + $exitStatus = 0; + passthru( 'mwdumper ' . + wfEscapeShellArg( + "--output=file:$filteredXmlFileName", + "--filter=revlist:$revFileName", + $xml + ), $exitStatus + ); + + if ( $exitStatus ) { + echo "mwdumper died with exit status $exitStatus\n"; + + return; + } + + $file = fopen( $filteredXmlFileName, 'r' ); + if ( !$file ) { + echo "Unable to open filtered XML file\n"; + + return; + } + + $dbr = wfGetDB( DB_REPLICA ); + $dbw = wfGetDB( DB_MASTER ); + $dbr->ping(); + $dbw->ping(); + + $source = new ImportStreamSource( $file ); + $importer = new WikiImporter( + $source, + MediaWikiServices::getInstance()->getMainConfig() + ); + $importer->setRevisionCallback( [ $this, 'importRevision' ] ); + $importer->doImport(); + } + + function importRevision( &$revision, &$importer ) { + $id = $revision->getID(); + $content = $revision->getContent( Revision::RAW ); + $id = $id ? $id : ''; + + if ( $content === null ) { + echo "Revision $id is broken, we have no content available\n"; + + return; + } + + $text = $content->serialize(); + if ( $text === '' ) { + // This is what happens if the revision was broken at the time the + // dump was made. Unfortunately, it also happens if the revision was + // legitimately blank, so there's no way to tell the difference. To + // be safe, we'll skip it and leave it broken + + echo "Revision $id is blank in the dump, may have been broken before export\n"; + + return; + } + + if ( !$id ) { + // No ID, can't import + echo "No id tag in revision, can't import\n"; + + return; + } + + // Find text row again + $dbr = wfGetDB( DB_REPLICA ); + $oldId = $dbr->selectField( 'revision', 'rev_text_id', [ 'rev_id' => $id ], __METHOD__ ); + if ( !$oldId ) { + echo "Missing revision row for rev_id $id\n"; + + return; + } + + // Compress the text + $flags = Revision::compressRevisionText( $text ); + + // Update the text row + $dbw = wfGetDB( DB_MASTER ); + $dbw->update( 'text', + [ 'old_flags' => $flags, 'old_text' => $text ], + [ 'old_id' => $oldId ], + __METHOD__, [ 'LIMIT' => 1 ] + ); + + // Remove it from the unfixed list and add it to the fixed list + unset( $this->errors['restore text'][$id] ); + $this->errors['fixed'][$id] = true; + } +} diff --git a/www/wiki/maintenance/storage/compressOld.php b/www/wiki/maintenance/storage/compressOld.php new file mode 100644 index 00000000..c17ce99c --- /dev/null +++ b/www/wiki/maintenance/storage/compressOld.php @@ -0,0 +1,476 @@ +<?php +/** + * Compress the text of a wiki. + * + * Usage: + * + * Non-wikimedia + * php compressOld.php [options...] + * + * Wikimedia + * php compressOld.php <database> [options...] + * + * Options are: + * -t <type> set compression type to either: + * gzip: compress revisions independently + * concat: concatenate revisions and compress in chunks (default) + * -c <chunk-size> maximum number of revisions in a concat chunk + * -b <begin-date> earliest date to check for uncompressed revisions + * -e <end-date> latest revision date to compress + * -s <startid> the id to start from (referring to the text table for + * type gzip, and to the page table for type concat) + * -n <endid> the page_id to stop at (only when using concat compression type) + * --extdb <cluster> store specified revisions in an external cluster (untested) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance ExternalStorage + */ + +require_once __DIR__ . '/../Maintenance.php'; + +/** + * Maintenance script that compress the text of a wiki. + * + * @ingroup Maintenance ExternalStorage + */ +class CompressOld extends Maintenance { + /** + * Option to load each revision individually. + */ + const LS_INDIVIDUAL = 0; + + /** + * Option to load revisions in chunks. + */ + const LS_CHUNKED = 1; + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Compress the text of a wiki' ); + $this->addOption( 'type', 'Set compression type to either: gzip|concat', false, true, 't' ); + $this->addOption( + 'chunksize', + 'Maximum number of revisions in a concat chunk', + false, + true, + 'c' + ); + $this->addOption( + 'begin-date', + 'Earliest date to check for uncompressed revisions', + false, + true, + 'b' + ); + $this->addOption( 'end-date', 'Latest revision date to compress', false, true, 'e' ); + $this->addOption( + 'startid', + 'The id to start from (gzip -> text table, concat -> page table)', + false, + true, + 's' + ); + $this->addOption( + 'extdb', + 'Store specified revisions in an external cluster (untested)', + false, + true + ); + $this->addOption( + 'endid', + 'The page_id to stop at (only when using concat compression type)', + false, + true, + 'n' + ); + } + + public function execute() { + global $wgDBname; + if ( !function_exists( "gzdeflate" ) ) { + $this->error( "You must enable zlib support in PHP to compress old revisions!\n" . + "Please see http://www.php.net/manual/en/ref.zlib.php\n", true ); + } + + $type = $this->getOption( 'type', 'concat' ); + $chunkSize = $this->getOption( 'chunksize', 20 ); + $startId = $this->getOption( 'startid', 0 ); + $beginDate = $this->getOption( 'begin-date', '' ); + $endDate = $this->getOption( 'end-date', '' ); + $extDB = $this->getOption( 'extdb', '' ); + $endId = $this->getOption( 'endid', false ); + + if ( $type != 'concat' && $type != 'gzip' ) { + $this->error( "Type \"{$type}\" not supported" ); + } + + if ( $extDB != '' ) { + $this->output( "Compressing database {$wgDBname} to external cluster {$extDB}\n" + . str_repeat( '-', 76 ) . "\n\n" ); + } else { + $this->output( "Compressing database {$wgDBname}\n" + . str_repeat( '-', 76 ) . "\n\n" ); + } + + $success = true; + if ( $type == 'concat' ) { + $success = $this->compressWithConcat( $startId, $chunkSize, $beginDate, + $endDate, $extDB, $endId ); + } else { + $this->compressOldPages( $startId, $extDB ); + } + + if ( $success ) { + $this->output( "Done.\n" ); + } + } + + /** + * Fetch the text row-by-row to 'compressPage' function for compression. + * + * @param int $start + * @param string $extdb + */ + private function compressOldPages( $start = 0, $extdb = '' ) { + $chunksize = 50; + $this->output( "Starting from old_id $start...\n" ); + $dbw = $this->getDB( DB_MASTER ); + do { + $res = $dbw->select( + 'text', + [ 'old_id', 'old_flags', 'old_text' ], + "old_id>=$start", + __METHOD__, + [ 'ORDER BY' => 'old_id', 'LIMIT' => $chunksize, 'FOR UPDATE' ] + ); + + if ( $res->numRows() == 0 ) { + break; + } + + $last = $start; + + foreach ( $res as $row ) { + # print " {$row->old_id} - {$row->old_namespace}:{$row->old_title}\n"; + $this->compressPage( $row, $extdb ); + $last = $row->old_id; + } + + $start = $last + 1; # Deletion may leave long empty stretches + $this->output( "$start...\n" ); + } while ( true ); + } + + /** + * Compress the text in gzip format. + * + * @param stdClass $row + * @param string $extdb + * @return bool + */ + private function compressPage( $row, $extdb ) { + if ( false !== strpos( $row->old_flags, 'gzip' ) + || false !== strpos( $row->old_flags, 'object' ) + ) { + # print "Already compressed row {$row->old_id}\n"; + return false; + } + $dbw = $this->getDB( DB_MASTER ); + $flags = $row->old_flags ? "{$row->old_flags},gzip" : "gzip"; + $compress = gzdeflate( $row->old_text ); + + # Store in external storage if required + if ( $extdb !== '' ) { + $storeObj = new ExternalStoreDB; + $compress = $storeObj->store( $extdb, $compress ); + if ( $compress === false ) { + $this->error( "Unable to store object" ); + + return false; + } + } + + # Update text row + $dbw->update( 'text', + [ /* SET */ + 'old_flags' => $flags, + 'old_text' => $compress + ], [ /* WHERE */ + 'old_id' => $row->old_id + ], __METHOD__, + [ 'LIMIT' => 1 ] + ); + + return true; + } + + /** + * Compress the text in chunks after concatenating the revisions. + * + * @param int $startId + * @param int $maxChunkSize + * @param string $beginDate + * @param string $endDate + * @param string $extdb + * @param bool|int $maxPageId + * @return bool + */ + private function compressWithConcat( $startId, $maxChunkSize, $beginDate, + $endDate, $extdb = "", $maxPageId = false + ) { + $loadStyle = self::LS_CHUNKED; + + $dbr = $this->getDB( DB_REPLICA ); + $dbw = $this->getDB( DB_MASTER ); + + # Set up external storage + if ( $extdb != '' ) { + $storeObj = new ExternalStoreDB; + } + + # Get all articles by page_id + if ( !$maxPageId ) { + $maxPageId = $dbr->selectField( 'page', 'max(page_id)', '', __METHOD__ ); + } + $this->output( "Starting from $startId of $maxPageId\n" ); + $pageConds = []; + + /* + if ( $exclude_ns0 ) { + print "Excluding main namespace\n"; + $pageConds[] = 'page_namespace<>0'; + } + if ( $queryExtra ) { + $pageConds[] = $queryExtra; + } + */ + + # For each article, get a list of revisions which fit the criteria + + # No recompression, use a condition on old_flags + # Don't compress object type entities, because that might produce data loss when + # overwriting bulk storage concat rows. Don't compress external references, because + # the script doesn't yet delete rows from external storage. + $conds = [ + 'old_flags NOT ' . $dbr->buildLike( $dbr->anyString(), 'object', $dbr->anyString() ) + . ' AND old_flags NOT ' + . $dbr->buildLike( $dbr->anyString(), 'external', $dbr->anyString() ) + ]; + + if ( $beginDate ) { + if ( !preg_match( '/^\d{14}$/', $beginDate ) ) { + $this->error( "Invalid begin date \"$beginDate\"\n" ); + + return false; + } + $conds[] = "rev_timestamp>'" . $beginDate . "'"; + } + if ( $endDate ) { + if ( !preg_match( '/^\d{14}$/', $endDate ) ) { + $this->error( "Invalid end date \"$endDate\"\n" ); + + return false; + } + $conds[] = "rev_timestamp<'" . $endDate . "'"; + } + if ( $loadStyle == self::LS_CHUNKED ) { + $tables = [ 'revision', 'text' ]; + $fields = [ 'rev_id', 'rev_text_id', 'old_flags', 'old_text' ]; + $conds[] = 'rev_text_id=old_id'; + $revLoadOptions = 'FOR UPDATE'; + } else { + $tables = [ 'revision' ]; + $fields = [ 'rev_id', 'rev_text_id' ]; + $revLoadOptions = []; + } + + # Don't work with current revisions + # Don't lock the page table for update either -- TS 2006-04-04 + # $tables[] = 'page'; + # $conds[] = 'page_id=rev_page AND rev_id != page_latest'; + + for ( $pageId = $startId; $pageId <= $maxPageId; $pageId++ ) { + wfWaitForSlaves(); + + # Wake up + $dbr->ping(); + + # Get the page row + $pageRes = $dbr->select( 'page', + [ 'page_id', 'page_namespace', 'page_title', 'page_latest' ], + $pageConds + [ 'page_id' => $pageId ], __METHOD__ ); + if ( $pageRes->numRows() == 0 ) { + continue; + } + $pageRow = $dbr->fetchObject( $pageRes ); + + # Display progress + $titleObj = Title::makeTitle( $pageRow->page_namespace, $pageRow->page_title ); + $this->output( "$pageId\t" . $titleObj->getPrefixedDBkey() . " " ); + + # Load revisions + $revRes = $dbw->select( $tables, $fields, + array_merge( [ + 'rev_page' => $pageRow->page_id, + # Don't operate on the current revision + # Use < instead of <> in case the current revision has changed + # since the page select, which wasn't locking + 'rev_id < ' . $pageRow->page_latest + ], $conds ), + __METHOD__, + $revLoadOptions + ); + $revs = []; + foreach ( $revRes as $revRow ) { + $revs[] = $revRow; + } + + if ( count( $revs ) < 2 ) { + # No revisions matching, no further processing + $this->output( "\n" ); + continue; + } + + # For each chunk + $i = 0; + while ( $i < count( $revs ) ) { + if ( $i < count( $revs ) - $maxChunkSize ) { + $thisChunkSize = $maxChunkSize; + } else { + $thisChunkSize = count( $revs ) - $i; + } + + $chunk = new ConcatenatedGzipHistoryBlob(); + $stubs = []; + $this->beginTransaction( $dbw, __METHOD__ ); + $usedChunk = false; + $primaryOldid = $revs[$i]->rev_text_id; + + // @codingStandardsIgnoreStart Ignore avoid function calls in a FOR loop test part warning + # Get the text of each revision and add it to the object + for ( $j = 0; $j < $thisChunkSize && $chunk->isHappy(); $j++ ) { + // @codingStandardsIgnoreEnd + $oldid = $revs[$i + $j]->rev_text_id; + + # Get text + if ( $loadStyle == self::LS_INDIVIDUAL ) { + $textRow = $dbw->selectRow( 'text', + [ 'old_flags', 'old_text' ], + [ 'old_id' => $oldid ], + __METHOD__, + 'FOR UPDATE' + ); + $text = Revision::getRevisionText( $textRow ); + } else { + $text = Revision::getRevisionText( $revs[$i + $j] ); + } + + if ( $text === false ) { + $this->error( "\nError, unable to get text in old_id $oldid" ); + # $dbw->delete( 'old', [ 'old_id' => $oldid ] ); + } + + if ( $extdb == "" && $j == 0 ) { + $chunk->setText( $text ); + $this->output( '.' ); + } else { + # Don't make a stub if it's going to be longer than the article + # Stubs are typically about 100 bytes + if ( strlen( $text ) < 120 ) { + $stub = false; + $this->output( 'x' ); + } else { + $stub = new HistoryBlobStub( $chunk->addItem( $text ) ); + $stub->setLocation( $primaryOldid ); + $stub->setReferrer( $oldid ); + $this->output( '.' ); + $usedChunk = true; + } + $stubs[$j] = $stub; + } + } + $thisChunkSize = $j; + + # If we couldn't actually use any stubs because the pages were too small, do nothing + if ( $usedChunk ) { + if ( $extdb != "" ) { + # Move blob objects to External Storage + $stored = $storeObj->store( $extdb, serialize( $chunk ) ); + if ( $stored === false ) { + $this->error( "Unable to store object" ); + + return false; + } + # Store External Storage URLs instead of Stub placeholders + foreach ( $stubs as $stub ) { + if ( $stub === false ) { + continue; + } + # $stored should provide base path to a BLOB + $url = $stored . "/" . $stub->getHash(); + $dbw->update( 'text', + [ /* SET */ + 'old_text' => $url, + 'old_flags' => 'external,utf-8', + ], [ /* WHERE */ + 'old_id' => $stub->getReferrer(), + ] + ); + } + } else { + # Store the main object locally + $dbw->update( 'text', + [ /* SET */ + 'old_text' => serialize( $chunk ), + 'old_flags' => 'object,utf-8', + ], [ /* WHERE */ + 'old_id' => $primaryOldid + ] + ); + + # Store the stub objects + for ( $j = 1; $j < $thisChunkSize; $j++ ) { + # Skip if not compressing and don't overwrite the first revision + if ( $stubs[$j] !== false && $revs[$i + $j]->rev_text_id != $primaryOldid ) { + $dbw->update( 'text', + [ /* SET */ + 'old_text' => serialize( $stubs[$j] ), + 'old_flags' => 'object,utf-8', + ], [ /* WHERE */ + 'old_id' => $revs[$i + $j]->rev_text_id + ] + ); + } + } + } + } + # Done, next + $this->output( "/" ); + $this->commitTransaction( $dbw, __METHOD__ ); + $i += $thisChunkSize; + wfWaitForSlaves(); + } + $this->output( "\n" ); + } + + return true; + } +} + +$maintClass = 'CompressOld'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/storage/drop_content_model_info.sql b/www/wiki/maintenance/storage/drop_content_model_info.sql new file mode 100644 index 00000000..7bd9aba9 --- /dev/null +++ b/www/wiki/maintenance/storage/drop_content_model_info.sql @@ -0,0 +1,7 @@ +ALTER TABLE /*$wgDBprefix*/archive DROP COLUMN ar_content_model; +ALTER TABLE /*$wgDBprefix*/archive DROP COLUMN ar_content_format; + +ALTER TABLE /*$wgDBprefix*/revision DROP COLUMN rev_content_model; +ALTER TABLE /*$wgDBprefix*/revision DROP COLUMN rev_content_format; + +ALTER TABLE /*$wgDBprefix*/page DROP COLUMN page_content_model; diff --git a/www/wiki/maintenance/storage/dumpRev.php b/www/wiki/maintenance/storage/dumpRev.php new file mode 100644 index 00000000..437bfcde --- /dev/null +++ b/www/wiki/maintenance/storage/dumpRev.php @@ -0,0 +1,88 @@ +<?php +/** + * Get the text of a revision, resolving external storage if needed. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance ExternalStorage + */ + +require_once __DIR__ . '/../Maintenance.php'; + +/** + * Maintenance script that gets the text of a revision, + * resolving external storage if needed. + * + * @ingroup Maintenance ExternalStorage + */ +class DumpRev extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addArg( 'rev-id', 'Revision ID', true ); + } + + public function execute() { + $dbr = $this->getDB( DB_REPLICA ); + $row = $dbr->selectRow( + [ 'text', 'revision' ], + [ 'old_flags', 'old_text' ], + [ 'old_id=rev_text_id', 'rev_id' => $this->getArg() ] + ); + if ( !$row ) { + $this->error( "Row not found", true ); + } + + $flags = explode( ',', $row->old_flags ); + $text = $row->old_text; + if ( in_array( 'external', $flags ) ) { + $this->output( "External $text\n" ); + if ( preg_match( '!^DB://(\w+)/(\w+)/(\w+)$!', $text, $m ) ) { + $es = ExternalStore::getStoreObject( 'DB' ); + $blob = $es->fetchBlob( $m[1], $m[2], $m[3] ); + if ( strtolower( get_class( $blob ) ) == 'concatenatedgziphistoryblob' ) { + $this->output( "Found external CGZ\n" ); + $blob->uncompress(); + $this->output( "Items: (" . implode( ', ', array_keys( $blob->mItems ) ) . ")\n" ); + $text = $blob->getItem( $m[3] ); + } else { + $this->output( "CGZ expected at $text, got " . gettype( $blob ) . "\n" ); + $text = $blob; + } + } else { + $this->output( "External plain $text\n" ); + $text = ExternalStore::fetchFromURL( $text ); + } + } + if ( in_array( 'gzip', $flags ) ) { + $text = gzinflate( $text ); + } + if ( in_array( 'object', $flags ) ) { + $obj = unserialize( $text ); + $text = $obj->getText(); + } + + if ( is_object( $text ) ) { + $this->error( "Unexpectedly got object of type: " . get_class( $text ) ); + } else { + $this->output( "Text length: " . strlen( $text ) . "\n" ); + $this->output( substr( $text, 0, 100 ) . "\n" ); + } + } +} + +$maintClass = "DumpRev"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/storage/fixBug20757.php b/www/wiki/maintenance/storage/fixBug20757.php new file mode 100644 index 00000000..0ea52cab --- /dev/null +++ b/www/wiki/maintenance/storage/fixBug20757.php @@ -0,0 +1,351 @@ +<?php +/** + * Script to fix bug 20757. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance ExternalStorage + */ + +require_once __DIR__ . '/../Maintenance.php'; + +/** + * Maintenance script to fix bug 20757. + * + * @ingroup Maintenance ExternalStorage + */ +class FixBug20757 extends Maintenance { + public $batchSize = 10000; + public $mapCache = []; + public $mapCacheSize = 0; + public $maxMapCacheSize = 1000000; + + function __construct() { + parent::__construct(); + $this->addDescription( 'Script to fix bug 20757 assuming that blob_tracking is intact' ); + $this->addOption( 'dry-run', 'Report only' ); + $this->addOption( 'start', 'old_id to start at', false, true ); + } + + function execute() { + $dbr = $this->getDB( DB_SLAVE ); + $dbw = $this->getDB( DB_MASTER ); + + $dryRun = $this->getOption( 'dry-run' ); + if ( $dryRun ) { + print "Dry run only.\n"; + } + + $startId = $this->getOption( 'start', 0 ); + $numGood = 0; + $numFixed = 0; + $numBad = 0; + + $totalRevs = $dbr->selectField( 'text', 'MAX(old_id)', false, __METHOD__ ); + + if ( $dbr->getType() == 'mysql' ) { + // In MySQL 4.1+, the binary field old_text has a non-working LOWER() function + $lowerLeft = 'LOWER(CONVERT(LEFT(old_text,22) USING latin1))'; + } + + while ( true ) { + print "ID: $startId / $totalRevs\r"; + + $res = $dbr->select( + 'text', + [ 'old_id', 'old_flags', 'old_text' ], + [ + 'old_id > ' . intval( $startId ), + 'old_flags LIKE \'%object%\' AND old_flags NOT LIKE \'%external%\'', + "$lowerLeft = 'o:15:\"historyblobstub\"'", + ], + __METHOD__, + [ + 'ORDER BY' => 'old_id', + 'LIMIT' => $this->batchSize, + ] + ); + + if ( !$res->numRows() ) { + break; + } + + $secondaryIds = []; + $stubs = []; + + foreach ( $res as $row ) { + $startId = $row->old_id; + + // Basic sanity checks + $obj = unserialize( $row->old_text ); + if ( $obj === false ) { + print "{$row->old_id}: unrecoverable: cannot unserialize\n"; + ++$numBad; + continue; + } + + if ( !is_object( $obj ) ) { + print "{$row->old_id}: unrecoverable: unserialized to type " . + gettype( $obj ) . ", possible double-serialization\n"; + ++$numBad; + continue; + } + + if ( strtolower( get_class( $obj ) ) !== 'historyblobstub' ) { + print "{$row->old_id}: unrecoverable: unexpected object class " . + get_class( $obj ) . "\n"; + ++$numBad; + continue; + } + + // Process flags + $flags = explode( ',', $row->old_flags ); + if ( in_array( 'utf-8', $flags ) || in_array( 'utf8', $flags ) ) { + $legacyEncoding = false; + } else { + $legacyEncoding = true; + } + + // Queue the stub for future batch processing + $id = intval( $obj->mOldId ); + $secondaryIds[] = $id; + $stubs[$row->old_id] = [ + 'legacyEncoding' => $legacyEncoding, + 'secondaryId' => $id, + 'hash' => $obj->mHash, + ]; + } + + $secondaryIds = array_unique( $secondaryIds ); + + if ( !count( $secondaryIds ) ) { + continue; + } + + // Run the batch query on blob_tracking + $res = $dbr->select( + 'blob_tracking', + '*', + [ + 'bt_text_id' => $secondaryIds, + ], + __METHOD__ + ); + $trackedBlobs = []; + foreach ( $res as $row ) { + $trackedBlobs[$row->bt_text_id] = $row; + } + + // Process the stubs + foreach ( $stubs as $primaryId => $stub ) { + $secondaryId = $stub['secondaryId']; + if ( !isset( $trackedBlobs[$secondaryId] ) ) { + // No tracked blob. Work out what went wrong + $secondaryRow = $dbr->selectRow( + 'text', + [ 'old_flags', 'old_text' ], + [ 'old_id' => $secondaryId ], + __METHOD__ + ); + if ( !$secondaryRow ) { + print "$primaryId: unrecoverable: secondary row is missing\n"; + ++$numBad; + } elseif ( $this->isUnbrokenStub( $stub, $secondaryRow ) ) { + // Not broken yet, and not in the tracked clusters so it won't get + // broken by the current RCT run. + ++$numGood; + } elseif ( strpos( $secondaryRow->old_flags, 'external' ) !== false ) { + print "$primaryId: unrecoverable: secondary gone to {$secondaryRow->old_text}\n"; + ++$numBad; + } else { + print "$primaryId: unrecoverable: miscellaneous corruption of secondary row\n"; + ++$numBad; + } + unset( $stubs[$primaryId] ); + continue; + } + $trackRow = $trackedBlobs[$secondaryId]; + + // Check that the specified text really is available in the tracked source row + $url = "DB://{$trackRow->bt_cluster}/{$trackRow->bt_blob_id}/{$stub['hash']}"; + $text = ExternalStore::fetchFromURL( $url ); + if ( $text === false ) { + print "$primaryId: unrecoverable: source text missing\n"; + ++$numBad; + unset( $stubs[$primaryId] ); + continue; + } + if ( md5( $text ) !== $stub['hash'] ) { + print "$primaryId: unrecoverable: content hashes do not match\n"; + ++$numBad; + unset( $stubs[$primaryId] ); + continue; + } + + // Find the page_id and rev_id + // The page is probably the same as the page of the secondary row + $pageId = intval( $trackRow->bt_page ); + if ( !$pageId ) { + $revId = $pageId = 0; + } else { + $revId = $this->findTextIdInPage( $pageId, $primaryId ); + if ( !$revId ) { + // Actually an orphan + $pageId = $revId = 0; + } + } + + $newFlags = $stub['legacyEncoding'] ? 'external' : 'external,utf-8'; + + if ( !$dryRun ) { + // Reset the text row to point to the original copy + $this->beginTransaction( $dbw, __METHOD__ ); + $dbw->update( + 'text', + // SET + [ + 'old_flags' => $newFlags, + 'old_text' => $url + ], + // WHERE + [ 'old_id' => $primaryId ], + __METHOD__ + ); + + // Add a blob_tracking row so that the new reference can be recompressed + // without needing to run trackBlobs.php again + $dbw->insert( 'blob_tracking', + [ + 'bt_page' => $pageId, + 'bt_rev_id' => $revId, + 'bt_text_id' => $primaryId, + 'bt_cluster' => $trackRow->bt_cluster, + 'bt_blob_id' => $trackRow->bt_blob_id, + 'bt_cgz_hash' => $stub['hash'], + 'bt_new_url' => null, + 'bt_moved' => 0, + ], + __METHOD__ + ); + $this->commitTransaction( $dbw, __METHOD__ ); + $this->waitForSlaves(); + } + + print "$primaryId: resolved to $url\n"; + ++$numFixed; + } + } + + print "\n"; + print "Fixed: $numFixed\n"; + print "Unrecoverable: $numBad\n"; + print "Good stubs: $numGood\n"; + } + + function waitForSlaves() { + static $iteration = 0; + ++$iteration; + if ( ++$iteration > 50 == 0 ) { + wfWaitForSlaves(); + $iteration = 0; + } + } + + function findTextIdInPage( $pageId, $textId ) { + $ids = $this->getRevTextMap( $pageId ); + if ( !isset( $ids[$textId] ) ) { + return null; + } else { + return $ids[$textId]; + } + } + + function getRevTextMap( $pageId ) { + if ( !isset( $this->mapCache[$pageId] ) ) { + // Limit cache size + while ( $this->mapCacheSize > $this->maxMapCacheSize ) { + $key = key( $this->mapCache ); + $this->mapCacheSize -= count( $this->mapCache[$key] ); + unset( $this->mapCache[$key] ); + } + + $dbr = $this->getDB( DB_SLAVE ); + $map = []; + $res = $dbr->select( 'revision', + [ 'rev_id', 'rev_text_id' ], + [ 'rev_page' => $pageId ], + __METHOD__ + ); + foreach ( $res as $row ) { + $map[$row->rev_text_id] = $row->rev_id; + } + $this->mapCache[$pageId] = $map; + $this->mapCacheSize += count( $map ); + } + + return $this->mapCache[$pageId]; + } + + /** + * This is based on part of HistoryBlobStub::getText(). + * Determine if the text can be retrieved from the row in the normal way. + * @param array $stub + * @param stdClass $secondaryRow + * @return bool + */ + function isUnbrokenStub( $stub, $secondaryRow ) { + $flags = explode( ',', $secondaryRow->old_flags ); + $text = $secondaryRow->old_text; + if ( in_array( 'external', $flags ) ) { + $url = $text; + MediaWiki\suppressWarnings(); + list( /* $proto */, $path ) = explode( '://', $url, 2 ); + MediaWiki\restoreWarnings(); + + if ( $path == "" ) { + return false; + } + $text = ExternalStore::fetchFromURL( $url ); + } + if ( !in_array( 'object', $flags ) ) { + return false; + } + + if ( in_array( 'gzip', $flags ) ) { + $obj = unserialize( gzinflate( $text ) ); + } else { + $obj = unserialize( $text ); + } + + if ( !is_object( $obj ) ) { + // Correct for old double-serialization bug. + $obj = unserialize( $obj ); + } + + if ( !is_object( $obj ) ) { + return false; + } + + $obj->uncompress(); + $text = $obj->getItem( $stub['hash'] ); + + return $text !== false; + } +} + +$maintClass = 'FixBug20757'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/storage/fixT22757.php b/www/wiki/maintenance/storage/fixT22757.php new file mode 100644 index 00000000..e8bd23d4 --- /dev/null +++ b/www/wiki/maintenance/storage/fixT22757.php @@ -0,0 +1,349 @@ +<?php +/** + * Script to fix T22757. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance ExternalStorage + */ + +require_once __DIR__ . '/../Maintenance.php'; + +/** + * Maintenance script to fix T22757. + * + * @ingroup Maintenance ExternalStorage + */ +class FixT22757 extends Maintenance { + public $batchSize = 10000; + public $mapCache = []; + public $mapCacheSize = 0; + public $maxMapCacheSize = 1000000; + + function __construct() { + parent::__construct(); + $this->addDescription( 'Script to fix T22757 assuming that blob_tracking is intact' ); + $this->addOption( 'dry-run', 'Report only' ); + $this->addOption( 'start', 'old_id to start at', false, true ); + } + + function execute() { + $dbr = $this->getDB( DB_REPLICA ); + $dbw = $this->getDB( DB_MASTER ); + + $dryRun = $this->getOption( 'dry-run' ); + if ( $dryRun ) { + print "Dry run only.\n"; + } + + $startId = $this->getOption( 'start', 0 ); + $numGood = 0; + $numFixed = 0; + $numBad = 0; + + $totalRevs = $dbr->selectField( 'text', 'MAX(old_id)', false, __METHOD__ ); + + // In MySQL 4.1+, the binary field old_text has a non-working LOWER() function + $lowerLeft = 'LOWER(CONVERT(LEFT(old_text,22) USING latin1))'; + + while ( true ) { + print "ID: $startId / $totalRevs\r"; + + $res = $dbr->select( + 'text', + [ 'old_id', 'old_flags', 'old_text' ], + [ + 'old_id > ' . intval( $startId ), + 'old_flags LIKE \'%object%\' AND old_flags NOT LIKE \'%external%\'', + "$lowerLeft = 'o:15:\"historyblobstub\"'", + ], + __METHOD__, + [ + 'ORDER BY' => 'old_id', + 'LIMIT' => $this->batchSize, + ] + ); + + if ( !$res->numRows() ) { + break; + } + + $secondaryIds = []; + $stubs = []; + + foreach ( $res as $row ) { + $startId = $row->old_id; + + // Basic sanity checks + $obj = unserialize( $row->old_text ); + if ( $obj === false ) { + print "{$row->old_id}: unrecoverable: cannot unserialize\n"; + ++$numBad; + continue; + } + + if ( !is_object( $obj ) ) { + print "{$row->old_id}: unrecoverable: unserialized to type " . + gettype( $obj ) . ", possible double-serialization\n"; + ++$numBad; + continue; + } + + if ( strtolower( get_class( $obj ) ) !== 'historyblobstub' ) { + print "{$row->old_id}: unrecoverable: unexpected object class " . + get_class( $obj ) . "\n"; + ++$numBad; + continue; + } + + // Process flags + $flags = explode( ',', $row->old_flags ); + if ( in_array( 'utf-8', $flags ) || in_array( 'utf8', $flags ) ) { + $legacyEncoding = false; + } else { + $legacyEncoding = true; + } + + // Queue the stub for future batch processing + $id = intval( $obj->mOldId ); + $secondaryIds[] = $id; + $stubs[$row->old_id] = [ + 'legacyEncoding' => $legacyEncoding, + 'secondaryId' => $id, + 'hash' => $obj->mHash, + ]; + } + + $secondaryIds = array_unique( $secondaryIds ); + + if ( !count( $secondaryIds ) ) { + continue; + } + + // Run the batch query on blob_tracking + $res = $dbr->select( + 'blob_tracking', + '*', + [ + 'bt_text_id' => $secondaryIds, + ], + __METHOD__ + ); + $trackedBlobs = []; + foreach ( $res as $row ) { + $trackedBlobs[$row->bt_text_id] = $row; + } + + // Process the stubs + foreach ( $stubs as $primaryId => $stub ) { + $secondaryId = $stub['secondaryId']; + if ( !isset( $trackedBlobs[$secondaryId] ) ) { + // No tracked blob. Work out what went wrong + $secondaryRow = $dbr->selectRow( + 'text', + [ 'old_flags', 'old_text' ], + [ 'old_id' => $secondaryId ], + __METHOD__ + ); + if ( !$secondaryRow ) { + print "$primaryId: unrecoverable: secondary row is missing\n"; + ++$numBad; + } elseif ( $this->isUnbrokenStub( $stub, $secondaryRow ) ) { + // Not broken yet, and not in the tracked clusters so it won't get + // broken by the current RCT run. + ++$numGood; + } elseif ( strpos( $secondaryRow->old_flags, 'external' ) !== false ) { + print "$primaryId: unrecoverable: secondary gone to {$secondaryRow->old_text}\n"; + ++$numBad; + } else { + print "$primaryId: unrecoverable: miscellaneous corruption of secondary row\n"; + ++$numBad; + } + unset( $stubs[$primaryId] ); + continue; + } + $trackRow = $trackedBlobs[$secondaryId]; + + // Check that the specified text really is available in the tracked source row + $url = "DB://{$trackRow->bt_cluster}/{$trackRow->bt_blob_id}/{$stub['hash']}"; + $text = ExternalStore::fetchFromURL( $url ); + if ( $text === false ) { + print "$primaryId: unrecoverable: source text missing\n"; + ++$numBad; + unset( $stubs[$primaryId] ); + continue; + } + if ( md5( $text ) !== $stub['hash'] ) { + print "$primaryId: unrecoverable: content hashes do not match\n"; + ++$numBad; + unset( $stubs[$primaryId] ); + continue; + } + + // Find the page_id and rev_id + // The page is probably the same as the page of the secondary row + $pageId = intval( $trackRow->bt_page ); + if ( !$pageId ) { + $revId = $pageId = 0; + } else { + $revId = $this->findTextIdInPage( $pageId, $primaryId ); + if ( !$revId ) { + // Actually an orphan + $pageId = $revId = 0; + } + } + + $newFlags = $stub['legacyEncoding'] ? 'external' : 'external,utf-8'; + + if ( !$dryRun ) { + // Reset the text row to point to the original copy + $this->beginTransaction( $dbw, __METHOD__ ); + $dbw->update( + 'text', + // SET + [ + 'old_flags' => $newFlags, + 'old_text' => $url + ], + // WHERE + [ 'old_id' => $primaryId ], + __METHOD__ + ); + + // Add a blob_tracking row so that the new reference can be recompressed + // without needing to run trackBlobs.php again + $dbw->insert( 'blob_tracking', + [ + 'bt_page' => $pageId, + 'bt_rev_id' => $revId, + 'bt_text_id' => $primaryId, + 'bt_cluster' => $trackRow->bt_cluster, + 'bt_blob_id' => $trackRow->bt_blob_id, + 'bt_cgz_hash' => $stub['hash'], + 'bt_new_url' => null, + 'bt_moved' => 0, + ], + __METHOD__ + ); + $this->commitTransaction( $dbw, __METHOD__ ); + $this->waitForSlaves(); + } + + print "$primaryId: resolved to $url\n"; + ++$numFixed; + } + } + + print "\n"; + print "Fixed: $numFixed\n"; + print "Unrecoverable: $numBad\n"; + print "Good stubs: $numGood\n"; + } + + function waitForSlaves() { + static $iteration = 0; + ++$iteration; + if ( ++$iteration > 50 == 0 ) { + wfWaitForSlaves(); + $iteration = 0; + } + } + + function findTextIdInPage( $pageId, $textId ) { + $ids = $this->getRevTextMap( $pageId ); + if ( !isset( $ids[$textId] ) ) { + return null; + } else { + return $ids[$textId]; + } + } + + function getRevTextMap( $pageId ) { + if ( !isset( $this->mapCache[$pageId] ) ) { + // Limit cache size + while ( $this->mapCacheSize > $this->maxMapCacheSize ) { + $key = key( $this->mapCache ); + $this->mapCacheSize -= count( $this->mapCache[$key] ); + unset( $this->mapCache[$key] ); + } + + $dbr = $this->getDB( DB_REPLICA ); + $map = []; + $res = $dbr->select( 'revision', + [ 'rev_id', 'rev_text_id' ], + [ 'rev_page' => $pageId ], + __METHOD__ + ); + foreach ( $res as $row ) { + $map[$row->rev_text_id] = $row->rev_id; + } + $this->mapCache[$pageId] = $map; + $this->mapCacheSize += count( $map ); + } + + return $this->mapCache[$pageId]; + } + + /** + * This is based on part of HistoryBlobStub::getText(). + * Determine if the text can be retrieved from the row in the normal way. + * @param array $stub + * @param stdClass $secondaryRow + * @return bool + */ + function isUnbrokenStub( $stub, $secondaryRow ) { + $flags = explode( ',', $secondaryRow->old_flags ); + $text = $secondaryRow->old_text; + if ( in_array( 'external', $flags ) ) { + $url = $text; + MediaWiki\suppressWarnings(); + list( /* $proto */, $path ) = explode( '://', $url, 2 ); + MediaWiki\restoreWarnings(); + + if ( $path == "" ) { + return false; + } + $text = ExternalStore::fetchFromURL( $url ); + } + if ( !in_array( 'object', $flags ) ) { + return false; + } + + if ( in_array( 'gzip', $flags ) ) { + $obj = unserialize( gzinflate( $text ) ); + } else { + $obj = unserialize( $text ); + } + + if ( !is_object( $obj ) ) { + // Correct for old double-serialization bug. + $obj = unserialize( $obj ); + } + + if ( !is_object( $obj ) ) { + return false; + } + + $obj->uncompress(); + $text = $obj->getItem( $stub['hash'] ); + + return $text !== false; + } +} + +$maintClass = 'FixT22757'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/storage/make-blobs b/www/wiki/maintenance/storage/make-blobs new file mode 100755 index 00000000..16dcb672 --- /dev/null +++ b/www/wiki/maintenance/storage/make-blobs @@ -0,0 +1,14 @@ +#!/bin/bash + +if [ -z $2 ];then + echo 'Usage: make-blobs <server> <db> [<table name>]' + exit 1 +fi +if [ -z $3 ]; then + table=blobs +else + table=$3 +fi + +echo "CREATE DATABASE $2" | mysql -u wikiadmin -p`wikiadmin_pass` -h $1 && \ +sed "s/blobs\>/$table/" blobs.sql | mysql -u wikiadmin -p`wikiadmin_pass` -h $1 $2 diff --git a/www/wiki/maintenance/storage/moveToExternal.php b/www/wiki/maintenance/storage/moveToExternal.php new file mode 100644 index 00000000..e1179920 --- /dev/null +++ b/www/wiki/maintenance/storage/moveToExternal.php @@ -0,0 +1,126 @@ +<?php +/** + * Move revision's text to external storage + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance ExternalStorage + */ + +define( 'REPORTING_INTERVAL', 1 ); + +if ( !defined( 'MEDIAWIKI' ) ) { + $optionsWithArgs = [ 'e', 's' ]; + require_once __DIR__ . '/../commandLine.inc'; + require_once 'resolveStubs.php'; + + $fname = 'moveToExternal'; + + if ( !isset( $args[0] ) ) { + print "Usage: php moveToExternal.php [-s <startid>] [-e <endid>] <cluster>\n"; + exit; + } + + $cluster = $args[0]; + $dbw = wfGetDB( DB_MASTER ); + + if ( isset( $options['e'] ) ) { + $maxID = $options['e']; + } else { + $maxID = $dbw->selectField( 'text', 'MAX(old_id)', false, $fname ); + } + $minID = isset( $options['s'] ) ? $options['s'] : 1; + + moveToExternal( $cluster, $maxID, $minID ); +} + +function moveToExternal( $cluster, $maxID, $minID = 1 ) { + $fname = 'moveToExternal'; + $dbw = wfGetDB( DB_MASTER ); + $dbr = wfGetDB( DB_REPLICA ); + + $count = $maxID - $minID + 1; + $blockSize = 1000; + $numBlocks = ceil( $count / $blockSize ); + print "Moving text rows from $minID to $maxID to external storage\n"; + $ext = new ExternalStoreDB; + $numMoved = 0; + + for ( $block = 0; $block < $numBlocks; $block++ ) { + $blockStart = $block * $blockSize + $minID; + $blockEnd = $blockStart + $blockSize - 1; + + if ( !( $block % REPORTING_INTERVAL ) ) { + print "oldid=$blockStart, moved=$numMoved\n"; + wfWaitForSlaves(); + } + + $res = $dbr->select( 'text', [ 'old_id', 'old_flags', 'old_text' ], + [ + "old_id BETWEEN $blockStart AND $blockEnd", + 'old_flags NOT ' . $dbr->buildLike( $dbr->anyString(), 'external', $dbr->anyString() ), + ], $fname ); + foreach ( $res as $row ) { + # Resolve stubs + $text = $row->old_text; + $id = $row->old_id; + if ( $row->old_flags === '' ) { + $flags = 'external'; + } else { + $flags = "{$row->old_flags},external"; + } + + if ( strpos( $flags, 'object' ) !== false ) { + $obj = unserialize( $text ); + $className = strtolower( get_class( $obj ) ); + if ( $className == 'historyblobstub' ) { + # resolveStub( $id, $row->old_text, $row->old_flags ); + # $numStubs++; + continue; + } elseif ( $className == 'historyblobcurstub' ) { + $text = gzdeflate( $obj->getText() ); + $flags = 'utf-8,gzip,external'; + } elseif ( $className == 'concatenatedgziphistoryblob' ) { + // Do nothing + } else { + print "Warning: unrecognised object class \"$className\"\n"; + continue; + } + } else { + $className = false; + } + + if ( strlen( $text ) < 100 && $className === false ) { + // Don't move tiny revisions + continue; + } + + # print "Storing " . strlen( $text ) . " bytes to $url\n"; + # print "old_id=$id\n"; + + $url = $ext->store( $cluster, $text ); + if ( !$url ) { + print "Error writing to external storage\n"; + exit; + } + $dbw->update( 'text', + [ 'old_flags' => $flags, 'old_text' => $url ], + [ 'old_id' => $id ], $fname ); + $numMoved++; + } + } +} diff --git a/www/wiki/maintenance/storage/orphanStats.php b/www/wiki/maintenance/storage/orphanStats.php new file mode 100644 index 00000000..d7d0b84c --- /dev/null +++ b/www/wiki/maintenance/storage/orphanStats.php @@ -0,0 +1,84 @@ +<?php +/** + * Show some statistics on the blob_orphans table, created with trackBlobs.php. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance ExternalStorage + */ + +require_once __DIR__ . '/../Maintenance.php'; + +/** + * Maintenance script that shows some statistics on the blob_orphans table, + * created with trackBlobs.php. + * + * @ingroup Maintenance ExternalStorage + */ +class OrphanStats extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( + "Show some statistics on the blob_orphans table, created with trackBlobs.php" ); + } + + protected function &getDB( $cluster, $groups = [], $wiki = false ) { + $lb = wfGetLBFactory()->getExternalLB( $cluster ); + + return $lb->getConnection( DB_REPLICA ); + } + + public function execute() { + $dbr = $this->getDB( DB_REPLICA ); + if ( !$dbr->tableExists( 'blob_orphans' ) ) { + $this->error( "blob_orphans doesn't seem to exist, need to run trackBlobs.php first", true ); + } + $res = $dbr->select( 'blob_orphans', '*', false, __METHOD__ ); + + $num = 0; + $totalSize = 0; + $hashes = []; + $maxSize = 0; + + foreach ( $res as $boRow ) { + $extDB = $this->getDB( $boRow->bo_cluster ); + $blobRow = $extDB->selectRow( + 'blobs', + '*', + [ 'blob_id' => $boRow->bo_blob_id ], + __METHOD__ + ); + + $num++; + $size = strlen( $blobRow->blob_text ); + $totalSize += $size; + $hashes[sha1( $blobRow->blob_text )] = true; + $maxSize = max( $size, $maxSize ); + } + unset( $res ); + + $this->output( "Number of orphans: $num\n" ); + if ( $num > 0 ) { + $this->output( "Average size: " . round( $totalSize / $num, 0 ) . " bytes\n" . + "Max size: $maxSize\n" . + "Number of unique texts: " . count( $hashes ) . "\n" ); + } + } +} + +$maintClass = "OrphanStats"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/storage/recompressTracked.php b/www/wiki/maintenance/storage/recompressTracked.php new file mode 100644 index 00000000..c5dd53b1 --- /dev/null +++ b/www/wiki/maintenance/storage/recompressTracked.php @@ -0,0 +1,841 @@ +<?php +/** + * Moves blobs indexed by trackBlobs.php to a specified list of destination + * clusters, and recompresses them in the process. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance ExternalStorage + */ + +use MediaWiki\Logger\LegacyLogger; +use MediaWiki\MediaWikiServices; + +$optionsWithArgs = RecompressTracked::getOptionsWithArgs(); +require __DIR__ . '/../commandLine.inc'; + +if ( count( $args ) < 1 ) { + echo "Usage: php recompressTracked.php [options] <cluster> [... <cluster>...] +Moves blobs indexed by trackBlobs.php to a specified list of destination clusters, +and recompresses them in the process. Restartable. + +Options: + --procs <procs> Set the number of child processes (default 1) + --copy-only Copy only, do not update the text table. Restart + without this option to complete. + --debug-log <file> Log debugging data to the specified file + --info-log <file> Log progress messages to the specified file + --critical-log <file> Log error messages to the specified file +"; + exit( 1 ); +} + +$job = RecompressTracked::newFromCommandLine( $args, $options ); +$job->execute(); + +/** + * Maintenance script that moves blobs indexed by trackBlobs.php to a specified + * list of destination clusters, and recompresses them in the process. + * + * @ingroup Maintenance ExternalStorage + */ +class RecompressTracked { + public $destClusters; + public $batchSize = 1000; + public $orphanBatchSize = 1000; + public $reportingInterval = 10; + public $numProcs = 1; + public $numBatches = 0; + public $pageBlobClass, $orphanBlobClass; + public $replicaPipes, $replicaProcs, $prevReplicaId; + public $copyOnly = false; + public $isChild = false; + public $replicaId = false; + public $noCount = false; + public $debugLog, $infoLog, $criticalLog; + public $store; + + private static $optionsWithArgs = [ + 'procs', + 'replica-id', + 'debug-log', + 'info-log', + 'critical-log' + ]; + + private static $cmdLineOptionMap = [ + 'no-count' => 'noCount', + 'procs' => 'numProcs', + 'copy-only' => 'copyOnly', + 'child' => 'isChild', + 'replica-id' => 'replicaId', + 'debug-log' => 'debugLog', + 'info-log' => 'infoLog', + 'critical-log' => 'criticalLog', + ]; + + static function getOptionsWithArgs() { + return self::$optionsWithArgs; + } + + static function newFromCommandLine( $args, $options ) { + $jobOptions = [ 'destClusters' => $args ]; + foreach ( self::$cmdLineOptionMap as $cmdOption => $classOption ) { + if ( isset( $options[$cmdOption] ) ) { + $jobOptions[$classOption] = $options[$cmdOption]; + } + } + + return new self( $jobOptions ); + } + + function __construct( $options ) { + foreach ( $options as $name => $value ) { + $this->$name = $value; + } + $this->store = new ExternalStoreDB; + if ( !$this->isChild ) { + $GLOBALS['wgDebugLogPrefix'] = "RCT M: "; + } elseif ( $this->replicaId !== false ) { + $GLOBALS['wgDebugLogPrefix'] = "RCT {$this->replicaId}: "; + } + $this->pageBlobClass = function_exists( 'xdiff_string_bdiff' ) ? + 'DiffHistoryBlob' : 'ConcatenatedGzipHistoryBlob'; + $this->orphanBlobClass = 'ConcatenatedGzipHistoryBlob'; + } + + function debug( $msg ) { + wfDebug( "$msg\n" ); + if ( $this->debugLog ) { + $this->logToFile( $msg, $this->debugLog ); + } + } + + function info( $msg ) { + echo "$msg\n"; + if ( $this->infoLog ) { + $this->logToFile( $msg, $this->infoLog ); + } + } + + function critical( $msg ) { + echo "$msg\n"; + if ( $this->criticalLog ) { + $this->logToFile( $msg, $this->criticalLog ); + } + } + + function logToFile( $msg, $file ) { + $header = '[' . date( 'd\TH:i:s' ) . '] ' . wfHostname() . ' ' . posix_getpid(); + if ( $this->replicaId !== false ) { + $header .= "({$this->replicaId})"; + } + $header .= ' ' . wfWikiID(); + LegacyLogger::emit( sprintf( "%-50s %s\n", $header, $msg ), $file ); + } + + /** + * Wait until the selected replica DB has caught up to the master. + * This allows us to use the replica DB for things that were committed in a + * previous part of this batch process. + */ + function syncDBs() { + $dbw = wfGetDB( DB_MASTER ); + $dbr = wfGetDB( DB_REPLICA ); + $pos = $dbw->getMasterPos(); + $dbr->masterPosWait( $pos, 100000 ); + } + + /** + * Execute parent or child depending on the isChild option + */ + function execute() { + if ( $this->isChild ) { + $this->executeChild(); + } else { + $this->executeParent(); + } + } + + /** + * Execute the parent process + */ + function executeParent() { + if ( !$this->checkTrackingTable() ) { + return; + } + + $this->syncDBs(); + $this->startReplicaProcs(); + $this->doAllPages(); + $this->doAllOrphans(); + $this->killReplicaProcs(); + } + + /** + * Make sure the tracking table exists and isn't empty + * @return bool + */ + function checkTrackingTable() { + $dbr = wfGetDB( DB_REPLICA ); + if ( !$dbr->tableExists( 'blob_tracking' ) ) { + $this->critical( "Error: blob_tracking table does not exist" ); + + return false; + } + $row = $dbr->selectRow( 'blob_tracking', '*', '', __METHOD__ ); + if ( !$row ) { + $this->info( "Warning: blob_tracking table contains no rows, skipping this wiki." ); + + return false; + } + + return true; + } + + /** + * Start the worker processes. + * These processes will listen on stdin for commands. + * This necessary because text recompression is slow: loading, compressing and + * writing are all slow. + */ + function startReplicaProcs() { + $cmd = 'php ' . wfEscapeShellArg( __FILE__ ); + foreach ( self::$cmdLineOptionMap as $cmdOption => $classOption ) { + if ( $cmdOption == 'replica-id' ) { + continue; + } elseif ( in_array( $cmdOption, self::$optionsWithArgs ) && isset( $this->$classOption ) ) { + $cmd .= " --$cmdOption " . wfEscapeShellArg( $this->$classOption ); + } elseif ( $this->$classOption ) { + $cmd .= " --$cmdOption"; + } + } + $cmd .= ' --child' . + ' --wiki ' . wfEscapeShellArg( wfWikiID() ) . + ' ' . call_user_func_array( 'wfEscapeShellArg', $this->destClusters ); + + $this->replicaPipes = $this->replicaProcs = []; + for ( $i = 0; $i < $this->numProcs; $i++ ) { + $pipes = []; + $spec = [ + [ 'pipe', 'r' ], + [ 'file', 'php://stdout', 'w' ], + [ 'file', 'php://stderr', 'w' ] + ]; + MediaWiki\suppressWarnings(); + $proc = proc_open( "$cmd --replica-id $i", $spec, $pipes ); + MediaWiki\restoreWarnings(); + if ( !$proc ) { + $this->critical( "Error opening replica DB process: $cmd" ); + exit( 1 ); + } + $this->replicaProcs[$i] = $proc; + $this->replicaPipes[$i] = $pipes[0]; + } + $this->prevReplicaId = -1; + } + + /** + * Gracefully terminate the child processes + */ + function killReplicaProcs() { + $this->info( "Waiting for replica DB processes to finish..." ); + for ( $i = 0; $i < $this->numProcs; $i++ ) { + $this->dispatchToReplica( $i, 'quit' ); + } + for ( $i = 0; $i < $this->numProcs; $i++ ) { + $status = proc_close( $this->replicaProcs[$i] ); + if ( $status ) { + $this->critical( "Warning: child #$i exited with status $status" ); + } + } + $this->info( "Done." ); + } + + /** + * Dispatch a command to the next available replica DB. + * This may block until a replica DB finishes its work and becomes available. + */ + function dispatch( /*...*/ ) { + $args = func_get_args(); + $pipes = $this->replicaPipes; + $numPipes = stream_select( $x = [], $pipes, $y = [], 3600 ); + if ( !$numPipes ) { + $this->critical( "Error waiting to write to replica DBs. Aborting" ); + exit( 1 ); + } + for ( $i = 0; $i < $this->numProcs; $i++ ) { + $replicaId = ( $i + $this->prevReplicaId + 1 ) % $this->numProcs; + if ( isset( $pipes[$replicaId] ) ) { + $this->prevReplicaId = $replicaId; + $this->dispatchToReplica( $replicaId, $args ); + + return; + } + } + $this->critical( "Unreachable" ); + exit( 1 ); + } + + /** + * Dispatch a command to a specified replica DB + * @param int $replicaId + * @param array|string $args + */ + function dispatchToReplica( $replicaId, $args ) { + $args = (array)$args; + $cmd = implode( ' ', $args ); + fwrite( $this->replicaPipes[$replicaId], "$cmd\n" ); + } + + /** + * Move all tracked pages to the new clusters + */ + function doAllPages() { + $dbr = wfGetDB( DB_REPLICA ); + $i = 0; + $startId = 0; + if ( $this->noCount ) { + $numPages = '[unknown]'; + } else { + $numPages = $dbr->selectField( 'blob_tracking', + 'COUNT(DISTINCT bt_page)', + # A condition is required so that this query uses the index + [ 'bt_moved' => 0 ], + __METHOD__ + ); + } + if ( $this->copyOnly ) { + $this->info( "Copying pages..." ); + } else { + $this->info( "Moving pages..." ); + } + while ( true ) { + $res = $dbr->select( 'blob_tracking', + [ 'bt_page' ], + [ + 'bt_moved' => 0, + 'bt_page > ' . $dbr->addQuotes( $startId ) + ], + __METHOD__, + [ + 'DISTINCT', + 'ORDER BY' => 'bt_page', + 'LIMIT' => $this->batchSize, + ] + ); + if ( !$res->numRows() ) { + break; + } + foreach ( $res as $row ) { + $startId = $row->bt_page; + $this->dispatch( 'doPage', $row->bt_page ); + $i++; + } + $this->report( 'pages', $i, $numPages ); + } + $this->report( 'pages', $i, $numPages ); + if ( $this->copyOnly ) { + $this->info( "All page copies queued." ); + } else { + $this->info( "All page moves queued." ); + } + } + + /** + * Display a progress report + * @param string $label + * @param int $current + * @param int $end + */ + function report( $label, $current, $end ) { + $this->numBatches++; + if ( $current == $end || $this->numBatches >= $this->reportingInterval ) { + $this->numBatches = 0; + $this->info( "$label: $current / $end" ); + MediaWikiServices::getInstance()->getDBLoadBalancerFactory()->waitForReplication(); + } + } + + /** + * Move all orphan text to the new clusters + */ + function doAllOrphans() { + $dbr = wfGetDB( DB_REPLICA ); + $startId = 0; + $i = 0; + if ( $this->noCount ) { + $numOrphans = '[unknown]'; + } else { + $numOrphans = $dbr->selectField( 'blob_tracking', + 'COUNT(DISTINCT bt_text_id)', + [ 'bt_moved' => 0, 'bt_page' => 0 ], + __METHOD__ ); + if ( !$numOrphans ) { + return; + } + } + if ( $this->copyOnly ) { + $this->info( "Copying orphans..." ); + } else { + $this->info( "Moving orphans..." ); + } + + while ( true ) { + $res = $dbr->select( 'blob_tracking', + [ 'bt_text_id' ], + [ + 'bt_moved' => 0, + 'bt_page' => 0, + 'bt_text_id > ' . $dbr->addQuotes( $startId ) + ], + __METHOD__, + [ + 'DISTINCT', + 'ORDER BY' => 'bt_text_id', + 'LIMIT' => $this->batchSize + ] + ); + if ( !$res->numRows() ) { + break; + } + $ids = []; + foreach ( $res as $row ) { + $startId = $row->bt_text_id; + $ids[] = $row->bt_text_id; + $i++; + } + // Need to send enough orphan IDs to the child at a time to fill a blob, + // so orphanBatchSize needs to be at least ~100. + // batchSize can be smaller or larger. + while ( count( $ids ) > $this->orphanBatchSize ) { + $args = array_slice( $ids, 0, $this->orphanBatchSize ); + $ids = array_slice( $ids, $this->orphanBatchSize ); + array_unshift( $args, 'doOrphanList' ); + call_user_func_array( [ $this, 'dispatch' ], $args ); + } + if ( count( $ids ) ) { + $args = $ids; + array_unshift( $args, 'doOrphanList' ); + call_user_func_array( [ $this, 'dispatch' ], $args ); + } + + $this->report( 'orphans', $i, $numOrphans ); + } + $this->report( 'orphans', $i, $numOrphans ); + $this->info( "All orphans queued." ); + } + + /** + * Main entry point for worker processes + */ + function executeChild() { + $this->debug( 'starting' ); + $this->syncDBs(); + + while ( !feof( STDIN ) ) { + $line = rtrim( fgets( STDIN ) ); + if ( $line == '' ) { + continue; + } + $this->debug( $line ); + $args = explode( ' ', $line ); + $cmd = array_shift( $args ); + switch ( $cmd ) { + case 'doPage': + $this->doPage( intval( $args[0] ) ); + break; + case 'doOrphanList': + $this->doOrphanList( array_map( 'intval', $args ) ); + break; + case 'quit': + return; + } + MediaWikiServices::getInstance()->getDBLoadBalancerFactory()->waitForReplication(); + } + } + + /** + * Move tracked text in a given page + * + * @param int $pageId + */ + function doPage( $pageId ) { + $title = Title::newFromID( $pageId ); + if ( $title ) { + $titleText = $title->getPrefixedText(); + } else { + $titleText = '[deleted]'; + } + $dbr = wfGetDB( DB_REPLICA ); + + // Finish any incomplete transactions + if ( !$this->copyOnly ) { + $this->finishIncompleteMoves( [ 'bt_page' => $pageId ] ); + $this->syncDBs(); + } + + $startId = 0; + $trx = new CgzCopyTransaction( $this, $this->pageBlobClass ); + + $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory(); + while ( true ) { + $res = $dbr->select( + [ 'blob_tracking', 'text' ], + '*', + [ + 'bt_page' => $pageId, + 'bt_text_id > ' . $dbr->addQuotes( $startId ), + 'bt_moved' => 0, + 'bt_new_url IS NULL', + 'bt_text_id=old_id', + ], + __METHOD__, + [ + 'ORDER BY' => 'bt_text_id', + 'LIMIT' => $this->batchSize + ] + ); + if ( !$res->numRows() ) { + break; + } + + $lastTextId = 0; + foreach ( $res as $row ) { + $startId = $row->bt_text_id; + if ( $lastTextId == $row->bt_text_id ) { + // Duplicate (null edit) + continue; + } + $lastTextId = $row->bt_text_id; + // Load the text + $text = Revision::getRevisionText( $row ); + if ( $text === false ) { + $this->critical( "Error loading {$row->bt_rev_id}/{$row->bt_text_id}" ); + continue; + } + + // Queue it + if ( !$trx->addItem( $text, $row->bt_text_id ) ) { + $this->debug( "$titleText: committing blob with " . $trx->getSize() . " items" ); + $trx->commit(); + $trx = new CgzCopyTransaction( $this, $this->pageBlobClass ); + $lbFactory->waitForReplication(); + } + } + } + + $this->debug( "$titleText: committing blob with " . $trx->getSize() . " items" ); + $trx->commit(); + } + + /** + * Atomic move operation. + * + * Write the new URL to the text table and set the bt_moved flag. + * + * This is done in a single transaction to provide restartable behavior + * without data loss. + * + * The transaction is kept short to reduce locking. + * + * @param int $textId + * @param string $url + */ + function moveTextRow( $textId, $url ) { + if ( $this->copyOnly ) { + $this->critical( "Internal error: can't call moveTextRow() in --copy-only mode" ); + exit( 1 ); + } + $dbw = wfGetDB( DB_MASTER ); + $dbw->begin( __METHOD__ ); + $dbw->update( 'text', + [ // set + 'old_text' => $url, + 'old_flags' => 'external,utf-8', + ], + [ // where + 'old_id' => $textId + ], + __METHOD__ + ); + $dbw->update( 'blob_tracking', + [ 'bt_moved' => 1 ], + [ 'bt_text_id' => $textId ], + __METHOD__ + ); + $dbw->commit( __METHOD__ ); + } + + /** + * Moves are done in two phases: bt_new_url and then bt_moved. + * - bt_new_url indicates that the text has been copied to the new cluster. + * - bt_moved indicates that the text table has been updated. + * + * This function completes any moves that only have done bt_new_url. This + * can happen when the script is interrupted, or when --copy-only is used. + * + * @param array $conds + */ + function finishIncompleteMoves( $conds ) { + $dbr = wfGetDB( DB_REPLICA ); + $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory(); + + $startId = 0; + $conds = array_merge( $conds, [ + 'bt_moved' => 0, + 'bt_new_url IS NOT NULL' + ] ); + while ( true ) { + $res = $dbr->select( 'blob_tracking', + '*', + array_merge( $conds, [ 'bt_text_id > ' . $dbr->addQuotes( $startId ) ] ), + __METHOD__, + [ + 'ORDER BY' => 'bt_text_id', + 'LIMIT' => $this->batchSize, + ] + ); + if ( !$res->numRows() ) { + break; + } + $this->debug( 'Incomplete: ' . $res->numRows() . ' rows' ); + foreach ( $res as $row ) { + $startId = $row->bt_text_id; + $this->moveTextRow( $row->bt_text_id, $row->bt_new_url ); + if ( $row->bt_text_id % 10 == 0 ) { + $lbFactory->waitForReplication(); + } + } + } + } + + /** + * Returns the name of the next target cluster + * @return string + */ + function getTargetCluster() { + $cluster = next( $this->destClusters ); + if ( $cluster === false ) { + $cluster = reset( $this->destClusters ); + } + + return $cluster; + } + + /** + * Gets a DB master connection for the given external cluster name + * @param string $cluster + * @return Database + */ + function getExtDB( $cluster ) { + $lb = wfGetLBFactory()->getExternalLB( $cluster ); + + return $lb->getConnection( DB_MASTER ); + } + + /** + * Move an orphan text_id to the new cluster + * + * @param array $textIds + */ + function doOrphanList( $textIds ) { + // Finish incomplete moves + if ( !$this->copyOnly ) { + $this->finishIncompleteMoves( [ 'bt_text_id' => $textIds ] ); + $this->syncDBs(); + } + + $trx = new CgzCopyTransaction( $this, $this->orphanBlobClass ); + + $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory(); + $res = wfGetDB( DB_REPLICA )->select( + [ 'text', 'blob_tracking' ], + [ 'old_id', 'old_text', 'old_flags' ], + [ + 'old_id' => $textIds, + 'bt_text_id=old_id', + 'bt_moved' => 0, + ], + __METHOD__, + [ 'DISTINCT' ] + ); + + foreach ( $res as $row ) { + $text = Revision::getRevisionText( $row ); + if ( $text === false ) { + $this->critical( "Error: cannot load revision text for old_id={$row->old_id}" ); + continue; + } + + if ( !$trx->addItem( $text, $row->old_id ) ) { + $this->debug( "[orphan]: committing blob with " . $trx->getSize() . " rows" ); + $trx->commit(); + $trx = new CgzCopyTransaction( $this, $this->orphanBlobClass ); + $lbFactory->waitForReplication(); + } + } + $this->debug( "[orphan]: committing blob with " . $trx->getSize() . " rows" ); + $trx->commit(); + } +} + +/** + * Class to represent a recompression operation for a single CGZ blob + */ +class CgzCopyTransaction { + /** @var RecompressTracked */ + public $parent; + public $blobClass; + /** @var ConcatenatedGzipHistoryBlob */ + public $cgz; + public $referrers; + + /** + * Create a transaction from a RecompressTracked object + * @param RecompressTracked $parent + * @param string $blobClass + */ + function __construct( $parent, $blobClass ) { + $this->blobClass = $blobClass; + $this->cgz = false; + $this->texts = []; + $this->parent = $parent; + } + + /** + * Add text. + * Returns false if it's ready to commit. + * @param string $text + * @param int $textId + * @return bool + */ + function addItem( $text, $textId ) { + if ( !$this->cgz ) { + $class = $this->blobClass; + $this->cgz = new $class; + } + $hash = $this->cgz->addItem( $text ); + $this->referrers[$textId] = $hash; + $this->texts[$textId] = $text; + + return $this->cgz->isHappy(); + } + + function getSize() { + return count( $this->texts ); + } + + /** + * Recompress text after some aberrant modification + */ + function recompress() { + $class = $this->blobClass; + $this->cgz = new $class; + $this->referrers = []; + foreach ( $this->texts as $textId => $text ) { + $hash = $this->cgz->addItem( $text ); + $this->referrers[$textId] = $hash; + } + } + + /** + * Commit the blob. + * Does nothing if no text items have been added. + * May skip the move if --copy-only is set. + */ + function commit() { + $originalCount = count( $this->texts ); + if ( !$originalCount ) { + return; + } + + /* Check to see if the target text_ids have been moved already. + * + * We originally read from the replica DB, so this can happen when a single + * text_id is shared between multiple pages. It's rare, but possible + * if a delete/move/undelete cycle splits up a null edit. + * + * We do a locking read to prevent closer-run race conditions. + */ + $dbw = wfGetDB( DB_MASTER ); + $dbw->begin( __METHOD__ ); + $res = $dbw->select( 'blob_tracking', + [ 'bt_text_id', 'bt_moved' ], + [ 'bt_text_id' => array_keys( $this->referrers ) ], + __METHOD__, [ 'FOR UPDATE' ] ); + $dirty = false; + foreach ( $res as $row ) { + if ( $row->bt_moved ) { + # This row has already been moved, remove it + $this->parent->debug( "TRX: conflict detected in old_id={$row->bt_text_id}" ); + unset( $this->texts[$row->bt_text_id] ); + $dirty = true; + } + } + + // Recompress the blob if necessary + if ( $dirty ) { + if ( !count( $this->texts ) ) { + // All have been moved already + if ( $originalCount > 1 ) { + // This is suspcious, make noise + $this->parent->critical( + "Warning: concurrent operation detected, are there two conflicting " . + "processes running, doing the same job?" ); + } + + return; + } + $this->recompress(); + } + + // Insert the data into the destination cluster + $targetCluster = $this->parent->getTargetCluster(); + $store = $this->parent->store; + $targetDB = $store->getMaster( $targetCluster ); + $targetDB->clearFlag( DBO_TRX ); // we manage the transactions + $targetDB->begin( __METHOD__ ); + $baseUrl = $this->parent->store->store( $targetCluster, serialize( $this->cgz ) ); + + // Write the new URLs to the blob_tracking table + foreach ( $this->referrers as $textId => $hash ) { + $url = $baseUrl . '/' . $hash; + $dbw->update( 'blob_tracking', + [ 'bt_new_url' => $url ], + [ + 'bt_text_id' => $textId, + 'bt_moved' => 0, # Check for concurrent conflicting update + ], + __METHOD__ + ); + } + + $targetDB->commit( __METHOD__ ); + // Critical section here: interruption at this point causes blob duplication + // Reversing the order of the commits would cause data loss instead + $dbw->commit( __METHOD__ ); + + // Write the new URLs to the text table and set the moved flag + if ( !$this->parent->copyOnly ) { + foreach ( $this->referrers as $textId => $hash ) { + $url = $baseUrl . '/' . $hash; + $this->parent->moveTextRow( $textId, $url ); + } + } + } +} diff --git a/www/wiki/maintenance/storage/resolveStubs.php b/www/wiki/maintenance/storage/resolveStubs.php new file mode 100644 index 00000000..8ca8bb29 --- /dev/null +++ b/www/wiki/maintenance/storage/resolveStubs.php @@ -0,0 +1,119 @@ +<?php +/** + * Convert history stubs that point to an external row to direct external + * pointers. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance ExternalStorage + */ + +if ( !defined( 'MEDIAWIKI' ) ) { + $optionsWithArgs = [ 'm' ]; + + require_once __DIR__ . '/../commandLine.inc'; + + resolveStubs(); +} + +/** + * Convert history stubs that point to an external row to direct + * external pointers + */ +function resolveStubs() { + $fname = 'resolveStubs'; + + $dbr = wfGetDB( DB_REPLICA ); + $maxID = $dbr->selectField( 'text', 'MAX(old_id)', false, $fname ); + $blockSize = 10000; + $numBlocks = intval( $maxID / $blockSize ) + 1; + + for ( $b = 0; $b < $numBlocks; $b++ ) { + wfWaitForSlaves(); + + printf( "%5.2f%%\n", $b / $numBlocks * 100 ); + $start = intval( $maxID / $numBlocks ) * $b + 1; + $end = intval( $maxID / $numBlocks ) * ( $b + 1 ); + + $res = $dbr->select( 'text', [ 'old_id', 'old_text', 'old_flags' ], + "old_id>=$start AND old_id<=$end " . + "AND old_flags LIKE '%object%' AND old_flags NOT LIKE '%external%' " . + 'AND LOWER(CONVERT(LEFT(old_text,22) USING latin1)) = \'o:15:"historyblobstub"\'', + $fname ); + foreach ( $res as $row ) { + resolveStub( $row->old_id, $row->old_text, $row->old_flags ); + } + } + print "100%\n"; +} + +/** + * Resolve a history stub + * @param int $id + * @param string $stubText + * @param string $flags + */ +function resolveStub( $id, $stubText, $flags ) { + $fname = 'resolveStub'; + + $stub = unserialize( $stubText ); + $flags = explode( ',', $flags ); + + $dbr = wfGetDB( DB_REPLICA ); + $dbw = wfGetDB( DB_MASTER ); + + if ( strtolower( get_class( $stub ) ) !== 'historyblobstub' ) { + print "Error found object of class " . get_class( $stub ) . ", expecting historyblobstub\n"; + + return; + } + + # Get the (maybe) external row + $externalRow = $dbr->selectRow( + 'text', + [ 'old_text' ], + [ + 'old_id' => $stub->mOldId, + 'old_flags' . $dbr->buildLike( $dbr->anyString(), 'external', $dbr->anyString() ) + ], + $fname + ); + + if ( !$externalRow ) { + # Object wasn't external + return; + } + + # Preserve the legacy encoding flag, but switch from object to external + if ( in_array( 'utf-8', $flags ) ) { + $newFlags = 'external,utf-8'; + } else { + $newFlags = 'external'; + } + + # Update the row + # print "oldid=$id\n"; + $dbw->update( 'text', + [ /* SET */ + 'old_flags' => $newFlags, + 'old_text' => $externalRow->old_text . '/' . $stub->mHash + ], + [ /* WHERE */ + 'old_id' => $id + ], $fname + ); +} diff --git a/www/wiki/maintenance/storage/storageTypeStats.php b/www/wiki/maintenance/storage/storageTypeStats.php new file mode 100644 index 00000000..c23f5086 --- /dev/null +++ b/www/wiki/maintenance/storage/storageTypeStats.php @@ -0,0 +1,115 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance ExternalStorage + */ + +require_once __DIR__ . '/../Maintenance.php'; + +class StorageTypeStats extends Maintenance { + function execute() { + $dbr = $this->getDB( DB_REPLICA ); + + $endId = $dbr->selectField( 'text', 'MAX(old_id)', false, __METHOD__ ); + if ( !$endId ) { + echo "No text rows!\n"; + exit( 1 ); + } + + $binSize = intval( pow( 10, floor( log10( $endId ) ) - 3 ) ); + if ( $binSize < 100 ) { + $binSize = 100; + } + echo "Using bin size of $binSize\n"; + + $stats = []; + + $classSql = <<<SQL + IF(old_flags LIKE '%external%', + IF(old_text REGEXP '^DB://[[:alnum:]]+/[0-9]+/[0-9a-f]{32}$', + 'CGZ pointer', + IF(old_text REGEXP '^DB://[[:alnum:]]+/[0-9]+/[0-9]{1,6}$', + 'DHB pointer', + IF(old_text REGEXP '^DB://[[:alnum:]]+/[0-9]+$', + 'simple pointer', + 'UNKNOWN pointer' + ) + ) + ), + IF(old_flags LIKE '%object%', + TRIM('"' FROM SUBSTRING_INDEX(SUBSTRING_INDEX(old_text, ':', 3), ':', -1)), + '[none]' + ) + ) +SQL; + + for ( $rangeStart = 0; $rangeStart < $endId; $rangeStart += $binSize ) { + if ( $rangeStart / $binSize % 10 == 0 ) { + echo "$rangeStart\r"; + } + $res = $dbr->select( + 'text', + [ + 'old_flags', + "$classSql AS class", + 'COUNT(*) as count', + ], + [ + 'old_id >= ' . intval( $rangeStart ), + 'old_id < ' . intval( $rangeStart + $binSize ) + ], + __METHOD__, + [ 'GROUP BY' => 'old_flags, class' ] + ); + + foreach ( $res as $row ) { + $flags = $row->old_flags; + if ( $flags === '' ) { + $flags = '[none]'; + } + $class = $row->class; + $count = $row->count; + if ( !isset( $stats[$flags][$class] ) ) { + $stats[$flags][$class] = [ + 'count' => 0, + 'first' => $rangeStart, + 'last' => 0 + ]; + } + $entry =& $stats[$flags][$class]; + $entry['count'] += $count; + $entry['last'] = max( $entry['last'], $rangeStart + $binSize ); + unset( $entry ); + } + } + echo "\n\n"; + + $format = "%-29s %-39s %-19s %-29s\n"; + printf( $format, "Flags", "Class", "Count", "old_id range" ); + echo str_repeat( '-', 120 ) . "\n"; + foreach ( $stats as $flags => $flagStats ) { + foreach ( $flagStats as $class => $entry ) { + printf( $format, $flags, $class, $entry['count'], + sprintf( "%-13d - %-13d", $entry['first'], $entry['last'] ) ); + } + } + } +} + +$maintClass = 'StorageTypeStats'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/storage/testCompression.php b/www/wiki/maintenance/storage/testCompression.php new file mode 100644 index 00000000..90d8d031 --- /dev/null +++ b/www/wiki/maintenance/storage/testCompression.php @@ -0,0 +1,102 @@ +<?php +/** + * Test revision text compression and decompression. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance ExternalStorage + */ + +$optionsWithArgs = [ 'start', 'limit', 'type' ]; +require __DIR__ . '/../commandLine.inc'; + +if ( !isset( $args[0] ) ) { + echo "Usage: php testCompression.php [--type=<type>] [--start=<start-date>] " . + "[--limit=<num-revs>] <page-title>\n"; + exit( 1 ); +} + +$lang = Language::factory( 'en' ); +$title = Title::newFromText( $args[0] ); +if ( isset( $options['start'] ) ) { + $start = wfTimestamp( TS_MW, strtotime( $options['start'] ) ); + echo "Starting from " . $lang->timeanddate( $start ) . "\n"; +} else { + $start = '19700101000000'; +} +if ( isset( $options['limit'] ) ) { + $limit = $options['limit']; + $untilHappy = false; +} else { + $limit = 1000; + $untilHappy = true; +} +$type = isset( $options['type'] ) ? $options['type'] : 'ConcatenatedGzipHistoryBlob'; + +$dbr = $this->getDB( DB_REPLICA ); +$res = $dbr->select( + [ 'page', 'revision', 'text' ], + '*', + [ + 'page_namespace' => $title->getNamespace(), + 'page_title' => $title->getDBkey(), + 'page_id=rev_page', + 'rev_timestamp > ' . $dbr->addQuotes( $dbr->timestamp( $start ) ), + 'rev_text_id=old_id' + ], __FILE__, [ 'LIMIT' => $limit ] +); + +$blob = new $type; +$hashes = []; +$keys = []; +$uncompressedSize = 0; +$t = -microtime( true ); +foreach ( $res as $row ) { + $revision = new Revision( $row ); + $text = $revision->getSerializedData(); + $uncompressedSize += strlen( $text ); + $hashes[$row->rev_id] = md5( $text ); + $keys[$row->rev_id] = $blob->addItem( $text ); + if ( $untilHappy && !$blob->isHappy() ) { + break; + } +} + +$serialized = serialize( $blob ); +$t += microtime( true ); +# print_r( $blob->mDiffMap ); + +printf( "%s\nCompression ratio for %d revisions: %5.2f, %s -> %d\n", + $type, + count( $hashes ), + $uncompressedSize / strlen( $serialized ), + $lang->formatSize( $uncompressedSize ), + strlen( $serialized ) +); +printf( "Compression time: %5.2f ms\n", $t * 1000 ); + +$t = -microtime( true ); +$blob = unserialize( $serialized ); +foreach ( $keys as $id => $key ) { + $text = $blob->getItem( $key ); + if ( md5( $text ) != $hashes[$id] ) { + echo "Content hash mismatch for rev_id $id\n"; + # var_dump( $text ); + } +} +$t += microtime( true ); +printf( "Decompression time: %5.2f ms\n", $t * 1000 ); diff --git a/www/wiki/maintenance/storage/trackBlobs.php b/www/wiki/maintenance/storage/trackBlobs.php new file mode 100644 index 00000000..b4514ecb --- /dev/null +++ b/www/wiki/maintenance/storage/trackBlobs.php @@ -0,0 +1,400 @@ +<?php +/** + * Adds blobs from a given external storage cluster to the blob_tracking table. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @see wfWaitForSlaves() + */ + +use Wikimedia\Rdbms\DBConnectionError; + +require __DIR__ . '/../commandLine.inc'; + +if ( count( $args ) < 1 ) { + echo "Usage: php trackBlobs.php <cluster> [... <cluster>]\n"; + echo "Adds blobs from a given ES cluster to the blob_tracking table\n"; + echo "Automatically deletes the tracking table and starts from the start again when restarted.\n"; + + exit( 1 ); +} +$tracker = new TrackBlobs( $args ); +$tracker->run(); +echo "All done.\n"; + +class TrackBlobs { + public $clusters, $textClause; + public $doBlobOrphans; + public $trackedBlobs = []; + + public $batchSize = 1000; + public $reportingInterval = 10; + + function __construct( $clusters ) { + $this->clusters = $clusters; + if ( extension_loaded( 'gmp' ) ) { + $this->doBlobOrphans = true; + foreach ( $clusters as $cluster ) { + $this->trackedBlobs[$cluster] = gmp_init( 0 ); + } + } else { + echo "Warning: the gmp extension is needed to find orphan blobs\n"; + } + } + + function run() { + $this->checkIntegrity(); + $this->initTrackingTable(); + $this->trackRevisions(); + $this->trackOrphanText(); + if ( $this->doBlobOrphans ) { + $this->findOrphanBlobs(); + } + } + + function checkIntegrity() { + echo "Doing integrity check...\n"; + $dbr = wfGetDB( DB_REPLICA ); + + // Scan for HistoryBlobStub objects in the text table (T22757) + + $exists = $dbr->selectField( 'text', 1, + 'old_flags LIKE \'%object%\' AND old_flags NOT LIKE \'%external%\' ' . + 'AND LOWER(CONVERT(LEFT(old_text,22) USING latin1)) = \'o:15:"historyblobstub"\'', + __METHOD__ + ); + + if ( $exists ) { + echo "Integrity check failed: found HistoryBlobStub objects in your text table.\n" . + "This script could destroy these objects if it continued. Run resolveStubs.php\n" . + "to fix this.\n"; + exit( 1 ); + } + + // Scan the archive table for HistoryBlobStub objects or external flags (T24624) + $flags = $dbr->selectField( 'archive', 'ar_flags', + 'ar_flags LIKE \'%external%\' OR (' . + 'ar_flags LIKE \'%object%\' ' . + 'AND LOWER(CONVERT(LEFT(ar_text,22) USING latin1)) = \'o:15:"historyblobstub"\' )', + __METHOD__ + ); + + if ( strpos( $flags, 'external' ) !== false ) { + echo "Integrity check failed: found external storage pointers in your archive table.\n" . + "Run normaliseArchiveTable.php to fix this.\n"; + exit( 1 ); + } elseif ( $flags ) { + echo "Integrity check failed: found HistoryBlobStub objects in your archive table.\n" . + "These objects are probably already broken, continuing would make them\n" . + "unrecoverable. Run \"normaliseArchiveTable.php --fix-cgz-bug\" to fix this.\n"; + exit( 1 ); + } + + echo "Integrity check OK\n"; + } + + function initTrackingTable() { + $dbw = wfGetDB( DB_MASTER ); + if ( $dbw->tableExists( 'blob_tracking' ) ) { + $dbw->query( 'DROP TABLE ' . $dbw->tableName( 'blob_tracking' ) ); + $dbw->query( 'DROP TABLE ' . $dbw->tableName( 'blob_orphans' ) ); + } + $dbw->sourceFile( __DIR__ . '/blob_tracking.sql' ); + } + + function getTextClause() { + if ( !$this->textClause ) { + $dbr = wfGetDB( DB_REPLICA ); + $this->textClause = ''; + foreach ( $this->clusters as $cluster ) { + if ( $this->textClause != '' ) { + $this->textClause .= ' OR '; + } + $this->textClause .= 'old_text' . $dbr->buildLike( "DB://$cluster/", $dbr->anyString() ); + } + } + + return $this->textClause; + } + + function interpretPointer( $text ) { + if ( !preg_match( '!^DB://(\w+)/(\d+)(?:/([0-9a-fA-F]+)|)$!', $text, $m ) ) { + return false; + } + + return [ + 'cluster' => $m[1], + 'id' => intval( $m[2] ), + 'hash' => isset( $m[3] ) ? $m[3] : null + ]; + } + + /** + * Scan the revision table for rows stored in the specified clusters + */ + function trackRevisions() { + $dbw = wfGetDB( DB_MASTER ); + $dbr = wfGetDB( DB_REPLICA ); + + $textClause = $this->getTextClause(); + $startId = 0; + $endId = $dbr->selectField( 'revision', 'MAX(rev_id)', false, __METHOD__ ); + $batchesDone = 0; + $rowsInserted = 0; + + echo "Finding revisions...\n"; + + while ( true ) { + $res = $dbr->select( [ 'revision', 'text' ], + [ 'rev_id', 'rev_page', 'old_id', 'old_flags', 'old_text' ], + [ + 'rev_id > ' . $dbr->addQuotes( $startId ), + 'rev_text_id=old_id', + $textClause, + 'old_flags ' . $dbr->buildLike( $dbr->anyString(), 'external', $dbr->anyString() ), + ], + __METHOD__, + [ + 'ORDER BY' => 'rev_id', + 'LIMIT' => $this->batchSize + ] + ); + if ( !$res->numRows() ) { + break; + } + + $insertBatch = []; + foreach ( $res as $row ) { + $startId = $row->rev_id; + $info = $this->interpretPointer( $row->old_text ); + if ( !$info ) { + echo "Invalid DB:// URL in rev_id {$row->rev_id}\n"; + continue; + } + if ( !in_array( $info['cluster'], $this->clusters ) ) { + echo "Invalid cluster returned in SQL query: {$info['cluster']}\n"; + continue; + } + $insertBatch[] = [ + 'bt_page' => $row->rev_page, + 'bt_rev_id' => $row->rev_id, + 'bt_text_id' => $row->old_id, + 'bt_cluster' => $info['cluster'], + 'bt_blob_id' => $info['id'], + 'bt_cgz_hash' => $info['hash'] + ]; + if ( $this->doBlobOrphans ) { + gmp_setbit( $this->trackedBlobs[$info['cluster']], $info['id'] ); + } + } + $dbw->insert( 'blob_tracking', $insertBatch, __METHOD__ ); + $rowsInserted += count( $insertBatch ); + + ++$batchesDone; + if ( $batchesDone >= $this->reportingInterval ) { + $batchesDone = 0; + echo "$startId / $endId\n"; + wfWaitForSlaves(); + } + } + echo "Found $rowsInserted revisions\n"; + } + + /** + * Scan the text table for orphan text + * Orphan text here does not imply DB corruption -- deleted text tracked by the + * archive table counts as orphan for our purposes. + */ + function trackOrphanText() { + # Wait until the blob_tracking table is available in the replica DB + $dbw = wfGetDB( DB_MASTER ); + $dbr = wfGetDB( DB_REPLICA ); + $pos = $dbw->getMasterPos(); + $dbr->masterPosWait( $pos, 100000 ); + + $textClause = $this->getTextClause( $this->clusters ); + $startId = 0; + $endId = $dbr->selectField( 'text', 'MAX(old_id)', false, __METHOD__ ); + $rowsInserted = 0; + $batchesDone = 0; + + echo "Finding orphan text...\n"; + + # Scan the text table for orphan text + while ( true ) { + $res = $dbr->select( [ 'text', 'blob_tracking' ], + [ 'old_id', 'old_flags', 'old_text' ], + [ + 'old_id>' . $dbr->addQuotes( $startId ), + $textClause, + 'old_flags ' . $dbr->buildLike( $dbr->anyString(), 'external', $dbr->anyString() ), + 'bt_text_id IS NULL' + ], + __METHOD__, + [ + 'ORDER BY' => 'old_id', + 'LIMIT' => $this->batchSize + ], + [ 'blob_tracking' => [ 'LEFT JOIN', 'bt_text_id=old_id' ] ] + ); + $ids = []; + foreach ( $res as $row ) { + $ids[] = $row->old_id; + } + + if ( !$res->numRows() ) { + break; + } + + $insertBatch = []; + foreach ( $res as $row ) { + $startId = $row->old_id; + $info = $this->interpretPointer( $row->old_text ); + if ( !$info ) { + echo "Invalid DB:// URL in old_id {$row->old_id}\n"; + continue; + } + if ( !in_array( $info['cluster'], $this->clusters ) ) { + echo "Invalid cluster returned in SQL query\n"; + continue; + } + + $insertBatch[] = [ + 'bt_page' => 0, + 'bt_rev_id' => 0, + 'bt_text_id' => $row->old_id, + 'bt_cluster' => $info['cluster'], + 'bt_blob_id' => $info['id'], + 'bt_cgz_hash' => $info['hash'] + ]; + if ( $this->doBlobOrphans ) { + gmp_setbit( $this->trackedBlobs[$info['cluster']], $info['id'] ); + } + } + $dbw->insert( 'blob_tracking', $insertBatch, __METHOD__ ); + + $rowsInserted += count( $insertBatch ); + ++$batchesDone; + if ( $batchesDone >= $this->reportingInterval ) { + $batchesDone = 0; + echo "$startId / $endId\n"; + wfWaitForSlaves(); + } + } + echo "Found $rowsInserted orphan text rows\n"; + } + + /** + * Scan the blobs table for rows not registered in blob_tracking (and thus not + * registered in the text table). + * + * Orphan blobs are indicative of DB corruption. They are inaccessible and + * should probably be deleted. + */ + function findOrphanBlobs() { + if ( !extension_loaded( 'gmp' ) ) { + echo "Can't find orphan blobs, need bitfield support provided by GMP.\n"; + + return; + } + + $dbw = wfGetDB( DB_MASTER ); + + foreach ( $this->clusters as $cluster ) { + echo "Searching for orphan blobs in $cluster...\n"; + $lb = wfGetLBFactory()->getExternalLB( $cluster ); + try { + $extDB = $lb->getConnection( DB_REPLICA ); + } catch ( DBConnectionError $e ) { + if ( strpos( $e->error, 'Unknown database' ) !== false ) { + echo "No database on $cluster\n"; + } else { + echo "Error on $cluster: " . $e->getMessage() . "\n"; + } + continue; + } + $table = $extDB->getLBInfo( 'blobs table' ); + if ( is_null( $table ) ) { + $table = 'blobs'; + } + if ( !$extDB->tableExists( $table ) ) { + echo "No blobs table on cluster $cluster\n"; + continue; + } + $startId = 0; + $batchesDone = 0; + $actualBlobs = gmp_init( 0 ); + $endId = $extDB->selectField( $table, 'MAX(blob_id)', false, __METHOD__ ); + + // Build a bitmap of actual blob rows + while ( true ) { + $res = $extDB->select( $table, + [ 'blob_id' ], + [ 'blob_id > ' . $extDB->addQuotes( $startId ) ], + __METHOD__, + [ 'LIMIT' => $this->batchSize, 'ORDER BY' => 'blob_id' ] + ); + + if ( !$res->numRows() ) { + break; + } + + foreach ( $res as $row ) { + gmp_setbit( $actualBlobs, $row->blob_id ); + } + $startId = $row->blob_id; + + ++$batchesDone; + if ( $batchesDone >= $this->reportingInterval ) { + $batchesDone = 0; + echo "$startId / $endId\n"; + } + } + + // Find actual blobs that weren't tracked by the previous passes + // This is a set-theoretic difference A \ B, or in bitwise terms, A & ~B + $orphans = gmp_and( $actualBlobs, gmp_com( $this->trackedBlobs[$cluster] ) ); + + // Traverse the orphan list + $insertBatch = []; + $id = 0; + $numOrphans = 0; + while ( true ) { + $id = gmp_scan1( $orphans, $id ); + if ( $id == -1 ) { + break; + } + $insertBatch[] = [ + 'bo_cluster' => $cluster, + 'bo_blob_id' => $id + ]; + if ( count( $insertBatch ) > $this->batchSize ) { + $dbw->insert( 'blob_orphans', $insertBatch, __METHOD__ ); + $insertBatch = []; + } + + ++$id; + ++$numOrphans; + } + if ( $insertBatch ) { + $dbw->insert( 'blob_orphans', $insertBatch, __METHOD__ ); + } + echo "Found $numOrphans orphan(s) in $cluster\n"; + } + } +} diff --git a/www/wiki/maintenance/syncFileBackend.php b/www/wiki/maintenance/syncFileBackend.php new file mode 100644 index 00000000..82149a6d --- /dev/null +++ b/www/wiki/maintenance/syncFileBackend.php @@ -0,0 +1,307 @@ +<?php +/** + * Sync one file backend to another based on the journal of later. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that syncs one file backend to another based on + * the journal of later. + * + * @ingroup Maintenance + */ +class SyncFileBackend extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Sync one file backend with another using the journal' ); + $this->addOption( 'src', 'Name of backend to sync from', true, true ); + $this->addOption( 'dst', 'Name of destination backend to sync', false, true ); + $this->addOption( 'start', 'Starting journal ID', false, true ); + $this->addOption( 'end', 'Ending journal ID', false, true ); + $this->addOption( 'posdir', 'Directory to read/record journal positions', false, true ); + $this->addOption( 'posdump', 'Just dump current journal position into the position dir.' ); + $this->addOption( 'postime', 'For position dumps, get the ID at this time', false, true ); + $this->addOption( 'backoff', 'Stop at entries younger than this age (sec).', false, true ); + $this->addOption( 'verbose', 'Verbose mode', false, false, 'v' ); + $this->setBatchSize( 50 ); + } + + public function execute() { + $src = FileBackendGroup::singleton()->get( $this->getOption( 'src' ) ); + + $posDir = $this->getOption( 'posdir' ); + $posFile = $posDir ? $posDir . '/' . wfWikiID() : false; + + if ( $this->hasOption( 'posdump' ) ) { + // Just dump the current position into the specified position dir + if ( !$this->hasOption( 'posdir' ) ) { + $this->error( "Param posdir required!", 1 ); + } + if ( $this->hasOption( 'postime' ) ) { + $id = (int)$src->getJournal()->getPositionAtTime( $this->getOption( 'postime' ) ); + $this->output( "Requested journal position is $id.\n" ); + } else { + $id = (int)$src->getJournal()->getCurrentPosition(); + $this->output( "Current journal position is $id.\n" ); + } + if ( file_put_contents( $posFile, $id, LOCK_EX ) !== false ) { + $this->output( "Saved journal position file.\n" ); + } else { + $this->output( "Could not save journal position file.\n" ); + } + if ( $this->isQuiet() ) { + print $id; // give a single machine-readable number + } + + return; + } + + if ( !$this->hasOption( 'dst' ) ) { + $this->error( "Param dst required!", 1 ); + } + $dst = FileBackendGroup::singleton()->get( $this->getOption( 'dst' ) ); + + $start = $this->getOption( 'start', 0 ); + if ( !$start && $posFile && is_dir( $posDir ) ) { + $start = is_file( $posFile ) + ? (int)trim( file_get_contents( $posFile ) ) + : 0; + ++$start; // we already did this ID, start with the next one + $startFromPosFile = true; + } else { + $startFromPosFile = false; + } + + if ( $this->hasOption( 'backoff' ) ) { + $time = time() - $this->getOption( 'backoff', 0 ); + $end = (int)$src->getJournal()->getPositionAtTime( $time ); + } else { + $end = $this->getOption( 'end', INF ); + } + + $this->output( "Synchronizing backend '{$dst->getName()}' to '{$src->getName()}'...\n" ); + $this->output( "Starting journal position is $start.\n" ); + if ( is_finite( $end ) ) { + $this->output( "Ending journal position is $end.\n" ); + } + + // Periodically update the position file + $callback = function ( $pos ) use ( $startFromPosFile, $posFile, $start ) { + if ( $startFromPosFile && $pos >= $start ) { // successfully advanced + file_put_contents( $posFile, $pos, LOCK_EX ); + } + }; + + // Actually sync the dest backend with the reference backend + $lastOKPos = $this->syncBackends( $src, $dst, $start, $end, $callback ); + + // Update the sync position file + if ( $startFromPosFile && $lastOKPos >= $start ) { // successfully advanced + if ( file_put_contents( $posFile, $lastOKPos, LOCK_EX ) !== false ) { + $this->output( "Updated journal position file.\n" ); + } else { + $this->output( "Could not update journal position file.\n" ); + } + } + + if ( $lastOKPos === false ) { + if ( !$start ) { + $this->output( "No journal entries found.\n" ); + } else { + $this->output( "No new journal entries found.\n" ); + } + } else { + $this->output( "Stopped synchronization at journal position $lastOKPos.\n" ); + } + + if ( $this->isQuiet() ) { + print $lastOKPos; // give a single machine-readable number + } + } + + /** + * Sync $dst backend to $src backend based on the $src logs given after $start. + * Returns the journal entry ID this advanced to and handled (inclusive). + * + * @param FileBackend $src + * @param FileBackend $dst + * @param int $start Starting journal position + * @param int $end Starting journal position + * @param Closure $callback Callback to update any position file + * @return int|bool Journal entry ID or false if there are none + */ + protected function syncBackends( + FileBackend $src, FileBackend $dst, $start, $end, Closure $callback + ) { + $lastOKPos = 0; // failed + $first = true; // first batch + + if ( $start > $end ) { // sanity + $this->error( "Error: given starting ID greater than ending ID.", 1 ); + } + + $next = null; + do { + $limit = min( $this->mBatchSize, $end - $start + 1 ); // don't go pass ending ID + $this->output( "Doing id $start to " . ( $start + $limit - 1 ) . "...\n" ); + + $entries = $src->getJournal()->getChangeEntries( $start, $limit, $next ); + $start = $next; // start where we left off next time + if ( $first && !count( $entries ) ) { + return false; // nothing to do + } + $first = false; + + $lastPosInBatch = 0; + $pathsInBatch = []; // changed paths + foreach ( $entries as $entry ) { + if ( $entry['op'] !== 'null' ) { // null ops are just for reference + $pathsInBatch[$entry['path']] = 1; // remove duplicates + } + $lastPosInBatch = $entry['id']; + } + + $status = $this->syncFileBatch( array_keys( $pathsInBatch ), $src, $dst ); + if ( $status->isOK() ) { + $lastOKPos = max( $lastOKPos, $lastPosInBatch ); + $callback( $lastOKPos ); // update position file + } else { + $this->error( print_r( $status->getErrorsArray(), true ) ); + break; // no gaps; everything up to $lastPos must be OK + } + + if ( !$start ) { + $this->output( "End of journal entries.\n" ); + } + } while ( $start && $start <= $end ); + + return $lastOKPos; + } + + /** + * Sync particular files of backend $src to the corresponding $dst backend files + * + * @param array $paths + * @param FileBackend $src + * @param FileBackend $dst + * @return Status + */ + protected function syncFileBatch( array $paths, FileBackend $src, FileBackend $dst ) { + $status = Status::newGood(); + if ( !count( $paths ) ) { + return $status; // nothing to do + } + + // Source: convert internal backend names (FileBackendMultiWrite) to the public one + $sPaths = $this->replaceNamePaths( $paths, $src ); + // Destination: get corresponding path name + $dPaths = $this->replaceNamePaths( $paths, $dst ); + + // Lock the live backend paths from modification + $sLock = $src->getScopedFileLocks( $sPaths, LockManager::LOCK_UW, $status ); + $eLock = $dst->getScopedFileLocks( $dPaths, LockManager::LOCK_EX, $status ); + if ( !$status->isOK() ) { + return $status; + } + + $src->preloadFileStat( [ 'srcs' => $sPaths, 'latest' => 1 ] ); + $dst->preloadFileStat( [ 'srcs' => $dPaths, 'latest' => 1 ] ); + + $ops = []; + $fsFiles = []; + foreach ( $sPaths as $i => $sPath ) { + $dPath = $dPaths[$i]; // destination + $sExists = $src->fileExists( [ 'src' => $sPath, 'latest' => 1 ] ); + if ( $sExists === true ) { // exists in source + if ( $this->filesAreSame( $src, $dst, $sPath, $dPath ) ) { + continue; // avoid local copies for non-FS backends + } + // Note: getLocalReference() is fast for FS backends + $fsFile = $src->getLocalReference( [ 'src' => $sPath, 'latest' => 1 ] ); + if ( !$fsFile ) { + $this->error( "Unable to sync '$dPath': could not get local copy." ); + $status->fatal( 'backend-fail-internal', $src->getName() ); + + return $status; + } + $fsFiles[] = $fsFile; // keep TempFSFile objects alive as needed + // Note: prepare() is usually fast for key/value backends + $status->merge( $dst->prepare( [ + 'dir' => dirname( $dPath ), 'bypassReadOnly' => 1 ] ) ); + if ( !$status->isOK() ) { + return $status; + } + $ops[] = [ 'op' => 'store', + 'src' => $fsFile->getPath(), 'dst' => $dPath, 'overwrite' => 1 ]; + } elseif ( $sExists === false ) { // does not exist in source + $ops[] = [ 'op' => 'delete', 'src' => $dPath, 'ignoreMissingSource' => 1 ]; + } else { // error + $this->error( "Unable to sync '$dPath': could not stat file." ); + $status->fatal( 'backend-fail-internal', $src->getName() ); + + return $status; + } + } + + $t_start = microtime( true ); + $status = $dst->doQuickOperations( $ops, [ 'bypassReadOnly' => 1 ] ); + if ( !$status->isOK() ) { + sleep( 10 ); // wait and retry copy again + $status = $dst->doQuickOperations( $ops, [ 'bypassReadOnly' => 1 ] ); + } + $elapsed_ms = floor( ( microtime( true ) - $t_start ) * 1000 ); + if ( $status->isOK() && $this->getOption( 'verbose' ) ) { + $this->output( "Synchronized these file(s) [{$elapsed_ms}ms]:\n" . + implode( "\n", $dPaths ) . "\n" ); + } + + return $status; + } + + /** + * Substitute the backend name of storage paths with that of a given one + * + * @param array|string $paths List of paths or single string path + * @param FileBackend $backend + * @return array|string + */ + protected function replaceNamePaths( $paths, FileBackend $backend ) { + return preg_replace( + '!^mwstore://([^/]+)!', + StringUtils::escapeRegexReplacement( "mwstore://" . $backend->getName() ), + $paths // string or array + ); + } + + protected function filesAreSame( FileBackend $src, FileBackend $dst, $sPath, $dPath ) { + return ( + ( $src->getFileSize( [ 'src' => $sPath ] ) + === $dst->getFileSize( [ 'src' => $dPath ] ) // short-circuit + ) && ( $src->getFileSha1Base36( [ 'src' => $sPath ] ) + === $dst->getFileSha1Base36( [ 'src' => $dPath ] ) + ) + ); + } +} + +$maintClass = "SyncFileBackend"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/tables.sql b/www/wiki/maintenance/tables.sql new file mode 100644 index 00000000..1813f6cd --- /dev/null +++ b/www/wiki/maintenance/tables.sql @@ -0,0 +1,1823 @@ +-- SQL to create the initial tables for the MediaWiki database. +-- This is read and executed by the install script; you should +-- not have to run it by itself unless doing a manual install. + +-- This is a shared schema file used for both MySQL and SQLite installs. +-- +-- For more documentation on the database schema, see +-- https://www.mediawiki.org/wiki/Manual:Database_layout +-- +-- General notes: +-- +-- If possible, create tables as InnoDB to benefit from the +-- superior resiliency against crashes and ability to read +-- during writes (and write during reads!) +-- +-- Only the 'searchindex' table requires MyISAM due to the +-- requirement for fulltext index support, which is missing +-- from InnoDB. +-- +-- +-- The MySQL table backend for MediaWiki currently uses +-- 14-character BINARY or VARBINARY fields to store timestamps. +-- The format is YYYYMMDDHHMMSS, which is derived from the +-- text format of MySQL's TIMESTAMP fields. +-- +-- Historically TIMESTAMP fields were used, but abandoned +-- in early 2002 after a lot of trouble with the fields +-- auto-updating. +-- +-- The Postgres backend uses TIMESTAMPTZ fields for timestamps, +-- and we will migrate the MySQL definitions at some point as +-- well. +-- +-- +-- The /*_*/ comments in this and other files are +-- replaced with the defined table prefix by the installer +-- and updater scripts. If you are installing or running +-- updates manually, you will need to manually insert the +-- table prefix if any when running these scripts. +-- + + +-- +-- The user table contains basic account information, +-- authentication keys, etc. +-- +-- Some multi-wiki sites may share a single central user table +-- between separate wikis using the $wgSharedDB setting. +-- +-- Note that when a external authentication plugin is used, +-- user table entries still need to be created to store +-- preferences and to key tracking information in the other +-- tables. +-- +CREATE TABLE /*_*/user ( + user_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + + -- Usernames must be unique, must not be in the form of + -- an IP address. _Shouldn't_ allow slashes or case + -- conflicts. Spaces are allowed, and are _not_ converted + -- to underscores like titles. See the User::newFromName() for + -- the specific tests that usernames have to pass. + user_name varchar(255) binary NOT NULL default '', + + -- Optional 'real name' to be displayed in credit listings + user_real_name varchar(255) binary NOT NULL default '', + + -- Password hashes, see User::crypt() and User::comparePasswords() + -- in User.php for the algorithm + user_password tinyblob NOT NULL, + + -- When using 'mail me a new password', a random + -- password is generated and the hash stored here. + -- The previous password is left in place until + -- someone actually logs in with the new password, + -- at which point the hash is moved to user_password + -- and the old password is invalidated. + user_newpassword tinyblob NOT NULL, + + -- Timestamp of the last time when a new password was + -- sent, for throttling and expiring purposes + -- Emailed passwords will expire $wgNewPasswordExpiry + -- (a week) after being set. If user_newpass_time is NULL + -- (eg. created by mail) it doesn't expire. + user_newpass_time binary(14), + + -- Note: email should be restricted, not public info. + -- Same with passwords. + user_email tinytext NOT NULL, + + -- If the browser sends an If-Modified-Since header, a 304 response is + -- suppressed if the value in this field for the current user is later than + -- the value in the IMS header. That is, this field is an invalidation timestamp + -- for the browser cache of logged-in users. Among other things, it is used + -- to prevent pages generated for a previously logged in user from being + -- displayed after a session expiry followed by a fresh login. + user_touched binary(14) NOT NULL default '', + + -- A pseudorandomly generated value that is stored in + -- a cookie when the "remember password" feature is + -- used (previously, a hash of the password was used, but + -- this was vulnerable to cookie-stealing attacks) + user_token binary(32) NOT NULL default '', + + -- Initially NULL; when a user's e-mail address has been + -- validated by returning with a mailed token, this is + -- set to the current timestamp. + user_email_authenticated binary(14), + + -- Randomly generated token created when the e-mail address + -- is set and a confirmation test mail sent. + user_email_token binary(32), + + -- Expiration date for the user_email_token + user_email_token_expires binary(14), + + -- Timestamp of account registration. + -- Accounts predating this schema addition may contain NULL. + user_registration binary(14), + + -- Count of edits and edit-like actions. + -- + -- *NOT* intended to be an accurate copy of COUNT(*) WHERE rev_user=user_id + -- May contain NULL for old accounts if batch-update scripts haven't been + -- run, as well as listing deleted edits and other myriad ways it could be + -- out of sync. + -- + -- Meant primarily for heuristic checks to give an impression of whether + -- the account has been used much. + -- + user_editcount int, + + -- Expiration date for user password. + user_password_expires varbinary(14) DEFAULT NULL + +) /*$wgDBTableOptions*/; + +CREATE UNIQUE INDEX /*i*/user_name ON /*_*/user (user_name); +CREATE INDEX /*i*/user_email_token ON /*_*/user (user_email_token); +CREATE INDEX /*i*/user_email ON /*_*/user (user_email(50)); + + +-- +-- User permissions have been broken out to a separate table; +-- this allows sites with a shared user table to have different +-- permissions assigned to a user in each project. +-- +-- This table replaces the old user_rights field which used a +-- comma-separated blob. +-- +CREATE TABLE /*_*/user_groups ( + -- Key to user_id + ug_user int unsigned NOT NULL default 0, + + -- Group names are short symbolic string keys. + -- The set of group names is open-ended, though in practice + -- only some predefined ones are likely to be used. + -- + -- At runtime $wgGroupPermissions will associate group keys + -- with particular permissions. A user will have the combined + -- permissions of any group they're explicitly in, plus + -- the implicit '*' and 'user' groups. + ug_group varbinary(255) NOT NULL default '', + + -- Time at which the user group membership will expire. Set to + -- NULL for a non-expiring (infinite) membership. + ug_expiry varbinary(14) NULL default NULL, + + PRIMARY KEY (ug_user, ug_group) +) /*$wgDBTableOptions*/; + +CREATE INDEX /*i*/ug_group ON /*_*/user_groups (ug_group); +CREATE INDEX /*i*/ug_expiry ON /*_*/user_groups (ug_expiry); + +-- Stores the groups the user has once belonged to. +-- The user may still belong to these groups (check user_groups). +-- Users are not autopromoted to groups from which they were removed. +CREATE TABLE /*_*/user_former_groups ( + -- Key to user_id + ufg_user int unsigned NOT NULL default 0, + ufg_group varbinary(255) NOT NULL default '', + PRIMARY KEY (ufg_user,ufg_group) +) /*$wgDBTableOptions*/; + +-- +-- Stores notifications of user talk page changes, for the display +-- of the "you have new messages" box +-- +CREATE TABLE /*_*/user_newtalk ( + -- Key to user.user_id + user_id int unsigned NOT NULL default 0, + -- If the user is an anonymous user their IP address is stored here + -- since the user_id of 0 is ambiguous + user_ip varbinary(40) NOT NULL default '', + -- The highest timestamp of revisions of the talk page viewed + -- by this user + user_last_timestamp varbinary(14) NULL default NULL +) /*$wgDBTableOptions*/; + +-- Indexes renamed for SQLite in 1.14 +CREATE INDEX /*i*/un_user_id ON /*_*/user_newtalk (user_id); +CREATE INDEX /*i*/un_user_ip ON /*_*/user_newtalk (user_ip); + + +-- +-- User preferences and perhaps other fun stuff. :) +-- Replaces the old user.user_options blob, with a couple nice properties: +-- +-- 1) We only store non-default settings, so changes to the defauls +-- are now reflected for everybody, not just new accounts. +-- 2) We can more easily do bulk lookups, statistics, or modifications of +-- saved options since it's a sane table structure. +-- +CREATE TABLE /*_*/user_properties ( + -- Foreign key to user.user_id + up_user int unsigned NOT NULL, + + -- Name of the option being saved. This is indexed for bulk lookup. + up_property varbinary(255) NOT NULL, + + -- Property value as a string. + up_value blob, + PRIMARY KEY (up_user,up_property) +) /*$wgDBTableOptions*/; + +CREATE INDEX /*i*/user_properties_property ON /*_*/user_properties (up_property); + +-- +-- This table contains a user's bot passwords: passwords that allow access to +-- the account via the API with limited rights. +-- +CREATE TABLE /*_*/bot_passwords ( + -- User ID obtained from CentralIdLookup. + bp_user int unsigned NOT NULL, + + -- Application identifier + bp_app_id varbinary(32) NOT NULL, + + -- Password hashes, like user.user_password + bp_password tinyblob NOT NULL, + + -- Like user.user_token + bp_token binary(32) NOT NULL default '', + + -- JSON blob for MWRestrictions + bp_restrictions blob NOT NULL, + + -- Grants allowed to the account when authenticated with this bot-password + bp_grants blob NOT NULL, + + PRIMARY KEY ( bp_user, bp_app_id ) +) /*$wgDBTableOptions*/; + +-- +-- Core of the wiki: each page has an entry here which identifies +-- it by title and contains some essential metadata. +-- +CREATE TABLE /*_*/page ( + -- Unique identifier number. The page_id will be preserved across + -- edits and rename operations, but not deletions and recreations. + page_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + + -- A page name is broken into a namespace and a title. + -- The namespace keys are UI-language-independent constants, + -- defined in includes/Defines.php + page_namespace int NOT NULL, + + -- The rest of the title, as text. + -- Spaces are transformed into underscores in title storage. + page_title varchar(255) binary NOT NULL, + + -- Comma-separated set of permission keys indicating who + -- can move or edit the page. + page_restrictions tinyblob NOT NULL, + + -- 1 indicates the article is a redirect. + page_is_redirect tinyint unsigned NOT NULL default 0, + + -- 1 indicates this is a new entry, with only one edit. + -- Not all pages with one edit are new pages. + page_is_new tinyint unsigned NOT NULL default 0, + + -- Random value between 0 and 1, used for Special:Randompage + page_random real unsigned NOT NULL, + + -- This timestamp is updated whenever the page changes in + -- a way requiring it to be re-rendered, invalidating caches. + -- Aside from editing this includes permission changes, + -- creation or deletion of linked pages, and alteration + -- of contained templates. + page_touched binary(14) NOT NULL default '', + + -- This timestamp is updated whenever a page is re-parsed and + -- it has all the link tracking tables updated for it. This is + -- useful for de-duplicating expensive backlink update jobs. + page_links_updated varbinary(14) NULL default NULL, + + -- Handy key to revision.rev_id of the current revision. + -- This may be 0 during page creation, but that shouldn't + -- happen outside of a transaction... hopefully. + page_latest int unsigned NOT NULL, + + -- Uncompressed length in bytes of the page's current source text. + page_len int unsigned NOT NULL, + + -- content model, see CONTENT_MODEL_XXX constants + page_content_model varbinary(32) DEFAULT NULL, + + -- Page content language + page_lang varbinary(35) DEFAULT NULL +) /*$wgDBTableOptions*/; + +-- The title index. Care must be taken to always specify a namespace when +-- by title, so that the index is used. Even listing all known namespaces +-- with IN() is better than omitting page_namespace from the WHERE clause. +CREATE UNIQUE INDEX /*i*/name_title ON /*_*/page (page_namespace,page_title); + +-- The index for Special:Random +CREATE INDEX /*i*/page_random ON /*_*/page (page_random); + +-- Questionable utility, used by ProofreadPage, possibly DynamicPageList. +-- ApiQueryAllPages unconditionally filters on namespace and so hopefully does +-- not use it. +CREATE INDEX /*i*/page_len ON /*_*/page (page_len); + +-- The index for Special:Shortpages and Special:Longpages. Also SiteStats::articles() +-- in 'comma' counting mode, MessageCache::loadFromDB(). +CREATE INDEX /*i*/page_redirect_namespace_len ON /*_*/page (page_is_redirect, page_namespace, page_len); + +-- +-- Every edit of a page creates also a revision row. +-- This stores metadata about the revision, and a reference +-- to the text storage backend. +-- +CREATE TABLE /*_*/revision ( + -- Unique ID to identify each revision + rev_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + + -- Key to page_id. This should _never_ be invalid. + rev_page int unsigned NOT NULL, + + -- Key to text.old_id, where the actual bulk text is stored. + -- It's possible for multiple revisions to use the same text, + -- for instance revisions where only metadata is altered + -- or a rollback to a previous version. + rev_text_id int unsigned NOT NULL, + + -- Text comment summarizing the change. Deprecated in favor of + -- revision_comment_temp.revcomment_comment_id. + rev_comment varbinary(767) NOT NULL default '', + + -- Key to user.user_id of the user who made this edit. + -- Stores 0 for anonymous edits and for some mass imports. + rev_user int unsigned NOT NULL default 0, + + -- Text username or IP address of the editor. + rev_user_text varchar(255) binary NOT NULL default '', + + -- Timestamp of when revision was created + rev_timestamp binary(14) NOT NULL default '', + + -- Records whether the user marked the 'minor edit' checkbox. + -- Many automated edits are marked as minor. + rev_minor_edit tinyint unsigned NOT NULL default 0, + + -- Restrictions on who can access this revision + rev_deleted tinyint unsigned NOT NULL default 0, + + -- Length of this revision in bytes + rev_len int unsigned, + + -- Key to revision.rev_id + -- This field is used to add support for a tree structure (The Adjacency List Model) + rev_parent_id int unsigned default NULL, + + -- SHA-1 text content hash in base-36 + rev_sha1 varbinary(32) NOT NULL default '', + + -- content model, see CONTENT_MODEL_XXX constants + rev_content_model varbinary(32) DEFAULT NULL, + + -- content format, see CONTENT_FORMAT_XXX constants + rev_content_format varbinary(64) DEFAULT NULL + +) /*$wgDBTableOptions*/ MAX_ROWS=10000000 AVG_ROW_LENGTH=1024; +-- In case tables are created as MyISAM, use row hints for MySQL <5.0 to avoid 4GB limit + +-- The index is proposed for removal, do not use it in new code: T163532. +-- Used for ordering revisions within a page by rev_id, which is usually +-- incorrect, since rev_timestamp is normally the correct order. It can also +-- be used by dumpBackup.php, if a page and rev_id range is specified. +CREATE INDEX /*i*/rev_page_id ON /*_*/revision (rev_page, rev_id); + +-- Used by ApiQueryAllRevisions +CREATE INDEX /*i*/rev_timestamp ON /*_*/revision (rev_timestamp); + +-- History index +CREATE INDEX /*i*/page_timestamp ON /*_*/revision (rev_page,rev_timestamp); + +-- Logged-in user contributions index +CREATE INDEX /*i*/user_timestamp ON /*_*/revision (rev_user,rev_timestamp); + +-- Anonymous user countributions index +CREATE INDEX /*i*/usertext_timestamp ON /*_*/revision (rev_user_text,rev_timestamp); + +-- Credits index. This is scanned in order to compile credits lists for pages, +-- in ApiQueryContributors. Also for ApiQueryRevisions if rvuser is specified +-- and is a logged-in user. +CREATE INDEX /*i*/page_user_timestamp ON /*_*/revision (rev_page,rev_user,rev_timestamp); + +-- +-- Temporary table to avoid blocking on an alter of revision. +-- +-- On large wikis like the English Wikipedia, altering the revision table is a +-- months-long process. This table is being created to avoid such an alter, and +-- will be merged back into revision in the future. +-- +CREATE TABLE /*_*/revision_comment_temp ( + -- Key to rev_id + revcomment_rev int unsigned NOT NULL, + -- Key to comment_id + revcomment_comment_id bigint unsigned NOT NULL, + PRIMARY KEY (revcomment_rev, revcomment_comment_id) +) /*$wgDBTableOptions*/; +-- Ensure uniqueness +CREATE UNIQUE INDEX /*i*/revcomment_rev ON /*_*/revision_comment_temp (revcomment_rev); + +-- +-- Every time an edit by a logged out user is saved, +-- a row is created in ip_changes. This stores +-- the IP as a hex representation so that we can more +-- easily find edits within an IP range. +-- +CREATE TABLE /*_*/ip_changes ( + -- Foreign key to the revision table, also serves as the unique primary key + ipc_rev_id int unsigned NOT NULL PRIMARY KEY DEFAULT '0', + + -- The timestamp of the revision + ipc_rev_timestamp binary(14) NOT NULL DEFAULT '', + + -- Hex representation of the IP address, as returned by IP::toHex() + -- For IPv4 it will resemble: ABCD1234 + -- For IPv6: v6-ABCD1234000000000000000000000000 + -- BETWEEN is then used to identify revisions within a given range + ipc_hex varbinary(35) NOT NULL DEFAULT '' + +) /*$wgDBTableOptions*/; + +CREATE INDEX /*i*/ipc_rev_timestamp ON /*_*/ip_changes (ipc_rev_timestamp); +CREATE INDEX /*i*/ipc_hex_time ON /*_*/ip_changes (ipc_hex,ipc_rev_timestamp); + +-- +-- Holds text of individual page revisions. +-- +-- Field names are a holdover from the 'old' revisions table in +-- MediaWiki 1.4 and earlier: an upgrade will transform that +-- table into the 'text' table to minimize unnecessary churning +-- and downtime. If upgrading, the other fields will be left unused. +-- +CREATE TABLE /*_*/text ( + -- Unique text storage key number. + -- Note that the 'oldid' parameter used in URLs does *not* + -- refer to this number anymore, but to rev_id. + -- + -- revision.rev_text_id is a key to this column + old_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + + -- Depending on the contents of the old_flags field, the text + -- may be convenient plain text, or it may be funkily encoded. + old_text mediumblob NOT NULL, + + -- Comma-separated list of flags: + -- gzip: text is compressed with PHP's gzdeflate() function. + -- utf-8: text was stored as UTF-8. + -- If $wgLegacyEncoding option is on, rows *without* this flag + -- will be converted to UTF-8 transparently at load time. Note + -- that due to a bug in a maintenance script, this flag may + -- have been stored as 'utf8' in some cases (T18841). + -- object: text field contained a serialized PHP object. + -- The object either contains multiple versions compressed + -- together to achieve a better compression ratio, or it refers + -- to another row where the text can be found. + -- external: text was stored in an external location specified by old_text. + -- Any additional flags apply to the data stored at that URL, not + -- the URL itself. The 'object' flag is *not* set for URLs of the + -- form 'DB://cluster/id/itemid', because the external storage + -- system itself decompresses these. + old_flags tinyblob NOT NULL +) /*$wgDBTableOptions*/ MAX_ROWS=10000000 AVG_ROW_LENGTH=10240; +-- In case tables are created as MyISAM, use row hints for MySQL <5.0 to avoid 4GB limit + + +-- +-- Edits, blocks, and other actions typically have a textual comment describing +-- the action. They are stored here to reduce the size of the main tables, and +-- to allow for deduplication. +-- +-- Deduplication is currently best-effort to avoid locking on inserts that +-- would be required for strict deduplication. There MAY be multiple rows with +-- the same comment_text and comment_data. +-- +CREATE TABLE /*_*/comment ( + -- Unique ID to identify each comment + comment_id bigint unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + + -- Hash of comment_text and comment_data, for deduplication + comment_hash INT NOT NULL, + + -- Text comment summarizing the change. + -- This text is shown in the history and other changes lists, + -- rendered in a subset of wiki markup by Linker::formatComment() + -- Size limits are enforced at the application level, and should + -- take care to crop UTF-8 strings appropriately. + comment_text BLOB NOT NULL, + + -- JSON data, intended for localizing auto-generated comments. + -- This holds structured data that is intended to be used to provide + -- localized versions of automatically-generated comments. When not empty, + -- comment_text should be the generated comment localized using the wiki's + -- content language. + comment_data BLOB +) /*$wgDBTableOptions*/; +-- Index used for deduplication. +CREATE INDEX /*i*/comment_hash ON /*_*/comment (comment_hash); + + +-- +-- Holding area for deleted articles, which may be viewed +-- or restored by admins through the Special:Undelete interface. +-- The fields generally correspond to the page, revision, and text +-- fields, with several caveats. +-- +CREATE TABLE /*_*/archive ( + -- Primary key + ar_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + ar_namespace int NOT NULL default 0, + ar_title varchar(255) binary NOT NULL default '', + + -- Newly deleted pages will not store text in this table, + -- but will reference the separately existing text rows. + -- This field is retained for backwards compatibility, + -- so old archived pages will remain accessible after + -- upgrading from 1.4 to 1.5. + -- Text may be gzipped or otherwise funky. + ar_text mediumblob NOT NULL, + + -- Basic revision stuff... + ar_comment varbinary(767) NOT NULL default '', -- Deprecated in favor of ar_comment_id + ar_comment_id bigint unsigned NOT NULL DEFAULT 0, -- ("DEFAULT 0" is temporary, signaling that ar_comment should be used) + ar_user int unsigned NOT NULL default 0, + ar_user_text varchar(255) binary NOT NULL, + ar_timestamp binary(14) NOT NULL default '', + ar_minor_edit tinyint NOT NULL default 0, + + -- See ar_text note. + ar_flags tinyblob NOT NULL, + + -- When revisions are deleted, their unique rev_id is stored + -- here so it can be retained after undeletion. This is necessary + -- to retain permalinks to given revisions after accidental delete + -- cycles or messy operations like history merges. + -- + -- Old entries from 1.4 will be NULL here, and a new rev_id will + -- be created on undeletion for those revisions. + ar_rev_id int unsigned, + + -- For newly deleted revisions, this is the text.old_id key to the + -- actual stored text. To avoid breaking the block-compression scheme + -- and otherwise making storage changes harder, the actual text is + -- *not* deleted from the text table, merely hidden by removal of the + -- page and revision entries. + -- + -- Old entries deleted under 1.2-1.4 will have NULL here, and their + -- ar_text and ar_flags fields will be used to create a new text + -- row upon undeletion. + ar_text_id int unsigned, + + -- rev_deleted for archives + ar_deleted tinyint unsigned NOT NULL default 0, + + -- Length of this revision in bytes + ar_len int unsigned, + + -- Reference to page_id. Useful for sysadmin fixing of large pages + -- merged together in the archives, or for cleanly restoring a page + -- at its original ID number if possible. + -- + -- Will be NULL for pages deleted prior to 1.11. + ar_page_id int unsigned, + + -- Original previous revision + ar_parent_id int unsigned default NULL, + + -- SHA-1 text content hash in base-36 + ar_sha1 varbinary(32) NOT NULL default '', + + -- content model, see CONTENT_MODEL_XXX constants + ar_content_model varbinary(32) DEFAULT NULL, + + -- content format, see CONTENT_FORMAT_XXX constants + ar_content_format varbinary(64) DEFAULT NULL +) /*$wgDBTableOptions*/; + +-- Index for Special:Undelete to page through deleted revisions +CREATE INDEX /*i*/name_title_timestamp ON /*_*/archive (ar_namespace,ar_title,ar_timestamp); + +-- Index for Special:DeletedContributions +CREATE INDEX /*i*/ar_usertext_timestamp ON /*_*/archive (ar_user_text,ar_timestamp); + +-- Index for linking archive rows with tables that normally link with revision +-- rows, such as change_tag. +CREATE INDEX /*i*/ar_revid ON /*_*/archive (ar_rev_id); + + +-- +-- Track page-to-page hyperlinks within the wiki. +-- +CREATE TABLE /*_*/pagelinks ( + -- Key to the page_id of the page containing the link. + pl_from int unsigned NOT NULL default 0, + -- Namespace for this page + pl_from_namespace int NOT NULL default 0, + + -- Key to page_namespace/page_title of the target page. + -- The target page may or may not exist, and due to renames + -- and deletions may refer to different page records as time + -- goes by. + pl_namespace int NOT NULL default 0, + pl_title varchar(255) binary NOT NULL default '', + PRIMARY KEY (pl_from,pl_namespace,pl_title) +) /*$wgDBTableOptions*/; + +-- Reverse index, for Special:Whatlinkshere +CREATE INDEX /*i*/pl_namespace ON /*_*/pagelinks (pl_namespace,pl_title,pl_from); + +-- Index for Special:Whatlinkshere with namespace filter +CREATE INDEX /*i*/pl_backlinks_namespace ON /*_*/pagelinks (pl_from_namespace,pl_namespace,pl_title,pl_from); + + +-- +-- Track template inclusions. +-- +CREATE TABLE /*_*/templatelinks ( + -- Key to the page_id of the page containing the link. + tl_from int unsigned NOT NULL default 0, + -- Namespace for this page + tl_from_namespace int NOT NULL default 0, + + -- Key to page_namespace/page_title of the target page. + -- The target page may or may not exist, and due to renames + -- and deletions may refer to different page records as time + -- goes by. + tl_namespace int NOT NULL default 0, + tl_title varchar(255) binary NOT NULL default '', + PRIMARY KEY (tl_from,tl_namespace,tl_title) +) /*$wgDBTableOptions*/; + +-- Reverse index, for Special:Whatlinkshere +CREATE INDEX /*i*/tl_namespace ON /*_*/templatelinks (tl_namespace,tl_title,tl_from); + +-- Index for Special:Whatlinkshere with namespace filter +CREATE INDEX /*i*/tl_backlinks_namespace ON /*_*/templatelinks (tl_from_namespace,tl_namespace,tl_title,tl_from); + + +-- +-- Track links to images *used inline* +-- We don't distinguish live from broken links here, so +-- they do not need to be changed on upload/removal. +-- +CREATE TABLE /*_*/imagelinks ( + -- Key to page_id of the page containing the image / media link. + il_from int unsigned NOT NULL default 0, + -- Namespace for this page + il_from_namespace int NOT NULL default 0, + + -- Filename of target image. + -- This is also the page_title of the file's description page; + -- all such pages are in namespace 6 (NS_FILE). + il_to varchar(255) binary NOT NULL default '', + PRIMARY KEY (il_from,il_to) +) /*$wgDBTableOptions*/; + +-- Reverse index, for Special:Whatlinkshere and file description page local usage +CREATE INDEX /*i*/il_to ON /*_*/imagelinks (il_to,il_from); + +-- Index for Special:Whatlinkshere with namespace filter +CREATE INDEX /*i*/il_backlinks_namespace ON /*_*/imagelinks (il_from_namespace,il_to,il_from); + + +-- +-- Track category inclusions *used inline* +-- This tracks a single level of category membership +-- +CREATE TABLE /*_*/categorylinks ( + -- Key to page_id of the page defined as a category member. + cl_from int unsigned NOT NULL default 0, + + -- Name of the category. + -- This is also the page_title of the category's description page; + -- all such pages are in namespace 14 (NS_CATEGORY). + cl_to varchar(255) binary NOT NULL default '', + + -- A binary string obtained by applying a sortkey generation algorithm + -- (Collation::getSortKey()) to page_title, or cl_sortkey_prefix . "\n" + -- . page_title if cl_sortkey_prefix is nonempty. + cl_sortkey varbinary(230) NOT NULL default '', + + -- A prefix for the raw sortkey manually specified by the user, either via + -- [[Category:Foo|prefix]] or {{defaultsort:prefix}}. If nonempty, it's + -- concatenated with a line break followed by the page title before the sortkey + -- conversion algorithm is run. We store this so that we can update + -- collations without reparsing all pages. + -- Note: If you change the length of this field, you also need to change + -- code in LinksUpdate.php. See T27254. + cl_sortkey_prefix varchar(255) binary NOT NULL default '', + + -- This isn't really used at present. Provided for an optional + -- sorting method by approximate addition time. + cl_timestamp timestamp NOT NULL, + + -- Stores $wgCategoryCollation at the time cl_sortkey was generated. This + -- can be used to install new collation versions, tracking which rows are not + -- yet updated. '' means no collation, this is a legacy row that needs to be + -- updated by updateCollation.php. In the future, it might be possible to + -- specify different collations per category. + cl_collation varbinary(32) NOT NULL default '', + + -- Stores whether cl_from is a category, file, or other page, so we can + -- paginate the three categories separately. This never has to be updated + -- after the page is created, since none of these page types can be moved to + -- any other. + cl_type ENUM('page', 'subcat', 'file') NOT NULL default 'page', + PRIMARY KEY (cl_from,cl_to) +) /*$wgDBTableOptions*/; + + +-- We always sort within a given category, and within a given type. FIXME: +-- Formerly this index didn't cover cl_type (since that didn't exist), so old +-- callers won't be using an index: fix this? +CREATE INDEX /*i*/cl_sortkey ON /*_*/categorylinks (cl_to,cl_type,cl_sortkey,cl_from); + +-- Used by the API (and some extensions) +CREATE INDEX /*i*/cl_timestamp ON /*_*/categorylinks (cl_to,cl_timestamp); + +-- Used when updating collation (e.g. updateCollation.php) +CREATE INDEX /*i*/cl_collation_ext ON /*_*/categorylinks (cl_collation, cl_to, cl_type, cl_from); + +-- +-- Track all existing categories. Something is a category if 1) it has an entry +-- somewhere in categorylinks, or 2) it has a description page. Categories +-- might not have corresponding pages, so they need to be tracked separately. +-- +CREATE TABLE /*_*/category ( + -- Primary key + cat_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + + -- Name of the category, in the same form as page_title (with underscores). + -- If there is a category page corresponding to this category, by definition, + -- it has this name (in the Category namespace). + cat_title varchar(255) binary NOT NULL, + + -- The numbers of member pages (including categories and media), subcatego- + -- ries, and Image: namespace members, respectively. These are signed to + -- make underflow more obvious. We make the first number include the second + -- two for better sorting: subtracting for display is easy, adding for order- + -- ing is not. + cat_pages int signed NOT NULL default 0, + cat_subcats int signed NOT NULL default 0, + cat_files int signed NOT NULL default 0 +) /*$wgDBTableOptions*/; + +CREATE UNIQUE INDEX /*i*/cat_title ON /*_*/category (cat_title); + +-- For Special:Mostlinkedcategories +CREATE INDEX /*i*/cat_pages ON /*_*/category (cat_pages); + + +-- +-- Track links to external URLs +-- +CREATE TABLE /*_*/externallinks ( + -- Primary key + el_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + + -- page_id of the referring page + el_from int unsigned NOT NULL default 0, + + -- The URL + el_to blob NOT NULL, + + -- In the case of HTTP URLs, this is the URL with any username or password + -- removed, and with the labels in the hostname reversed and converted to + -- lower case. An extra dot is added to allow for matching of either + -- example.com or *.example.com in a single scan. + -- Example: + -- http://user:password@sub.example.com/page.html + -- becomes + -- http://com.example.sub./page.html + -- which allows for fast searching for all pages under example.com with the + -- clause: + -- WHERE el_index LIKE 'http://com.example.%' + el_index blob NOT NULL, + + -- This is el_index truncated to 60 bytes to allow for sortable queries that + -- aren't supported by a partial index. + -- @todo Drop the default once this is deployed everywhere and code is populating it. + el_index_60 varbinary(60) NOT NULL default '' +) /*$wgDBTableOptions*/; + +-- Forward index, for page edit, save +CREATE INDEX /*i*/el_from ON /*_*/externallinks (el_from, el_to(40)); + +-- Index for Special:LinkSearch exact search +CREATE INDEX /*i*/el_to ON /*_*/externallinks (el_to(60), el_from); + +-- For Special:LinkSearch wildcard search +CREATE INDEX /*i*/el_index ON /*_*/externallinks (el_index(60)); + +-- For Special:LinkSearch wildcard search with efficient paging by el_id +CREATE INDEX /*i*/el_index_60 ON /*_*/externallinks (el_index_60, el_id); +CREATE INDEX /*i*/el_from_index_60 ON /*_*/externallinks (el_from, el_index_60, el_id); + +-- +-- Track interlanguage links +-- +CREATE TABLE /*_*/langlinks ( + -- page_id of the referring page + ll_from int unsigned NOT NULL default 0, + + -- Language code of the target + ll_lang varbinary(20) NOT NULL default '', + + -- Title of the target, including namespace + ll_title varchar(255) binary NOT NULL default '', + PRIMARY KEY (ll_from,ll_lang) +) /*$wgDBTableOptions*/; + +-- Index for ApiQueryLangbacklinks +CREATE INDEX /*i*/ll_lang ON /*_*/langlinks (ll_lang, ll_title); + + +-- +-- Track inline interwiki links +-- +CREATE TABLE /*_*/iwlinks ( + -- page_id of the referring page + iwl_from int unsigned NOT NULL default 0, + + -- Interwiki prefix code of the target + iwl_prefix varbinary(20) NOT NULL default '', + + -- Title of the target, including namespace + iwl_title varchar(255) binary NOT NULL default '', + PRIMARY KEY (iwl_from,iwl_prefix,iwl_title) +) /*$wgDBTableOptions*/; + +-- Index for ApiQueryIWBacklinks +CREATE INDEX /*i*/iwl_prefix_title_from ON /*_*/iwlinks (iwl_prefix, iwl_title, iwl_from); + +-- Index for ApiQueryIWLinks +CREATE INDEX /*i*/iwl_prefix_from_title ON /*_*/iwlinks (iwl_prefix, iwl_from, iwl_title); + + +-- +-- Contains a single row with some aggregate info +-- on the state of the site. +-- +CREATE TABLE /*_*/site_stats ( + -- The single row should contain 1 here. + ss_row_id int unsigned NOT NULL PRIMARY KEY, + + -- Total number of edits performed. + ss_total_edits bigint unsigned default 0, + + -- An approximate count of pages matching the following criteria: + -- * in namespace 0 + -- * not a redirect + -- * contains the text '[[' + -- See Article::isCountable() in includes/Article.php + ss_good_articles bigint unsigned default 0, + + -- Total pages, theoretically equal to SELECT COUNT(*) FROM page; except faster + ss_total_pages bigint default '-1', + + -- Number of users, theoretically equal to SELECT COUNT(*) FROM user; + ss_users bigint default '-1', + + -- Number of users that still edit + ss_active_users bigint default '-1', + + -- Number of images, equivalent to SELECT COUNT(*) FROM image + ss_images int default 0 +) /*$wgDBTableOptions*/; + +-- +-- The internet is full of jerks, alas. Sometimes it's handy +-- to block a vandal or troll account. +-- +CREATE TABLE /*_*/ipblocks ( + -- Primary key, introduced for privacy. + ipb_id int NOT NULL PRIMARY KEY AUTO_INCREMENT, + + -- Blocked IP address in dotted-quad form or user name. + ipb_address tinyblob NOT NULL, + + -- Blocked user ID or 0 for IP blocks. + ipb_user int unsigned NOT NULL default 0, + + -- User ID who made the block. + ipb_by int unsigned NOT NULL default 0, + + -- User name of blocker + ipb_by_text varchar(255) binary NOT NULL default '', + + -- Text comment made by blocker. Deprecated in favor of ipb_reason_id + ipb_reason varbinary(767) NOT NULL default '', + + -- Key to comment_id. Text comment made by blocker. + -- ("DEFAULT 0" is temporary, signaling that ipb_reason should be used) + ipb_reason_id bigint unsigned NOT NULL DEFAULT 0, + + -- Creation (or refresh) date in standard YMDHMS form. + -- IP blocks expire automatically. + ipb_timestamp binary(14) NOT NULL default '', + + -- Indicates that the IP address was banned because a banned + -- user accessed a page through it. If this is 1, ipb_address + -- will be hidden, and the block identified by block ID number. + ipb_auto bool NOT NULL default 0, + + -- If set to 1, block applies only to logged-out users + ipb_anon_only bool NOT NULL default 0, + + -- Block prevents account creation from matching IP addresses + ipb_create_account bool NOT NULL default 1, + + -- Block triggers autoblocks + ipb_enable_autoblock bool NOT NULL default '1', + + -- Time at which the block will expire. + -- May be "infinity" + ipb_expiry varbinary(14) NOT NULL default '', + + -- Start and end of an address range, in hexadecimal + -- Size chosen to allow IPv6 + -- FIXME: these fields were originally blank for single-IP blocks, + -- but now they are populated. No migration was ever done. They + -- should be fixed to be blank again for such blocks (T51504). + ipb_range_start tinyblob NOT NULL, + ipb_range_end tinyblob NOT NULL, + + -- Flag for entries hidden from users and Sysops + ipb_deleted bool NOT NULL default 0, + + -- Block prevents user from accessing Special:Emailuser + ipb_block_email bool NOT NULL default 0, + + -- Block allows user to edit their own talk page + ipb_allow_usertalk bool NOT NULL default 0, + + -- ID of the block that caused this block to exist + -- Autoblocks set this to the original block + -- so that the original block being deleted also + -- deletes the autoblocks + ipb_parent_block_id int default NULL + +) /*$wgDBTableOptions*/; + +-- Unique index to support "user already blocked" messages +-- Any new options which prevent collisions should be included +CREATE UNIQUE INDEX /*i*/ipb_address ON /*_*/ipblocks (ipb_address(255), ipb_user, ipb_auto, ipb_anon_only); + +-- For querying whether a logged-in user is blocked +CREATE INDEX /*i*/ipb_user ON /*_*/ipblocks (ipb_user); + +-- For querying whether an IP address is in any range +CREATE INDEX /*i*/ipb_range ON /*_*/ipblocks (ipb_range_start(8), ipb_range_end(8)); + +-- Index for Special:BlockList +CREATE INDEX /*i*/ipb_timestamp ON /*_*/ipblocks (ipb_timestamp); + +-- Index for table pruning +CREATE INDEX /*i*/ipb_expiry ON /*_*/ipblocks (ipb_expiry); + +-- Index for removing autoblocks when a parent block is removed +CREATE INDEX /*i*/ipb_parent_block_id ON /*_*/ipblocks (ipb_parent_block_id); + + +-- +-- Uploaded images and other files. +-- +CREATE TABLE /*_*/image ( + -- Filename. + -- This is also the title of the associated description page, + -- which will be in namespace 6 (NS_FILE). + img_name varchar(255) binary NOT NULL default '' PRIMARY KEY, + + -- File size in bytes. + img_size int unsigned NOT NULL default 0, + + -- For images, size in pixels. + img_width int NOT NULL default 0, + img_height int NOT NULL default 0, + + -- Extracted Exif metadata stored as a serialized PHP array. + img_metadata mediumblob NOT NULL, + + -- For images, bits per pixel if known. + img_bits int NOT NULL default 0, + + -- Media type as defined by the MEDIATYPE_xxx constants + img_media_type ENUM("UNKNOWN", "BITMAP", "DRAWING", "AUDIO", "VIDEO", "MULTIMEDIA", "OFFICE", "TEXT", "EXECUTABLE", "ARCHIVE", "3D") default NULL, + + -- major part of a MIME media type as defined by IANA + -- see https://www.iana.org/assignments/media-types/ + -- for "chemical" cf. http://dx.doi.org/10.1021/ci9803233 by the ACS + img_major_mime ENUM("unknown", "application", "audio", "image", "text", "video", "message", "model", "multipart", "chemical") NOT NULL default "unknown", + + -- minor part of a MIME media type as defined by IANA + -- the minor parts are not required to adher to any standard + -- but should be consistent throughout the database + -- see https://www.iana.org/assignments/media-types/ + img_minor_mime varbinary(100) NOT NULL default "unknown", + + -- Description field as entered by the uploader. + -- This is displayed in image upload history and logs. + -- Deprecated in favor of image_comment_temp.imgcomment_description_id. + img_description varbinary(767) NOT NULL default '', + + -- user_id and user_name of uploader. + img_user int unsigned NOT NULL default 0, + img_user_text varchar(255) binary NOT NULL, + + -- Time of the upload. + img_timestamp varbinary(14) NOT NULL default '', + + -- SHA-1 content hash in base-36 + img_sha1 varbinary(32) NOT NULL default '' +) /*$wgDBTableOptions*/; + +-- Used by Special:Newimages and ApiQueryAllImages +CREATE INDEX /*i*/img_user_timestamp ON /*_*/image (img_user,img_timestamp); +CREATE INDEX /*i*/img_usertext_timestamp ON /*_*/image (img_user_text,img_timestamp); +-- Used by Special:ListFiles for sort-by-size +CREATE INDEX /*i*/img_size ON /*_*/image (img_size); +-- Used by Special:Newimages and Special:ListFiles +CREATE INDEX /*i*/img_timestamp ON /*_*/image (img_timestamp); +-- Used in API and duplicate search +CREATE INDEX /*i*/img_sha1 ON /*_*/image (img_sha1(10)); +-- Used to get media of one type +CREATE INDEX /*i*/img_media_mime ON /*_*/image (img_media_type,img_major_mime,img_minor_mime); + +-- +-- Temporary table to avoid blocking on an alter of image. +-- +-- On large wikis like Wikimedia Commons, altering the image table is a +-- months-long process. This table is being created to avoid such an alter, and +-- will be merged back into image in the future. +-- +CREATE TABLE /*_*/image_comment_temp ( + -- Key to img_name (ugh) + imgcomment_name varchar(255) binary NOT NULL, + -- Key to comment_id + imgcomment_description_id bigint unsigned NOT NULL, + PRIMARY KEY (imgcomment_name, imgcomment_description_id) +) /*$wgDBTableOptions*/; +-- Ensure uniqueness +CREATE UNIQUE INDEX /*i*/imgcomment_name ON /*_*/image_comment_temp (imgcomment_name); + + +-- +-- Previous revisions of uploaded files. +-- Awkwardly, image rows have to be moved into +-- this table at re-upload time. +-- +CREATE TABLE /*_*/oldimage ( + -- Base filename: key to image.img_name + oi_name varchar(255) binary NOT NULL default '', + + -- Filename of the archived file. + -- This is generally a timestamp and '!' prepended to the base name. + oi_archive_name varchar(255) binary NOT NULL default '', + + -- Other fields as in image... + oi_size int unsigned NOT NULL default 0, + oi_width int NOT NULL default 0, + oi_height int NOT NULL default 0, + oi_bits int NOT NULL default 0, + oi_description varbinary(767) NOT NULL default '', -- Deprecated. + oi_description_id bigint unsigned NOT NULL DEFAULT 0, -- ("DEFAULT 0" is temporary, signaling that oi_description should be used) + oi_user int unsigned NOT NULL default 0, + oi_user_text varchar(255) binary NOT NULL, + oi_timestamp binary(14) NOT NULL default '', + + oi_metadata mediumblob NOT NULL, + oi_media_type ENUM("UNKNOWN", "BITMAP", "DRAWING", "AUDIO", "VIDEO", "MULTIMEDIA", "OFFICE", "TEXT", "EXECUTABLE", "ARCHIVE", "3D") default NULL, + oi_major_mime ENUM("unknown", "application", "audio", "image", "text", "video", "message", "model", "multipart", "chemical") NOT NULL default "unknown", + oi_minor_mime varbinary(100) NOT NULL default "unknown", + oi_deleted tinyint unsigned NOT NULL default 0, + oi_sha1 varbinary(32) NOT NULL default '' +) /*$wgDBTableOptions*/; + +CREATE INDEX /*i*/oi_usertext_timestamp ON /*_*/oldimage (oi_user_text,oi_timestamp); +CREATE INDEX /*i*/oi_name_timestamp ON /*_*/oldimage (oi_name,oi_timestamp); +-- oi_archive_name truncated to 14 to avoid key length overflow +CREATE INDEX /*i*/oi_name_archive_name ON /*_*/oldimage (oi_name,oi_archive_name(14)); +CREATE INDEX /*i*/oi_sha1 ON /*_*/oldimage (oi_sha1(10)); + + +-- +-- Record of deleted file data +-- +CREATE TABLE /*_*/filearchive ( + -- Unique row id + fa_id int NOT NULL PRIMARY KEY AUTO_INCREMENT, + + -- Original base filename; key to image.img_name, page.page_title, etc + fa_name varchar(255) binary NOT NULL default '', + + -- Filename of archived file, if an old revision + fa_archive_name varchar(255) binary default '', + + -- Which storage bin (directory tree or object store) the file data + -- is stored in. Should be 'deleted' for files that have been deleted; + -- any other bin is not yet in use. + fa_storage_group varbinary(16), + + -- SHA-1 of the file contents plus extension, used as a key for storage. + -- eg 8f8a562add37052a1848ff7771a2c515db94baa9.jpg + -- + -- If NULL, the file was missing at deletion time or has been purged + -- from the archival storage. + fa_storage_key varbinary(64) default '', + + -- Deletion information, if this file is deleted. + fa_deleted_user int, + fa_deleted_timestamp binary(14) default '', + fa_deleted_reason varbinary(767) default '', -- Deprecated + fa_deleted_reason_id bigint unsigned NOT NULL DEFAULT 0, -- ("DEFAULT 0" is temporary, signaling that fa_deleted_reason should be used) + + -- Duped fields from image + fa_size int unsigned default 0, + fa_width int default 0, + fa_height int default 0, + fa_metadata mediumblob, + fa_bits int default 0, + fa_media_type ENUM("UNKNOWN", "BITMAP", "DRAWING", "AUDIO", "VIDEO", "MULTIMEDIA", "OFFICE", "TEXT", "EXECUTABLE", "ARCHIVE", "3D") default NULL, + fa_major_mime ENUM("unknown", "application", "audio", "image", "text", "video", "message", "model", "multipart", "chemical") default "unknown", + fa_minor_mime varbinary(100) default "unknown", + fa_description varbinary(767) default '', -- Deprecated + fa_description_id bigint unsigned NOT NULL DEFAULT 0, -- ("DEFAULT 0" is temporary, signaling that fa_description should be used) + fa_user int unsigned default 0, + fa_user_text varchar(255) binary, + fa_timestamp binary(14) default '', + + -- Visibility of deleted revisions, bitfield + fa_deleted tinyint unsigned NOT NULL default 0, + + -- sha1 hash of file content + fa_sha1 varbinary(32) NOT NULL default '' +) /*$wgDBTableOptions*/; + +-- pick out by image name +CREATE INDEX /*i*/fa_name ON /*_*/filearchive (fa_name, fa_timestamp); +-- pick out dupe files +CREATE INDEX /*i*/fa_storage_group ON /*_*/filearchive (fa_storage_group, fa_storage_key); +-- sort by deletion time +CREATE INDEX /*i*/fa_deleted_timestamp ON /*_*/filearchive (fa_deleted_timestamp); +-- sort by uploader +CREATE INDEX /*i*/fa_user_timestamp ON /*_*/filearchive (fa_user_text,fa_timestamp); +-- find file by sha1, 10 bytes will be enough for hashes to be indexed +CREATE INDEX /*i*/fa_sha1 ON /*_*/filearchive (fa_sha1(10)); + + +-- +-- Store information about newly uploaded files before they're +-- moved into the actual filestore +-- +CREATE TABLE /*_*/uploadstash ( + us_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + + -- the user who uploaded the file. + us_user int unsigned NOT NULL, + + -- file key. this is how applications actually search for the file. + -- this might go away, or become the primary key. + us_key varchar(255) NOT NULL, + + -- the original path + us_orig_path varchar(255) NOT NULL, + + -- the temporary path at which the file is actually stored + us_path varchar(255) NOT NULL, + + -- which type of upload the file came from (sometimes) + us_source_type varchar(50), + + -- the date/time on which the file was added + us_timestamp varbinary(14) NOT NULL, + + us_status varchar(50) NOT NULL, + + -- chunk counter starts at 0, current offset is stored in us_size + us_chunk_inx int unsigned NULL, + + -- Serialized file properties from FSFile::getProps() + us_props blob, + + -- file size in bytes + us_size int unsigned NOT NULL, + -- this hash comes from FSFile::getSha1Base36(), and is 31 characters + us_sha1 varchar(31) NOT NULL, + us_mime varchar(255), + -- Media type as defined by the MEDIATYPE_xxx constants, should duplicate definition in the image table + us_media_type ENUM("UNKNOWN", "BITMAP", "DRAWING", "AUDIO", "VIDEO", "MULTIMEDIA", "OFFICE", "TEXT", "EXECUTABLE", "ARCHIVE", "3D") default NULL, + -- image-specific properties + us_image_width int unsigned, + us_image_height int unsigned, + us_image_bits smallint unsigned + +) /*$wgDBTableOptions*/; + +-- sometimes there's a delete for all of a user's stuff. +CREATE INDEX /*i*/us_user ON /*_*/uploadstash (us_user); +-- pick out files by key, enforce key uniqueness +CREATE UNIQUE INDEX /*i*/us_key ON /*_*/uploadstash (us_key); +-- the abandoned upload cleanup script needs this +CREATE INDEX /*i*/us_timestamp ON /*_*/uploadstash (us_timestamp); + + +-- +-- Primarily a summary table for Special:Recentchanges, +-- this table contains some additional info on edits from +-- the last few days, see Article::editUpdates() +-- +CREATE TABLE /*_*/recentchanges ( + rc_id int NOT NULL PRIMARY KEY AUTO_INCREMENT, + rc_timestamp varbinary(14) NOT NULL default '', + + -- As in revision + rc_user int unsigned NOT NULL default 0, + rc_user_text varchar(255) binary NOT NULL, + + -- When pages are renamed, their RC entries do _not_ change. + rc_namespace int NOT NULL default 0, + rc_title varchar(255) binary NOT NULL default '', + + -- as in revision... + rc_comment varbinary(767) NOT NULL default '', -- Deprecated. + rc_comment_id bigint unsigned NOT NULL DEFAULT 0, -- ("DEFAULT 0" is temporary, signaling that rc_comment should be used) + rc_minor tinyint unsigned NOT NULL default 0, + + -- Edits by user accounts with the 'bot' rights key are + -- marked with a 1 here, and will be hidden from the + -- default view. + rc_bot tinyint unsigned NOT NULL default 0, + + -- Set if this change corresponds to a page creation + rc_new tinyint unsigned NOT NULL default 0, + + -- Key to page_id (was cur_id prior to 1.5). + -- This will keep links working after moves while + -- retaining the at-the-time name in the changes list. + rc_cur_id int unsigned NOT NULL default 0, + + -- rev_id of the given revision + rc_this_oldid int unsigned NOT NULL default 0, + + -- rev_id of the prior revision, for generating diff links. + rc_last_oldid int unsigned NOT NULL default 0, + + -- The type of change entry (RC_EDIT,RC_NEW,RC_LOG,RC_EXTERNAL) + rc_type tinyint unsigned NOT NULL default 0, + + -- The source of the change entry (replaces rc_type) + -- default of '' is temporary, needed for initial migration + rc_source varchar(16) binary not null default '', + + -- If the Recent Changes Patrol option is enabled, + -- users may mark edits as having been reviewed to + -- remove a warning flag on the RC list. + -- A value of 1 indicates the page has been reviewed. + rc_patrolled tinyint unsigned NOT NULL default 0, + + -- Recorded IP address the edit was made from, if the + -- $wgPutIPinRC option is enabled. + rc_ip varbinary(40) NOT NULL default '', + + -- Text length in characters before + -- and after the edit + rc_old_len int, + rc_new_len int, + + -- Visibility of recent changes items, bitfield + rc_deleted tinyint unsigned NOT NULL default 0, + + -- Value corresponding to log_id, specific log entries + rc_logid int unsigned NOT NULL default 0, + -- Store log type info here, or null + rc_log_type varbinary(255) NULL default NULL, + -- Store log action or null + rc_log_action varbinary(255) NULL default NULL, + -- Log params + rc_params blob NULL +) /*$wgDBTableOptions*/; + +-- Special:Recentchanges +CREATE INDEX /*i*/rc_timestamp ON /*_*/recentchanges (rc_timestamp); + +-- Special:Watchlist +CREATE INDEX /*i*/rc_namespace_title ON /*_*/recentchanges (rc_namespace, rc_title); + +-- Special:Recentchangeslinked when finding changes in pages linked from a page +CREATE INDEX /*i*/rc_cur_id ON /*_*/recentchanges (rc_cur_id); + +-- Special:Newpages +CREATE INDEX /*i*/new_name_timestamp ON /*_*/recentchanges (rc_new,rc_namespace,rc_timestamp); + +-- Blank unless $wgPutIPinRC=true (false at WMF), possibly used by extensions, +-- but mostly replaced by CheckUser. +CREATE INDEX /*i*/rc_ip ON /*_*/recentchanges (rc_ip); + +-- Probably intended for Special:NewPages namespace filter +CREATE INDEX /*i*/rc_ns_usertext ON /*_*/recentchanges (rc_namespace, rc_user_text); + +-- SiteStats active user count, Special:ActiveUsers, Special:NewPages user filter +CREATE INDEX /*i*/rc_user_text ON /*_*/recentchanges (rc_user_text, rc_timestamp); + +-- ApiQueryRecentChanges (T140108) +CREATE INDEX /*i*/rc_name_type_patrolled_timestamp ON /*_*/recentchanges (rc_namespace, rc_type, rc_patrolled, rc_timestamp); + + +CREATE TABLE /*_*/watchlist ( + wl_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + -- Key to user.user_id + wl_user int unsigned NOT NULL, + + -- Key to page_namespace/page_title + -- Note that users may watch pages which do not exist yet, + -- or existed in the past but have been deleted. + wl_namespace int NOT NULL default 0, + wl_title varchar(255) binary NOT NULL default '', + + -- Timestamp used to send notification e-mails and show "updated since last visit" markers on + -- history and recent changes / watchlist. Set to NULL when the user visits the latest revision + -- of the page, which means that they should be sent an e-mail on the next change. + wl_notificationtimestamp varbinary(14) + +) /*$wgDBTableOptions*/; + +-- Special:Watchlist +CREATE UNIQUE INDEX /*i*/wl_user ON /*_*/watchlist (wl_user, wl_namespace, wl_title); + +-- Special:Movepage (WatchedItemStore::duplicateEntry) +CREATE INDEX /*i*/namespace_title ON /*_*/watchlist (wl_namespace, wl_title); + +-- ApiQueryWatchlistRaw changed filter +CREATE INDEX /*i*/wl_user_notificationtimestamp ON /*_*/watchlist (wl_user, wl_notificationtimestamp); + + +-- +-- When using the default MySQL search backend, page titles +-- and text are munged to strip markup, do Unicode case folding, +-- and prepare the result for MySQL's fulltext index. +-- +-- This table must be MyISAM; InnoDB does not support the needed +-- fulltext index. +-- +CREATE TABLE /*_*/searchindex ( + -- Key to page_id + si_page int unsigned NOT NULL, + + -- Munged version of title + si_title varchar(255) NOT NULL default '', + + -- Munged version of body text + si_text mediumtext NOT NULL +) ENGINE=MyISAM DEFAULT CHARSET=utf8; + +CREATE UNIQUE INDEX /*i*/si_page ON /*_*/searchindex (si_page); +CREATE FULLTEXT INDEX /*i*/si_title ON /*_*/searchindex (si_title); +CREATE FULLTEXT INDEX /*i*/si_text ON /*_*/searchindex (si_text); + + +-- +-- Recognized interwiki link prefixes +-- +CREATE TABLE /*_*/interwiki ( + -- The interwiki prefix, (e.g. "Meatball", or the language prefix "de") + iw_prefix varchar(32) NOT NULL, + + -- The URL of the wiki, with "$1" as a placeholder for an article name. + -- Any spaces in the name will be transformed to underscores before + -- insertion. + iw_url blob NOT NULL, + + -- The URL of the file api.php + iw_api blob NOT NULL, + + -- The name of the database (for a connection to be established with wfGetLB( 'wikiid' )) + iw_wikiid varchar(64) NOT NULL, + + -- A boolean value indicating whether the wiki is in this project + -- (used, for example, to detect redirect loops) + iw_local bool NOT NULL, + + -- Boolean value indicating whether interwiki transclusions are allowed. + iw_trans tinyint NOT NULL default 0 +) /*$wgDBTableOptions*/; + +CREATE UNIQUE INDEX /*i*/iw_prefix ON /*_*/interwiki (iw_prefix); + + +-- +-- Used for caching expensive grouped queries +-- +CREATE TABLE /*_*/querycache ( + -- A key name, generally the base name of of the special page. + qc_type varbinary(32) NOT NULL, + + -- Some sort of stored value. Sizes, counts... + qc_value int unsigned NOT NULL default 0, + + -- Target namespace+title + qc_namespace int NOT NULL default 0, + qc_title varchar(255) binary NOT NULL default '' +) /*$wgDBTableOptions*/; + +CREATE INDEX /*i*/qc_type ON /*_*/querycache (qc_type,qc_value); + + +-- +-- For a few generic cache operations if not using Memcached +-- +CREATE TABLE /*_*/objectcache ( + keyname varbinary(255) NOT NULL default '' PRIMARY KEY, + value mediumblob, + exptime datetime +) /*$wgDBTableOptions*/; +CREATE INDEX /*i*/exptime ON /*_*/objectcache (exptime); + + +-- +-- Cache of interwiki transclusion +-- +CREATE TABLE /*_*/transcache ( + tc_url varbinary(255) NOT NULL PRIMARY KEY, + tc_contents text, + tc_time binary(14) NOT NULL +) /*$wgDBTableOptions*/; + + +CREATE TABLE /*_*/logging ( + -- Log ID, for referring to this specific log entry, probably for deletion and such. + log_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + + -- Symbolic keys for the general log type and the action type + -- within the log. The output format will be controlled by the + -- action field, but only the type controls categorization. + log_type varbinary(32) NOT NULL default '', + log_action varbinary(32) NOT NULL default '', + + -- Timestamp. Duh. + log_timestamp binary(14) NOT NULL default '19700101000000', + + -- The user who performed this action; key to user_id + log_user int unsigned NOT NULL default 0, + + -- Name of the user who performed this action + log_user_text varchar(255) binary NOT NULL default '', + + -- Key to the page affected. Where a user is the target, + -- this will point to the user page. + log_namespace int NOT NULL default 0, + log_title varchar(255) binary NOT NULL default '', + log_page int unsigned NULL, + + -- Freeform text. Interpreted as edit history comments. + -- Deprecated in favor of log_comment_id. + log_comment varbinary(767) NOT NULL default '', + + -- Key to comment_id. Comment summarizing the change. + -- ("DEFAULT 0" is temporary, signaling that log_comment should be used) + log_comment_id bigint unsigned NOT NULL DEFAULT 0, + + -- miscellaneous parameters: + -- LF separated list (old system) or serialized PHP array (new system) + log_params blob NOT NULL, + + -- rev_deleted for logs + log_deleted tinyint unsigned NOT NULL default 0 +) /*$wgDBTableOptions*/; + +-- Special:Log type filter +CREATE INDEX /*i*/type_time ON /*_*/logging (log_type, log_timestamp); + +-- Special:Log performer filter +CREATE INDEX /*i*/user_time ON /*_*/logging (log_user, log_timestamp); + +-- Special:Log title filter, log extract +CREATE INDEX /*i*/page_time ON /*_*/logging (log_namespace, log_title, log_timestamp); + +-- Special:Log unfiltered +CREATE INDEX /*i*/times ON /*_*/logging (log_timestamp); + +-- Special:Log filter by performer and type +CREATE INDEX /*i*/log_user_type_time ON /*_*/logging (log_user, log_type, log_timestamp); + +-- Apparently just used for a few maintenance pages (findMissingFiles.php, Flow). +-- Could be removed? +CREATE INDEX /*i*/log_page_id_time ON /*_*/logging (log_page,log_timestamp); + +-- Special:Log action filter +CREATE INDEX /*i*/type_action ON /*_*/logging (log_type, log_action, log_timestamp); + +-- Special:Log filter by type and anonymous performer +CREATE INDEX /*i*/log_user_text_type_time ON /*_*/logging (log_user_text, log_type, log_timestamp); + +-- Special:Log filter by anonymous performer +CREATE INDEX /*i*/log_user_text_time ON /*_*/logging (log_user_text, log_timestamp); + + +CREATE TABLE /*_*/log_search ( + -- The type of ID (rev ID, log ID, rev timestamp, username) + ls_field varbinary(32) NOT NULL, + -- The value of the ID + ls_value varchar(255) NOT NULL, + -- Key to log_id + ls_log_id int unsigned NOT NULL default 0, + PRIMARY KEY (ls_field,ls_value,ls_log_id) +) /*$wgDBTableOptions*/; +CREATE INDEX /*i*/ls_log_id ON /*_*/log_search (ls_log_id); + + +-- Jobs performed by parallel apache threads or a command-line daemon +CREATE TABLE /*_*/job ( + job_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + + -- Command name + -- Limited to 60 to prevent key length overflow + job_cmd varbinary(60) NOT NULL default '', + + -- Namespace and title to act on + -- Should be 0 and '' if the command does not operate on a title + job_namespace int NOT NULL, + job_title varchar(255) binary NOT NULL, + + -- Timestamp of when the job was inserted + -- NULL for jobs added before addition of the timestamp + job_timestamp varbinary(14) NULL default NULL, + + -- Any other parameters to the command + -- Stored as a PHP serialized array, or an empty string if there are no parameters + job_params blob NOT NULL, + + -- Random, non-unique, number used for job acquisition (for lock concurrency) + job_random integer unsigned NOT NULL default 0, + + -- The number of times this job has been locked + job_attempts integer unsigned NOT NULL default 0, + + -- Field that conveys process locks on rows via process UUIDs + job_token varbinary(32) NOT NULL default '', + + -- Timestamp when the job was locked + job_token_timestamp varbinary(14) NULL default NULL, + + -- Base 36 SHA1 of the job parameters relevant to detecting duplicates + job_sha1 varbinary(32) NOT NULL default '' +) /*$wgDBTableOptions*/; + +CREATE INDEX /*i*/job_sha1 ON /*_*/job (job_sha1); +CREATE INDEX /*i*/job_cmd_token ON /*_*/job (job_cmd,job_token,job_random); +CREATE INDEX /*i*/job_cmd_token_id ON /*_*/job (job_cmd,job_token,job_id); +CREATE INDEX /*i*/job_cmd ON /*_*/job (job_cmd, job_namespace, job_title, job_params(128)); +CREATE INDEX /*i*/job_timestamp ON /*_*/job (job_timestamp); + + +-- Details of updates to cached special pages +CREATE TABLE /*_*/querycache_info ( + -- Special page name + -- Corresponds to a qc_type value + qci_type varbinary(32) NOT NULL default '' PRIMARY KEY, + + -- Timestamp of last update + qci_timestamp binary(14) NOT NULL default '19700101000000' +) /*$wgDBTableOptions*/; + + +-- For each redirect, this table contains exactly one row defining its target +CREATE TABLE /*_*/redirect ( + -- Key to the page_id of the redirect page + rd_from int unsigned NOT NULL default 0 PRIMARY KEY, + + -- Key to page_namespace/page_title of the target page. + -- The target page may or may not exist, and due to renames + -- and deletions may refer to different page records as time + -- goes by. + rd_namespace int NOT NULL default 0, + rd_title varchar(255) binary NOT NULL default '', + rd_interwiki varchar(32) default NULL, + rd_fragment varchar(255) binary default NULL +) /*$wgDBTableOptions*/; + +CREATE INDEX /*i*/rd_ns_title ON /*_*/redirect (rd_namespace,rd_title,rd_from); + + +-- Used for caching expensive grouped queries that need two links (for example double-redirects) +CREATE TABLE /*_*/querycachetwo ( + -- A key name, generally the base name of of the special page. + qcc_type varbinary(32) NOT NULL, + + -- Some sort of stored value. Sizes, counts... + qcc_value int unsigned NOT NULL default 0, + + -- Target namespace+title + qcc_namespace int NOT NULL default 0, + qcc_title varchar(255) binary NOT NULL default '', + + -- Target namespace+title2 + qcc_namespacetwo int NOT NULL default 0, + qcc_titletwo varchar(255) binary NOT NULL default '' +) /*$wgDBTableOptions*/; + +CREATE INDEX /*i*/qcc_type ON /*_*/querycachetwo (qcc_type,qcc_value); +CREATE INDEX /*i*/qcc_title ON /*_*/querycachetwo (qcc_type,qcc_namespace,qcc_title); +CREATE INDEX /*i*/qcc_titletwo ON /*_*/querycachetwo (qcc_type,qcc_namespacetwo,qcc_titletwo); + + +-- Used for storing page restrictions (i.e. protection levels) +CREATE TABLE /*_*/page_restrictions ( + -- Field for an ID for this restrictions row (sort-key for Special:ProtectedPages) + pr_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + -- Page to apply restrictions to (Foreign Key to page). + pr_page int NOT NULL, + -- The protection type (edit, move, etc) + pr_type varbinary(60) NOT NULL, + -- The protection level (Sysop, autoconfirmed, etc) + pr_level varbinary(60) NOT NULL, + -- Whether or not to cascade the protection down to pages transcluded. + pr_cascade tinyint NOT NULL, + -- Field for future support of per-user restriction. + pr_user int unsigned NULL, + -- Field for time-limited protection. + pr_expiry varbinary(14) NULL +) /*$wgDBTableOptions*/; + +CREATE UNIQUE INDEX /*i*/pr_pagetype ON /*_*/page_restrictions (pr_page,pr_type); +CREATE INDEX /*i*/pr_typelevel ON /*_*/page_restrictions (pr_type,pr_level); +CREATE INDEX /*i*/pr_level ON /*_*/page_restrictions (pr_level); +CREATE INDEX /*i*/pr_cascade ON /*_*/page_restrictions (pr_cascade); + + +-- Protected titles - nonexistent pages that have been protected +CREATE TABLE /*_*/protected_titles ( + pt_namespace int NOT NULL, + pt_title varchar(255) binary NOT NULL, + pt_user int unsigned NOT NULL, + pt_reason varbinary(767) default '', -- Deprecated. + pt_reason_id bigint unsigned NOT NULL DEFAULT 0, -- ("DEFAULT 0" is temporary, signaling that pt_reason should be used) + pt_timestamp binary(14) NOT NULL, + pt_expiry varbinary(14) NOT NULL default '', + pt_create_perm varbinary(60) NOT NULL +) /*$wgDBTableOptions*/; + +CREATE UNIQUE INDEX /*i*/pt_namespace_title ON /*_*/protected_titles (pt_namespace,pt_title); +CREATE INDEX /*i*/pt_timestamp ON /*_*/protected_titles (pt_timestamp); + + +-- Name/value pairs indexed by page_id +CREATE TABLE /*_*/page_props ( + pp_page int NOT NULL, + pp_propname varbinary(60) NOT NULL, + pp_value blob NOT NULL, + pp_sortkey float DEFAULT NULL +) /*$wgDBTableOptions*/; + +CREATE UNIQUE INDEX /*i*/pp_page_propname ON /*_*/page_props (pp_page,pp_propname); +CREATE UNIQUE INDEX /*i*/pp_propname_page ON /*_*/page_props (pp_propname,pp_page); +CREATE UNIQUE INDEX /*i*/pp_propname_sortkey_page ON /*_*/page_props (pp_propname,pp_sortkey,pp_page); + +-- A table to log updates, one text key row per update. +CREATE TABLE /*_*/updatelog ( + ul_key varchar(255) NOT NULL PRIMARY KEY, + ul_value blob +) /*$wgDBTableOptions*/; + + +-- A table to track tags for revisions, logs and recent changes. +CREATE TABLE /*_*/change_tag ( + ct_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + -- RCID for the change + ct_rc_id int NULL, + -- LOGID for the change + ct_log_id int unsigned NULL, + -- REVID for the change + ct_rev_id int unsigned NULL, + -- Tag applied + ct_tag varchar(255) NOT NULL, + -- Parameters for the tag, presently unused + ct_params blob NULL +) /*$wgDBTableOptions*/; + +CREATE UNIQUE INDEX /*i*/change_tag_rc_tag ON /*_*/change_tag (ct_rc_id,ct_tag); +CREATE UNIQUE INDEX /*i*/change_tag_log_tag ON /*_*/change_tag (ct_log_id,ct_tag); +CREATE UNIQUE INDEX /*i*/change_tag_rev_tag ON /*_*/change_tag (ct_rev_id,ct_tag); +-- Covering index, so we can pull all the info only out of the index. +CREATE INDEX /*i*/change_tag_tag_id ON /*_*/change_tag (ct_tag,ct_rc_id,ct_rev_id,ct_log_id); + + +-- Rollup table to pull a LIST of tags simply without ugly GROUP_CONCAT +-- that only works on MySQL 4.1+ +CREATE TABLE /*_*/tag_summary ( + ts_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + -- RCID for the change + ts_rc_id int NULL, + -- LOGID for the change + ts_log_id int unsigned NULL, + -- REVID for the change + ts_rev_id int unsigned NULL, + -- Comma-separated list of tags + ts_tags blob NOT NULL +) /*$wgDBTableOptions*/; + +CREATE UNIQUE INDEX /*i*/tag_summary_rc_id ON /*_*/tag_summary (ts_rc_id); +CREATE UNIQUE INDEX /*i*/tag_summary_log_id ON /*_*/tag_summary (ts_log_id); +CREATE UNIQUE INDEX /*i*/tag_summary_rev_id ON /*_*/tag_summary (ts_rev_id); + + +CREATE TABLE /*_*/valid_tag ( + vt_tag varchar(255) NOT NULL PRIMARY KEY +) /*$wgDBTableOptions*/; + +-- Table for storing localisation data +CREATE TABLE /*_*/l10n_cache ( + -- Language code + lc_lang varbinary(32) NOT NULL, + -- Cache key + lc_key varchar(255) NOT NULL, + -- Value + lc_value mediumblob NOT NULL, + PRIMARY KEY (lc_lang, lc_key) +) /*$wgDBTableOptions*/; + +-- Table caching which local files a module depends on that aren't +-- registered directly, used for fast retrieval of file dependency. +-- Currently only used for tracking images that CSS depends on +CREATE TABLE /*_*/module_deps ( + -- Module name + md_module varbinary(255) NOT NULL, + -- Module context vary (includes skin and language; called "md_skin" for legacy reasons) + md_skin varbinary(32) NOT NULL, + -- JSON blob with file dependencies + md_deps mediumblob NOT NULL, + PRIMARY KEY (md_module,md_skin) +) /*$wgDBTableOptions*/; + +-- Holds all the sites known to the wiki. +CREATE TABLE /*_*/sites ( + -- Numeric id of the site + site_id INT UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT, + + -- Global identifier for the site, ie 'enwiktionary' + site_global_key varbinary(32) NOT NULL, + + -- Type of the site, ie 'mediawiki' + site_type varbinary(32) NOT NULL, + + -- Group of the site, ie 'wikipedia' + site_group varbinary(32) NOT NULL, + + -- Source of the site data, ie 'local', 'wikidata', 'my-magical-repo' + site_source varbinary(32) NOT NULL, + + -- Language code of the sites primary language. + site_language varbinary(32) NOT NULL, + + -- Protocol of the site, ie 'http://', 'irc://', '//' + -- This field is an index for lookups and is build from type specific data in site_data. + site_protocol varbinary(32) NOT NULL, + + -- Domain of the site in reverse order, ie 'org.mediawiki.www.' + -- This field is an index for lookups and is build from type specific data in site_data. + site_domain VARCHAR(255) NOT NULL, + + -- Type dependent site data. + site_data BLOB NOT NULL, + + -- If site.tld/path/key:pageTitle should forward users to the page on + -- the actual site, where "key" is the local identifier. + site_forward bool NOT NULL, + + -- Type dependent site config. + -- For instance if template transclusion should be allowed if it's a MediaWiki. + site_config BLOB NOT NULL +) /*$wgDBTableOptions*/; + +CREATE UNIQUE INDEX /*i*/sites_global_key ON /*_*/sites (site_global_key); +CREATE INDEX /*i*/sites_type ON /*_*/sites (site_type); +CREATE INDEX /*i*/sites_group ON /*_*/sites (site_group); +CREATE INDEX /*i*/sites_source ON /*_*/sites (site_source); +CREATE INDEX /*i*/sites_language ON /*_*/sites (site_language); +CREATE INDEX /*i*/sites_protocol ON /*_*/sites (site_protocol); +CREATE INDEX /*i*/sites_domain ON /*_*/sites (site_domain); +CREATE INDEX /*i*/sites_forward ON /*_*/sites (site_forward); + +-- Links local site identifiers to their corresponding site. +CREATE TABLE /*_*/site_identifiers ( + -- Key on site.site_id + si_site INT UNSIGNED NOT NULL, + + -- local key type, ie 'interwiki' or 'langlink' + si_type varbinary(32) NOT NULL, + + -- local key value, ie 'en' or 'wiktionary' + si_key varbinary(32) NOT NULL +) /*$wgDBTableOptions*/; + +CREATE UNIQUE INDEX /*i*/site_ids_type ON /*_*/site_identifiers (si_type, si_key); +CREATE INDEX /*i*/site_ids_site ON /*_*/site_identifiers (si_site); +CREATE INDEX /*i*/site_ids_key ON /*_*/site_identifiers (si_key); + +-- vim: sw=2 sts=2 et diff --git a/www/wiki/maintenance/term/MWTerm.php b/www/wiki/maintenance/term/MWTerm.php new file mode 100644 index 00000000..d90d0695 --- /dev/null +++ b/www/wiki/maintenance/term/MWTerm.php @@ -0,0 +1,72 @@ +<?php +/** + * Set of classes to help with test output and such. Right now pretty specific + * to the parser tests but could be more useful one day :) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance Testing + * @todo Fixme: Make this more generic + */ + +/** + * Terminal that supports ANSI escape sequences. + * + * @ingroup Maintenance Testing + */ +class AnsiTermColorer { + function __construct() { + } + + /** + * Return ANSI terminal escape code for changing text attribs/color + * + * @param string $color Semicolon-separated list of attribute/color codes + * @return string + */ + public function color( $color ) { + global $wgCommandLineDarkBg; + + $light = $wgCommandLineDarkBg ? "1;" : "0;"; + + return "\x1b[{$light}{$color}m"; + } + + /** + * Return ANSI terminal escape code for restoring default text attributes + * + * @return string + */ + public function reset() { + return $this->color( 0 ); + } +} + +/** + * A colour-less terminal + * + * @ingroup Maintenance Testing + */ +class DummyTermColorer { + public function color( $color ) { + return ''; + } + + public function reset() { + return ''; + } +} diff --git a/www/wiki/maintenance/tidyUpBug37714.php b/www/wiki/maintenance/tidyUpBug37714.php new file mode 100644 index 00000000..9d7cc0e9 --- /dev/null +++ b/www/wiki/maintenance/tidyUpBug37714.php @@ -0,0 +1,48 @@ +<?php +require_once __DIR__ . '/Maintenance.php'; + +/** + * Fixes all rows affected by https://bugzilla.wikimedia.org/show_bug.cgi?id=37714 + */ +class TidyUpBug37714 extends Maintenance { + public function execute() { + // Search for all log entries which are about changing the visability of other log entries. + $result = $this->getDB( DB_REPLICA )->select( + 'logging', + [ 'log_id', 'log_params' ], + [ + 'log_type' => [ 'suppress', 'delete' ], + 'log_action' => 'event', + 'log_namespace' => NS_SPECIAL, + 'log_title' => SpecialPage::getTitleFor( 'Log' )->getText() + ], + __METHOD__ + ); + + foreach ( $result as $row ) { + $ids = explode( ',', explode( "\n", $row->log_params )[0] ); + $result = $this->getDB( DB_REPLICA )->select( // Work out what log entries were changed here. + 'logging', + 'log_type', + [ 'log_id' => $ids ], + __METHOD__, + 'DISTINCT' + ); + if ( $result->numRows() === 1 ) { + // If there's only one type, the target title can be set to include it. + $logTitle = SpecialPage::getTitleFor( 'Log', $result->current()->log_type )->getText(); + $this->output( 'Set log_title to "' . $logTitle . '" for log entry ' . $row->log_id . ".\n" ); + $this->getDB( DB_MASTER )->update( + 'logging', + [ 'log_title' => $logTitle ], + [ 'log_id' => $row->log_id ], + __METHOD__ + ); + wfWaitForSlaves(); + } + } + } +} + +$maintClass = 'TidyUpBug37714'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/undelete.php b/www/wiki/maintenance/undelete.php new file mode 100644 index 00000000..c2d5c2c2 --- /dev/null +++ b/www/wiki/maintenance/undelete.php @@ -0,0 +1,62 @@ +<?php +/** + * Undelete a page by fetching it from the archive table + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +class Undelete extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Undelete a page' ); + $this->addOption( 'user', 'The user to perform the undeletion', false, true, 'u' ); + $this->addOption( 'reason', 'The reason to undelete', false, true, 'r' ); + $this->addArg( 'pagename', 'Page to undelete' ); + } + + public function execute() { + global $wgUser; + + $user = $this->getOption( 'user', false ); + $reason = $this->getOption( 'reason', '' ); + $pageName = $this->getArg(); + + $title = Title::newFromText( $pageName ); + if ( !$title ) { + $this->error( "Invalid title", true ); + } + if ( $user === false ) { + $wgUser = User::newSystemUser( 'Command line script', [ 'steal' => true ] ); + } else { + $wgUser = User::newFromName( $user ); + } + if ( !$wgUser ) { + $this->error( "Invalid username", true ); + } + $archive = new PageArchive( $title, RequestContext::getMain()->getConfig() ); + $this->output( "Undeleting " . $title->getPrefixedDBkey() . '...' ); + $archive->undelete( [], $reason ); + $this->output( "done\n" ); + } +} + +$maintClass = "Undelete"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/update-keys.sql b/www/wiki/maintenance/update-keys.sql new file mode 100644 index 00000000..dfbb67ea --- /dev/null +++ b/www/wiki/maintenance/update-keys.sql @@ -0,0 +1,29 @@ +-- SQL to insert update keys into the initial tables after a +-- fresh installation of MediaWiki's database. +-- This is read and executed by the install script; you should +-- not have to run it by itself unless doing a manual install. +-- Insert keys here if either the unnecessary would cause heavy +-- processing or could potentially cause trouble by lowering field +-- sizes, adding constraints, etc. +-- When adjusting field sizes, it is recommended removing old +-- patches but to play safe, update keys should also inserted here. + +-- This is a shared file used for both MySQL and SQLite installs. +-- Therefore inserting multiple values is not possible using the +-- INSERT INTO VALUES syntax. +-- +-- +-- The /*_*/ comments in this and other files are +-- replaced with the defined table prefix by the installer +-- and updater scripts. If you are installing or running +-- updates manually, you will need to manually insert the +-- table prefix if any when running these scripts. +-- + +INSERT IGNORE INTO /*_*/updatelog + SELECT 'filearchive-fa_major_mime-patch-fa_major_mime-chemical.sql' AS ul_key, null as ul_value + UNION SELECT 'image-img_major_mime-patch-img_major_mime-chemical.sql', null + UNION SELECT 'oldimage-oi_major_mime-patch-oi_major_mime-chemical.sql', null + UNION SELECT 'user_groups-ug_group-patch-ug_group-length-increase-255.sql', null + UNION SELECT 'user_former_groups-ufg_group-patch-ufg_group-length-increase-255.sql', null + UNION SELECT 'user_properties-up_property-patch-up_property.sql', null;
\ No newline at end of file diff --git a/www/wiki/maintenance/update.php b/www/wiki/maintenance/update.php new file mode 100755 index 00000000..52e90175 --- /dev/null +++ b/www/wiki/maintenance/update.php @@ -0,0 +1,249 @@ +#!/usr/bin/env php +<?php +/** + * Run all updaters. + * + * This is used when the database schema is modified and we need to apply patches. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @todo document + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +use Wikimedia\Rdbms\IMaintainableDatabase; + +/** + * Maintenance script to run database schema updates. + * + * @ingroup Maintenance + */ +class UpdateMediaWiki extends Maintenance { + function __construct() { + parent::__construct(); + $this->addDescription( 'MediaWiki database updater' ); + $this->addOption( 'skip-compat-checks', 'Skips compatibility checks, mostly for developers' ); + $this->addOption( 'quick', 'Skip 5 second countdown before starting' ); + $this->addOption( 'doshared', 'Also update shared tables' ); + $this->addOption( 'nopurge', 'Do not purge the objectcache table after updates' ); + $this->addOption( 'noschema', 'Only do the updates that are not done during schema updates' ); + $this->addOption( + 'schema', + 'Output SQL to do the schema updates instead of doing them. Works ' + . 'even when $wgAllowSchemaUpdates is false', + false, + true + ); + $this->addOption( 'force', 'Override when $wgAllowSchemaUpdates disables this script' ); + $this->addOption( + 'skip-external-dependencies', + 'Skips checking whether external dependencies are up to date, mostly for developers' + ); + } + + function getDbType() { + return Maintenance::DB_ADMIN; + } + + function compatChecks() { + $minimumPcreVersion = Installer::MINIMUM_PCRE_VERSION; + + list( $pcreVersion ) = explode( ' ', PCRE_VERSION, 2 ); + if ( version_compare( $pcreVersion, $minimumPcreVersion, '<' ) ) { + $this->error( + "PCRE $minimumPcreVersion or later is required.\n" . + "Your PHP binary is linked with PCRE $pcreVersion.\n\n" . + "More information:\n" . + "https://www.mediawiki.org/wiki/Manual:Errors_and_symptoms/PCRE\n\n" . + "ABORTING.\n", + true ); + } + + $test = new PhpXmlBugTester(); + if ( !$test->ok ) { + $this->error( + "Your system has a combination of PHP and libxml2 versions that is buggy\n" . + "and can cause hidden data corruption in MediaWiki and other web apps.\n" . + "Upgrade to libxml2 2.7.3 or later.\n" . + "ABORTING (see https://bugs.php.net/bug.php?id=45996).\n", + true ); + } + } + + function execute() { + global $wgVersion, $wgLang, $wgAllowSchemaUpdates; + + if ( !$wgAllowSchemaUpdates + && !( $this->hasOption( 'force' ) + || $this->hasOption( 'schema' ) + || $this->hasOption( 'noschema' ) ) + ) { + $this->error( "Do not run update.php on this wiki. If you're seeing this you should\n" + . "probably ask for some help in performing your schema updates or use\n" + . "the --noschema and --schema options to get an SQL file for someone\n" + . "else to inspect and run.\n\n" + . "If you know what you are doing, you can continue with --force\n", true ); + } + + $this->fileHandle = null; + if ( substr( $this->getOption( 'schema' ), 0, 2 ) === "--" ) { + $this->error( "The --schema option requires a file as an argument.\n", true ); + } elseif ( $this->hasOption( 'schema' ) ) { + $file = $this->getOption( 'schema' ); + $this->fileHandle = fopen( $file, "w" ); + if ( $this->fileHandle === false ) { + $err = error_get_last(); + $this->error( "Problem opening the schema file for writing: $file\n\t{$err['message']}", true ); + } + } + + $lang = Language::factory( 'en' ); + // Set global language to ensure localised errors are in English (T22633) + RequestContext::getMain()->setLanguage( $lang ); + $wgLang = $lang; // BackCompat + + define( 'MW_UPDATER', true ); + + $this->output( "MediaWiki {$wgVersion} Updater\n\n" ); + + wfWaitForSlaves(); + + if ( !$this->hasOption( 'skip-compat-checks' ) ) { + $this->compatChecks(); + } else { + $this->output( "Skipping compatibility checks, proceed at your own risk (Ctrl+C to abort)\n" ); + wfCountDown( 5 ); + } + + // Check external dependencies are up to date + if ( !$this->hasOption( 'skip-external-dependencies' ) ) { + $composerLockUpToDate = $this->runChild( 'CheckComposerLockUpToDate' ); + $composerLockUpToDate->execute(); + } else { + $this->output( + "Skipping checking whether external dependencies are up to date, proceed at your own risk\n" + ); + } + + # Attempt to connect to the database as a privileged user + # This will vomit up an error if there are permissions problems + $db = $this->getDB( DB_MASTER ); + + # Check to see whether the database server meets the minimum requirements + /** @var DatabaseInstaller $dbInstallerClass */ + $dbInstallerClass = Installer::getDBInstallerClass( $db->getType() ); + $status = $dbInstallerClass::meetsMinimumRequirement( $db->getServerVersion() ); + if ( !$status->isOK() ) { + // This might output some wikitext like <strong> but it should be comprehensible + $text = $status->getWikiText(); + $this->error( $text, 1 ); + } + + $this->output( "Going to run database updates for " . wfWikiID() . "\n" ); + if ( $db->getType() === 'sqlite' ) { + /** @var IMaintainableDatabase|DatabaseSqlite $db */ + $this->output( "Using SQLite file: '{$db->getDbFilePath()}'\n" ); + } + $this->output( "Depending on the size of your database this may take a while!\n" ); + + if ( !$this->hasOption( 'quick' ) ) { + $this->output( "Abort with control-c in the next five seconds " + . "(skip this countdown with --quick) ... " ); + wfCountDown( 5 ); + } + + $time1 = microtime( true ); + + $badPhpUnit = dirname( __DIR__ ) . '/vendor/phpunit/phpunit/src/Util/PHP/eval-stdin.php'; + if ( file_exists( $badPhpUnit ) ) { + // @codingStandardsIgnoreStart Generic.Files.LineLength.TooLong + // Bad versions of the file are: + // https://raw.githubusercontent.com/sebastianbergmann/phpunit/c820f915bfae34e5a836f94967a2a5ea5ef34f21/src/Util/PHP/eval-stdin.php + // https://raw.githubusercontent.com/sebastianbergmann/phpunit/3aaddb1c5bd9b9b8d070b4cf120e71c36fd08412/src/Util/PHP/eval-stdin.php + // @codingStandardsIgnoreEnd + $md5 = md5_file( $badPhpUnit ); + if ( $md5 === '120ac49800671dc383b6f3709c25c099' + || $md5 === '28af792cb38fc9a1b236b91c1aad2876' + ) { + $success = unlink( $badPhpUnit ); + if ( $success ) { + $this->output( "Removed PHPUnit eval-stdin.php to protect against CVE-2017-9841\n" ); + } else { + $this->error( "Unable to remove $badPhpUnit, you should manually. See CVE-2017-9841" ); + } + } + } + + $shared = $this->hasOption( 'doshared' ); + + $updates = [ 'core', 'extensions' ]; + if ( !$this->hasOption( 'schema' ) ) { + if ( $this->hasOption( 'noschema' ) ) { + $updates[] = 'noschema'; + } + $updates[] = 'stats'; + } + + $updater = DatabaseUpdater::newForDB( $db, $shared, $this ); + $updater->doUpdates( $updates ); + + foreach ( $updater->getPostDatabaseUpdateMaintenance() as $maint ) { + $child = $this->runChild( $maint ); + + // LoggedUpdateMaintenance is checking the updatelog itself + $isLoggedUpdate = $child instanceof LoggedUpdateMaintenance; + + if ( !$isLoggedUpdate && $updater->updateRowExists( $maint ) ) { + continue; + } + + $child->execute(); + if ( !$isLoggedUpdate ) { + $updater->insertUpdateRow( $maint ); + } + } + + $updater->setFileAccess(); + if ( !$this->hasOption( 'nopurge' ) ) { + $updater->purgeCache(); + } + + $time2 = microtime( true ); + + $timeDiff = $lang->formatTimePeriod( $time2 - $time1 ); + $this->output( "\nDone in $timeDiff.\n" ); + } + + function afterFinalSetup() { + global $wgLocalisationCacheConf; + + # Don't try to access the database + # This needs to be disabled early since extensions will try to use the l10n + # cache from $wgExtensionFunctions (T22471) + $wgLocalisationCacheConf = [ + 'class' => 'LocalisationCache', + 'storeClass' => 'LCStoreNull', + 'storeDirectory' => false, + 'manualRecache' => false, + ]; + } +} + +$maintClass = 'UpdateMediaWiki'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/updateArticleCount.php b/www/wiki/maintenance/updateArticleCount.php new file mode 100644 index 00000000..213195df --- /dev/null +++ b/www/wiki/maintenance/updateArticleCount.php @@ -0,0 +1,73 @@ +<?php +/** + * Provide a better count of the number of articles + * and update the site statistics table, if desired. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @author Rob Church <robchur@gmail.com> + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to provide a better count of the number of articles + * and update the site statistics table, if desired. + * + * @ingroup Maintenance + */ +class UpdateArticleCount extends Maintenance { + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Count of the number of articles and update the site statistics table' ); + $this->addOption( 'update', 'Update the site_stats table with the new count' ); + $this->addOption( 'use-master', 'Count using the master database' ); + } + + public function execute() { + $this->output( "Counting articles..." ); + + if ( $this->hasOption( 'use-master' ) ) { + $dbr = $this->getDB( DB_MASTER ); + } else { + $dbr = $this->getDB( DB_REPLICA, 'vslow' ); + } + $counter = new SiteStatsInit( $dbr ); + $result = $counter->articles(); + + $this->output( "found {$result}.\n" ); + if ( $this->hasOption( 'update' ) ) { + $this->output( "Updating site statistics table... " ); + $dbw = $this->getDB( DB_MASTER ); + $dbw->update( + 'site_stats', + [ 'ss_good_articles' => $result ], + [ 'ss_row_id' => 1 ], + __METHOD__ + ); + $this->output( "done.\n" ); + } else { + $this->output( "To update the site statistics table, run the script " + . "with the --update option.\n" ); + } + } +} + +$maintClass = "UpdateArticleCount"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/updateCollation.php b/www/wiki/maintenance/updateCollation.php new file mode 100644 index 00000000..84fc2d20 --- /dev/null +++ b/www/wiki/maintenance/updateCollation.php @@ -0,0 +1,348 @@ +<?php +/** + * Find all rows in the categorylinks table whose collation is out-of-date + * (cl_collation != $wgCategoryCollation) and repopulate cl_sortkey + * using the page title and cl_sortkey_prefix. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @author Aryeh Gregor (Simetrical) + */ + +require_once __DIR__ . '/Maintenance.php'; + +use Wikimedia\Rdbms\IDatabase; + +/** + * Maintenance script that will find all rows in the categorylinks table + * whose collation is out-of-date. + * + * @ingroup Maintenance + */ +class UpdateCollation extends Maintenance { + const BATCH_SIZE = 100; // Number of rows to process in one batch + const SYNC_INTERVAL = 5; // Wait for replica DBs after this many batches + + public $sizeHistogram = []; + + public function __construct() { + parent::__construct(); + + global $wgCategoryCollation; + $this->addDescription( <<<TEXT +This script will find all rows in the categorylinks table whose collation is +out-of-date (cl_collation != '$wgCategoryCollation') and repopulate cl_sortkey +using the page title and cl_sortkey_prefix. If all collations are +up-to-date, it will do nothing. +TEXT + ); + + $this->addOption( 'force', 'Run on all rows, even if the collation is ' . + 'supposed to be up-to-date.', false, false, 'f' ); + $this->addOption( 'previous-collation', 'Set the previous value of ' . + '$wgCategoryCollation here to speed up this script, especially if your ' . + 'categorylinks table is large. This will only update rows with that ' . + 'collation, though, so it may miss out-of-date rows with a different, ' . + 'even older collation.', false, true ); + $this->addOption( 'target-collation', 'Set this to the new collation type to ' . + 'use instead of $wgCategoryCollation. Usually you should not use this, ' . + 'you should just update $wgCategoryCollation in LocalSettings.php.', + false, true ); + $this->addOption( 'dry-run', 'Don\'t actually change the collations, just ' . + 'compile statistics.' ); + $this->addOption( 'verbose-stats', 'Show more statistics.' ); + } + + public function execute() { + global $wgCategoryCollation; + + $dbw = $this->getDB( DB_MASTER ); + $dbr = $this->getDB( DB_REPLICA ); + $force = $this->getOption( 'force' ); + $dryRun = $this->getOption( 'dry-run' ); + $verboseStats = $this->getOption( 'verbose-stats' ); + if ( $this->hasOption( 'target-collation' ) ) { + $collationName = $this->getOption( 'target-collation' ); + $collation = Collation::factory( $collationName ); + } else { + $collationName = $wgCategoryCollation; + $collation = Collation::singleton(); + } + + // Collation sanity check: in some cases the constructor will work, + // but this will raise an exception, breaking all category pages + $collation->getFirstLetter( 'MediaWiki' ); + + // Locally at least, (my local is a rather old version of mysql) + // mysql seems to filesort if there is both an equality + // (but not for an inequality) condition on cl_collation in the + // WHERE and it is also the first item in the ORDER BY. + if ( $this->hasOption( 'previous-collation' ) ) { + $orderBy = 'cl_to, cl_type, cl_from'; + } else { + $orderBy = 'cl_collation, cl_to, cl_type, cl_from'; + } + $options = [ + 'LIMIT' => self::BATCH_SIZE, + 'ORDER BY' => $orderBy, + 'STRAIGHT_JOIN' // per T58041 + ]; + + if ( $force ) { + $collationConds = []; + } else { + if ( $this->hasOption( 'previous-collation' ) ) { + $collationConds['cl_collation'] = $this->getOption( 'previous-collation' ); + } else { + $collationConds = [ 0 => + 'cl_collation != ' . $dbw->addQuotes( $collationName ) + ]; + } + + $count = $dbr->estimateRowCount( + 'categorylinks', + '*', + $collationConds, + __METHOD__ + ); + // Improve estimate if feasible + if ( $count < 1000000 ) { + $count = $dbr->selectField( + 'categorylinks', + 'COUNT(*)', + $collationConds, + __METHOD__ + ); + } + if ( $count == 0 ) { + $this->output( "Collations up-to-date.\n" ); + + return; + } + if ( $dryRun ) { + $this->output( "$count rows would be updated.\n" ); + } else { + $this->output( "Fixing collation for $count rows.\n" ); + } + wfWaitForSlaves(); + } + $count = 0; + $batchCount = 0; + $batchConds = []; + do { + $this->output( "Selecting next " . self::BATCH_SIZE . " rows..." ); + + // cl_type must be selected as a number for proper paging because + // enums suck. + if ( $dbw->getType() === 'mysql' ) { + $clType = 'cl_type+0 AS "cl_type_numeric"'; + } else { + $clType = 'cl_type'; + } + $res = $dbw->select( + [ 'categorylinks', 'page' ], + [ 'cl_from', 'cl_to', 'cl_sortkey_prefix', 'cl_collation', + 'cl_sortkey', $clType, + 'page_namespace', 'page_title' + ], + array_merge( $collationConds, $batchConds, [ 'cl_from = page_id' ] ), + __METHOD__, + $options + ); + $this->output( " processing..." ); + + if ( !$dryRun ) { + $this->beginTransaction( $dbw, __METHOD__ ); + } + foreach ( $res as $row ) { + $title = Title::newFromRow( $row ); + if ( !$row->cl_collation ) { + # This is an old-style row, so the sortkey needs to be + # converted. + if ( $row->cl_sortkey == $title->getText() + || $row->cl_sortkey == $title->getPrefixedText() + ) { + $prefix = ''; + } else { + # Custom sortkey, use it as a prefix + $prefix = $row->cl_sortkey; + } + } else { + $prefix = $row->cl_sortkey_prefix; + } + # cl_type will be wrong for lots of pages if cl_collation is 0, + # so let's update it while we're here. + if ( $title->getNamespace() == NS_CATEGORY ) { + $type = 'subcat'; + } elseif ( $title->getNamespace() == NS_FILE ) { + $type = 'file'; + } else { + $type = 'page'; + } + $newSortKey = $collation->getSortKey( + $title->getCategorySortkey( $prefix ) ); + if ( $verboseStats ) { + $this->updateSortKeySizeHistogram( $newSortKey ); + } + + if ( !$dryRun ) { + $dbw->update( + 'categorylinks', + [ + 'cl_sortkey' => $newSortKey, + 'cl_sortkey_prefix' => $prefix, + 'cl_collation' => $collationName, + 'cl_type' => $type, + 'cl_timestamp = cl_timestamp', + ], + [ 'cl_from' => $row->cl_from, 'cl_to' => $row->cl_to ], + __METHOD__ + ); + } + if ( $row ) { + $batchConds = [ $this->getBatchCondition( $row, $dbw ) ]; + } + } + if ( !$dryRun ) { + $this->commitTransaction( $dbw, __METHOD__ ); + } + + $count += $res->numRows(); + $this->output( "$count done.\n" ); + + if ( !$dryRun && ++$batchCount % self::SYNC_INTERVAL == 0 ) { + $this->output( "Waiting for replica DBs ... " ); + wfWaitForSlaves(); + $this->output( "done\n" ); + } + } while ( $res->numRows() == self::BATCH_SIZE ); + + $this->output( "$count rows processed\n" ); + + if ( $verboseStats ) { + $this->output( "\n" ); + $this->showSortKeySizeHistogram(); + } + } + + /** + * Return an SQL expression selecting rows which sort above the given row, + * assuming an ordering of cl_collation, cl_to, cl_type, cl_from + * @param stdClass $row + * @param IDatabase $dbw + * @return string + */ + function getBatchCondition( $row, $dbw ) { + if ( $this->hasOption( 'previous-collation' ) ) { + $fields = [ 'cl_to', 'cl_type', 'cl_from' ]; + } else { + $fields = [ 'cl_collation', 'cl_to', 'cl_type', 'cl_from' ]; + } + $first = true; + $cond = false; + $prefix = false; + foreach ( $fields as $field ) { + if ( $dbw->getType() === 'mysql' && $field === 'cl_type' ) { + // Range conditions with enums are weird in mysql + // This must be a numeric literal, or it won't work. + $encValue = intval( $row->cl_type_numeric ); + } else { + $encValue = $dbw->addQuotes( $row->$field ); + } + $inequality = "$field > $encValue"; + $equality = "$field = $encValue"; + if ( $first ) { + $cond = $inequality; + $prefix = $equality; + $first = false; + } else { + $cond .= " OR ($prefix AND $inequality)"; + $prefix .= " AND $equality"; + } + } + + return $cond; + } + + function updateSortKeySizeHistogram( $key ) { + $length = strlen( $key ); + if ( !isset( $this->sizeHistogram[$length] ) ) { + $this->sizeHistogram[$length] = 0; + } + $this->sizeHistogram[$length]++; + } + + function showSortKeySizeHistogram() { + $maxLength = max( array_keys( $this->sizeHistogram ) ); + if ( $maxLength == 0 ) { + return; + } + $numBins = 20; + $coarseHistogram = array_fill( 0, $numBins, 0 ); + $coarseBoundaries = []; + $boundary = 0; + for ( $i = 0; $i < $numBins - 1; $i++ ) { + $boundary += $maxLength / $numBins; + $coarseBoundaries[$i] = round( $boundary ); + } + $coarseBoundaries[$numBins - 1] = $maxLength + 1; + $raw = ''; + for ( $i = 0; $i <= $maxLength; $i++ ) { + if ( $raw !== '' ) { + $raw .= ', '; + } + if ( !isset( $this->sizeHistogram[$i] ) ) { + $val = 0; + } else { + $val = $this->sizeHistogram[$i]; + } + for ( $coarseIndex = 0; $coarseIndex < $numBins - 1; $coarseIndex++ ) { + if ( $coarseBoundaries[$coarseIndex] > $i ) { + $coarseHistogram[$coarseIndex] += $val; + break; + } + } + if ( $coarseIndex == $numBins - 1 ) { + $coarseHistogram[$coarseIndex] += $val; + } + $raw .= $val; + } + + $this->output( "Sort key size histogram\nRaw data: $raw\n\n" ); + + $maxBinVal = max( $coarseHistogram ); + $scale = 60 / $maxBinVal; + $prevBoundary = 0; + for ( $coarseIndex = 0; $coarseIndex < $numBins; $coarseIndex++ ) { + if ( !isset( $coarseHistogram[$coarseIndex] ) ) { + $val = 0; + } else { + $val = $coarseHistogram[$coarseIndex]; + } + $boundary = $coarseBoundaries[$coarseIndex]; + $this->output( sprintf( "%-10s %-10d |%s\n", + $prevBoundary . '-' . ( $boundary - 1 ) . ': ', + $val, + str_repeat( '*', $scale * $val ) ) ); + $prevBoundary = $boundary; + } + } +} + +$maintClass = "UpdateCollation"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/updateCredits.php b/www/wiki/maintenance/updateCredits.php new file mode 100644 index 00000000..b7e8c1cc --- /dev/null +++ b/www/wiki/maintenance/updateCredits.php @@ -0,0 +1,80 @@ +<?php +/** + * Update the CREDITS list by merging in the list of git commit authors. + * + * The contents of the existing contributors list will be preserved. If a name + * needs to be removed for some reason that must be done manually before or + * after running this script. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + */ + +if ( PHP_SAPI != 'cli' ) { + die( "This script can only be run from the command line.\n" ); +} + +$CREDITS = 'CREDITS'; +$START_CONTRIBUTORS = '<!-- BEGIN CONTRIBUTOR LIST -->'; +$END_CONTRIBUTORS = '<!-- END CONTRIBUTOR LIST -->'; + +$inHeader = true; +$inFooter = false; +$header = []; +$contributors = []; +$footer = []; + +if ( !file_exists( $CREDITS ) ) { + exit( 'No CREDITS file found. Are you running this script in the right directory?' ); +} + +$lines = explode( "\n", file_get_contents( $CREDITS ) ); +foreach ( $lines as $line ) { + if ( $inHeader ) { + $header[] = $line; + $inHeader = $line !== $START_CONTRIBUTORS; + } elseif ( $inFooter ) { + $footer[] = $line; + } elseif ( $line == $END_CONTRIBUTORS ) { + $inFooter = true; + $footer[] = $line; + } else { + $name = substr( $line, 2 ); + $contributors[$name] = true; + } +} +unset( $lines ); + +$lines = explode( "\n", shell_exec( 'git log --format="%aN"' ) ); +foreach ( $lines as $line ) { + if ( empty( $line ) ) { + continue; + } + if ( substr( $line, 0, 5 ) === '[BOT]' ) { + continue; + } + $contributors[$line] = true; +} + +$contributors = array_keys( $contributors ); +$collator = Collator::create( 'root' ); +$collator->setAttribute( Collator::NUMERIC_COLLATION, Collator::ON ); +$collator->sort( $contributors ); +array_walk( $contributors, function ( &$v, $k ) { + $v = "* {$v}"; +} ); + +file_put_contents( $CREDITS, + implode( "\n", array_merge( $header, $contributors, $footer ) ) ); diff --git a/www/wiki/maintenance/updateDoubleWidthSearch.php b/www/wiki/maintenance/updateDoubleWidthSearch.php new file mode 100644 index 00000000..cb2f125e --- /dev/null +++ b/www/wiki/maintenance/updateDoubleWidthSearch.php @@ -0,0 +1,81 @@ +<?php +/** + * Normalize double-byte latin UTF-8 characters + * + * Usage: php updateDoubleWidthSearch.php + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to normalize double-byte latin UTF-8 characters. + * + * @ingroup Maintenance + */ +class UpdateDoubleWidthSearch extends Maintenance { + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Script to normalize double-byte latin UTF-8 characters' ); + $this->addOption( 'q', 'quiet', false, true ); + $this->addOption( + 'l', + 'How long the searchindex and revision tables will be locked for', + false, + true + ); + } + + public function getDbType() { + return Maintenance::DB_ADMIN; + } + + public function execute() { + $maxLockTime = $this->getOption( 'l', 20 ); + + $dbw = $this->getDB( DB_MASTER ); + if ( $dbw->getType() !== 'mysql' ) { + $this->error( "This change is only needed on MySQL, quitting.\n", true ); + } + + $res = $this->findRows( $dbw ); + $this->updateSearchIndex( $maxLockTime, [ $this, 'searchIndexUpdateCallback' ], $dbw, $res ); + + $this->output( "Done\n" ); + } + + public function searchIndexUpdateCallback( $dbw, $row ) { + return $this->updateSearchIndexForPage( $dbw, $row->si_page ); + } + + private function findRows( $dbw ) { + $searchindex = $dbw->tableName( 'searchindex' ); + $regexp = '[[:<:]]u8efbd([89][1-9a]|8[b-f]|90)[[:>:]]'; + $sql = "SELECT si_page FROM $searchindex + WHERE ( si_text RLIKE '$regexp' ) + OR ( si_title RLIKE '$regexp' )"; + + return $dbw->query( $sql, __METHOD__ ); + } +} + +$maintClass = "UpdateDoubleWidthSearch"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/updateExtensionJsonSchema.php b/www/wiki/maintenance/updateExtensionJsonSchema.php new file mode 100644 index 00000000..427769f1 --- /dev/null +++ b/www/wiki/maintenance/updateExtensionJsonSchema.php @@ -0,0 +1,69 @@ +<?php + +require_once __DIR__ . '/Maintenance.php'; + +class UpdateExtensionJsonSchema extends Maintenance { + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Updates extension.json files to the latest manifest_version' ); + $this->addArg( 'path', 'Location to the extension.json or skin.json you wish to convert', + /* $required = */ true ); + } + + public function execute() { + $filename = $this->getArg( 0 ); + if ( !is_readable( $filename ) ) { + $this->error( "Error: Unable to read $filename", 1 ); + } + + $json = FormatJson::decode( file_get_contents( $filename ), true ); + if ( $json === null ) { + $this->error( "Error: Invalid JSON", 1 ); + } + + if ( !isset( $json['manifest_version'] ) ) { + $json['manifest_version'] = 1; + } + + if ( $json['manifest_version'] == ExtensionRegistry::MANIFEST_VERSION ) { + $this->output( "Already at the latest version: {$json['manifest_version']}\n" ); + return; + } + + while ( $json['manifest_version'] !== ExtensionRegistry::MANIFEST_VERSION ) { + $json['manifest_version'] += 1; + $func = "updateTo{$json['manifest_version']}"; + $this->$func( $json ); + } + + file_put_contents( $filename, FormatJson::encode( $json, "\t", FormatJson::ALL_OK ) . "\n" ); + $this->output( "Updated to {$json['manifest_version']}...\n" ); + } + + protected function updateTo2( &$json ) { + if ( isset( $json['config'] ) ) { + $config = $json['config']; + $json['config'] = []; + if ( isset( $config['_prefix'] ) ) { + $json = wfArrayInsertAfter( $json, [ + 'config_prefix' => $config['_prefix'] + ], 'config' ); + unset( $config['_prefix'] ); + } + + foreach ( $config as $name => $value ) { + if ( $name[0] !== '@' ) { + $json['config'][$name] = [ 'value' => $value ]; + if ( isset( $value[ExtensionRegistry::MERGE_STRATEGY] ) ) { + $json['config'][$name]['merge_strategy'] = $value[ExtensionRegistry::MERGE_STRATEGY]; + unset( $value[ExtensionRegistry::MERGE_STRATEGY] ); + } + } + } + } + } +} + +$maintClass = 'UpdateExtensionJsonSchema'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/updateRestrictions.php b/www/wiki/maintenance/updateRestrictions.php new file mode 100644 index 00000000..2f3fc365 --- /dev/null +++ b/www/wiki/maintenance/updateRestrictions.php @@ -0,0 +1,128 @@ +<?php +/** + * Makes the required database updates for Special:ProtectedPages + * to show all protected pages, even ones before the page restrictions + * schema change. All remaining page_restriction column values are moved + * to the new table. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script that updates page_restrictions table from + * old page_restriction column. + * + * @ingroup Maintenance + */ +class UpdateRestrictions extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Updates page_restrictions table from old page_restriction column' ); + $this->setBatchSize( 1000 ); + } + + public function execute() { + $db = $this->getDB( DB_MASTER ); + if ( !$db->tableExists( 'page_restrictions' ) ) { + $this->error( "page_restrictions table does not exist", true ); + } + + $start = $db->selectField( 'page', 'MIN(page_id)', false, __METHOD__ ); + if ( !$start ) { + $this->error( "Nothing to do.", true ); + } + $end = $db->selectField( 'page', 'MAX(page_id)', false, __METHOD__ ); + + # Do remaining chunk + $end += $this->mBatchSize - 1; + $blockStart = $start; + $blockEnd = $start + $this->mBatchSize - 1; + $encodedExpiry = 'infinity'; + while ( $blockEnd <= $end ) { + $this->output( "...doing page_id from $blockStart to $blockEnd out of $end\n" ); + $cond = "page_id BETWEEN $blockStart AND $blockEnd AND page_restrictions !=''"; + $res = $db->select( + 'page', + [ 'page_id', 'page_namespace', 'page_restrictions' ], + $cond, + __METHOD__ + ); + $batch = []; + foreach ( $res as $row ) { + $oldRestrictions = []; + foreach ( explode( ':', trim( $row->page_restrictions ) ) as $restrict ) { + $temp = explode( '=', trim( $restrict ) ); + // Make sure we are not settings restrictions to "" + if ( count( $temp ) == 1 && $temp[0] ) { + // old old format should be treated as edit/move restriction + $oldRestrictions["edit"] = trim( $temp[0] ); + $oldRestrictions["move"] = trim( $temp[0] ); + } elseif ( $temp[1] ) { + $oldRestrictions[$temp[0]] = trim( $temp[1] ); + } + } + # Clear invalid columns + if ( $row->page_namespace == NS_MEDIAWIKI ) { + $db->update( 'page', [ 'page_restrictions' => '' ], + [ 'page_id' => $row->page_id ], __FUNCTION__ ); + $this->output( "...removed dead page_restrictions column for page {$row->page_id}\n" ); + } + # Update restrictions table + foreach ( $oldRestrictions as $action => $restrictions ) { + $batch[] = [ + 'pr_page' => $row->page_id, + 'pr_type' => $action, + 'pr_level' => $restrictions, + 'pr_cascade' => 0, + 'pr_expiry' => $encodedExpiry + ]; + } + } + # We use insert() and not replace() as Article.php replaces + # page_restrictions with '' when protected in the restrictions table + if ( count( $batch ) ) { + $ok = $db->deadlockLoop( [ $db, 'insert' ], 'page_restrictions', + $batch, __FUNCTION__, [ 'IGNORE' ] ); + if ( !$ok ) { + throw new MWException( "Deadlock loop failed wtf :(" ); + } + } + $blockStart += $this->mBatchSize - 1; + $blockEnd += $this->mBatchSize - 1; + wfWaitForSlaves(); + } + $this->output( "...removing dead rows from page_restrictions\n" ); + // Kill any broken rows from previous imports + $db->delete( 'page_restrictions', [ 'pr_level' => '' ] ); + // Kill other invalid rows + $db->deleteJoin( + 'page_restrictions', + 'page', + 'pr_page', + 'page_id', + [ 'page_namespace' => NS_MEDIAWIKI ] + ); + $this->output( "...Done!\n" ); + } +} + +$maintClass = "UpdateRestrictions"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/updateSearchIndex.php b/www/wiki/maintenance/updateSearchIndex.php new file mode 100644 index 00000000..cdb7d9f7 --- /dev/null +++ b/www/wiki/maintenance/updateSearchIndex.php @@ -0,0 +1,125 @@ +<?php +/** + * Periodic off-peak updating of the search index. + * + * Usage: php updateSearchIndex.php [-s START] [-e END] [-p POSFILE] [-l LOCKTIME] [-q] + * Where START is the starting timestamp + * END is the ending timestamp + * POSFILE is a file to load timestamps from and save them to, searchUpdate.WIKI_ID.pos by default + * LOCKTIME is how long the searchindex and revision tables will be locked for + * -q means quiet + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script for periodic off-peak updating of the search index. + * + * @ingroup Maintenance + */ +class UpdateSearchIndex extends Maintenance { + + public function __construct() { + parent::__construct(); + $this->addDescription( 'Script for periodic off-peak updating of the search index' ); + $this->addOption( 's', 'starting timestamp', false, true ); + $this->addOption( 'e', 'Ending timestamp', false, true ); + $this->addOption( + 'p', + 'File for saving/loading timestamps, searchUpdate.WIKI_ID.pos by default', + false, + true + ); + $this->addOption( + 'l', + 'How long the searchindex and revision tables will be locked for', + false, + true + ); + } + + public function getDbType() { + return Maintenance::DB_ADMIN; + } + + public function execute() { + $posFile = $this->getOption( 'p', 'searchUpdate.' . wfWikiID() . '.pos' ); + $end = $this->getOption( 'e', wfTimestampNow() ); + if ( $this->hasOption( 's' ) ) { + $start = $this->getOption( 's' ); + } elseif ( is_readable( 'searchUpdate.pos' ) ) { + # B/c to the old position file name which was hardcoded + # We can safely delete the file when we're done though. + $start = file_get_contents( 'searchUpdate.pos' ); + unlink( 'searchUpdate.pos' ); + } elseif ( is_readable( $posFile ) ) { + $start = file_get_contents( $posFile ); + } else { + $start = wfTimestamp( TS_MW, time() - 86400 ); + } + $lockTime = $this->getOption( 'l', 20 ); + + $this->doUpdateSearchIndex( $start, $end, $lockTime ); + if ( is_writable( dirname( realpath( $posFile ) ) ) ) { + $file = fopen( $posFile, 'w' ); + if ( $file !== false ) { + fwrite( $file, $end ); + fclose( $file ); + } else { + $this->error( "*** Couldn't write to the $posFile!\n" ); + } + } else { + $this->error( "*** Couldn't write to the $posFile!\n" ); + } + } + + private function doUpdateSearchIndex( $start, $end, $maxLockTime ) { + global $wgDisableSearchUpdate; + + $wgDisableSearchUpdate = false; + + $dbw = $this->getDB( DB_MASTER ); + $recentchanges = $dbw->tableName( 'recentchanges' ); + + $this->output( "Updating searchindex between $start and $end\n" ); + + # Select entries from recentchanges which are on top and between the specified times + $start = $dbw->timestamp( $start ); + $end = $dbw->timestamp( $end ); + + $page = $dbw->tableName( 'page' ); + $sql = "SELECT rc_cur_id FROM $recentchanges + JOIN $page ON rc_cur_id=page_id AND rc_this_oldid=page_latest + WHERE rc_type != " . RC_LOG . " AND rc_timestamp BETWEEN '$start' AND '$end'"; + $res = $dbw->query( $sql, __METHOD__ ); + + $this->updateSearchIndex( $maxLockTime, [ $this, 'searchIndexUpdateCallback' ], $dbw, $res ); + + $this->output( "Done\n" ); + } + + public function searchIndexUpdateCallback( $dbw, $row ) { + $this->updateSearchIndexForPage( $dbw, $row->rc_cur_id ); + } +} + +$maintClass = "UpdateSearchIndex"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/updateSpecialPages.php b/www/wiki/maintenance/updateSpecialPages.php new file mode 100644 index 00000000..5ea38282 --- /dev/null +++ b/www/wiki/maintenance/updateSpecialPages.php @@ -0,0 +1,164 @@ +<?php +/** + * Update for cached special pages. + * Run this script periodically if you have miser mode enabled. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to update cached special pages. + * + * @ingroup Maintenance + */ +class UpdateSpecialPages extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addOption( 'list', 'List special page names' ); + $this->addOption( 'only', 'Only update "page"; case sensitive, ' . + 'check correct case by calling this script with --list. ' . + 'Ex: --only=BrokenRedirects', false, true ); + $this->addOption( 'override', 'Also update pages that have updates disabled' ); + } + + public function execute() { + global $wgQueryCacheLimit, $wgDisableQueryPageUpdate; + + $dbw = $this->getDB( DB_MASTER ); + + $this->doSpecialPageCacheUpdates( $dbw ); + + foreach ( QueryPage::getPages() as $page ) { + list( $class, $special ) = $page; + $limit = isset( $page[2] ) ? $page[2] : null; + + # --list : just show the name of pages + if ( $this->hasOption( 'list' ) ) { + $this->output( "$special [QueryPage]\n" ); + continue; + } + + if ( !$this->hasOption( 'override' ) + && $wgDisableQueryPageUpdate && in_array( $special, $wgDisableQueryPageUpdate ) + ) { + $this->output( sprintf( "%-30s [QueryPage] disabled\n", $special ) ); + continue; + } + + $specialObj = SpecialPageFactory::getPage( $special ); + if ( !$specialObj ) { + $this->output( "No such special page: $special\n" ); + exit; + } + if ( $specialObj instanceof QueryPage ) { + $queryPage = $specialObj; + } else { + $class = get_class( $specialObj ); + $this->error( "$class is not an instance of QueryPage.\n", 1 ); + die; + } + + if ( !$this->hasOption( 'only' ) || $this->getOption( 'only' ) == $queryPage->getName() ) { + $this->output( sprintf( '%-30s [QueryPage] ', $special ) ); + if ( $queryPage->isExpensive() ) { + $t1 = microtime( true ); + # Do the query + $num = $queryPage->recache( $limit === null ? $wgQueryCacheLimit : $limit ); + $t2 = microtime( true ); + if ( $num === false ) { + $this->output( "FAILED: database error\n" ); + } else { + $this->output( "got $num rows in " ); + + $elapsed = $t2 - $t1; + $hours = intval( $elapsed / 3600 ); + $minutes = intval( $elapsed % 3600 / 60 ); + $seconds = $elapsed - $hours * 3600 - $minutes * 60; + if ( $hours ) { + $this->output( $hours . 'h ' ); + } + if ( $minutes ) { + $this->output( $minutes . 'm ' ); + } + $this->output( sprintf( "%.2fs\n", $seconds ) ); + } + # Reopen any connections that have closed + if ( !wfGetLB()->pingAll() ) { + $this->output( "\n" ); + do { + $this->error( "Connection failed, reconnecting in 10 seconds..." ); + sleep( 10 ); + } while ( !wfGetLB()->pingAll() ); + $this->output( "Reconnected\n\n" ); + } + # Wait for the replica DB to catch up + wfWaitForSlaves(); + } else { + $this->output( "cheap, skipped\n" ); + } + if ( $this->hasOption( 'only' ) ) { + break; + } + } + } + } + + public function doSpecialPageCacheUpdates( $dbw ) { + global $wgSpecialPageCacheUpdates; + + foreach ( $wgSpecialPageCacheUpdates as $special => $call ) { + # --list : just show the name of pages + if ( $this->hasOption( 'list' ) ) { + $this->output( "$special [callback]\n" ); + continue; + } + + if ( !$this->hasOption( 'only' ) || $this->getOption( 'only' ) == $special ) { + if ( !is_callable( $call ) ) { + $this->error( "Uncallable function $call!" ); + continue; + } + $this->output( sprintf( '%-30s [callback] ', $special ) ); + $t1 = microtime( true ); + call_user_func( $call, $dbw ); + $t2 = microtime( true ); + + $this->output( "completed in " ); + $elapsed = $t2 - $t1; + $hours = intval( $elapsed / 3600 ); + $minutes = intval( $elapsed % 3600 / 60 ); + $seconds = $elapsed - $hours * 3600 - $minutes * 60; + if ( $hours ) { + $this->output( $hours . 'h ' ); + } + if ( $minutes ) { + $this->output( $minutes . 'm ' ); + } + $this->output( sprintf( "%.2fs\n", $seconds ) ); + # Wait for the replica DB to catch up + wfWaitForSlaves(); + } + } + } +} + +$maintClass = "UpdateSpecialPages"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/userDupes.inc b/www/wiki/maintenance/userDupes.inc new file mode 100644 index 00000000..69c92658 --- /dev/null +++ b/www/wiki/maintenance/userDupes.inc @@ -0,0 +1,297 @@ +<?php +/** + * Helper class for update.php. + * + * Copyright © 2005 Brion Vibber <brion@pobox.com> + * https://www.mediawiki.org/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +/** + * Look for duplicate user table entries and optionally prune them. + * + * This is still used by our MysqlUpdater at: + * includes/installer/MysqlUpdater.php + * + * @ingroup Maintenance + */ +class UserDupes { + private $db; + private $reassigned; + private $trimmed; + private $failed; + private $outputCallback; + + function __construct( &$database, $outputCallback ) { + $this->db = $database; + $this->outputCallback = $outputCallback; + } + + /** + * Output some text via the output callback provided + * @param string $str Text to print + */ + private function out( $str ) { + call_user_func( $this->outputCallback, $str ); + } + + /** + * Check if this database's user table has already had a unique + * user_name index applied. + * @return bool + */ + function hasUniqueIndex() { + $info = $this->db->indexInfo( 'user', 'user_name', __METHOD__ ); + if ( !$info ) { + $this->out( "WARNING: doesn't seem to have user_name index at all!\n" ); + + return false; + } + + # Confusingly, 'Non_unique' is 0 for *unique* indexes, + # and 1 for *non-unique* indexes. Pass the crack, MySQL, + # it's obviously some good stuff! + return ( $info[0]->Non_unique == 0 ); + } + + /** + * Checks the database for duplicate user account records + * and remove them in preparation for application of a unique + * index on the user_name field. Returns true if the table is + * clean or if duplicates have been resolved automatically. + * + * May return false if there are unresolvable problems. + * Status information will be echo'd to stdout. + * + * @return bool + */ + function clearDupes() { + return $this->checkDupes( true ); + } + + /** + * Checks the database for duplicate user account records + * in preparation for application of a unique index on the + * user_name field. Returns true if the table is clean or + * if duplicates can be resolved automatically. + * + * Returns false if there are duplicates and resolution was + * not requested. (If doing resolution, edits may be reassigned.) + * Status information will be echo'd to stdout. + * + * @param bool $doDelete Pass true to actually remove things + * from the database; false to just check. + * @return bool + */ + function checkDupes( $doDelete = false ) { + if ( $this->hasUniqueIndex() ) { + echo wfWikiID() . " already has a unique index on its user table.\n"; + + return true; + } + + $this->lock(); + + $this->out( "Checking for duplicate accounts...\n" ); + $dupes = $this->getDupes(); + $count = count( $dupes ); + + $this->out( "Found $count accounts with duplicate records on " . wfWikiID() . ".\n" ); + $this->trimmed = 0; + $this->reassigned = 0; + $this->failed = 0; + foreach ( $dupes as $name ) { + $this->examine( $name, $doDelete ); + } + + $this->unlock(); + + $this->out( "\n" ); + + if ( $this->reassigned > 0 ) { + if ( $doDelete ) { + $this->out( "$this->reassigned duplicate accounts had edits " + . "reassigned to a canonical record id.\n" ); + } else { + $this->out( "$this->reassigned duplicate accounts need to have edits reassigned.\n" ); + } + } + + if ( $this->trimmed > 0 ) { + if ( $doDelete ) { + $this->out( "$this->trimmed duplicate user records were deleted from " + . wfWikiID() . ".\n" ); + } else { + $this->out( "$this->trimmed duplicate user accounts were found on " + . wfWikiID() . " which can be removed safely.\n" ); + } + } + + if ( $this->failed > 0 ) { + $this->out( "Something terribly awry; $this->failed duplicate accounts were not removed.\n" ); + + return false; + } + + if ( $this->trimmed == 0 || $doDelete ) { + $this->out( "It is now safe to apply the unique index on user_name.\n" ); + + return true; + } else { + $this->out( "Run this script again with the --fix option to automatically delete them.\n" ); + + return false; + } + } + + /** + * We don't want anybody to mess with our stuff... + * @access private + */ + function lock() { + $set = [ 'user', 'revision' ]; + $names = array_map( [ $this, 'lockTable' ], $set ); + $tables = implode( ',', $names ); + + $this->db->query( "LOCK TABLES $tables", __METHOD__ ); + } + + function lockTable( $table ) { + return $this->db->tableName( $table ) . ' WRITE'; + } + + /** + * @access private + */ + function unlock() { + $this->db->query( "UNLOCK TABLES", __METHOD__ ); + } + + /** + * Grab usernames for which multiple records are present in the database. + * @return array + * @access private + */ + function getDupes() { + $user = $this->db->tableName( 'user' ); + $result = $this->db->query( + "SELECT user_name,COUNT(*) AS n + FROM $user + GROUP BY user_name + HAVING n > 1", __METHOD__ ); + + $list = []; + foreach ( $result as $row ) { + $list[] = $row->user_name; + } + + return $list; + } + + /** + * Examine user records for the given name. Try to see which record + * will be the one that actually gets used, then check remaining records + * for edits. If the dupes have no edits, we can safely remove them. + * @param string $name + * @param bool $doDelete + * @access private + */ + function examine( $name, $doDelete ) { + $result = $this->db->select( 'user', + [ 'user_id' ], + [ 'user_name' => $name ], + __METHOD__ ); + + $firstRow = $this->db->fetchObject( $result ); + $firstId = $firstRow->user_id; + $this->out( "Record that will be used for '$name' is user_id=$firstId\n" ); + + foreach ( $result as $row ) { + $dupeId = $row->user_id; + $this->out( "... dupe id $dupeId: " ); + $edits = $this->editCount( $dupeId ); + if ( $edits > 0 ) { + $this->reassigned++; + $this->out( "has $edits edits! " ); + if ( $doDelete ) { + $this->reassignEdits( $dupeId, $firstId ); + $newEdits = $this->editCount( $dupeId ); + if ( $newEdits == 0 ) { + $this->out( "confirmed cleaned. " ); + } else { + $this->failed++; + $this->out( "WARNING! $newEdits remaining edits for $dupeId; NOT deleting user.\n" ); + continue; + } + } else { + $this->out( "(will need to reassign edits on fix)" ); + } + } else { + $this->out( "ok, no edits. " ); + } + $this->trimmed++; + if ( $doDelete ) { + $this->trimAccount( $dupeId ); + } + $this->out( "\n" ); + } + } + + /** + * Count the number of edits attributed to this user. + * Does not currently check log table or other things + * where it might show up... + * @param int $userid + * @return int + * @access private + */ + function editCount( $userid ) { + return intval( $this->db->selectField( + 'revision', + 'COUNT(*)', + [ 'rev_user' => $userid ], + __METHOD__ ) ); + } + + /** + * @param int $from + * @param int $to + * @access private + */ + function reassignEdits( $from, $to ) { + $this->out( 'reassigning... ' ); + $this->db->update( 'revision', + [ 'rev_user' => $to ], + [ 'rev_user' => $from ], + __METHOD__ ); + $this->out( "ok. " ); + } + + /** + * Remove a user account line. + * @param int $userid + * @access private + */ + function trimAccount( $userid ) { + $this->out( "deleting..." ); + $this->db->delete( 'user', [ 'user_id' => $userid ], __METHOD__ ); + $this->out( " ok" ); + } +} diff --git a/www/wiki/maintenance/userOptions.inc b/www/wiki/maintenance/userOptions.inc new file mode 100644 index 00000000..8ac7f919 --- /dev/null +++ b/www/wiki/maintenance/userOptions.inc @@ -0,0 +1,292 @@ +<?php +/** + * Helper class for userOptions.php script. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +// Options we will use +$options = [ 'list', 'nowarn', 'quiet', 'usage', 'dry' ]; +$optionsWithArgs = [ 'old', 'new' ]; + +require_once __DIR__ . '/commandLine.inc'; + +/** + * @ingroup Maintenance + */ +class UserOptions { + public $mQuick; + public $mQuiet; + public $mDry; + public $mAnOption; + public $mOldValue; + public $mNewValue; + + private $mMode, $mReady; + + /** + * Constructor. Will show usage and exit if script options are not correct + * @param array $opts + * @param array $args + */ + function __construct( $opts, $args ) { + if ( !$this->checkOpts( $opts, $args ) ) { + self::showUsageAndExit(); + } else { + $this->mReady = $this->initializeOpts( $opts, $args ); + } + } + + /** + * This is used to check options. Only needed on construction + * + * @param array $opts + * @param array $args + * + * @return bool + */ + private function checkOpts( $opts, $args ) { + // The three possible ways to run the script: + $list = isset( $opts['list'] ); + $usage = isset( $opts['usage'] ) && ( count( $args ) <= 1 ); + $change = isset( $opts['old'] ) && isset( $opts['new'] ) && ( count( $args ) <= 1 ); + + // We want only one of them + $isValid = ( ( $list + $usage + $change ) == 1 ); + + return $isValid; + } + + /** + * load script options in the object + * + * @param array $opts + * @param array $args + * + * @return bool + */ + private function initializeOpts( $opts, $args ) { + $this->mQuick = isset( $opts['nowarn'] ); + $this->mQuiet = isset( $opts['quiet'] ); + $this->mDry = isset( $opts['dry'] ); + + // Set object properties, specially 'mMode' used by run() + if ( isset( $opts['list'] ) ) { + $this->mMode = 'LISTER'; + } elseif ( isset( $opts['usage'] ) ) { + $this->mMode = 'USAGER'; + $this->mAnOption = isset( $args[0] ) ? $args[0] : false; + } elseif ( isset( $opts['old'] ) && isset( $opts['new'] ) ) { + $this->mMode = 'CHANGER'; + $this->mOldValue = $opts['old']; + $this->mNewValue = $opts['new']; + $this->mAnOption = $args[0]; + } else { + die( "There is a bug in the software, this should never happen\n" ); + } + + return true; + } + + /** + * Dumb stuff to run a mode. + * @return bool + */ + public function run() { + if ( !$this->mReady ) { + return false; + } + + $this->{$this->mMode}(); + + return true; + } + + /** + * List default options and their value + */ + private function LISTER() { + $def = User::getDefaultOptions(); + ksort( $def ); + $maxOpt = 0; + foreach ( $def as $opt => $value ) { + $maxOpt = max( $maxOpt, strlen( $opt ) ); + } + foreach ( $def as $opt => $value ) { + printf( "%-{$maxOpt}s: %s\n", $opt, $value ); + } + } + + /** + * List options usage + */ + private function USAGER() { + $ret = []; + $defaultOptions = User::getDefaultOptions(); + + // We list user by user_id from one of the replica DBs + $dbr = wfGetDB( DB_REPLICA ); + $result = $dbr->select( 'user', + [ 'user_id' ], + [], + __METHOD__ + ); + + foreach ( $result as $id ) { + $user = User::newFromId( $id->user_id ); + + // Get the options and update stats + if ( $this->mAnOption ) { + if ( !array_key_exists( $this->mAnOption, $defaultOptions ) ) { + print "Invalid user option. Use --list to see valid choices\n"; + exit; + } + + $userValue = $user->getOption( $this->mAnOption ); + if ( $userValue <> $defaultOptions[$this->mAnOption] ) { + // @codingStandardsIgnoreStart Ignore silencing errors is discouraged warning + @$ret[$this->mAnOption][$userValue]++; + // @codingStandardsIgnoreEnd + } + } else { + + foreach ( $defaultOptions as $name => $defaultValue ) { + $userValue = $user->getOption( $name ); + if ( $userValue <> $defaultValue ) { + // @codingStandardsIgnoreStart Ignore silencing errors is discouraged warning + @$ret[$name][$userValue]++; + // @codingStandardsIgnoreEnd + } + } + } + } + + foreach ( $ret as $optionName => $usageStats ) { + print "Usage for <$optionName> (default: '{$defaultOptions[$optionName]}'):\n"; + foreach ( $usageStats as $value => $count ) { + print " $count user(s): '$value'\n"; + } + print "\n"; + } + } + + /** + * Change our users options + */ + private function CHANGER() { + $this->warn(); + + // We list user by user_id from one of the replica DBs + $dbr = wfGetDB( DB_REPLICA ); + $result = $dbr->select( 'user', + [ 'user_id' ], + [], + __METHOD__ + ); + + foreach ( $result as $id ) { + $user = User::newFromId( $id->user_id ); + + $curValue = $user->getOption( $this->mAnOption ); + $username = $user->getName(); + + if ( $curValue == $this->mOldValue ) { + if ( !$this->mQuiet ) { + print "Setting {$this->mAnOption} for $username from '{$this->mOldValue}' " . + "to '{$this->mNewValue}'): "; + } + + // Change value + $user->setOption( $this->mAnOption, $this->mNewValue ); + + // Will not save the settings if run with --dry + if ( !$this->mDry ) { + $user->saveSettings(); + } + if ( !$this->mQuiet ) { + print " OK\n"; + } + } elseif ( !$this->mQuiet ) { + print "Not changing '$username' using <{$this->mAnOption}> = '$curValue'\n"; + } + } + } + + /** + * Return an array of option names + * @return array + */ + public static function getDefaultOptionsNames() { + $def = User::getDefaultOptions(); + $ret = []; + foreach ( $def as $optname => $defaultValue ) { + array_push( $ret, $optname ); + } + + return $ret; + } + + public static function showUsageAndExit() { + print <<<USAGE + +This script pass through all users and change one of their options. +The new option is NOT validated. + +Usage: + php userOptions.php --list + php userOptions.php [user option] --usage + php userOptions.php [options] <user option> --old <old value> --new <new value> + +Switchs: + --list : list available user options and their default value + + --usage : report all options statistics or just one if you specify it. + + --old <old value> : the value to look for + --new <new value> : new value to update users with + +Options: + --nowarn: hides the 5 seconds warning + --quiet : do not print what is happening + --dry : do not save user settings back to database + +USAGE; + exit( 0 ); + } + + /** + * The warning message and countdown + * @return bool + */ + public function warn() { + if ( $this->mQuick ) { + return true; + } + + print <<<WARN +The script is about to change the skin for ALL USERS in the database. +Users with option <$this->mAnOption> = '$this->mOldValue' will be made to use '$this->mNewValue'. + +Abort with control-c in the next five seconds.... +WARN; + wfCountDown( 5 ); + + return true; + } +} diff --git a/www/wiki/maintenance/userOptions.php b/www/wiki/maintenance/userOptions.php new file mode 100644 index 00000000..53db48cd --- /dev/null +++ b/www/wiki/maintenance/userOptions.php @@ -0,0 +1,35 @@ +<?php +/** + * Script to change users preferences on the fly. + * + * Made on an original idea by Fooey (freenode) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @author Antoine Musso <hashar at free dot fr> + */ + +// This is a command line script, load tools and parse args +require_once 'userOptions.inc'; + +// Load up our tool system, exit with usage() if options are not fine +$uo = new UserOptions( $options, $args ); + +$uo->run(); + +print "Done.\n"; diff --git a/www/wiki/maintenance/validateRegistrationFile.php b/www/wiki/maintenance/validateRegistrationFile.php new file mode 100644 index 00000000..aa1f668d --- /dev/null +++ b/www/wiki/maintenance/validateRegistrationFile.php @@ -0,0 +1,26 @@ +<?php + +require_once __DIR__ . '/Maintenance.php'; + +class ValidateRegistrationFile extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addArg( 'path', 'Path to extension.json/skin.json file.', true ); + } + public function execute() { + $validator = new ExtensionJsonValidator( function ( $msg ) { + $this->error( $msg, 1 ); + } ); + $validator->checkDependencies(); + $path = $this->getArg( 0 ); + try { + $validator->validate( $path ); + $this->output( "$path validates against the schema!\n" ); + } catch ( ExtensionJsonValidationError $e ) { + $this->error( $e->getMessage(), 1 ); + } + } +} + +$maintClass = 'ValidateRegistrationFile'; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/view.php b/www/wiki/maintenance/view.php new file mode 100644 index 00000000..af7eb2d9 --- /dev/null +++ b/www/wiki/maintenance/view.php @@ -0,0 +1,59 @@ +<?php +/** + * Show page contents. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to show page contents. + * + * @ingroup Maintenance + */ +class ViewCLI extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Show article contents on the command line' ); + $this->addArg( 'title', 'Title of article to view' ); + } + + public function execute() { + $title = Title::newFromText( $this->getArg() ); + if ( !$title ) { + $this->error( "Invalid title", true ); + } + + $page = WikiPage::factory( $title ); + + $content = $page->getContent( Revision::RAW ); + if ( !$content ) { + $this->error( "Page has no content", true ); + } + if ( !$content instanceof TextContent ) { + $this->error( "Non-text content models not supported", true ); + } + + $this->output( $content->getNativeData() ); + } +} + +$maintClass = "ViewCLI"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/www/wiki/maintenance/wrapOldPasswords.php b/www/wiki/maintenance/wrapOldPasswords.php new file mode 100644 index 00000000..1dbad184 --- /dev/null +++ b/www/wiki/maintenance/wrapOldPasswords.php @@ -0,0 +1,125 @@ +<?php + +use MediaWiki\MediaWikiServices; + +/** + * Maintenance script to wrap all old-style passwords in a layered type + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ +require_once __DIR__ . '/Maintenance.php'; + +/** + * Maintenance script to wrap all passwords of a certain type in a specified layered + * type that wraps around the old type. + * + * @since 1.24 + * @ingroup Maintenance + */ +class WrapOldPasswords extends Maintenance { + public function __construct() { + parent::__construct(); + $this->addDescription( 'Wrap all passwords of a certain type in a new layered type' ); + $this->addOption( 'type', + 'Password type to wrap passwords in (must inherit LayeredParameterizedPassword)', true, true ); + $this->addOption( 'verbose', 'Enables verbose output', false, false, 'v' ); + $this->setBatchSize( 100 ); + } + + public function execute() { + $passwordFactory = new PasswordFactory(); + $passwordFactory->init( RequestContext::getMain()->getConfig() ); + + $typeInfo = $passwordFactory->getTypes(); + $layeredType = $this->getOption( 'type' ); + + // Check that type exists and is a layered type + if ( !isset( $typeInfo[$layeredType] ) ) { + $this->error( 'Undefined password type', true ); + } + + $passObj = $passwordFactory->newFromType( $layeredType ); + if ( !$passObj instanceof LayeredParameterizedPassword ) { + $this->error( 'Layered parameterized password type must be used.', true ); + } + + // Extract the first layer type + $typeConfig = $typeInfo[$layeredType]; + $firstType = $typeConfig['types'][0]; + + // Get a list of password types that are applicable + $dbw = $this->getDB( DB_MASTER ); + $typeCond = 'user_password' . $dbw->buildLike( ":$firstType:", $dbw->anyString() ); + + $minUserId = 0; + $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory(); + do { + $this->beginTransaction( $dbw, __METHOD__ ); + + $res = $dbw->select( 'user', + [ 'user_id', 'user_name', 'user_password' ], + [ + 'user_id > ' . $dbw->addQuotes( $minUserId ), + $typeCond + ], + __METHOD__, + [ + 'ORDER BY' => 'user_id', + 'LIMIT' => $this->mBatchSize, + 'LOCK IN SHARE MODE', + ] + ); + + /** @var User[] $updateUsers */ + $updateUsers = []; + foreach ( $res as $row ) { + if ( $this->hasOption( 'verbose' ) ) { + $this->output( "Updating password for user {$row->user_name} ({$row->user_id}).\n" ); + } + + $user = User::newFromId( $row->user_id ); + /** @var ParameterizedPassword $password */ + $password = $passwordFactory->newFromCiphertext( $row->user_password ); + /** @var LayeredParameterizedPassword $layeredPassword */ + $layeredPassword = $passwordFactory->newFromType( $layeredType ); + $layeredPassword->partialCrypt( $password ); + + $updateUsers[] = $user; + $dbw->update( 'user', + [ 'user_password' => $layeredPassword->toString() ], + [ 'user_id' => $row->user_id ], + __METHOD__ + ); + + $minUserId = $row->user_id; + } + + $this->commitTransaction( $dbw, __METHOD__ ); + $lbFactory->waitForReplication(); + + // Clear memcached so old passwords are wiped out + foreach ( $updateUsers as $user ) { + $user->clearSharedCache(); + } + } while ( $res->numRows() ); + } +} + +$maintClass = "WrapOldPasswords"; +require_once RUN_MAINTENANCE_IF_MAIN; |