User:Alterego/ExtensionMatrix/Source code
Extension Matrix Source
[edit]This is the source code for User:Alterego/ExtensionMatrix
username = 'test'
password = 'test'
from re import sub
from sys import path
from dateutil.parser import parse
import datetime
path.append('/usr/local/mwclient')
path.append('/usr/local/mwclient/simplejson')
import client as mwclient
site = mwclient.Site('www.mediawiki.org', path='/w/')
site.login(username,password)
all_extensions = site.categories["All extensions"]
extensions, extensions_dicts, extensions_by_type, extensions_by_status = {}, {}, {}, {}
extensions_by_mw_version, extensions_by_creation_date = {}, {}
recently_edited, recently_discussed, recently_updated, recently_created = [], [], [], []
# converts an extension dict back into template format
def BuildTemplate(extension_dict):
template = '{{ExtensionMatrix\n'
keys = extension_dict.keys()
for key in keys:
# This guy giving me a hard time for some reason
if '<!-' in extension_dict[key] or '-->' in extension_dict[key]:
continue
# Build this line of the template
template += '|' + key + '=' + extension_dict[key] + '\n'
template += '}}\n'
return template
##########################################
# Download the template for each extension
##########################################
for this_extension in all_extensions:
try:
extension_name = this_extension.name.split(':')[1]
# Keep empty vals around to create a list of poorly formatted extensions
extensions[extension_name] = ''
# Extract the wikitext. Normally wouldn't be this simple but
# the extensions are well formatted, each ending with
# \n}}. Could recursively look for sub templates to be more
# sure we're at the end.
wikitext = site.Pages['Extension:' + extension_name].edit()
template_start = wikitext.find('{{Extension')
if template_start == -1:
template_start = wikitext.find('{{extension')
if template_start == -1:
raise
template_end = template_start + wikitext[template_start:].find('\n}}')
template = wikitext[template_start:template_end+3]
extensions[extension_name] = template
except:
# If someone did something stupid, not worth breaking the bot
continue
# With just a little work we can turn the template into a dictionary
# and then do some cleanup processing of its parameters. This bot
# is definitely relying on the fact that the template ends with \n}}
for extension in extensions.keys():
extension_dict = {}
hooks, tags, types = [], [], []
template = extensions[extension]
# Some people like to have funky spacing. Double up just in case
template = template.replace(' |','|').replace(' |','|').replace('| ','|').replace('| ','|')
# This hacks off {{Extension and }}, and has the convenient side effect
# of nuking |templatemode= when it shows up on the first line
template = template.split('\n')[1:-1]
# Can't allow newlines - saw way too many crazy template values. In order for
# this to be sane the template must have a pipe as the first non whitespace char
# on each line
filtered_template = []
for line in template:
if len(line):
if line[0] == '|':
filtered_template.append(line)
template = filtered_template
# Don't allow subpage extensions
if '/' in extension:
continue
for param in template:
try:
param = param.split('=',1)
key = param[0].replace('|','').strip()
value = param[1].strip()
except:
continue # Can't do this? Not my fault.
if not len(value.strip()):
continue
if 'name' in key:
# Sometimes the name field doesn't contain the actual name of the extension
value = extension
if key.find('hook') is not -1:
hooks.append(value)
continue
if key.find('tag') is not -1:
tags.append(value)
continue
if key.find('type') is not -1:
types.append(value)
continue
# Have a look at LocalisationUpdate for nested templateness that is just not ok.
if '{{' in value and not '}}' in value:
continue
if '}}' in value and not '{{' in value:
continue
# These always turn out to be copy/paste jobs from the prototype template
if '<!-' in value or '-->' in value:
continue
if '<ref>' in value:
value = value.replace('<ref>',' ')
if '</ref>' in value:
value = value.replace('</ref>',' ')
extension_dict[key] = value
if hooks:
hooks.sort()
hooks = '<br/>'.join(hooks)
extension_dict['hooks'] = hooks
if tags:
tags.sort()
tags = '<br/>'.join(tags)
extension_dict['tags'] = tags
if types:
types.sort()
types = '<br/>'.join(types)
extension_dict['types'] = types
# Sometimes the name isn't specified at all
if not extension_dict.has_key('name'):
extension_dict['name'] = extension
# If there is a newline in a template parameter, that's probably
# going to mess things up
keys = extension_dict.keys()
# Don't allow empty templates, or templates with just one parameter
if not len(keys) or len(keys) == 1:
continue
extensions_dicts[extension] = extension_dict
# sorted list of the full matrix for later use
sorted_matrix = extensions_dicts.keys()
sorted_matrix.sort()
# convert all parseable dates into a common wikitable-sortable format
months = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec']
for extension in extensions_dicts.keys():
if extensions_dicts[extension].has_key('update'):
try:
this_date = parse(extensions_dicts[extension]['update'])
this_day = this_date.day
this_month = months[this_date.month-1]
this_year = this_date.year
extensions_dicts[extension]['update'] = str(this_day) + ' ' + \
str(this_month) + ' ' + \
str(this_year)
except:
del extensions_dicts[extension]['update']
##########################################
# figure out what versions of mediawiki this extension works on
# this just looks for a string match of the version. i personally
# don't trust the +,>=,etc.. sign people like to use, for example, 1.12+.
# that generally means that they tested it on 1.12, but not the
# versions that came afterwards.
##########################################
for extension in extensions_dicts.keys():
if extensions_dicts[extension].has_key('mediawiki'):
supported_versions = []
version_text = extensions_dicts[extension]['mediawiki']
for major_version in xrange(1,3):
for version in xrange(0,20):
this_version = str(major_version) + '.' + str(version)
# TODO: Hacky. This still fails on i.e.
# - 1.11.0+ in that it puts it in 1.0 because it matches on "1.0+"
# - mediawiki 1.16 <= accesscontrol 1.3 because it matches on "1.3 "
# - 1.6.x, 1.8.x, 1.9.x or higher (not tested by author on most recent MW versions - i.e. > 1.12) because it matches on "1.12)"
if version_text.find(this_version + " ") != -1 or \
version_text.find(this_version + "\n") != -1 or \
version_text.find(this_version + "(") != -1 or \
version_text.find(this_version + ".") != -1 or \
version_text.find(this_version + "+") != -1:
supported_versions.append(this_version)
if not extensions_by_mw_version.has_key(this_version):
extensions_by_mw_version[this_version] = [extension]
else:
extensions_by_mw_version[this_version].append(extension)
extensions_dicts[extension]['mediawiki'] = ', '.join(supported_versions)
##########################################
# Get the last day that each extension and its talk page were edited
# and the creation date of the extension
##########################################
for extension in extensions_dicts.keys():
this_extension = site.Pages["Extension:" + extension]
if this_extension.exists: # should never fail!
this_date = this_extension.touched
this_day = this_date.tm_mday
this_month = months[this_date.tm_mon-1]
this_year = this_date.tm_year
this_date = str(this_day) + ' ' + \
str(this_month) + ' ' + \
str(this_year)
extensions_dicts[extension]['lastupdated'] = this_date
first_edit_timestamp = list(this_extension.revisions())[-1]['timestamp']
first_edit_year = first_edit_timestamp.tm_year
first_edit_month = months[first_edit_timestamp.tm_mon-1]
first_edit_day = first_edit_timestamp.tm_mday
first_edit_date = str(first_edit_day) + ' ' + \
str(first_edit_month) + ' ' + \
str(first_edit_year)
extensions_dicts[extension]['created'] = first_edit_date
this_extension = site.Pages["Extension_talk:" + extension]
if this_extension.exists:
this_date = this_extension.touched
this_day = this_date.tm_mday
this_month = months[this_date.tm_mon-1]
this_year = this_date.tm_year
this_date = str(this_day) + ' ' + \
str(this_month) + ' ' + \
str(this_year)
extensions_dicts[extension]['lastupdatedtalk'] = this_date
##########################################
# Create lists the most recently edited, discussed, updated and created extensions
# A bit redundant with above code, but its more clear to break it out
# Key to sorting by date is a tuple with (year,month,day). easy peasy.
##########################################
for extension in extensions_dicts.keys():
if extensions_dicts[extension].has_key('lastupdated'):
this_date = parse(extensions_dicts[extension]['lastupdated'])
recently_edited.append((this_date.year, this_date.month, this_date.day, extension))
if extensions_dicts[extension].has_key('lastupdatedtalk'):
this_date = parse(extensions_dicts[extension]['lastupdatedtalk'])
recently_discussed.append((this_date.year, this_date.month, this_date.day, extension))
if extensions_dicts[extension].has_key('update'):
this_date = parse(extensions_dicts[extension]['update'])
recently_updated.append((this_date.year, this_date.month, this_date.day, extension))
if extensions_dicts[extension].has_key('created'):
this_date = parse(extensions_dicts[extension]['created'])
recently_created.append((this_date.year, this_date.month, this_date.day, extension))
recently_edited.sort()
recently_discussed.sort()
recently_updated.sort()
recently_created.sort()
recently_edited.reverse()
recently_discussed.reverse()
recently_updated.reverse()
recently_created.reverse()
##########################################
# extensions by type
##########################################
for extension in extensions_dicts.keys():
this_extension = extensions_dicts[extension]
if this_extension.has_key('types'):
this_extensions_types = this_extension['types'].split('<br/>')
for this_type in this_extensions_types:
this_type = this_type.lower()
if '--' in this_type:
this_type = this_type.split('<!--')[0]
if not extensions_by_type.has_key(this_type):
if '--' in this_type:
this_type = this_type.split('<!--')[0]
extensions_by_type[this_type] = [extension]
else:
extensions_by_type[this_type].append(extension)
else:
if not extensions_by_type.has_key('notype'):
extensions_by_type['notype'] = [extension]
else:
extensions_by_type['notype'].append(extension)
for this_type in extensions_by_type.keys():
if not len(extensions_by_type[this_type]) >= 5:
del extensions_by_type[this_type]
##########################################
# extensions by status
##########################################
extensions_by_status = {}
for extension in extensions_dicts.keys():
this_extension = extensions_dicts[extension]
if this_extension.has_key('status'):
this_status = this_extension['status'].lower()
# Make sure this is a single word status - sanity check
if len(this_status.split(' ')) == 1:
if not extensions_by_status.has_key(this_status):
extensions_by_status[this_status] = [extension]
else:
extensions_by_status[this_status].append(extension)
##########################################
# Create main extension matrix output page
##########################################
extension_matrix = ''
prefix = 'Extension Matrix'
updated = 'Last updated: ' + \
datetime.datetime.now().strftime("%Y-%m-%d %H:%M") + ' MST. '
num_listed = 'Listing ' + str(len(extensions_dicts)) + \
' out of ' + str(len(extensions)) + \
' members of [[:Category:Extensions]]<br/>'
extension_matrix = updated + num_listed + '\n'
extension_matrix += '== Entire Extension Matrix ==\n'
extension_matrix += '* [[' + prefix + '/AllExtensions|View all extensions]] (very large!)\n'
# Create the entire extension matrix
entire_matrix = '{{ExtensionMatrixHeader}}'
for extension in sorted_matrix:
entire_matrix += BuildTemplate(extensions_dicts[extension])
entire_matrix += '{{ExtensionMatrixFooter}}'
page = site.Pages[prefix + "/AllExtensions"]
page.save(entire_matrix)
# One subpage for each version of mediawiki that has extensions which mention it
extension_matrix += '== By explicitly supported MediaWiki version ==\n* '
for major_version in xrange(1,3):
for version in xrange(0,20):
version = str(major_version) + '.' + str(version)
if extensions_by_mw_version.has_key(version):
num_extensions = str(len(extensions_by_mw_version[version]))
extension_matrix += '[[' + prefix + '/' + version + '|' + version + ']] (' + num_extensions + '), '
# Create an extension matrix for each version
this_version_matrix = '{{ExtensionMatrixHeader}}'
for extension in extensions_by_mw_version[version]:
this_version_matrix += BuildTemplate(extensions_dicts[extension])
this_version_matrix += '{{ExtensionMatrixFooter}}'
page = site.Pages[prefix + '/' + version]
page.save(this_version_matrix)
extension_matrix += '\n'
# One subpage for each type of status
extension_matrix += '== By status of extension ==\n*'
status_keys = extensions_by_status.keys()
status_keys.sort()
for this_status in status_keys:
num_extensions = str(len(extensions_by_status[this_status]))
extension_matrix += '[[' + prefix + '/' + this_status + '|' + this_status + ']] (' + num_extensions + '), '
this_status_matrix = '{{ExtensionMatrixHeader}}'
for extension in extensions_by_status[this_status]:
this_status_matrix += BuildTemplate(extensions_dicts[extension])
this_status_matrix += '{{ExtensionMatrixFooter}}'
page = site.Pages[prefix + '/' + this_status]
page.save(this_status_matrix)
extension_matrix += '\n'
# One subpage for each extension type
extension_matrix += '== By type of extension ==\n* '
type_keys = extensions_by_type.keys()
type_keys.sort()
for this_type in type_keys[1:]: # [1:] gets rid of weird 'Alterego/ExtensionMatrix' type
num_extensions = str(len(extensions_by_type[this_type]))
extension_matrix += '[[' + prefix + '/' + this_type + '|' + this_type + ']] (' + num_extensions + '), '
this_type_matrix = '{{ExtensionMatrixHeader}}'
for extension in extensions_by_type[this_type]:
this_type_matrix += BuildTemplate(extensions_dicts[extension])
this_type_matrix += '{{ExtensionMatrixFooter}}'
page = site.Pages[prefix + '/' + this_type]
page.save(this_type_matrix)
extension_matrix += '\n'
extension_matrix += '== 500 most recently created extensions ==\n* '
for extension in xrange(500):
extension_name = recently_created[extension][3]
extension_date = extensions_dicts[extension_name]['created']
extension_matrix += '[[Extension:' + extension_name + '|' + extension_name + ']] (' + extension_date + '), '
extension_matrix += '\n'
extension_matrix += '== 500 most recently edited extension pages ==\n* '
for extension in xrange(500):
extension_name = recently_edited[extension][3]
extension_date = extensions_dicts[extension_name]['lastupdated']
extension_matrix += '[[Extension:' + extension_name + '|' + extension_name + ']] (' + extension_date + '), '
extension_matrix += '\n'
extension_matrix += '== 500 most recently edited extension talk pages ==\n* '
for extension in xrange(500):
extension_name = recently_discussed[extension][3]
extension_date = extensions_dicts[extension_name]['lastupdatedtalk']
extension_matrix += '[[Extension_talk:' + extension_name + '|' + extension_name + ']] (' + extension_date + '), '
extension_matrix += '\n'
extension_matrix += '== 500 most recently updated extensions ==\n* '
for extension in xrange(500):
extension_name = recently_updated[extension][3]
extension_date = extensions_dicts[extension_name]['update']
extension_matrix += '[[Extension:' + extension_name + '|' + extension_name + ']] (' + extension_date + '), '
extension_matrix += '\n'
# '\n{{ExtensionMatrixFooter}}\n'
page = site.Pages[prefix]
page.save(extension_matrix)