User:WindBOT/Filters: Difference between revisions

From the Portal Wiki
Jump to navigation Jump to search
 
(23 intermediate revisions by 3 users not shown)
Line 4: Line 4:
If the bot is malfunctioning, chances are that the problem lies in one of these blocks of code. Thus, instead of shutting down the whole bot, it would be wiser to disable only the chunk of code that is misbehaving.
If the bot is malfunctioning, chances are that the problem lies in one of these blocks of code. Thus, instead of shutting down the whole bot, it would be wiser to disable only the chunk of code that is misbehaving.
To make the bot ignore a certain line, add a "#" in front of it:
To make the bot ignore a certain line, add a "#" in front of it:
  # This line will be ignored
# This line will be ignored
If there are multiple lines, wrap them inside triple-quotes ('''you still need to put the two spaces at the beginning of the line'''):
If there are multiple lines, wrap them inside triple-quotes ('''you still need to put the two spaces at the beginning of the line'''):
  """This line will be ignored
"""This line will be ignored
  and this one as well
and this one as well
  and this one is cake
and this one is cake
  and the previous one was a lie but it was still ignored"""
and the previous one was a lie but it was still ignored"""
If all else fails, you can simply delete the block from the page. The bot can't come up with code by itself yet, so it won't run anything.
If all else fails, you can simply delete the block from the page. The bot can't come up with code by itself yet, so it won't run anything.
Or, if the problem really is elsewhere, [{{fullurl:Special:Block|wpBlockAddress={{BASEPAGENAMEE}}&wpBlockExpiry=infinite&wpAnonOnly=0&wpEnableAutoblock=0&wpCreateAccount=0&wpBlockReason=Bot%20gone%20crazy:%20}} block the bot].
Or, if the problem really is elsewhere, [{{fullurl:Special:Block|wpBlockAddress={{BASEPAGENAMEE}}&wpBlockExpiry=infinite&wpAnonOnly=0&wpEnableAutoblock=0&wpCreateAccount=0&wpBlockReason=Bot%20gone%20crazy:%20}} block the bot].
Line 16: Line 16:


== Page filters ==
== Page filters ==
  addPageFilter(r'^user:', r'(?:talk|help|wiki|template):')
addPageFilter(r'^user:', r'(?:talk|help|wiki|template):')


== Semantic filters ==
== Semantic filters ==
Line 26: Line 26:
== Link filters ==
== Link filters ==
=== Wikipedia links filter ===
=== Wikipedia links filter ===
  def wikipediaLinks(link, **kwargs):
def wikipediaLinks(link, **kwargs):
      wikipediaRegex = compileRegex(r'^https?://(?:(\w+)\.)?wikipedia\.org/wiki/(\S+)')
    wikipediaRegex = compileRegex(r'^https?://(?:(\w+)\.)?wikipedia\.org/wiki/(\S+)')
      if link.getType() == u'external':
    if link.getType() == u'external':
          linkInfo = wikipediaRegex.search(link.getLink())
        linkInfo = wikipediaRegex.search(link.getLink())
          if linkInfo:
        if linkInfo:
              link.setType(u'internal')
            link.setType(u'internal')
              try:
            try:
                  wikiPage = urllib2.unquote(str(linkInfo.group(2))).decode('utf8', 'ignore').replace(u'_', ' ')
                wikiPage = urllib2.unquote(str(linkInfo.group(2))).decode('utf8', 'ignore').replace(u'_', ' ')
              except:
            except:
                  wikiPage = u(linkInfo.group(2)).replace(u'_', ' ')
                wikiPage = u(linkInfo.group(2)).replace(u'_', ' ')
              if not linkInfo.group(1) or linkInfo.group(1).lower() == u'en':
            if not linkInfo.group(1) or linkInfo.group(1).lower() == u'en':
                  link.setLink(u'Wikipedia:' + wikiPage) # English Wikipedia
                link.setLink(u'Wikipedia:' + wikiPage) # English Wikipedia
              else:
            else:
                  link.setLink(u'Wikipedia:' + linkInfo.group(1).lower() + u':' + wikiPage) # Non-english Wikipedia
                link.setLink(u'Wikipedia:' + linkInfo.group(1).lower() + u':' + wikiPage) # Non-english Wikipedia
              if link.getLabel() is None:
            if link.getLabel() is None:
                  link.setLabel(u'(Wikipedia)')
                link.setLabel(u'(Wikipedia)')
      return link
    return link
  addLinkFilter(wikipediaLinks)
addLinkFilter(wikipediaLinks)


=== Remove trailing slashes from internal links ===
=== HL Wiki to Combine Overwiki links filter ===
  def removeTrailingSlash(l, **kwargs):
def hlwikiLinks(link, **kwargs):
      if l.getType() != u'internal':
    hlwikiRegex1 = compileRegex(r'^https?://[-.\w]*half-life\.wikia\.com/wiki/(\S+)$')
          return l
    hlwikiRegex2 = compileRegex(r'^https?://[-.\w]*half-life\.wikia\.com/w[-_/\w]+?/([^/\s]+)$')
      if l.getLink()[-1] == '/':
    if link.getType() == 'external':
          l.setLink(l.getLink()[:-1])
        linkInfo = hlwikiRegex1.search(link.getLink())
      return l
        isMedia = False
  addLinkFilter(removeTrailingSlash)
        if not linkInfo:
            linkInfo = hlwikiRegex2.search(link.getLink())
            isMedia = True
        if linkInfo:
            link.setType('internal')
            try:
                wikiPage = u(urllib2.unquote(str(linkInfo.group(1))).decode('utf8', 'ignore').replace(u'_', ' '))
            except:
                wikiPage = u(linkInfo.group(1)).replace(u'_', ' ')
            label = wikiPage
            if isMedia:
                if wikiPage[-4:].lower() == '.wav':
                    wikiPage = 'Media:' + wikiPage
                else:
                    wikiPage = ':File:' + wikiPage
            link.setLink('hl2:' + wikiPage)
            if link.getLabel() is None:
                link.setLabel(label)
    return link
addLinkFilter(hlwikiLinks)


== Template filters ==
== Template filters ==
=== Template renaming ===
=== Template renaming ===
  def templateRenameMapping(t, **kwargs):
def templateRenameMapping(t, **kwargs):
      templateMap = {
    templateMap = {
          # Format goes like this (without the "#" in front obviously):
        # Format goes like this (without the "#" in front obviously):
          #'Good template name': ['Bad template lowercase name 1', 'Bad template lowercase name 2', 'Bad template lowercase name 3'],
        #'Good template name': ['Bad template lowercase name 1', 'Bad template lowercase name 2', 'Bad template lowercase name 3'],
          # Last line has no comma at the end
        # Last line has no comma at the end
          'Crush': ['pngcrush']
        'Crush': ['pngcrush']
      }
    }
      for n in templateMap:
    for n in templateMap:
          if t.getName().lower() in templateMap[n]:
        if t.getName().lower() in templateMap[n]:
              t.setName(n)
            t.setName(n)
      return t
    return t
  addTemplateFilter(templateRenameMapping)
addTemplateFilter(templateRenameMapping)


=== Remove useless templates ===
=== Remove useless templates ===
  def removeUselessTemplate(t, **kwargs):
def removeUselessTemplate(t, **kwargs):
      if t.getName().lower() in (u'targeted', u'languages'):
    if t.getName().lower() in (u'targeted', u'languages'):
          return None # Delete template
        return None # Delete template
      return t
    return t
  addTemplateFilter(removeUselessTemplate)
addTemplateFilter(removeUselessTemplate)


=== Filter parameters of certain templates ===
=== Filter parameters of certain templates ===
  def templateParamFilter(t, **kwargs):
def templateParamFilter(t, **kwargs):
      params = { # Map: 'lowercase template name': ['list', 'of', 'params', 'to', 'filter']
    params = { # Map: 'lowercase template name': ['list', 'of', 'params', 'to', 'filter']
          'patch layout': ['before', 'after', 'current'],
        'patch layout': ['before', 'after', 'current'],
          'item infobox': ['released']
        'item infobox': ['released']
      }
    }
      if t.getName().lower() not in params:
    if t.getName().lower() not in params:
          return t
        return t
      for p in params[t.getName().lower()]:
    for p in params[t.getName().lower()]:
          if t.getParam(p):
        if t.getParam(p):
              t.setParam(p, fixContent(t.getParam(p), **kwargs))
            t.setParam(p, fixContent(t.getParam(p), **kwargs))
      return t
    return t
  addTemplateFilter(templateParamFilter)
addTemplateFilter(templateParamFilter)


=== Remove obsolete parameters ===
=== Remove obsolete parameters ===
  def obsoleteParameterFilter(t, **kwargs):
def obsoleteParameterFilter(t, **kwargs):
      params = { # Map: 'lowercase template name': ['list', 'of', 'params', 'to', 'delete']
    params = { # Map: 'lowercase template name': ['list', 'of', 'params', 'to', 'delete']
      }
    }
      if t.getName().lower() not in params:
    if t.getName().lower() not in params:
          return t
        return t
      for p in params[t.getName().lower()]:
    for p in params[t.getName().lower()]:
          p = u(p)
        p = u(p)
          if p.find(u'#n') != -1:
        if p.find(u'#n') != -1:
              for i in range(10):
            for i in range(10):
                  t.delParam(p.replace(u'#n', str(i)))
                t.delParam(p.replace(u'#n', str(i)))
          else:
        else:
              t.delParam(p)
            t.delParam(p)
      return t
    return t
  addTemplateFilter(obsoleteParameterFilter)
addTemplateFilter(obsoleteParameterFilter)


=== Implement {{tl|Dictionary}} ===
=== Implement {{tl|Dictionary}} ===
  class DictionaryUpdater:
class DictionaryUpdater:
      def __init__(self):
    def __init__(self):
          self.subpageTemplateLang = <nowiki>"""{{#switch:{{{lang|{{SUBPAGENAME}}}}}|%options%}}<noinclude><hr style="margin: 1em 0em;" /><div style="font-size: 95%;">\n:[[File:Pictogram info.png|15px|text-top|link=]]&nbsp;'''Note''': Any changes made here will be automatically overwritten by a bot. Please ''do not'' make changes here as they will be lost. Edit '''[[:%dictionary%|the master page]]''' instead.\n:%missing%</div>[[Category:Template dictionary|%dictionaryname%/%keyname%]]</noinclude>"""</nowiki>
        self.subpageTemplateLang = <nowiki>"""{{#switch:{{{lang|{{SUBPAGENAME}}}}}|%options%}}"""</nowiki>
          self.subpageTemplateParam = <nowiki>"""{{#switch:{{{1|}}}|%options%}}<noinclude><hr style="margin: 1em 0em;" /><div style="font-size: 95%;">\n:[[File:Pictogram info.png|15px|text-top|link=]]&nbsp;'''Note''': Any changes made here will be automatically overwritten by a bot. Please ''do not'' make changes here as they will be lost. Edit '''[[:%dictionary%|the master page]]''' instead.</div>[[Category:Template dictionary|%dictionaryname%/%keyname%]]</noinclude>"""</nowiki>
        self.subpageTemplateParam = <nowiki>"""{{#switch:{{{1|}}}|%options%}}"""</nowiki>
          self.invalidParamError = <nowiki>"""<div style="font-size: 95%; color: #CC0000;">\n:[[File:Pictogram info.png|15px|text-top|link=]]&nbsp;'''Error''': Invalid parameter passed.</div>"""</nowiki>
        self.invalidParamError = <nowiki>"""<span class="error">Error: invalid param.</span>[[Category:ERROR]]"""</nowiki>
          self.subpageTemplateID = <nowiki>"""%string%<noinclude><hr style="margin: 1em 0em;" /><div style="font-size: 95%;">\n:[[File:Pictogram info.png|15px|text-top|link=]]&nbsp;'''Note''': Any changes made here will be automatically overwritten by a bot. Please ''do not'' make changes here as they will be lost. Edit '''[[:%dictionary%|the master page]]''' instead.</div>[[Category:Template dictionary|%dictionaryname%/%keyname%]]</noinclude>"""</nowiki>
        self.subpageTemplateID = <nowiki>"""%string%"""</nowiki>
          self.dictionaries = {
        self.partialUpdateThreshold = 750 # Update SyncData every n edits
              u'Template:Dictionary/items': { # Dictionary page
        self.dictionaries = {
                  'name': 'items', # Dictionary name (used for categorizing)
            u'Template:Dictionary/items': { # Dictionary page
                  'sync': 'Template:Dictionary/items/Special:SyncData' # Page holding last sync data
                'name': 'items', # Dictionary name (used for categorizing)
              },
                'sync': 'Template:Dictionary/items/Special:SyncData' # Page holding last sync data
              u'Template:Dictionary/common strings': { # Warning: no underscore
            },
                  'name': 'common strings',
            u'Template:Dictionary/common strings': { # Warning: no underscore
                  'sync': 'Template:Dictionary/common strings/Special:SyncData'
                'name': 'common strings',
              },
                'sync': 'Template:Dictionary/common strings/Special:SyncData'
              u'Template:Dictionary/price': {
            },
                  'name': 'price',
            u'Template:Dictionary/price': {
                  'sync': 'Template:Dictionary/price/Special:SyncData'
                'name': 'price',
              },
                'sync': 'Template:Dictionary/price/Special:SyncData',
              u'Template:Dictionary/mechanics': {
                'allTemplate': <nowiki>'{{{{{template|item price/fmt}}}|%options%|tt={{{tt|yes}}}}}'</nowiki>
                  'name': 'mechanics',
            },
                  'sync': 'Template:Dictionary/mechanics/Special:SyncData'
            u'Template:Dictionary/mechanics': {
              },
                'name': 'mechanics',
              u'Template:Dictionary/characters': {
                'sync': 'Template:Dictionary/mechanics/Special:SyncData'
                  'name': 'characters',
            },
                  'sync': 'Template:Dictionary/characters/Special:SyncData'
            u'Template:Dictionary/characters': {
              },
                'name': 'characters',
              u'Template:Dictionary/demonstration': {
                'sync': 'Template:Dictionary/characters/Special:SyncData'
                  'name': 'demonstration',
            },
                  'sync': 'Template:Dictionary/demonstration/Special:SyncData'
            u'Template:Dictionary/demonstration': {
              },
                'name': 'demonstration',
              u'Template:Dictionary/transcripts': {
                'sync': 'Template:Dictionary/demonstration/Special:SyncData'
                  'name': 'transcripts',
            },
                  'sync': 'Template:Dictionary/transcripts/Special:SyncData'
            u'Template:Dictionary/transcripts': {
              },
                'name': 'transcripts',
              u'Template:Dictionary/portal achievements': {
                'sync': 'Template:Dictionary/transcripts/Special:SyncData'
                  'name': 'portal achievements',
            },
                  'sync': 'Template:Dictionary/portal achievements/Special:SyncData'
            u'Template:Dictionary/portal achievements': {
              },
                'name': 'portal achievements',
              u'Template:Dictionary/portal: still alive achievements': {
                'sync': 'Template:Dictionary/portal achievements/Special:SyncData'
                  'name': 'portal: still alive achievements',
            },
                  'sync': 'Template:Dictionary/portal: still alive achievements/Special:SyncData'
            u'Template:Dictionary/portal: still alive achievements': {
              },
                'name': 'portal: still alive achievements',
              u'Template:Dictionary/portal 2 achievements': {
                'sync': 'Template:Dictionary/portal: still alive achievements/Special:SyncData'
                  'name': 'portal 2 achievements',
            },
                  'sync': 'Template:Dictionary/portal 2 achievements/Special:SyncData'
            u'Template:Dictionary/portal 2 achievements': {
              },
                'name': 'portal 2 achievements',
              u'Template:Dictionary/audio': {
                'sync': 'Template:Dictionary/portal 2 achievements/Special:SyncData'
                  'name': 'audio',
            },
                  'sync': 'Template:Dictionary/audio/Special:SyncData'
            u'Template:Dictionary/rexaura achievements': {
              }
                'name': 'rexaura achievements',
          }
                'sync': 'Template:Dictionary/rexaura achievements/Special:SyncData'
          self.subpageSeparator = u'/'
            },
          # List of supported languages, in prefered order
            u'Template:Dictionary/portal stories: mel achievements': {
          self.languages = [u'en', u'ar', u'cs', u'da', u'de', u'es', u'fi', u'fr', u'hu', u'it', u'ja', u'ko', u'nl', u'no', u'pl', u'pt', u'pt-br', u'ro', u'ru', u'sv', u'zh-hans', u'zh-hant']
                'name': 'portal stories: mel achievements',
          self.defaultLang = u'en'
                'sync': 'Template:Dictionary/portal stories: mel achievements/Special:SyncData'
          self.filterName = u'Your friendly neighborhood dictionary updater'
            },
          self.commentsExtract = compileRegex(r'<!--([\S\s]+?)-->')
            u'Template:Dictionary/portal pinball achievements': {
          self.stringsExtract = compileRegex(r'(?:^[ \t]*#[ \t]*([^\r\n]*?)[ \t]*$\s*)?^[ \t]*([^\r\n]+?[ \t]*(?:\|[ \t]*[^\r\n]+?[ \t]*)*):[ \t]*([^\r\n]+?[ \t]*$|\s*[\r\n]+(?:\s*[ \t]+[-\w]+[ \t]*:[ \t]*[^\r\n]+[ \t]*$)+)', re.IGNORECASE | re.MULTILINE)
                'name': 'portal pinball achievements',
          self.translationExtract = compileRegex(r'^[ \t]+([-\w]+)[ \t]*:[ \t]*([^\r\n]+)[ \t]*$', re.IGNORECASE | re.MULTILINE)
                'sync': 'Template:Dictionary/portal pinball achievements/Special:SyncData'
          addWhitelistPage(self.dictionaries.keys())
            },
      def generateSubpage(self, keyName, data, currentDict, syncData):
            u'Template:Dictionary/bridge constructor portal achievements': {
          h = hashlib.md5()
                'name': 'bridge constructor portal achievements',
          if type(data) is type({}): # Subkeys (translations or not)
                'sync': 'Template:Dictionary/bridge constructor portal achievements/Special:SyncData'
              isTranslation = True
            },
              subpage = u(self.subpageTemplateLang)
            u'Template:Dictionary/audio': {
              for k in data:
                'name': 'audio',
                  if k not in self.languages:
                'sync': 'Template:Dictionary/audio/Special:SyncData'
                      isTranslation = False
            },
                      subpage = u(self.subpageTemplateParam)
            u'Template:Dictionary/voice lines/Adventure core': {
                      break
                'name': 'voice lines/Adventure core',
              ordered = []
                'sync': 'Template:Dictionary/voice lines/Adventure core/Special:SyncData'
              if isTranslation:
            },
                  missing = []
            u'Template:Dictionary/voice lines/Announcer': {
                  for lang in self.languages:
                'name': 'voice lines/Announcer',
                      if lang in data:
                'sync': 'Template:Dictionary/voice lines/Announcer/Special:SyncData'
                          ordered.append(lang + u'=' + data[lang])
            },
                          h.update((lang + u'=' + data[lang]).encode('utf8'))
            u'Template:Dictionary/voice lines/Ap-Sap': {
                      else:
                'name': 'voice lines/Ap-Sap',
                          missing.append(lang)
                'sync': 'Template:Dictionary/voice lines/Ap-Sap/Special:SyncData'
                          h.update((u'null-' + lang).encode('utf8'))
            },
                  if self.defaultLang in data:
            u'Template:Dictionary/voice lines/Cave Johnson': {
                      ordered.append(u'#default=' + data[self.defaultLang])
                'name': 'voice lines/Cave Johnson',
                  if len(missing):
                'sync': 'Template:Dictionary/voice lines/Cave Johnson/Special:SyncData'
                      subpage = subpage.replace(u'%missing%', <nowiki>u"'''Languages missing''': "</nowiki> + u', '.join(missing))
            },
                  else:
            u'Template:Dictionary/voice lines/Defective Turret': {
                      subpage = subpage.replace(u'%missing%', <nowiki>u"'''Supported languages''': All"</nowiki>)
                'name': 'voice lines/Defective Turret',
              else: # Not a translation
                'sync': 'Template:Dictionary/voice lines/Defective Turret/Special:SyncData'
                  h.update('Any-')
            },
                  subkeys = data.keys()
            u'Template:Dictionary/voice lines/Fact core': {
                  subkeys.sort()
                'name': 'voice lines/Fact core',
                  for k in subkeys:
                'sync': 'Template:Dictionary/voice lines/Fact core/Special:SyncData'
                      ordered.append(k + u'=' + data[k])
            },
                      h.update((k + u'=' + data[k]).encode('utf8'))
            u'Template:Dictionary/voice lines/Floor Turret': {
                  #ordered.append(u'#default=' + u(self.invalidParamError))
                'name': 'voice lines/Floor Turret',
              subpage = subpage.replace(u'%options%', u'|'.join(ordered))
                'sync': 'Template:Dictionary/voice lines/Floor Turret/Special:SyncData'
          else: # No subkeys
            },
              data = u(data)
            u'Template:Dictionary/voice lines/GLaDOS': {
              subpage = self.subpageTemplateID
                'name': 'voice lines/GLaDOS',
              h.update(u(u'ID-' + data).encode('utf8'))
                'sync': 'Template:Dictionary/voice lines/GLaDOS/Special:SyncData'
              subpage = subpage.replace(u'%string%', data)
            },
          h = u(h.hexdigest())
            u'Template:Dictionary/voice lines/Space core': {
          if keyName in syncData and syncData[keyName] == h:
                'name': 'voice lines/Space core',
              return # Same hash
                'sync': 'Template:Dictionary/voice lines/Space core/Special:SyncData'
          syncData[keyName] = h # Update sync data
            },
          subpage = subpage.replace(u'%dictionary%', currentDict)
            u'Template:Dictionary/voice lines/Wheatley': {
          subpage = subpage.replace(u'%dictionaryname%', self.dictionaries[currentDict]['name'])
                'name': 'voice lines/Wheatley',
          subpage = subpage.replace(u'%keyname%', keyName)
                'sync': 'Template:Dictionary/voice lines/Wheatley/Special:SyncData'
          editPage(currentDict + self.subpageSeparator + keyName, subpage, summary=<nowiki>u'Pushed changes from [[:' + currentDict + u']] for string "' + keyName + u'".'</nowiki>, minor=True, nocreate=False)
            },
      def processComment(self, commentString, currentDict, definedStrings, syncData):
            u'Template:Dictionary/voice lines/Grady': {
          commentContents = []
                'name': 'voice lines/Grady',
          for extractedStr in self.stringsExtract.finditer(commentString):
                'sync': 'Template:Dictionary/voice lines/Grady/Special:SyncData'
              comment = u''
            },
              if extractedStr.group(1):
            u'Template:Dictionary/voice lines/Desk Job': {
                  comment = u'# ' + u(extractedStr.group(1)) + u'\n'
                'name': 'voice lines/Desk Job',
              dataString = u(extractedStr.group(3))
                'sync': 'Template:Dictionary/voice lines/Desk Job/Special:SyncData'
              if dataString.find(u'\r') == -1 and dataString.find(u'\n') == -1: # Assume no subkeys
            }
                  data = dataString.strip()
        }
                  dataWriteback = u' ' + data
        self.subpageSeparator = u'/'
              else: # There's subkeys; detect whether this is a translation or not
        # List of supported languages, in prefered order
                  data = {}
        self.languages = [u'en', u'ar', u'cs', u'cz', u'da', u'de', u'es', u'fi', u'fr', u'hu', u'it', u'ja', u'ka', u'ko', u'nl', u'no', u'pl', u'po', u'pt', u'pt-br', u'ro', u'ru', u'sv', u'sw', u'th', u'tr', u'tu', u'zh-hans', u'zh-hant']
                  isTranslation = True
        self.defaultLang = u'en'
                  for translation in self.translationExtract.finditer(dataString.rstrip()):
        self.allKeyName = u'_all_'
                      data[u(translation.group(1))] = u(translation.group(2))
        self.filterName = u'Your friendly neighborhood dictionary updater'
                      if u(translation.group(1)) not in self.languages:
        self.commentsExtract = compileRegex(r'<!--([\S\s]+?)-->')
                          isTranslation = False
        self.stringsExtract = compileRegex(r'(?:^[ \t]*#[ \t]*([^\r\n]*?)[ \t]*$\s*)?^[ \t]*([^\r\n]+?[ \t]*(?:\|[ \t]*[^\r\n]+?[ \t]*)*):[ \t]*([^\r\n]+?[ \t]*$|\s*[\r\n]+(?:\s*[ \t]+[-\w]+[ \t]*:[ \t]*[^\r\n]+[ \t]*$)+)', re.IGNORECASE | re.MULTILINE)
                  ordered = []
        self.translationExtract = compileRegex(r'^[ \t]+([-\w]+)[ \t]*:[ \t]*([^\r\n]+)[ \t]*$', re.IGNORECASE | re.MULTILINE)
                  if isTranslation:
        addWhitelistPage(self.dictionaries.keys())
                      for lang in self.languages:
        self.editCounts = {}
                          if lang in data:
    def updateSyncData(self, currentDict, syncData, note=''):
                              ordered.append(u' ' + lang + u': ' + data[lang])
        # Build syncdata string representation
                  else: # Not a translation, so order in alphabetical order
        syncKeys = syncData.keys()
                      subkeys = data.keys()
        syncKeys.sort()
                      subkeys.sort()
        syncLines = []
                      for subk in subkeys:
        for k in syncKeys:
                          ordered.append(u' ' + subk + u': ' + data[subk])
            syncLines.append(k + u':' + syncData[k])
                  dataWriteback = u'\n' + u'\n'.join(ordered)
        if note:
              keyNames = u(extractedStr.group(2)).lower().split(u'|')
            note = u' (' + u(note) + u')'
              validKeyNames = []
        editPage(self.dictionaries[currentDict]['sync'], u'\n'.join(syncLines), summary=<nowiki>u'Updated synchronization information for [[:' + currentDict + u']]' + note + u'.'</nowiki>, minor=True, nocreate=False)
              for keyName in keyNames:
    def generateSubpage(self, keyName, data, currentDict, syncData):
                  keyName = keyName.replace(u'_', u' ').strip()
        h = hashlib.md5()
                  if keyName in definedStrings:
        if type(data) is type({}): # Subkeys (translations or not)
                      continue # Duplicate key
            isTranslation = True
                  definedStrings.append(keyName)
            subpage = u(self.subpageTemplateLang)
                  validKeyNames.append(keyName)
            for k in data:
                  self.generateSubpage(keyName, data, currentDict, syncData)
                if 'blankString' in self.dictionaries[currentDict] and data[k] == self.dictionaries[currentDict]['blankString']:
              if len(validKeyNames):
                    data[k] = u''
                  commentContents.append(comment + u' | '.join(validKeyNames) + u':' + dataWriteback)
                if isTranslation and k not in self.languages:
          return u'\n\n'.join(commentContents)
                    isTranslation = False
      def __call__(self, content, **kwargs):
                    subpage = u(self.subpageTemplateParam)
          if 'article' not in kwargs:
            ordered = []
              return content
            unordered = {}
          if u(kwargs['article'].title) not in self.dictionaries:
            if isTranslation:
              return content
                missing = []
          currentDict = u(kwargs['article'].title)
                for lang in self.languages:
          syncPage = page(self.dictionaries[currentDict]['sync'])
                    if lang in data:
          try:
                        ordered.append(lang + u'=' + data[lang])
              syncDataText = u(syncPage.getWikiText()).split(u'\n')
                        unordered[lang] = data[lang]
          except: # Page probably doesn't exist
                        h.update((lang + u'=' + data[lang]).encode('utf8'))
              syncDataText = u''
                    else:
          syncData = {}
                        missing.append(lang)
          for sync in syncDataText:
                        h.update((u'null-' + lang).encode('utf8'))
              sync = u(sync.strip())
                if self.defaultLang in data:
              if not sync:
                    ordered.insert(0, u'#default=' + data[self.defaultLang])
                  continue
                if len(missing):
              sync = sync.split(u':', 2)
                    subpage = subpage.replace(u'%missing%', <nowiki>u"Languages missing: "</nowiki> + u', '.join(missing))
              if len(sync) == 2:
                else:
                  syncData[sync[0]] = sync[1]
                    subpage = subpage.replace(u'%missing%', <nowiki>u"Supported languages: all"</nowiki>)
          oldSyncData = syncData.copy()
            else: # Not a translation
          newContent = u''
                h.update('Any-')
          previousIndex = 0
                subkeys = data.keys()
          definedStrings = []
                subkeys.sort()
          for comment in self.commentsExtract.finditer(content):
                for k in subkeys:
              newContent += content[previousIndex:comment.start()]
                    ordered.append(k + u'=' + data[k])
              previousIndex = comment.end()
                    unordered[k] = data[k]
              # Process current comment
                    h.update((k + u'=' + data[k]).encode('utf8'))
              newContent += u'<!--\n\n' + self.processComment(u(comment.group(1)).strip(), currentDict, definedStrings, syncData) + u'\n\n-->'
            if 'allTemplate' in self.dictionaries[currentDict] and (len(unordered) or len(self.dictionaries[currentDict]['allTemplate']['params'])):
          newContent += content[previousIndex:]
                allKey = []
          # Check if we need to update sync data
                keys = unordered.keys()
          needUpdate = False
                keys.sort()
          for k in syncData:
                for k in keys:
              if k not in oldSyncData or oldSyncData[k] != syncData[k]:
                    allKey.append(k + u'=' + unordered[k])
                  needUpdate = True
                insertIndex = 0
                  break
                if isTranslation and self.defaultLang in data:
          # Check for deleted strings
                    insertIndex = 1
          for k in oldSyncData:
                ordered.insert(insertIndex, u(self.allKeyName) + u'=' + u(self.dictionaries[currentDict]['allTemplate'].replace(u'%options%', u'|'.join(allKey))))
              if k not in definedStrings:
            subpage = subpage.replace(u'%options%', u'|'.join(ordered))
                  try:
        else: # No subkeys
                      deletePage(currentDict + self.subpageSeparator + k, 'Removed deleted string "' + k + u'" from [[:' + currentDict + u']].')
            data = u(data)
                  except:
            subpage = self.subpageTemplateID
                      pass
            h.update(u(u'ID-' + data).encode('utf8'))
                  if k in syncData:
            subpage = subpage.replace(u'%string%', data)
                      del syncData[k]
        h = u(h.hexdigest())
                  needUpdate = True
        if keyName in syncData and syncData[keyName] == h:
          if needUpdate:
            return # Same hash
              # Build syncdata string representation
        subpage = subpage.replace(u'%dictionary%', currentDict)
              syncKeys = syncData.keys()
        subpage = subpage.replace(u'%dictionaryname%', self.dictionaries[currentDict]['name'])
              syncKeys.sort()
        subpage = subpage.replace(u'%keyname%', keyName)
              syncLines = []
        if editPage(currentDict + self.subpageSeparator + keyName, subpage, summary=<nowiki>u'Pushed changes from [[:' + currentDict + u']] for string "' + keyName + u'".'</nowiki>, minor=True, nocreate=False):
              for k in syncKeys:
            syncData[keyName] = h # Update sync data
                  syncLines.append(k + u':' + syncData[k])
            if currentDict not in self.editCounts:
              editPage(syncPage, u'\n'.join(syncLines), summary=<nowiki>u'Updated synchronization information for [[:' + currentDict + u']].'</nowiki>, minor=True, nocreate=False)
                self.editCounts[currentDict] = 0
          return newContent
            self.editCounts[currentDict] += 1
      def scheduledRun(self):
            if self.editCounts[currentDict] > self.partialUpdateThreshold:
          for d in self.dictionaries:
                self.editCounts[currentDict] = 0
              fixPage(d)
                self.updateSyncData(currentDict, syncData, 'Partial update')
  dictUpdater = DictionaryUpdater()
    def processComment(self, commentString, currentDict, definedStrings, syncData):
  addFilter(dictUpdater)
        commentContents = []
  scheduleTask(dictUpdater.scheduledRun, 3)
        for extractedStr in self.stringsExtract.finditer(commentString):
            comment = u''
            if extractedStr.group(1):
                comment = u'# ' + u(extractedStr.group(1)) + u'\n'
            dataString = u(extractedStr.group(3))
            if dataString.find(u'\r') == -1 and dataString.find(u'\n') == -1: # Assume no subkeys
                data = dataString.strip()
                dataWriteback = u' ' + data
            else: # There's subkeys; detect whether this is a translation or not
                data = {}
                isTranslation = True
                for translation in self.translationExtract.finditer(dataString.rstrip()):
                    data[u(translation.group(1))] = u(translation.group(2))
                    if u(translation.group(1)) not in self.languages:
                        isTranslation = False
                ordered = []
                if isTranslation:
                    for lang in self.languages:
                        if lang in data:
                            ordered.append(u'  ' + lang + u': ' + data[lang])
                else: # Not a translation, so order in alphabetical order
                    subkeys = data.keys()
                    subkeys.sort()
                    for subk in subkeys:
                        ordered.append(u'  ' + subk + u': ' + data[subk])
                dataWriteback = u'\n' + u'\n'.join(ordered)
            keyNames = u(extractedStr.group(2)).lower().split(u'|')
            validKeyNames = []
            for keyName in keyNames:
                keyName = keyName.replace(u'_', u' ').replace(u'#', u'').strip()
                if keyName in definedStrings:
                    continue # Duplicate key
                definedStrings.append(keyName)
                validKeyNames.append(keyName)
                self.generateSubpage(keyName, data, currentDict, syncData)
            if len(validKeyNames):
                commentContents.append(comment + u' | '.join(validKeyNames) + u':' + dataWriteback)
        return u'\n\n'.join(commentContents)
    def __call__(self, content, **kwargs):
        if 'article' not in kwargs:
            return content
        if u(kwargs['article'].title) not in self.dictionaries:
            return content
        currentDict = u(kwargs['article'].title)
        if random.randint(0, 50) == 0: # With probability 2%, ignore syncdata completely. Helps with stale syncdata and people overwriting things.
            syncDataText = u''
        else:
            try:
                syncDataText = u(page(self.dictionaries[currentDict]['sync']).getWikiText()).split(u'\n')
            except: # Page probably doesn't exist
                syncDataText = u''
        syncData = {}
        for sync in syncDataText:
            sync = u(sync.strip())
            if not sync:
                continue
            sync = sync.split(u':', 2)
            if len(sync) == 2:
                syncData[sync[0]] = sync[1]
        oldSyncData = syncData.copy()
        newContent = u''
        previousIndex = 0
        definedStrings = []
        for comment in self.commentsExtract.finditer(content):
            newContent += content[previousIndex:comment.start()]
            previousIndex = comment.end()
            # Process current comment
            newContent += u'<!--\n\n' + self.processComment(u(comment.group(1)).strip(), currentDict, definedStrings, syncData) + u'\n\n-->'
        newContent += content[previousIndex:]
        # Check for deleted strings
        for k in oldSyncData:
            if k not in definedStrings:
                try:
                    deletePage(currentDict + self.subpageSeparator + k, 'Removed deleted string "' + k + u'" from [[:' + currentDict + u']].')
                except:
                    pass
                if k in syncData:
                    del syncData[k]
        self.updateSyncData(currentDict, syncData, 'Full update')
        self.editCounts[currentDict] = 0
        return newContent
    def scheduledRun(self):
        for d in self.dictionaries:
            fixPage(d)
dictUpdater = DictionaryUpdater()
addFilter(dictUpdater)
scheduleTask(dictUpdater.scheduledRun, 3)


== File filters ==
=== Update checklists on [[User:WindBOT/Item checklists|list of subscribers]] ===
=== [http://en.wikipedia.org/wiki/Pngcrush PNGCrush]/[http://jpegclub.org/ jpegtran] all PNG/JPG images ===
def itemChecklists():
  class imageCrushFilter:
    game = 620
      def __init__(self):
    cleanItemName = compileRegex(r'^the +')
          self.minRatio = 10 # Compression ratio threshold
    def updateItemChecklist(checklist, schema, support):
          self.minByteDiff = 2048 # Byte difference threshold
        if not checklist.getParam('steamid'):
          self.jpgScanMap = u'0:  0  0 0 0 ;1 2: 0  0 0 0 ;0:  1  8 0 2 ;1:  1  8 0 0 ;2:  1  8 0 0 ;0:  9 63 0 2 ;0:  1 63 2 1 ;0:  1 63 1 0 ;1:  9 63 0 0 ;2:  9 63 0 0 ;'.replace(u';', u';\n')
            checklist.setParam('error', 'Unspecified Steam ID.')
          self.filterName = 'Saved crush information'
            return
          self.extractHash = compileRegex(r'\{\{(?:png)?crush\s*\|\s*(\w+?)\s*\|\s*(\w+?)\s*}}')
        supportedItems = {}
          try:
        for i in support:
              subprocess.call(['pngcrush', '-version'])
            supportedItems[i] = 0
              self.pngenabled = True
        try:
          except:
            steamUser = steam.user.profile(checklist.getParam('steamid')).id64
              print 'Warning: PNGCrush is not installed or not in $PATH'
        except steam.user.ProfileNotFoundError as e:
              self.pngenabled = False
            try:
          try:
                steamUser = steam.user.vanity_url(checklist.getParam('steamid')).id64
              subprocess.call(['jpegtran', '-h'])
            except Exception as e2:
              self.jpgenabled = True
                checklist.setParam('error', u'Cannot find profile: ' + u(e) + u' / ' + u(e2))
          except:
                return
              print 'Warning: jpegtran is not installed or not in $PATH'
        try:
              self.jpgenabled = False
            backpack = steam.items.inventory(game, steamUser, schema)
      def getRandBits(self):
        except Exception as e:
          return random.getrandbits(128)
            checklist.setParam('error', u'Cannot load inventory: ' + u(e))
      def getFileHash(self, filename):
            return
          h = hashlib.md5()
        for item in backpack:
          f = open(filename, 'rb')
            itemName = cleanItemName.sub(u'', u(item.name).lower())
          for i in f.readlines():
            if itemName in supportedItems:
              h.update(i)
                supportedItems[itemName] += 1
          f.close()
        for item in supportedItems:
          return u(h.hexdigest())
            if supportedItems[item] > 1:
      def deleteFile(self, *fs):
                checklist.setParam(item, supportedItems[item])
          for f in fs:
            elif supportedItems[item] == 1:
              try:
                checklist.setParam(item, 'yes')
                  os.remove(tempFile)
            else:
              except:
                p = checklist.getParam(item)
                  pass
                if p is not None:
      def __call__(self, content, article, **kwargs):
                    p = p.lower()
          title = u(article.title).lower()
                if p in (None, 'no', '0'):
          if title[-4:] == '.png':
                    checklist.setParam(item, 'no')
              isPNG = True
                elif p not in ('wanted', 'want', 'do not', 'anti', 'do not want'):
              if not self.pngenabled:
                    checklist.setParam(item, 'had')
                  return content
        return
          elif title[-5:] == '.jpeg' or title[-4:] == '.jpg':
    try:
              isPNG = False
        schema = steamGetGameSchema(game)
              if not self.jpgenabled:
        allItems = []
                  return content
        for item in schema:
          else:
            allItems.append(cleanItemName.sub(u'', u(item.name).lower()))
              return content
    except:
          try: # This is a high-risk filter, lots of I/O, so wrap it in a big try
        return # No schema means no fancy
              filePage = wikitools.wikifile.File(wiki(), article.title)
    support = []
              hashes = [u'', u'']
    templateParams = compileRegex(r'\{\{\{\s*(?:the +)?([^{}|]+?)\s*\|')
              hashResult = self.extractHash.search(content)
    templateCode = page('Template:Item checklist').getWikiText()
              hashTemplate = None
    res = templateParams.search(templateCode)
              if hashResult:
    while res:
                  hashes = [u(hashResult.group(1)), u(hashResult.group(2))]
        item = u(res.group(1)).lower()
                  hashTemplate = <nowiki>u'{{crush|' + hashes[0] + u'|' + hashes[1] + u'}}'</nowiki>
        if item not in support and item in allItems:
              tempFile = getTempFilename()
            support.append(item)
              filePage.download(location=tempFile, urlQuery=u('nocache'))
        templateCode = templateCode[res.end():]
              oldHash = self.getFileHash(tempFile)
        res = templateParams.search(templateCode)
              if oldHash in hashes:
    checkPage, checkLinks, checkKeys = linkExtract(page('User:WindBOT/Item_checklists').getWikiText())
                  return content # Already worked on that one
    linksLeft = checkLinks.values()[:]
              hashTemplate = <nowiki>u'{{crush|' + oldHash + u'|None}}'</nowiki>
    for i in range(12):
              tempOutput = getTempFilename()
        randLink = random.choice(linksLeft)
              if isPNG:
        linksLeft.remove(randLink)
                  result = subprocess.call(['pngcrush', '-rem', 'gAMA', '-rem', 'cHRM', '-rem', 'iCCP', '-rem', 'sRGB', '-brute', tempFile, tempOutput])
        checklist = page(randLink.getLink())
              else:
        print 'Updating', checklist
                  mapFile = getTempFilename()
        oldContent = u(checklist.getWikiText())
                  mapFileHandle = open(mapFile, 'wb')
        content, templatelist, templatekeys = templateExtract(oldContent)
                  mapFileHandle.write(self.jpgScanMap.encode('ascii')) # Onoz ASCII
        for t in templatelist.values():
                  mapFileHandle.close()
            if t.getName().lower().find(u'checklist') != -1:
                  result = subprocess.call(['jpegtran', '-o', '-scans', mapFile, '-copy', 'none', '-progressive', '-outfile', tempOutput, tempFile])
                updateItemChecklist(t, schema, support)
                  self.deleteFile(mapFile)
        content = templateRestore(content, templatelist, templatekeys)
              oldSize = os.path.getsize(tempFile)
        if oldContent != content:
              newSize = os.path.getsize(tempOutput)
            editPage(checklist, content, summary=<nowiki>u'Updated Item checklist [[:' + u(checklist.title) + u']]'</nowiki>, minor=True)
              self.deleteFile(tempFile)
scheduleTask(itemChecklists, 365)
              if not result and oldSize > newSize:
                  # Ready to upload... or are we?
                  ratio = int(round(100 * (1.0 - float(newSize) / float(oldSize))))
                  if ratio >= self.minRatio or oldSize - newSize >= self.minByteDiff:
                      newHash = self.getFileHash(tempOutput)
                      if newHash in hashes:
                          self.deleteFile(tempOutput)
                          return content # Already got that result, no need to reupload
                      hashTemplate = <nowiki>u'{{crush|' + oldHash + u'|' + newHash + u'}}'</nowiki>
                      uploadFile(tempOutput, u(article.title), u'Crushed version: ' + u(ratio) + u'% reduction / ' + u(oldSize - newSize) + u' bytes saved; from ' + u(oldSize) + u' to ' + u(newSize) + u' bytes.', overwrite=True, reupload=True)
                      hashes = [oldHash, newHash]
              if hashResult:
                  content = content[:hashResult.start()] + hashTemplate + content[hashResult.end():]
              else:
                  content = content.strip() + u'\n\n' + hashTemplate
              self.deleteFile(tempOutput)
          except:
              pass # Well, that didn't work
          return content
  addFileFilter(imageCrushFilter())

Latest revision as of 02:19, 11 March 2022

Filters Talk Blacklist Requests Documentation Source Backpack updater Edits
Filters Talk Blacklist Requests Documentation Source Backpack updater Edits

How to disable a filter

If the bot is malfunctioning, chances are that the problem lies in one of these blocks of code. Thus, instead of shutting down the whole bot, it would be wiser to disable only the chunk of code that is misbehaving. To make the bot ignore a certain line, add a "#" in front of it: 

# This line will be ignored

If there are multiple lines, wrap them inside triple-quotes (you still need to put the two spaces at the beginning of the line): 

"""This line will be ignored
and this one as well
and this one is cake
and the previous one was a lie but it was still ignored"""

If all else fails, you can simply delete the block from the page. The bot can't come up with code by itself yet, so it won't run anything. Or, if the problem really is elsewhere, block the bot.

Page filters

addPageFilter(r'^user:', r'(?:talk|help|wiki|template):')

Semantic filters

None yet~

Language-specific filters

None yet~

Link filters

Wikipedia links filter

def wikipediaLinks(link, **kwargs):
    wikipediaRegex = compileRegex(r'^https?://(?:(\w+)\.)?wikipedia\.org/wiki/(\S+)')
    if link.getType() == u'external':
        linkInfo = wikipediaRegex.search(link.getLink())
        if linkInfo:
            link.setType(u'internal')
            try:
                wikiPage = urllib2.unquote(str(linkInfo.group(2))).decode('utf8', 'ignore').replace(u'_', ' ')
            except:
                wikiPage = u(linkInfo.group(2)).replace(u'_', ' ')
            if not linkInfo.group(1) or linkInfo.group(1).lower() == u'en':
                link.setLink(u'Wikipedia:' + wikiPage) # English Wikipedia
            else:
                link.setLink(u'Wikipedia:' + linkInfo.group(1).lower() + u':' + wikiPage) # Non-english Wikipedia
            if link.getLabel() is None:
                link.setLabel(u'(Wikipedia)')
    return link
addLinkFilter(wikipediaLinks)

HL Wiki to Combine Overwiki links filter

def hlwikiLinks(link, **kwargs):
    hlwikiRegex1 = compileRegex(r'^https?://[-.\w]*half-life\.wikia\.com/wiki/(\S+)$')
    hlwikiRegex2 = compileRegex(r'^https?://[-.\w]*half-life\.wikia\.com/w[-_/\w]+?/([^/\s]+)$')
    if link.getType() == 'external':
        linkInfo = hlwikiRegex1.search(link.getLink())
        isMedia = False
        if not linkInfo:
            linkInfo = hlwikiRegex2.search(link.getLink())
            isMedia = True
        if linkInfo:
            link.setType('internal')
            try:
                wikiPage = u(urllib2.unquote(str(linkInfo.group(1))).decode('utf8', 'ignore').replace(u'_', ' '))
            except:
                wikiPage = u(linkInfo.group(1)).replace(u'_', ' ')
            label = wikiPage
            if isMedia:
                if wikiPage[-4:].lower() == '.wav':
                    wikiPage = 'Media:' + wikiPage
                else:
                    wikiPage = ':File:' + wikiPage
            link.setLink('hl2:' + wikiPage)
            if link.getLabel() is None:
                link.setLabel(label)
    return link
addLinkFilter(hlwikiLinks)

Template filters

Template renaming

def templateRenameMapping(t, **kwargs):
    templateMap = {
        # Format goes like this (without the "#" in front obviously):
        #'Good template name': ['Bad template lowercase name 1', 'Bad template lowercase name 2', 'Bad template lowercase name 3'],
        # Last line has no comma at the end
        'Crush': ['pngcrush']
    }
    for n in templateMap:
        if t.getName().lower() in templateMap[n]:
            t.setName(n)
    return t
addTemplateFilter(templateRenameMapping)

Remove useless templates

def removeUselessTemplate(t, **kwargs):
    if t.getName().lower() in (u'targeted', u'languages'):
        return None # Delete template
    return t
addTemplateFilter(removeUselessTemplate)

Filter parameters of certain templates

def templateParamFilter(t, **kwargs):
    params = { # Map: 'lowercase template name': ['list', 'of', 'params', 'to', 'filter']
        'patch layout': ['before', 'after', 'current'],
        'item infobox': ['released']
    }
    if t.getName().lower() not in params:
        return t
    for p in params[t.getName().lower()]:
        if t.getParam(p):
            t.setParam(p, fixContent(t.getParam(p), **kwargs))
    return t
addTemplateFilter(templateParamFilter)

Remove obsolete parameters

def obsoleteParameterFilter(t, **kwargs):
    params = { # Map: 'lowercase template name': ['list', 'of', 'params', 'to', 'delete']
    }
    if t.getName().lower() not in params:
        return t
    for p in params[t.getName().lower()]:
        p = u(p)
        if p.find(u'#n') != -1:
            for i in range(10):
                t.delParam(p.replace(u'#n', str(i)))
        else:
            t.delParam(p)
    return t
addTemplateFilter(obsoleteParameterFilter)

Implement {{Dictionary}}

class DictionaryUpdater:
    def __init__(self):
        self.subpageTemplateLang = """{{#switch:{{{lang|{{SUBPAGENAME}}}}}|%options%}}"""
        self.subpageTemplateParam = """{{#switch:{{{1|}}}|%options%}}"""
        self.invalidParamError = """<span class="error">Error: invalid param.</span>[[Category:ERROR]]"""
        self.subpageTemplateID = """%string%"""
        self.partialUpdateThreshold = 750 # Update SyncData every n edits
        self.dictionaries = {
            u'Template:Dictionary/items': { # Dictionary page
                'name': 'items', # Dictionary name (used for categorizing)
                'sync': 'Template:Dictionary/items/Special:SyncData' # Page holding last sync data
            },
            u'Template:Dictionary/common strings': { # Warning: no underscore
                'name': 'common strings',
                'sync': 'Template:Dictionary/common strings/Special:SyncData'
            },
            u'Template:Dictionary/price': {
                'name': 'price',
                'sync': 'Template:Dictionary/price/Special:SyncData',
                'allTemplate': '{{{{{template|item price/fmt}}}|%options%|tt={{{tt|yes}}}}}'
            },
            u'Template:Dictionary/mechanics': {
                'name': 'mechanics',
                'sync': 'Template:Dictionary/mechanics/Special:SyncData'
            },
            u'Template:Dictionary/characters': {
                'name': 'characters',
                'sync': 'Template:Dictionary/characters/Special:SyncData'
            },
            u'Template:Dictionary/demonstration': {
                'name': 'demonstration',
                'sync': 'Template:Dictionary/demonstration/Special:SyncData'
            },
            u'Template:Dictionary/transcripts': {
                'name': 'transcripts',
                'sync': 'Template:Dictionary/transcripts/Special:SyncData'
            },
            u'Template:Dictionary/portal achievements': {
                'name': 'portal achievements',
                'sync': 'Template:Dictionary/portal achievements/Special:SyncData'
            },
            u'Template:Dictionary/portal: still alive achievements': {
                'name': 'portal: still alive achievements',
                'sync': 'Template:Dictionary/portal: still alive achievements/Special:SyncData'
            },
            u'Template:Dictionary/portal 2 achievements': {
                'name': 'portal 2 achievements',
                'sync': 'Template:Dictionary/portal 2 achievements/Special:SyncData'
            },
            u'Template:Dictionary/rexaura achievements': {
                'name': 'rexaura achievements',
                'sync': 'Template:Dictionary/rexaura achievements/Special:SyncData'
            },
            u'Template:Dictionary/portal stories: mel achievements': {
                'name': 'portal stories: mel achievements',
                'sync': 'Template:Dictionary/portal stories: mel achievements/Special:SyncData'
            },
            u'Template:Dictionary/portal pinball achievements': {
                'name': 'portal pinball achievements',
                'sync': 'Template:Dictionary/portal pinball achievements/Special:SyncData'
            },
            u'Template:Dictionary/bridge constructor portal achievements': {
                'name': 'bridge constructor portal achievements',
                'sync': 'Template:Dictionary/bridge constructor portal achievements/Special:SyncData'
            },
            u'Template:Dictionary/audio': {
                'name': 'audio',
                'sync': 'Template:Dictionary/audio/Special:SyncData'
            },
            u'Template:Dictionary/voice lines/Adventure core': {
                'name': 'voice lines/Adventure core',
                'sync': 'Template:Dictionary/voice lines/Adventure core/Special:SyncData'
            },
            u'Template:Dictionary/voice lines/Announcer': {
                'name': 'voice lines/Announcer',
                'sync': 'Template:Dictionary/voice lines/Announcer/Special:SyncData'
            },
            u'Template:Dictionary/voice lines/Ap-Sap': {
                'name': 'voice lines/Ap-Sap',
                'sync': 'Template:Dictionary/voice lines/Ap-Sap/Special:SyncData'
            },
            u'Template:Dictionary/voice lines/Cave Johnson': {
                'name': 'voice lines/Cave Johnson',
                'sync': 'Template:Dictionary/voice lines/Cave Johnson/Special:SyncData'
            },
            u'Template:Dictionary/voice lines/Defective Turret': {
                'name': 'voice lines/Defective Turret',
                'sync': 'Template:Dictionary/voice lines/Defective Turret/Special:SyncData'
            },
            u'Template:Dictionary/voice lines/Fact core': {
                'name': 'voice lines/Fact core',
                'sync': 'Template:Dictionary/voice lines/Fact core/Special:SyncData'
            },
            u'Template:Dictionary/voice lines/Floor Turret': {
                'name': 'voice lines/Floor Turret',
                'sync': 'Template:Dictionary/voice lines/Floor Turret/Special:SyncData'
            },
            u'Template:Dictionary/voice lines/GLaDOS': {
                'name': 'voice lines/GLaDOS',
                'sync': 'Template:Dictionary/voice lines/GLaDOS/Special:SyncData'
            },
            u'Template:Dictionary/voice lines/Space core': {
                'name': 'voice lines/Space core',
                'sync': 'Template:Dictionary/voice lines/Space core/Special:SyncData'
            },
            u'Template:Dictionary/voice lines/Wheatley': {
                'name': 'voice lines/Wheatley',
                'sync': 'Template:Dictionary/voice lines/Wheatley/Special:SyncData'
            },
            u'Template:Dictionary/voice lines/Grady': {
                'name': 'voice lines/Grady',
                'sync': 'Template:Dictionary/voice lines/Grady/Special:SyncData'
            },
            u'Template:Dictionary/voice lines/Desk Job': {
                'name': 'voice lines/Desk Job',
                'sync': 'Template:Dictionary/voice lines/Desk Job/Special:SyncData'
            }
        }
        self.subpageSeparator = u'/'
        # List of supported languages, in prefered order
        self.languages = [u'en', u'ar', u'cs', u'cz', u'da', u'de', u'es', u'fi', u'fr', u'hu', u'it', u'ja', u'ka', u'ko', u'nl', u'no', u'pl', u'po', u'pt', u'pt-br', u'ro', u'ru', u'sv', u'sw', u'th', u'tr', u'tu', u'zh-hans', u'zh-hant']
        self.defaultLang = u'en'
        self.allKeyName = u'_all_'
        self.filterName = u'Your friendly neighborhood dictionary updater'
        self.commentsExtract = compileRegex(r)
        self.stringsExtract = compileRegex(r'(?:^[ \t]*#[ \t]*([^\r\n]*?)[ \t]*$\s*)?^[ \t]*([^\r\n]+?[ \t]*(?:\|[ \t]*[^\r\n]+?[ \t]*)*):[ \t]*([^\r\n]+?[ \t]*$|\s*[\r\n]+(?:\s*[ \t]+[-\w]+[ \t]*:[ \t]*[^\r\n]+[ \t]*$)+)', re.IGNORECASE | re.MULTILINE)
        self.translationExtract = compileRegex(r'^[ \t]+([-\w]+)[ \t]*:[ \t]*([^\r\n]+)[ \t]*$', re.IGNORECASE | re.MULTILINE)
        addWhitelistPage(self.dictionaries.keys())
        self.editCounts = {}
    def updateSyncData(self, currentDict, syncData, note=):
        # Build syncdata string representation
        syncKeys = syncData.keys()
        syncKeys.sort()
        syncLines = []
        for k in syncKeys:
            syncLines.append(k + u':' + syncData[k])
        if note:
            note = u' (' + u(note) + u')'
        editPage(self.dictionaries[currentDict]['sync'], u'\n'.join(syncLines), summary=u'Updated synchronization information for [[:' + currentDict + u']]' + note + u'.', minor=True, nocreate=False)
    def generateSubpage(self, keyName, data, currentDict, syncData):
        h = hashlib.md5()
        if type(data) is type({}): # Subkeys (translations or not)
            isTranslation = True
            subpage = u(self.subpageTemplateLang)
            for k in data:
                if 'blankString' in self.dictionaries[currentDict] and data[k] == self.dictionaries[currentDict]['blankString']:
                    data[k] = u
                if isTranslation and k not in self.languages:
                    isTranslation = False
                    subpage = u(self.subpageTemplateParam)
            ordered = []
            unordered = {}
            if isTranslation:
                missing = []
                for lang in self.languages:
                    if lang in data:
                        ordered.append(lang + u'=' + data[lang])
                        unordered[lang] = data[lang]
                        h.update((lang + u'=' + data[lang]).encode('utf8'))
                    else:
                        missing.append(lang)
                        h.update((u'null-' + lang).encode('utf8'))
                if self.defaultLang in data:
                    ordered.insert(0, u'#default=' + data[self.defaultLang])
                if len(missing):
                    subpage = subpage.replace(u'%missing%', u"Languages missing: " + u', '.join(missing))
                else:
                    subpage = subpage.replace(u'%missing%', u"Supported languages: all")
            else: # Not a translation
                h.update('Any-')
                subkeys = data.keys()
                subkeys.sort()
                for k in subkeys:
                    ordered.append(k + u'=' + data[k])
                    unordered[k] = data[k]
                    h.update((k + u'=' + data[k]).encode('utf8'))
            if 'allTemplate' in self.dictionaries[currentDict] and (len(unordered) or len(self.dictionaries[currentDict]['allTemplate']['params'])):
                allKey = []
                keys = unordered.keys()
                keys.sort()
                for k in keys:
                    allKey.append(k + u'=' + unordered[k])
                insertIndex = 0
                if isTranslation and self.defaultLang in data:
                    insertIndex = 1
                ordered.insert(insertIndex, u(self.allKeyName) + u'=' + u(self.dictionaries[currentDict]['allTemplate'].replace(u'%options%', u'|'.join(allKey))))
            subpage = subpage.replace(u'%options%', u'|'.join(ordered))
        else: # No subkeys
            data = u(data)
            subpage = self.subpageTemplateID
            h.update(u(u'ID-' + data).encode('utf8'))
            subpage = subpage.replace(u'%string%', data)
        h = u(h.hexdigest())
        if keyName in syncData and syncData[keyName] == h:
            return # Same hash
        subpage = subpage.replace(u'%dictionary%', currentDict)
        subpage = subpage.replace(u'%dictionaryname%', self.dictionaries[currentDict]['name'])
        subpage = subpage.replace(u'%keyname%', keyName)
        if editPage(currentDict + self.subpageSeparator + keyName, subpage, summary=u'Pushed changes from [[:' + currentDict + u']] for string "' + keyName + u'".', minor=True, nocreate=False):
            syncData[keyName] = h # Update sync data
            if currentDict not in self.editCounts:
                self.editCounts[currentDict] = 0
            self.editCounts[currentDict] += 1
            if self.editCounts[currentDict] > self.partialUpdateThreshold:
                self.editCounts[currentDict] = 0
                self.updateSyncData(currentDict, syncData, 'Partial update')
    def processComment(self, commentString, currentDict, definedStrings, syncData):
        commentContents = []
        for extractedStr in self.stringsExtract.finditer(commentString):
            comment = u
            if extractedStr.group(1):
                comment = u'# ' + u(extractedStr.group(1)) + u'\n'
            dataString = u(extractedStr.group(3))
            if dataString.find(u'\r') == -1 and dataString.find(u'\n') == -1: # Assume no subkeys
                data = dataString.strip()
                dataWriteback = u' ' + data
            else: # There's subkeys; detect whether this is a translation or not
                data = {}
                isTranslation = True
                for translation in self.translationExtract.finditer(dataString.rstrip()):
                    data[u(translation.group(1))] = u(translation.group(2))
                    if u(translation.group(1)) not in self.languages:
                        isTranslation = False
                ordered = []
                if isTranslation:
                    for lang in self.languages:
                        if lang in data:
                            ordered.append(u'  ' + lang + u': ' + data[lang])
                else: # Not a translation, so order in alphabetical order
                    subkeys = data.keys()
                    subkeys.sort()
                    for subk in subkeys:
                        ordered.append(u'  ' + subk + u': ' + data[subk])
                dataWriteback = u'\n' + u'\n'.join(ordered)
            keyNames = u(extractedStr.group(2)).lower().split(u'|')
            validKeyNames = []
            for keyName in keyNames:
                keyName = keyName.replace(u'_', u' ').replace(u'#', u).strip()
                if keyName in definedStrings:
                    continue # Duplicate key
                definedStrings.append(keyName)
                validKeyNames.append(keyName)
                self.generateSubpage(keyName, data, currentDict, syncData)
            if len(validKeyNames):
                commentContents.append(comment + u' | '.join(validKeyNames) + u':' + dataWriteback)
        return u'\n\n'.join(commentContents)
    def __call__(self, content, **kwargs):
        if 'article' not in kwargs:
            return content
        if u(kwargs['article'].title) not in self.dictionaries:
            return content
        currentDict = u(kwargs['article'].title)
        if random.randint(0, 50) == 0: # With probability 2%, ignore syncdata completely. Helps with stale syncdata and people overwriting things.
            syncDataText = u
        else:
            try:
                syncDataText = u(page(self.dictionaries[currentDict]['sync']).getWikiText()).split(u'\n')
            except: # Page probably doesn't exist
                syncDataText = u
        syncData = {}
        for sync in syncDataText:
            sync = u(sync.strip())
            if not sync:
                continue
            sync = sync.split(u':', 2)
            if len(sync) == 2:
                syncData[sync[0]] = sync[1]
        oldSyncData = syncData.copy()
        newContent = u
        previousIndex = 0
        definedStrings = []
        for comment in self.commentsExtract.finditer(content):
            newContent += content[previousIndex:comment.start()]
            previousIndex = comment.end()
            # Process current comment
            newContent += u
        newContent += content[previousIndex:]
        # Check for deleted strings
        for k in oldSyncData:
            if k not in definedStrings:
                try:
                    deletePage(currentDict + self.subpageSeparator + k, 'Removed deleted string "' + k + u'" from ' + currentDict + u'.')
                except:
                    pass
                if k in syncData:
                    del syncData[k]
        self.updateSyncData(currentDict, syncData, 'Full update')
        self.editCounts[currentDict] = 0
        return newContent
    def scheduledRun(self):
        for d in self.dictionaries:
            fixPage(d)
dictUpdater = DictionaryUpdater()
addFilter(dictUpdater)
scheduleTask(dictUpdater.scheduledRun, 3)

Update checklists on list of subscribers

def itemChecklists():
    game = 620
    cleanItemName = compileRegex(r'^the +')
    def updateItemChecklist(checklist, schema, support):
        if not checklist.getParam('steamid'):
            checklist.setParam('error', 'Unspecified Steam ID.')
            return
        supportedItems = {}
        for i in support:
            supportedItems[i] = 0
        try:
            steamUser = steam.user.profile(checklist.getParam('steamid')).id64
        except steam.user.ProfileNotFoundError as e:
            try:
                steamUser = steam.user.vanity_url(checklist.getParam('steamid')).id64
            except Exception as e2:
                checklist.setParam('error', u'Cannot find profile: ' + u(e) + u' / ' + u(e2))
                return
        try:
            backpack = steam.items.inventory(game, steamUser, schema)
        except Exception as e:
            checklist.setParam('error', u'Cannot load inventory: ' + u(e))
            return
        for item in backpack:
            itemName = cleanItemName.sub(u, u(item.name).lower())
            if itemName in supportedItems:
                supportedItems[itemName] += 1
        for item in supportedItems:
            if supportedItems[item] > 1:
                checklist.setParam(item, supportedItems[item])
            elif supportedItems[item] == 1:
                checklist.setParam(item, 'yes')
            else:
                p = checklist.getParam(item)
                if p is not None:
                    p = p.lower()
                if p in (None, 'no', '0'):
                    checklist.setParam(item, 'no')
                elif p not in ('wanted', 'want', 'do not', 'anti', 'do not want'):
                    checklist.setParam(item, 'had')
        return
    try:
        schema = steamGetGameSchema(game)
        allItems = []
        for item in schema:
            allItems.append(cleanItemName.sub(u, u(item.name).lower()))
    except:
        return # No schema means no fancy
    support = []
    templateParams = compileRegex(r'\{\{\{\s*(?:the +)?([^{}|]+?)\s*\|')
    templateCode = page('Template:Item checklist').getWikiText()
    res = templateParams.search(templateCode)
    while res:
        item = u(res.group(1)).lower()
        if item not in support and item in allItems:
            support.append(item)
        templateCode = templateCode[res.end():]
        res = templateParams.search(templateCode)
    checkPage, checkLinks, checkKeys = linkExtract(page('User:WindBOT/Item_checklists').getWikiText())
    linksLeft = checkLinks.values()[:]
    for i in range(12):
        randLink = random.choice(linksLeft)
        linksLeft.remove(randLink)
        checklist = page(randLink.getLink())
        print 'Updating', checklist
        oldContent = u(checklist.getWikiText())
        content, templatelist, templatekeys = templateExtract(oldContent)
        for t in templatelist.values():
            if t.getName().lower().find(u'checklist') != -1:
                updateItemChecklist(t, schema, support)
        content = templateRestore(content, templatelist, templatekeys)
        if oldContent != content:
            editPage(checklist, content, summary=u'Updated Item checklist [[:' + u(checklist.title) + u']]', minor=True)
scheduleTask(itemChecklists, 365)
Retrieved from "https://theportalwiki.com/w/index.php?title=User:WindBOT/Filters&oldid=177526"