1,816
edits
m (→Implement {{tl|Dictionary}}: fun times for windbot) |
|||
(9 intermediate revisions by 3 users not shown) | |||
Line 128: | Line 128: | ||
class DictionaryUpdater: | class DictionaryUpdater: | ||
def __init__(self): | def __init__(self): | ||
self.subpageTemplateLang = <nowiki>"""{{#switch:{{{lang|{{SUBPAGENAME}}}}}|%options%}} | self.subpageTemplateLang = <nowiki>"""{{#switch:{{{lang|{{SUBPAGENAME}}}}}|%options%}}"""</nowiki> | ||
self.subpageTemplateParam = <nowiki>"""{{#switch:{{{1|}}}|%options%}} | self.subpageTemplateParam = <nowiki>"""{{#switch:{{{1|}}}|%options%}}"""</nowiki> | ||
self.invalidParamError = <nowiki>"""< | self.invalidParamError = <nowiki>"""<span class="error">Error: invalid param.</span>[[Category:ERROR]]"""</nowiki> | ||
self.subpageTemplateID = <nowiki>"""%string% | self.subpageTemplateID = <nowiki>"""%string%"""</nowiki> | ||
self.partialUpdateThreshold = 750 # Update SyncData every n edits | |||
self.dictionaries = { | self.dictionaries = { | ||
u'Template:Dictionary/items': { # Dictionary page | u'Template:Dictionary/items': { # Dictionary page | ||
Line 173: | Line 174: | ||
'name': 'portal 2 achievements', | 'name': 'portal 2 achievements', | ||
'sync': 'Template:Dictionary/portal 2 achievements/Special:SyncData' | 'sync': 'Template:Dictionary/portal 2 achievements/Special:SyncData' | ||
}, | |||
u'Template:Dictionary/rexaura achievements': { | |||
'name': 'rexaura achievements', | |||
'sync': 'Template:Dictionary/rexaura achievements/Special:SyncData' | |||
}, | |||
u'Template:Dictionary/portal stories: mel achievements': { | |||
'name': 'portal stories: mel achievements', | |||
'sync': 'Template:Dictionary/portal stories: mel achievements/Special:SyncData' | |||
}, | |||
u'Template:Dictionary/portal pinball achievements': { | |||
'name': 'portal pinball achievements', | |||
'sync': 'Template:Dictionary/portal pinball achievements/Special:SyncData' | |||
}, | |||
u'Template:Dictionary/bridge constructor portal achievements': { | |||
'name': 'bridge constructor portal achievements', | |||
'sync': 'Template:Dictionary/bridge constructor portal achievements/Special:SyncData' | |||
}, | }, | ||
u'Template:Dictionary/audio': { | u'Template:Dictionary/audio': { | ||
Line 217: | Line 234: | ||
'name': 'voice lines/Wheatley', | 'name': 'voice lines/Wheatley', | ||
'sync': 'Template:Dictionary/voice lines/Wheatley/Special:SyncData' | 'sync': 'Template:Dictionary/voice lines/Wheatley/Special:SyncData' | ||
}, | |||
u'Template:Dictionary/voice lines/Grady': { | |||
'name': 'voice lines/Grady', | |||
'sync': 'Template:Dictionary/voice lines/Grady/Special:SyncData' | |||
}, | |||
u'Template:Dictionary/voice lines/Desk Job': { | |||
'name': 'voice lines/Desk Job', | |||
'sync': 'Template:Dictionary/voice lines/Desk Job/Special:SyncData' | |||
} | } | ||
} | } | ||
Line 228: | Line 253: | ||
self.stringsExtract = compileRegex(r'(?:^[ \t]*#[ \t]*([^\r\n]*?)[ \t]*$\s*)?^[ \t]*([^\r\n]+?[ \t]*(?:\|[ \t]*[^\r\n]+?[ \t]*)*):[ \t]*([^\r\n]+?[ \t]*$|\s*[\r\n]+(?:\s*[ \t]+[-\w]+[ \t]*:[ \t]*[^\r\n]+[ \t]*$)+)', re.IGNORECASE | re.MULTILINE) | self.stringsExtract = compileRegex(r'(?:^[ \t]*#[ \t]*([^\r\n]*?)[ \t]*$\s*)?^[ \t]*([^\r\n]+?[ \t]*(?:\|[ \t]*[^\r\n]+?[ \t]*)*):[ \t]*([^\r\n]+?[ \t]*$|\s*[\r\n]+(?:\s*[ \t]+[-\w]+[ \t]*:[ \t]*[^\r\n]+[ \t]*$)+)', re.IGNORECASE | re.MULTILINE) | ||
self.translationExtract = compileRegex(r'^[ \t]+([-\w]+)[ \t]*:[ \t]*([^\r\n]+)[ \t]*$', re.IGNORECASE | re.MULTILINE) | self.translationExtract = compileRegex(r'^[ \t]+([-\w]+)[ \t]*:[ \t]*([^\r\n]+)[ \t]*$', re.IGNORECASE | re.MULTILINE) | ||
addWhitelistPage(self.dictionaries.keys()) | addWhitelistPage(self.dictionaries.keys()) | ||
self.editCounts = {} | |||
def updateSyncData(self, currentDict, syncData, note=''): | |||
# Build syncdata string representation | |||
syncKeys = syncData.keys() | |||
syncKeys.sort() | |||
syncLines = [] | |||
for k in syncKeys: | |||
syncLines.append(k + u':' + syncData[k]) | |||
if note: | |||
note = u' (' + u(note) + u')' | |||
editPage(self.dictionaries[currentDict]['sync'], u'\n'.join(syncLines), summary=<nowiki>u'Updated synchronization information for [[:' + currentDict + u']]' + note + u'.'</nowiki>, minor=True, nocreate=False) | |||
def generateSubpage(self, keyName, data, currentDict, syncData): | def generateSubpage(self, keyName, data, currentDict, syncData): | ||
h = hashlib.md5() | h = hashlib.md5() | ||
Line 286: | Line 321: | ||
if keyName in syncData and syncData[keyName] == h: | if keyName in syncData and syncData[keyName] == h: | ||
return # Same hash | return # Same hash | ||
subpage = subpage.replace(u'%dictionary%', currentDict) | subpage = subpage.replace(u'%dictionary%', currentDict) | ||
subpage = subpage.replace(u'%dictionaryname%', self.dictionaries[currentDict]['name']) | subpage = subpage.replace(u'%dictionaryname%', self.dictionaries[currentDict]['name']) | ||
subpage = subpage.replace(u'%keyname%', keyName) | subpage = subpage.replace(u'%keyname%', keyName) | ||
if editPage(currentDict + self.subpageSeparator + keyName, subpage, summary=<nowiki>u'Pushed changes from [[:' + currentDict + u']] for string "' + keyName + u'".'</nowiki>, minor=True, nocreate=False): | |||
syncData[keyName] = h # Update sync data | |||
if currentDict not in self.editCounts: | |||
self.editCounts[currentDict] = 0 | |||
self.editCounts[currentDict] += 1 | |||
if self.editCounts[currentDict] > self.partialUpdateThreshold: | |||
self.editCounts[currentDict] = 0 | |||
self.updateSyncData(currentDict, syncData, 'Partial update') | |||
def processComment(self, commentString, currentDict, definedStrings, syncData): | def processComment(self, commentString, currentDict, definedStrings, syncData): | ||
commentContents = [] | commentContents = [] | ||
Line 322: | Line 363: | ||
validKeyNames = [] | validKeyNames = [] | ||
for keyName in keyNames: | for keyName in keyNames: | ||
keyName = keyName.replace(u'_', u' ').strip() | keyName = keyName.replace(u'_', u' ').replace(u'#', u'').strip() | ||
if keyName in definedStrings: | if keyName in definedStrings: | ||
continue # Duplicate key | continue # Duplicate key | ||
Line 330: | Line 371: | ||
if len(validKeyNames): | if len(validKeyNames): | ||
commentContents.append(comment + u' | '.join(validKeyNames) + u':' + dataWriteback) | commentContents.append(comment + u' | '.join(validKeyNames) + u':' + dataWriteback) | ||
return u'\n\n'.join(commentContents) | return u'\n\n'.join(commentContents) | ||
def __call__(self, content, **kwargs): | def __call__(self, content, **kwargs): | ||
Line 338: | Line 378: | ||
return content | return content | ||
currentDict = u(kwargs['article'].title) | currentDict = u(kwargs['article'].title) | ||
if random.randint(0, 50) == 0: # With probability 2%, ignore syncdata completely. Helps with stale syncdata and people overwriting things. | |||
syncDataText = u'' | syncDataText = u'' | ||
else: | |||
try: | |||
syncDataText = u(page(self.dictionaries[currentDict]['sync']).getWikiText()).split(u'\n') | |||
except: # Page probably doesn't exist | |||
syncDataText = u'' | |||
syncData = {} | syncData = {} | ||
for sync in syncDataText: | for sync in syncDataText: | ||
Line 361: | Line 403: | ||
newContent += u'<!--\n\n' + self.processComment(u(comment.group(1)).strip(), currentDict, definedStrings, syncData) + u'\n\n-->' | newContent += u'<!--\n\n' + self.processComment(u(comment.group(1)).strip(), currentDict, definedStrings, syncData) + u'\n\n-->' | ||
newContent += content[previousIndex:] | newContent += content[previousIndex:] | ||
# Check for deleted strings | # Check for deleted strings | ||
for k in oldSyncData: | for k in oldSyncData: | ||
Line 376: | Line 412: | ||
if k in syncData: | if k in syncData: | ||
del syncData[k] | del syncData[k] | ||
self.updateSyncData(currentDict, syncData, 'Full update') | |||
self.editCounts[currentDict] = 0 | |||
return newContent | return newContent | ||
def scheduledRun(self): | def scheduledRun(self): | ||
Line 395: | Line 424: | ||
=== Update checklists on [[User:WindBOT/Item checklists|list of subscribers]] === | === Update checklists on [[User:WindBOT/Item checklists|list of subscribers]] === | ||
def itemChecklists(): | def itemChecklists(): | ||
game = 620 | |||
cleanItemName = compileRegex(r'^the +') | |||
def updateItemChecklist(checklist, schema, support): | def updateItemChecklist(checklist, schema, support): | ||
if not checklist.getParam('steamid'): | if not checklist.getParam('steamid'): | ||
checklist.setParam('error', 'Unspecified Steam ID.') | checklist.setParam('error', 'Unspecified Steam ID.') | ||
return | return | ||
supportedItems = {} | supportedItems = {} | ||
for i in support: | for i in support: | ||
supportedItems[i] = 0 | supportedItems[i] = 0 | ||
try: | try: | ||
steamUser = steam.user.profile(checklist.getParam('steamid')) | steamUser = steam.user.profile(checklist.getParam('steamid')).id64 | ||
except steam.user.ProfileNotFoundError as e: | |||
try: | |||
steamUser = steam.user.vanity_url(checklist.getParam('steamid')).id64 | |||
except Exception as e2: | |||
checklist.setParam('error', u'Cannot find profile: ' + u(e) + u' / ' + u(e2)) | |||
return | |||
try: | |||
backpack = steam.items.inventory(game, steamUser, schema) | |||
except Exception as e: | except Exception as e: | ||
checklist.setParam('error', u(e)) | checklist.setParam('error', u'Cannot load inventory: ' + u(e)) | ||
return | return | ||
for item in backpack: | for item in backpack: | ||
itemName = u(item. | itemName = cleanItemName.sub(u'', u(item.name).lower()) | ||
if itemName in supportedItems: | if itemName in supportedItems: | ||
supportedItems[itemName] += 1 | supportedItems[itemName] += 1 | ||
Line 428: | Line 463: | ||
elif p not in ('wanted', 'want', 'do not', 'anti', 'do not want'): | elif p not in ('wanted', 'want', 'do not', 'anti', 'do not want'): | ||
checklist.setParam(item, 'had') | checklist.setParam(item, 'had') | ||
return | return | ||
try: | try: | ||
schema = steamGetGameSchema( | schema = steamGetGameSchema(game) | ||
allItems = [] | allItems = [] | ||
for item in schema: | for item in schema: | ||
allItems.append(u(item. | allItems.append(cleanItemName.sub(u'', u(item.name).lower())) | ||
except: | except: | ||
return # No schema means no fancy | return # No schema means no fancy | ||
support = [] | support = [] | ||
templateParams = compileRegex(r'\{\{\{\s*([^{}|]+?)\s*\|') | templateParams = compileRegex(r'\{\{\{\s*(?:the +)?([^{}|]+?)\s*\|') | ||
templateCode = page('Template:Item checklist').getWikiText() | templateCode = page('Template:Item checklist').getWikiText() | ||
res = templateParams.search(templateCode) | res = templateParams.search(templateCode) | ||
Line 453: | Line 488: | ||
checklist = page(randLink.getLink()) | checklist = page(randLink.getLink()) | ||
print 'Updating', checklist | print 'Updating', checklist | ||
oldContent = u(checklist.getWikiText()) | oldContent = u(checklist.getWikiText()) | ||
content, templatelist, templatekeys = templateExtract(oldContent) | content, templatelist, templatekeys = templateExtract(oldContent) | ||
for t in templatelist.values(): | for t in templatelist.values(): | ||
if t.getName().lower().find(u'checklist') != -1: | if t.getName().lower().find(u'checklist') != -1: | ||
updateItemChecklist(t, schema, support) | |||
content = templateRestore(content, templatelist, templatekeys) | content = templateRestore(content, templatelist, templatekeys) | ||
if | if oldContent != content: | ||
editPage(checklist, content, summary=<nowiki>u'Updated Item checklist [[:' + u(checklist.title) + u']]'</nowiki>, minor=True) | editPage(checklist, content, summary=<nowiki>u'Updated Item checklist [[:' + u(checklist.title) + u']]'</nowiki>, minor=True) | ||
scheduleTask(itemChecklists, | scheduleTask(itemChecklists, 365) |