User:P-bot/字词转换代码标准化

/pywikipedia/user-fixes.py

# -*- coding: utf-8  -*-
# @filename:user-fixes.py
import re

convreg=re.compile(ur"\s*[a-zA-Z-]+\s*:\s*[^;]*(;\s*[a-zA-Z-]+\s*:\s*[^;]+)*;?\s*")
# 其它语言另列
orderedvars=['zh','zh-hans','zh-hant','zh-cn','zh-tw','zh-hk','zh-sg']

# @return list [('zh',zhWord),('zh-hans',zhHansWord)......] ordered
def getOrderVariants(varscode):
	if not convreg.match(varscode):
		return varscode
	ret=[]
	vars={}
	for v in [varstr.split(':',1)  for varstr in varscode.split(';')]:
		if len(v)==2:
			vars[v[0].strip()]=v[1].strip()
	for vk in orderedvars:
		if vk in vars:
			ret.append((vk,vars[vk]))
			del vars[vk]
	for vk in vars:
		ret.append((vk,vars[vk]))
	return ret

# @return  tuple: (string flags, list variants)
# variants  see return of getOrderVariants function
def parseRulesCode(text,sep='|'):
	if text.startswith('\r\n{|'):
		return 'R',text
	try:
		flags,rules=text.split(sep,1)
	except ValueError:
		flags,rules='',text
	else:
		flags=flags.strip()
		if flags=='R':
			return 'R',rules
		rules=getOrderVariants(rules)
	return flags,rules

def getConvCodeStr(texts,codes):
	ret=''
	for i in range(len (codes)):
		code = codes[i]
		pre = '-{' if code[0]!='raw' else ''
		post = '}-' if code[0]!='raw' else ''
		ret += texts[i] + pre + getRulesStr(codes[i]) + post
	return ret

#  split 'xxx -{convcode }- yyy-{convcode }- ......' to  a generator:
# @return generator:
# yield <code>('xxx ',CODE1), (' yyy',CODE2),......</code>
# will insert virtual CODE at the end of text
# about CODE format see return of function parseRulesCode 
def splitConvCode(text):
	while (text):
		isRaw=False
		try:
			txt,rest=text.split('-{',1)
		except ValueError:
			isRaw,txt,code,text=True,text,'',''
		else:
			if rest:
				try:
					code,text=rest.split('}-',1)
				except ValueError:
					print (u"有语法错误:-{與}-不匹配")
					isRaw,txt,code,text=True,text,'',''
			else:
				print (u"有语法错误:-{與}-不匹配")
				isRaw,txt,code,text=True,text,'',''
		yield txt,('raw','') if isRaw else parseRulesCode(code)

def getRulesStr(code,sep='|'):
	if code[0]=='raw':
		return code[1]
	if type(code[1])==unicode or type(code[1])==str:
		return (code[0]+sep if code[0] else '')\
			+ (code[1] if code[0]=='R' else code[1].strip())
	return (code[0]+sep if code[0] else '') \
			+' '.join([vk+":"+vv+";" for (vk,vv) in code[1]])

def findend(sep,str):
	end=str.find(sep)
	while end>0:
		pre=str[:end]
		if pre.count('{{')-pre.count('}}')>0 \
			or pre.count('[[')-pre.count(']]')>0:
			end=str.find(sep,end+2)
		else:
			break
	return end

# @return generator (str paraname, list codes) , ......
def parseTempContent(text):
	while(text):
		end=findend('|',text)
		if end<0:
			para,text=text,''
		else:
			para,text=text[:end],text[(end+1):]
		yield parseRulesCode(para,'=')

def getTempStr(tname,paras):
	if type(paras)==str or type(paras)==unicode:
		return paras
	if tname=='none':
		return ''
	cr = u'\n' if tname=='noteTA' else u''
	return u'{{'+tname \
			+ ''.join(cr+'|'+getRulesStr(code,sep='=') for code in paras) \
			+cr+'}}'+cr
	
#  split 'xxx {{tempname|tempcode }} yyy {{tempname|tempcode }} ......' to  a generator:
# @return generator :
#    yield <code>{'text':'xxx ','tname':tempname,'temp':PARAS1} , {'text':'yyy ','tname':tempname,'temp':PARAS2} ,......</code>
#    about PARAS format see return of function parseTempContent
def splitTemp(text,cmd):
	#根据cmd分派
	if cmd.startswith(('noteTA','safemerge','merge')):
		creg=re.compile( ur'\{\{\s*[Nn]oteTA\s*\|' )
		tname=u'noteTA'
	elif cmd==u'CItem':
		creg=re.compile( ur'\{\{\s*[Cc]Item\s*\|' )
		tname=u'CItem'
	else:
		yield { 'text':text, 'tname':'none', 'temp':[] }
		return
	#分派结束
	while(text):
		hasTemp=False
		try:
			txt,rest=creg.split(text,1)
		except ValueError:
			txt,text=text,''
		else:
			if rest!='':
				end=findend("}}",rest)
				if end<0:
					print ("wiktext有语法错误")
					txt,text=text,''
				else:
					hasTemp=True
					parastr,text=rest[:end].strip(),rest[(end+2):]
					temp=list(parseTempContent(parastr))
			else:
				print ("wiktext有语法错误")
				txt,text=text,''
		#print txt
		yield { 'text':txt, 'tname':tname, 'temp':temp } if hasTemp \
				else { 'text':txt, 'tname':'none', 'temp':[] }

def getOneConv(rules):
	if type(rules)==str or type(rules)==unicode:
		return rules
	for vk,vv in rules:
		if vk in orderedvars:
			return vv

def mergeNoteTA(merge,item):
	merge[0]['temp']+=item['temp']
	merge[1]['text']+=item['text']
	if item['tname']=='noteTA':
		merge[0]['tname']='noteTA'
	return merge

def parseConversion(text,cmd):
	if not cmd in ['noteTA','CItem','merge','safemerge']:
		return text
	texts,codes=[],[]
	for seg in splitConvCode(text):
		texts.append(seg[0])
		codes.append(seg[1])
	convtag='<convcode/>'
	tarray=splitTemp(convtag.join(texts),cmd)
	if cmd.startswith(('noteTA','safemerge','merge')):
		# 合并多个noteTA为一个，并置顶
		tarray=reduce(mergeNoteTA,tarray,[
			{'text':'','tname':'none','temp':[]},
			{'text':'','tname':'none','temp':[]} ])
	else:
		tarray=list(tarray)

	if cmd=='safemerge' or cmd=='merge': 
		# 合并-{A|}-到noteTA (cmd=='merge' 下也合并-{}-)
		tarray[0]['temp']+=[ code
			for code in codes
			if code[0]=='A' or code[0]=='T' or (cmd=='merge' and code[0]=='')]
		if len(tarray[0]['temp']):
			tarray[0]['tname']='noteTA'
		codes=[ ('raw',getOneConv(code[1])) 
				if code[0]=='A' or (cmd=='merge' and code[0]=='')
				else ('raw','') if code[0]=='T' else code
				for code in codes]

	if cmd.startswith(('noteTA','safemerge','merge')):
		#  清除同类项/重列序号/消除空参数/
		newRules = []
		existconvs = ['']
		firstT = True
		index = 0
		for rule in tarray[0]['temp']:
			if rule[0]=='T' and firstT:
				firstT=False
				newRules.insert(0,rule)
			elif rule[0]=='A' or rule[0]=='' or re.match('\s*\d+\s*',rule[0]):
				rulestr=getRulesStr(('',rule[1])).strip()
				if rulestr not in existconvs:
					existconvs.append(rulestr)
					index+=1
					newRules.append(('%d'%index,rule[1]))
			else:
				newRules.append(rule)
		tarray[0]['temp']=newRules

	texts=''.join( 
			[t['text'].strip() + getTempStr(t['tname'],t['temp'])
			for t in tarray] 
			).split(convtag)

	return getConvCodeStr(texts,codes).strip()

# 以下用来调度命令，可以在fix的replacements作为第二个参数
def fixConversion(cmd):
	return lambda m:parseConversion(m.string,cmd)

fixes['zh-conversion'] = {
	'regex': True,
	'msg': {
		'zh': u'机器人：字词转换代码标准化',
	},
	'replacements': [
		# STEP 1: SORT
		(ur'[\s\S]+', fixConversion('noteTA') ),
	],
}

fixes['zh-conversion-to-noteta'] = {
	'regex': True,
	'msg': {
		'zh': u'机器人：字词转换代码NoteTA化（全自动安全模式）',
	},
	'replacements': [
		# STEP 1: replace to standard noteTA template
		(ur'\{\{\s*([nN]ote[tT]?[aA][tT]?|[tT]A)(?![a-zA-Z0-9]*)\s*', ur'{{NoteTA'),
		# STEP 2: to NOTETA
		(ur'[\s\S]+', fixConversion('safemerge')),
	],
}

# !!! this fix is unsafe!!!
# replace.py -page:珍妮佛·康納莉 -fix:zh-conversion-to-noteta-unsafe
fixes['zh-conversion-to-noteta-unsafe'] = {
	'regex': True,
	'msg': {
		'zh': u'机器人：字词转换代码NoteTA化（半人工非安全模式）',
	},
	'replacements': [
		# STEP 1: replace to standard noteTA template
		(ur'\{\{\s*([nN]ote[tT]?[aA][tT]?|[tT]A)(?![a-zA-Z0-9]*)\s*', ur'{{NoteTA'),
		# STEP 2: to NOTETA
		(ur'[\s\S]+', fixConversion('merge')),
	],
}