PCART  v1.4
Automated Repair of Python API Parameter Compatibility Issues
repair.py
Go to the documentation of this file.
1 
13 
14 
15 
16 import os
17 import ast
18 import subprocess
19 from Tool.tool import getAst,getFileName,getParameter,getLastAPIParameter,resolvePythonExecutable
20 from API.LibApi import Parameter
21 from Change.changeAnalyze import para2Obj
22 
23 
24 
25 
34 def mapPos(pos,dic):
35  for key in dic:
36  try:
37  if key[1]==pos:
38  return dic[key]
39  except:
40  print(f"mapPos error, repairDict:{dic}")
41  return {}
42 
43 
44 
45 
54 def mapName(name,dic):
55  for key in dic:
56  if key[0]==name:
57  return dic[key]
58  return {}
59 
60 
61 
62 
68 def findName(pos,dic):
69  for key in dic.keys():
70  if key[1]==pos:
71  return key[0] #返回参数名
72  print(f"findName error:{pos}")
73  return ''
74 
75 
76 
77 
83 def mirrorAPI(fixedAPI,dic):
84  paraStr=getLastAPIParameter(fixedAPI)
85  paraObjLst=[] #保存参数对象
86  paraStr=paraStr.replace(' ','') #去空格
87  if paraStr:
88  lst=getParameter(paraStr,space=0)
89  else:
90  lst=[]
91  for i in range(len(lst)):
92  para=lst[i]
93  nameFlag=0
94  parameter=Parameter()
95  parameter.position=i #修复后参数的真实位置
96  if '=' in para:
97  pos=para.find('=')
98  if '(' not in para[0:pos] and "'" not in para[0:pos] and '"' not in para[0:pos]: #等号前面也不能出现引号,比如f('x=2',y=1)
99  nameFlag=1
100  parameter.name=para[0:pos]
101  else:
102  nameFlag=0
103  else:
104  nameFlag=0
105 
106  if not nameFlag: #到修复字典中找到修复后的参数名
107  flag=0
108  for k,subDict in dic.items():
109  if 'posChange' in subDict:
110  if subDict['posChange']==parameter.position: #(a,b)-->(b,a)
111  parameter.name=k[0]
112  flag=1
113  break
114  if flag==0:
115  for k,subDict in dic.items():
116  if parameter.position==k[1]:
117  if 'rename' in subDict:
118  parameter.name=subDict['rename']
119  elif 'replace' in subDict:
120  pos=subDict['replace'].find('=')
121  parameter.name=subDict['replace'][0:pos]
122  else:
123  parameter.name=k[0]
124 
125 
126  if nameFlag:
127  paraObjLst.append((parameter,1)) #1表示使用时带了参数名
128  else:
129  paraObjLst.append((parameter,0)) #0表示使用时没有带参数名
130  return paraObjLst
131 
132 
133 
134 
142 def fix(callAPI,repairDict,node,starFlag,twoStarFlag):
143  for n in ast.iter_child_nodes(node):
144  fix(callAPI,repairDict,n,starFlag,twoStarFlag)
145 
146  if isinstance(node,ast.Call):
147  #因为ast在对代码还原时会自动把原本的双引号解析为单引号,所以这里修正一下
148  callState=ast.unparse(node).replace(' ','').replace('"','').replace("'",'') #确保函数名和参数都要一致
149  if callState==callAPI.replace(' ','').replace('"','').replace("'",''):
150  posLst=node.args
151  keyLst=node.keywords
152  newPosLst=[]
153  newKeyLst=[]
154  index=0
155  insertParas=[] #记录待插入的参数,(insertPos, moveFlag, para)
156  #step1: 先对位置参数进行处理
157  for para in posLst:
158  opDict=mapPos(index,repairDict)
159  for op,v in opDict.items():
160  s=ast.unparse(para)
161  if op=='delete':
162  if starFlag:
163  if len(opDict)==1:
164  insertParas.append((index,-1,para))
165  break
166 
167  if op=='typeChange': #若有类型变化的话就无法修复
168  if len(opDict)==1: #typeChange的同时,可能还有其它变更,避免参数重复加入
169  insertParas.append((index,-1,para))
170  break
171 
172  if op=='rename': #当位置参数不带名字使用发生了rename,则无需改动
173  insertParas.append((index,-1,para))
174  break
175 
176  if op=='pos2key':
177  paraNode = ast.keyword(arg=v,value=ast.Name(id=s))
178  newKeyLst.append(paraNode)
179  break
180 
181  if op=='replace':
182  insertParas.append((index,-1,ast.Name(id=v.split('=')[-1])))
183 
184  if op=='posChange':
185  insertParas.append((v,index,para))
186 
187  if len(opDict)==0:
188  insertParas.append((index,-1,para))
189  index+=1
190  #最后还要判断是否有新增的位置参数,新增的位置参数也需要具体的值,新增带默认值的位置参数可以不填
191  # for key,value in repairDict.items():
192  # if 'addPos' in value:
193  # insertParas.append((key[1],ast.Name(id=value['addPos'].split('=')[-1])))
194  insertParas.sort(key=lambda it:it[0]) #按照插入的位置进行从小到大排序
195  for pos,moveFlag,para in insertParas:
196  if pos<=len(newPosLst):
197  newPosLst.insert(pos,para)
198  else:
199  if moveFlag==-1:
200  paraName=findName(pos,repairDict) #即使一个参数移到了其它位置,也要用它原本位置的名字
201  else:
202  paraName=findName(moveFlag,repairDict)
203  s=ast.unparse(para)
204  if paraName:
205  paraNode=ast.keyword(arg=paraName,value=ast.Name(id=s))
206  newKeyLst.append(paraNode)
207 
208  #保留API调用中存在的单独self或者cls参数
209  try:
210  if 'self' == posLst[0].id or 'cls' == posLst[0].id:
211  newPosLst.insert(0,posLst[0])
212  except Exception as e:
213  pass
214  node.args=newPosLst
215 
216  #再对带参数名使用的参数进行处理(位置参数或关键字参数)
217  for para in keyLst:
218  k=para.arg
219  opDict=mapName(k,repairDict)
220  for op,v in opDict.items():
221 
222  if op=='delete':
223  if twoStarFlag:
224  if len(opDict)==1:
225  newKeyLst.append(para)
226  break
227 
228  if op=='typeChange':
229  if len(opDict)==1:#typeChange的同时,还可能有其他变更(posChange),避免重复加入
230  newKeyLst.append(para)
231  break
232 
233  if op=='posChange': #若带参数名使用的参数位置发生改变,则无需修改,直接添加到newKeyLst中即可
234  newKeyLst.append(para)
235 
236  # if op=='replace':
237  # paraName=v.split('=',1)[0]
238  # if '=' in v:
239  # paraNode=ast.keyword(arg=paraName,value=ast.Name(id=v.split('=',1)[-1]))
240  # else:
241  # paraVal=ast.unparse(para).split('=',1)[-1]
242  # paraNode=ast.keyword(arg=paraName,value=ast.Name(id=paraVal))
243  # newKeyLst.append(paraNode)
244 
245  if op=='rename':
246  if not twoStarFlag:
247  para.arg=v #para.arg存放的就是参数名
248  newKeyLst.append(para)
249 
250 
251  if op=='pos2key' or op=='key2pos':
252  newKeyLst.append(para)
253 
254 
255  if len(opDict)==0:
256  newKeyLst.append(para)
257 
258  #最后再判断新增关键字,但新增的关键字参数一般不会导致兼容性问题,所以暂不添加
259  # for key,value in repairDict.items():
260  # if 'addKey' in value:
261  # newKeyLst.append(ast.keyword(arg=k,value=ast.Name(id=val)))
262  node.keywords=newKeyLst #更新修改后的关键字参数
263 
264 
265 
266 
278 def validateByRun(callAPI,apiWithValue,projName,virtualEnv,runPath,runCommand,callKey,*,runtimePaths):
279  if apiWithValue=='':
280  return None
281  copyRoot=runtimePaths['copy_root']
282  dynamicRoot=runtimePaths['dynamic_root']
283  pklName=getFileName(callKey,'.pkl')
284  # 验证阶段复用候选pkl顺序,保证修复验证和动态匹配读取的是同一类接收者
285  pklCandidates=[
286  'new_'+pklName[:-4]+'__object.pkl',
287  'new_'+pklName[:-4]+'__expr.pkl',
288  'new_'+pklName,
289  pklName[:-4]+'__object.pkl',
290  pklName[:-4]+'__expr.pkl',
291  pklName,
292  ]
293  pklPath=''
294  for candidateName in pklCandidates:
295  candidatePath=os.path.join(copyRoot,'pkl',candidateName)
296  if os.path.exists(candidatePath):
297  pklPath=candidatePath
298  break
299  if not pklPath:
300  return None
301 
302  #当runPath不在runCommand中时,需要切换到运行文件所在的目录执行命令
303  #而文件操作的相对路径就是相对于命令执行的路径
304  pythonPath = resolvePythonExecutable(virtualEnv)
305 
306  if runPath and runPath not in runCommand:
307  cwd = os.path.join(dynamicRoot, projName, runPath)
308  script = 'verifySingle.py'
309  elif runPath:
310  cwd = os.path.join(dynamicRoot, projName)
311  script = os.path.join(runPath, 'verifySingle.py')
312  else:
313  cwd = os.path.join(dynamicRoot, projName)
314  script = 'verifySingle.py'
315  result = subprocess.run([pythonPath, script, pklPath, apiWithValue],cwd=cwd, capture_output=True, text=True, encoding='utf-8')
316  return result
317 
318 
319 
320 
329 def validateByStr(fixedAPI,repairDict,targetAPIDefinition,starFlag,twoStarFlag):
330  paraObjLst=mirrorAPI(fixedAPI,repairDict)
331  posLst,keyLst=para2Obj(targetAPIDefinition)
332  targetPara=posLst+keyLst
333 
334  for para,nameFlag in paraObjLst:
335  compatible=0 #假设每个参数都不兼容
336  for it in targetPara:
337  if nameFlag: #带参数名使用的,要求修复完之后能在目标版本的参数定义中找到相同名字的参数
338  if para.name==it.name:
339  compatible=1
340  break
341  else: #不带参数使用的要求修复完之后,位置和参数名都要能找到对应的
342  if para.position==it.position and para.name==it.name:
343  compatible=1
344  break
345 
346 
347  if not compatible:
348  if nameFlag and not twoStarFlag:
349  return False
350  if not nameFlag and not starFlag:
351  return False
352 
353  return True
354 
355 
356 
357 
380 def repairTask(root,callAPI,apiWithValue,projName,runPath,runCommand,repairLst,virtualEnv,errLst,callKey,*,runtimePaths):
381  #静态修复,pkl加载失败,只能进入静态修复
382  repairCandidates=[]
383  if apiWithValue=='':
384  for repairDict,targetPara in repairLst:
385  starFlag=0 #判断目标版本的参数定义中是否含有*args
386  twoStarFlag=0 #判断目标版本的参数定义中是否含有*kwargs
387  if '*args' in targetPara:
388  starFlag=1
389  if '**' in targetPara:
390  twoStarFlag=1
391  apiRoot=getAst(callAPI,1)
392  fix(callAPI,repairDict,apiRoot,starFlag,twoStarFlag)
393  fixedAPI=ast.unparse(apiRoot)
394  if validateByStr(fixedAPI,repairDict,targetPara,starFlag,twoStarFlag):
395  if fixedAPI not in repairCandidates:
396  repairCandidates.append(fixedAPI)
397 
398  # repairCandidates=list(set(repairCandidates))
399  if len(repairCandidates)==0:
400  return str(repairCandidates), 'Unknown', 'Unknown'
401  elif len(repairCandidates)==1:
402  fixedAPI=repairCandidates[0]
403  if callAPI.replace(' ','').replace('"','').replace("'",'')==fixedAPI.replace(' ','').replace('"','').replace("'",''):
404  return repairCandidates[0],"Compatible","Unknown"
405  else:
406  return repairCandidates[0],"Incompatible","Unknown"
407  else:
408  return str(repairCandidates),"Unknown", "Unknown"
409 
410  #pkl加载成功,但匹配的结果也可能是多个,内置API只能静态匹配
411  failedLst=[]
412  for repairDict,targetPara in repairLst:
413  starFlag=0 #判断目标版本的参数定义中是否含有*args
414  twoStarFlag=0 #判断目标版本的参数定义中是否含有*kwargs
415  # 1.先进行静态验证
416  if '*args' in targetPara:
417  starFlag=1
418  if '**' in targetPara:
419  twoStarFlag=1
420  apiRoot=getAst(callAPI,1)
421  fix(callAPI,repairDict,apiRoot,starFlag,twoStarFlag)
422  fixedAPI=ast.unparse(apiRoot)
423  if not validateByStr(fixedAPI,repairDict,targetPara,starFlag,twoStarFlag):
424  continue
425  elif len(repairLst)==1:
426  repairCandidates.append(fixedAPI)
427 
428  #2. 动态验证
429  fixFlag='Failed'
430  apiRoot=getAst(apiWithValue,1)
431  fix(apiWithValue,repairDict,apiRoot,starFlag,twoStarFlag)
432  apiWithValueFixed=ast.unparse(apiRoot)
433  #1先通过动态运行,判断其是否修复成功
434  result=validateByRun(callAPI,apiWithValueFixed,projName,virtualEnv,runPath,runCommand,callKey,runtimePaths=runtimePaths)
435  if result==None:
436  if fixedAPI not in repairCandidates:
437  repairCandidates.append(fixedAPI)
438  fixFlag='Unknown'
439  elif result.returncode!=0:
440  errLst.append(f"{callAPI}, validate error: {result.stderr}\n")
441  failedLst.append(f"{callAPI}, validate error: {result.stderr}\n")
442  if 'dill' in result.stderr:
443  fixFlag='Unknown'
444  if fixedAPI not in repairCandidates:
445  repairCandidates.append(fixedAPI)
446  elif validateByStr(fixedAPI,repairDict,targetPara,starFlag,twoStarFlag):
447  fixFlag='Unknown'
448  if fixedAPI not in repairCandidates:
449  repairCandidates.append(fixedAPI)
450 
451  else:
452  fixFlag='Successful'
453  if fixedAPI not in repairCandidates:
454  repairCandidates.append(fixedAPI)
455  break
456 
457  # repairCandidates=list(set(repairCandidates))
458  if len(repairCandidates)==0:
459  return str(repairCandidates), 'Unknown' , 'Unknown'
460  elif len(repairCandidates)==1:
461  fixedAPI=repairCandidates[0]
462  if callAPI.replace(' ','').replace('"','').replace("'",'')==fixedAPI.replace(' ','').replace('"','').replace("'",''):
463  return repairCandidates[0],'Compatible',fixFlag
464  else:
465  return repairCandidates[0],'Incompatible',fixFlag
466  else:
467  return str(repairCandidates),'Unknown','Unknown'
def para2Obj(paraStr)
Convert parameter string to positional and keyword parameter objects 将参数字符串转换为位置和关键字参数对象
def getFileName(fileName, extension)
Normalize and sanitize file name 规范化并清理文件名
Definition: codeUtils.py:66
def getParameter(p_string, separator=',', space=1)
Split parameter string into list of separated parameters 将参数字符串拆分成单个的参数
Definition: codeUtils.py:244
def getLastAPIParameter(apiStr)
Get parameter(s) of the last API from the API call string 获取最后一个API参数
Definition: codeUtils.py:334
def mirrorAPI(fixedAPI, dic)
Create mirror for the fixed API by adding all parameter names 给修复后API的参数填上参数名
Definition: repair.py:83
def mapPos(pos, dic)
Get repair operation dictionary of positional parameter 获取位置参数的修复操作字典
Definition: repair.py:34
def repairTask(root, callAPI, apiWithValue, projName, runPath, runCommand, repairLst, virtualEnv, errLst, callKey, *runtimePaths)
Task of repairing parameter compatibility issues 参数兼容性问题修复任务
Definition: repair.py:380
def fix(callAPI, repairDict, node, starFlag, twoStarFlag)
Perform parameter repair operations through AST 通过AST执行参数修复操作
Definition: repair.py:142
def findName(pos, dic)
Find parameter name by its position 通过位置查找参数名
Definition: repair.py:68
def validateByRun(callAPI, apiWithValue, projName, virtualEnv, runPath, runCommand, callKey, *runtimePaths)
Dynamic validation 动态验证
Definition: repair.py:278
def validateByStr(fixedAPI, repairDict, targetAPIDefinition, starFlag, twoStarFlag)
Static validation 静态验证
Definition: repair.py:329
def mapName(name, dic)
Get repair operation dictionary of keyword parameter 获取关键字参数的修复操作字典
Definition: repair.py:54
def getAst(filePath, strFlag=0)
Get AST for code 将代码转化为Ast树
Definition: tool.py:33
def resolvePythonExecutable(envPath)
Resolve Python executable from a virtual environment root 从虚拟环境根目录解析 Python 解释器路径
Definition: tool.py:491