PCART  v1.4
Automated Repair of Python API Parameter Compatibility Issues
tool.py
Go to the documentation of this file.
1 
12 
13 
14 
15 import re
16 import os
17 import ast
18 import json
19 import platform
20 import shlex
21 import tokenize
22 from Path.getPath import Path
23 from Tool.codeUtils import getArtifactHash, getArtifactDisplayName, shortenArtifactFileName, getFileName, getParameter, removeParameter, getLastAPIParameter, departAPI, departAPI2
24 
25 
26 
27 
33 def getAst(filePath,strFlag=0): #若strFlag=1,则表明传进来的是一个api,而不是一个路径
34  if strFlag==0:
35  with tokenize.open(filePath) as f:
36  s=f.read()
37  root=ast.parse(s,filename='<unknown>',mode='exec')
38  return root
39  root=ast.parse(filePath,filename='<unknown>',mode='exec')
40  return root
41 
42 
43 
44 
49 def getImportLst(filePath):
50  importLst=[]
51  with open(filePath,'r',encoding='UTF-8') as f:
52  lstR=f.readlines()
53  for it in lstR:
54  #这个判断条件在后续应该将其设置为通用的,不同的库判断条件不同
55  if (it[0:6]=='import' or it[0:4]=='from') and 'torch' in it and 'torchvision' not in it and 'torch_' not in it and '.torch' not in it:
56  importLst.append(it)
57  return importLst
58 
59 
60 
61 
66 def isDynamic(dic):
67  lst=list(dic.values())
68  for it in lst:
69  if isinstance(it,str):
70  return 1
71  return 0 #空字典也看作是模糊匹配的字典
72 
73 
74 
75 
80 def getVersionLst(libPath):
81  obj=Path('D')
82  versionLst=[]
83  obj.getPath(libPath)
84  path=obj.path
85  for p in path:
86  p = os.path.basename(p)
87  index=-1
88  for i in range(len(p)):
89  if p[i].isdigit():
90  index=i
91  break
92  versionLst.append(p[index:])
93  versionLst.sort(key=lambda it:cmp(it))
94  return versionLst
95 
96 
97 
98 
108 def cmp(version):
109  index=len(version)
110  innerVersion=""
111  for i in range(len(version)):
112  if version[i].isalpha():
113  index=i
114  break
115  if index!=len(version):
116  innerVersion=re.findall(r'\d+',version[index:])[0]
117  Type=re.findall(r'[a-zA-Z]+',version[index:])[0] #type='rc' or type='b' or type='a'
118  if Type=='a': #alpha版
119  if len(innerVersion)==1:
120  innerVersion='0.00000'+innerVersion
121  else:
122  innerVersion='0.0000'+innerVersion #为了确保a10<b1
123  elif Type=='b':#beta版
124  if len(innerVersion)==1:
125  innerVersion='0.000'+innerVersion
126  else:
127  innerVersion='0.00'+innerVersion #为了确保b10<rc1
128  elif Type=='rc': #release版
129  if len(innerVersion)==1:
130  innerVersion='0.0'+innerVersion
131  else:
132  innerVersion='0.'+innerVersion
133 
134 
135  lst=version[0:index].split('.')
136  if len(lst)==2:
137  lst.append('0')
138  if len(lst[1])==1:
139  lst[1]='0'+lst[1]
140  if len(lst[2])==1:
141  lst[2]='0'+lst[2]
142  v=''.join(lst)
143  if innerVersion=="":
144  return float(v)
145  else:
146  return float(v)-(1-float(innerVersion)) #转化为浮点数形式
147 
148 
149 
150 
159 def writeLine(width,s,fw):
160  contentWidth=width-4
161  for start in range(0,max(len(s),1),contentWidth):
162  content=s[start:start+contentWidth]
163  tailSpaceNum=width-2-len(content)-1
164  fw.write('| '+content+' '*tailSpaceNum+'|'+'\n')
165 
166 
167 
168 
175 def save2txt(lst,libName,runCommand,savePath):
176  fw=open(savePath,'w',encoding='UTF-8')
177  totalFileNum=0
178  totalAPINum=0
179  compatibleNum=0
180  incompatibleNum=0
181  unknownCompatibleNum=0
182  notCoverNum=0
183  successRepairNum=0
184  failedRepairNum=0
185  unknownRepairNum=0
186  for Tuple in lst: #计数
187  totalFileNum+=1
188  totalAPINum+=Tuple[2]
189  dic=Tuple[0]
190  for callAPI,subDict in dic.items():
191  if subDict['Coverage']=='No':
192  notCoverNum+=1
193  continue
194 
195  if subDict['Compatible']=='Yes':
196  compatibleNum+=1
197  elif subDict['Compatible']=='No':
198  incompatibleNum+=1
199  else:
200  unknownCompatibleNum+=1
201 
202  if 'Repair <Successful>' in subDict:
203  successRepairNum+=1
204  elif 'Repair <Failed>' in subDict:
205  failedRepairNum+=1
206  elif 'Repair <Unknown>' in subDict:
207  unknownRepairNum+=1
208 
209  #写结果
210  libName=libName.capitalize()
211  fw.write(f"Run Command: {runCommand}\n")
212  fw.write(f"Total File Number: {totalFileNum}\n")
213  fw.write(f"Total {libName} Invoked API Number: {totalAPINum}\n")
214  fw.write(f"Not Covered {libName} Invoked API Number: {notCoverNum}/{totalAPINum}\n")
215  fw.write(f"Covered {libName} Invoked API Number: {totalAPINum-notCoverNum}/{totalAPINum}\n\n")
216 
217  fw.write(f"Compatible {libName} Invoked API Number: {compatibleNum}/{totalAPINum-notCoverNum}\n")
218  fw.write(f"Unknown Compatible {libName} Invoked API Number: {unknownCompatibleNum}/{totalAPINum-notCoverNum}\n\n")
219 
220  fw.write(f"Incompatible {libName} Invoked API Number: {incompatibleNum}/{totalAPINum-notCoverNum}\n")
221  fw.write(f"-> Successfully Repaired {libName} Invoked API number: {successRepairNum}/{incompatibleNum}\n")
222  fw.write(f"-> Failed to Repair {libName} Invoked API Number: {failedRepairNum}/{incompatibleNum}\n")
223  fw.write(f"-> Unknown Repair Status {libName} Invoked API Number: {unknownRepairNum}/{incompatibleNum}\n\n")
224  fileCount=0
225  for Tuple in lst:
226  dic=Tuple[0]
227  filePath=Tuple[1]
228  invokedAPINum=Tuple[2]
229  fileCount+=1
230  width=102 #设置列表总宽度175个字符
231  title=f"File #{fileCount}: {filePath} has {invokedAPINum} {libName}-Invoked API(s)"
232  line='='*width+'\n'
233  fontSpaceNum=(width-len(title)-2)//2
234  tailSpaceNum=width-2-fontSpaceNum-len(title)
235  title='|'+' '*fontSpaceNum+title+' '*tailSpaceNum+'|'+'\n'
236  fw.write(line)
237  fw.write(title)
238  fw.write(line)
239  index=1
240  for callAPI,subDict in dic.items():
241  displayCall=subDict.pop('Invoked API',callAPI)
242  tempStr1=f"Invoked API #{index}: {displayCall}"
243  writeLine(width,tempStr1,fw)
244  fw.write('|'+' '*(width-2)+'|'+'\n')
245  cnt=1
246  for k,v in subDict.items():
247  tempStr2=f"{k}: {v}"
248  writeLine(width,tempStr2,fw)
249  if cnt!=len(subDict):
250  fw.write('|'+' '*(width-2)+'|'+'\n')
251  cnt+=1
252 
253  fw.write('|'+' '*(width-2)+'|'+'\n')
254  fw.write('|'+'-'*(width-2)+'|'+'\n')
255  if index!=len(dic):
256  fw.write('|'+' '*(width-2)+'|'+'\n')
257  index+=1
258  fw.write('\n\n')
259 
260  fw.close()
261 
262 
263 
264 
269 def loadConfig(configPath):
270  with open(configPath,'r',encoding='UTF-8') as fr:
271  dic=json.load(fr)
272  for key in dic:
273  if isinstance(dic[key], str):
274  dic[key] = dic[key].strip()
275  runCommand=dic['runCommand']
276  return dic['projPath'],runCommand,dic['runFilePath'],dic['libName'],dic['currentVersion'],dic['targetVersion'],dic['currentEnv'],dic['targetEnv']
277 
278 
279 
285 def resolveConfigFilePath(config,repoRoot):
286  if os.path.isabs(config):
287  return config
288  if os.path.exists(config):
289  return os.path.abspath(config)
290  return os.path.join(repoRoot,'Configure',config)
291 
292 
293 
299 def resolveConfigValuePath(repoRoot,path):
300  if not path:
301  return path
302  expanded=os.path.expanduser(path)
303  if os.name=='nt' and expanded.startswith('/') and not expanded.startswith('//'):
304  return expanded
305  if os.path.isabs(expanded):
306  return os.path.abspath(expanded)
307  return os.path.abspath(os.path.join(repoRoot,expanded))
308 
309 
310 
314 class UnsupportedRunCommand(Exception):
315  """Raised when PCART cannot safely execute a configured run command."""
316 
317 
318 
323 def isPythonExecutable(command):
324  baseName=os.path.basename(command.replace('\\','/')).lower()
325  if baseName in ('python','python.exe'):
326  return True
327  if re.match(r'^python3(\.\d+)?(\.exe)?$',baseName):
328  return True
329  return False
330 
331 
332 
337 def isPyLauncher(command):
338  baseName=os.path.basename(command.replace('\\','/')).lower()
339  return baseName in ('py','py.exe')
340 
341 
342 
348  if tokens and re.match(r'^-\d(\.\d+)?(-\d+)?$',tokens[0]):
349  return tokens[1:]
350  return tokens
351 
352 
353 
360 def normalizeRunCommand(runCommand):
361  tokens=shlex.split(runCommand)
362  if not tokens:
363  return runCommand,'python'
364 
365  # 兼容未加引号的Windows Python路径,避免shlex处理反斜杠
366  simpleTokens=runCommand.split(maxsplit=1)
367  if simpleTokens and isPythonExecutable(simpleTokens[0]):
368  rest=simpleTokens[1].lstrip() if len(simpleTokens)>1 else ''
369  return rest,'python'
370  if simpleTokens and isPyLauncher(simpleTokens[0]):
371  rest=simpleTokens[1].lstrip() if len(simpleTokens)>1 else ''
372  restTokens=shlex.split(rest)
373  return shlex.join(stripPyLauncherOption(restTokens)),'python'
374 
375  if isPythonExecutable(tokens[0]):
376  return shlex.join(tokens[1:]),'python'
377  if isPyLauncher(tokens[0]):
378  return shlex.join(stripPyLauncherOption(tokens[1:])),'python'
379  if tokens[0]=='-m' or tokens[0].endswith('.py'):
380  return runCommand,'python'
381  return runCommand,'console'
382 
383 
384 
390 def resolveConsoleExecutable(envPath,commandName):
391  normalizedEnvPath=os.path.abspath(envPath)
392  candidates=[]
393  if platform.system()=='Windows':
394  scriptsPath=os.path.join(normalizedEnvPath,'Scripts')
395  candidates.extend([
396  os.path.join(scriptsPath,commandName),
397  os.path.join(scriptsPath,commandName+'.exe'),
398  os.path.join(scriptsPath,commandName+'.cmd'),
399  os.path.join(scriptsPath,commandName+'.bat'),
400  ])
401  else:
402  candidates.append(os.path.join(normalizedEnvPath,'bin',commandName))
403 
404  for candidate in candidates:
405  if os.path.exists(candidate):
406  return candidate
407 
408  candidateStr=', '.join(candidates)
409  raise FileNotFoundError(f"Cannot find console command '{commandName}' under virtual " f"environment root: {normalizedEnvPath}. Tried: {candidateStr}")
410 
411 
412 
418 def buildRunCommand(runCommand,envPath):
419  normalizedCommand,commandType=normalizeRunCommand(runCommand)
420  if commandType=='python':
421  return [resolvePythonExecutable(envPath)] + shlex.split(normalizedCommand)
422 
423  args=shlex.split(normalizedCommand)
424  if not args:
425  raise UnsupportedRunCommand('Empty run command')
426  try:
427  commandPath=resolveConsoleExecutable(envPath,args[0])
428  except FileNotFoundError as e:
429  raise UnsupportedRunCommand(str(e)) from e
430  return [commandPath] + args[1:]
431 
432 
433 
438 def getRunFile(runCommand):
439  tokens=shlex.split(runCommand)
440  for token in tokens:
441  candidate=token.split('::',1)[0]
442  if candidate.endswith('.py'):
443  return candidate
444  return ''
445 
446 
447 
448 
453 def findPythonDir(basePath):
454  if not os.path.exists(basePath):
455  raise FileNotFoundError(f"Cannot find Python directory: {basePath} does not exist")
456 
457  for entry in os.listdir(basePath):
458  full_path = os.path.join(basePath, entry)
459  if os.path.isdir(full_path):
460  if entry.startswith("python"):
461  return full_path
462 
463  raise FileNotFoundError(f"Cannot find Python directory under {basePath}/pythonxx.xx")
464 
465 
466 
472 def resolveLibSourceCodePath(envPath,libName):
473  if platform.system() == 'Windows':
474  sitePackages=os.path.join(envPath, "Lib", "site-packages")
475  else:
476  sitePackages=os.path.join(findPythonDir(os.path.join(envPath, "lib")), "site-packages")
477 
478  if libName=="tensorflow":
479  tensorflowCore=os.path.join(sitePackages, "tensorflow_core")
480  if os.path.isdir(tensorflowCore):
481  return tensorflowCore
482 
483  return os.path.join(sitePackages, libName)
484 
485 
486 
492  normalizedEnvPath = os.path.abspath(envPath)
493  if platform.system() == 'Windows':
494  candidates = [
495  os.path.join(normalizedEnvPath, 'python.exe'),
496  os.path.join(normalizedEnvPath, 'Scripts', 'python.exe'),
497  ]
498  else:
499  candidates = [
500  os.path.join(normalizedEnvPath, 'bin', 'python'),
501  ]
502 
503  for candidate in candidates:
504  if os.path.exists(candidate):
505  return candidate
506 
507  candidate_str = ', '.join(candidates)
508  raise FileNotFoundError(f"Cannot find Python executable under virtual environment root: " f"{normalizedEnvPath}. Tried: {candidate_str}")
509 
510 
511 
519 def getSourceCodePath(configPath,repoRoot=None):
520  if repoRoot is None:
521  repoRoot=os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
522  resolvedConfigPath=resolveConfigFilePath(configPath,repoRoot)
523  with open(resolvedConfigPath,'r',encoding='UTF-8') as fr:
524  dic=json.load(fr)
525  libName=dic['libName']
526  currentVersion=dic['currentVersion']
527  targetVersion=dic['targetVersion']
528  currentEnvPath=resolveConfigValuePath(repoRoot,dic['currentEnv'])
529  targetEnvPath=resolveConfigValuePath(repoRoot,dic['targetEnv'])
530 
531  currentSourceCodePath=resolveLibSourceCodePath(currentEnvPath, libName)
532  targetSourceCodePath=resolveLibSourceCodePath(targetEnvPath, libName)
533 
534  return currentVersion, targetVersion, currentSourceCodePath, targetSourceCodePath
535 
536 
537 
538 
540 class ConditionalReturnTransformer(ast.NodeTransformer):
541 
545  def visit_Return(self, node):
546  #检查return语句是否为单行条件语句(IfExp)
547  if isinstance(node, ast.Return) and isinstance(node.value, ast.IfExp):
548  ifExp = node.value
549  newIf = ast.If(test=ifExp.test,body=[ast.Return(value=ifExp.body)],orelse=[ast.Return(value=ifExp.orelse)])
550  return newIf
551  return node
Convert conditional return statement to multi-line if-else structure 展开单行条件返回语句为多行if-else结构
Definition: tool.py:540
def visit_Return(self, node)
Transform single-line conditional return into multi-line if-else structure 将单行条件返回语句转换为多行if-else结构
Definition: tool.py:545
Unsupported run command exception 不支持的运行命令异常
Definition: tool.py:314
def buildRunCommand(runCommand, envPath)
Build subprocess argv for project run command 为被测项目运行命令构造subprocess argv.
Definition: tool.py:418
def save2txt(lst, libName, runCommand, savePath)
Save PCART report 保存PCART报告
Definition: tool.py:175
def getAst(filePath, strFlag=0)
Get AST for code 将代码转化为Ast树
Definition: tool.py:33
def getImportLst(filePath)
Extract import statements from a source file (currently torch-specific) 从源码文件中提取import语句(目前仅筛选torch相关...
Definition: tool.py:49
def resolveLibSourceCodePath(envPath, libName)
Resolve the source package directory for a library in a virtual environment 从虚拟环境中解析库源码目录
Definition: tool.py:472
def cmp(version)
Normalize and compare version strings 归一化并比较版本号字符串
Definition: tool.py:108
def getSourceCodePath(configPath, repoRoot=None)
Get source code path of the lib 获取库源码路径
Definition: tool.py:519
def getVersionLst(libPath)
Get all version numbers from a library package directory 给定库的路径,获得该库的所有版本号
Definition: tool.py:80
def resolveConfigValuePath(repoRoot, path)
Resolve path value loaded from PCART config 解析PCART配置字段中的路径值
Definition: tool.py:299
def writeLine(width, s, fw)
Format output 格式化输出
Definition: tool.py:159
def isDynamic(dic)
Check if a match dictionary contains dynamic (string) match results 检查匹配字典是否包含动态(字符串)匹配结果
Definition: tool.py:66
def resolveConfigFilePath(config, repoRoot)
Resolve PCART config file path 解析PCART配置文件路径
Definition: tool.py:285
def resolveConsoleExecutable(envPath, commandName)
Resolve console script executable from a virtual environment 从虚拟环境中解析console script可执行文件
Definition: tool.py:390
def isPyLauncher(command)
Check whether command is Windows py launcher 判断命令是否为Windows py启动器
Definition: tool.py:337
def loadConfig(configPath)
Load PCART's configuration file 加载PCART配置文件
Definition: tool.py:269
def resolvePythonExecutable(envPath)
Resolve Python executable from a virtual environment root 从虚拟环境根目录解析 Python 解释器路径
Definition: tool.py:491
def normalizeRunCommand(runCommand)
Normalize project run command 归一化被测项目运行命令
Definition: tool.py:360
def getRunFile(runCommand)
Extract the .py script filename from a run command 从运行命令中提取.py脚本文件名
Definition: tool.py:438
def isPythonExecutable(command)
Check whether command is a Python executable name/path 判断命令是否为Python解释器名称或路径
Definition: tool.py:323
def stripPyLauncherOption(tokens)
Remove version option after py launcher 去除py启动器后的版本选项
Definition: tool.py:347
def findPythonDir(basePath)
Get Python interpreter path 获取Python解释器路径
Definition: tool.py:453