python代码词频分析
python实现词法分析器简单python词法分析器实现，供大家参考，具体内容如下
词法分析器状态转换图：
词法分析器总流程图：
预处理程序：
词法分析器：
词法分析器程序详细设计
详细代码实现:
|
|
#!/usr/bin/env python3.4# coding=utf-8import sysimport stringkeywards = {}# 关键字部分keywards['false'] = 101keywards['class'] = 102keywards['finally'] = 103keywards['is'] = 104keywards['return'] = 105keywards['none'] = 106keywards['continue'] = 107keywards['for'] = 108keywards['lambda'] = 109keywards['try'] = 110keywards['true'] = 111keywards['def'] = 112keywards['from'] = 113keywards['nonlocal'] = 114keywards['while'] = 115keywards['and'] = 116keywards['del'] = 117keywards['global'] = 118keywards['not'] = 119keywards['with'] = 120keywards['as'] = 121keywards['elif'] = 122keywards['if'] = 123keywards['or'] = 124keywards['yield'] = 125keywards['assert'] = 126keywards['else'] = 127keywards['import'] = 128keywards['pass'] = 129keywards['break'] = 130keywards['except'] = 131keywards['in'] = 132keywards['raise'] = 133# 符号keywards['+'] = 201keywards['-'] = 202keywards['*'] = 203keywards['/'] = 204keywards['='] = 205keywards[':'] = 206keywards['<'] = 207keywards['>'] = 208keywards['%'] = 209keywards['&'] = 210keywards['!'] = 211keywards['('] = 212keywards[')'] = 213keywards['['] = 214keywards[']'] = 215keywards['{'] = 216keywards['}'] = 217keywards['#'] = 218keywards['|'] = 219keywards[','] = 220# 变量# keywards['var'] = 301# 常量# keywards['const'] = 401# error# keywards['const'] = 501signlist = {}# 预处理函数，将文件中的空格，换行等无关字符处理掉def pretreatment(file_name): try: fp_read = open(file_name, 'r') fp_write = open('file.tmp', 'w') sign = 0 while true: read = fp_read.readline() if not read: break length = len(read) i = -1 while i < length - 1: i += 1 if sign == 0: if read[i] == ' ': continue if read[i] == '#': break elif read[i] == ' ': if sign == 1: continue else: sign = 1 fp_write.write(' ') elif read[i] == '\t': if sign == 1: continue else: sign = 1 fp_write.write(' ') elif read[i] == '\n': if sign == 1: continue else: fp_write.write(' ') sign = 1 elif read[i] == '"': fp_write.write(read[i]) i += 1 while i < length and read[i] != '"': fp_write.write(read[i]) i += 1 if i >= length: break fp_write.write(read[i]) elif read[i] == "'": fp_write.write(read[i]) i += 1 while i < length and read[i] != "'": fp_write.write(read[i]) i += 1 if i >= length: break fp_write.write(read[i]) else: sign = 3 fp_write.write(read[i]) except exception: print(file_name, ': this filename not found!')def save(string): if string in keywards.keys(): if string not in signlist.keys(): signlist[string] = keywards[string] else: try: float(string) save_const(string) except valueerror: save_var(string)def save_var(string): if string not in signlist.keys(): if len(string.strip()) < 1: pass else: if is_signal(string) == 1: signlist[string] = 301 else: signlist[string] = 501def save_const(string): if string not in signlist.keys(): signlist[string] = 401def save_error(string): if string not in signlist.keys(): signlist[string] = 501def is_signal(s): if s[0] == '_' or s[0] in string.ascii_letters: for i in s: if i in string.ascii_letters or i == '_' or i in string.digits: pass else: return 0 return 1 else: return 0def recognition(filename): try: fp_read = open(filename, 'r') string = "" sign = 0 while true: read = fp_read.read(1) if not read: break if read == ' ': if len(string.strip()) < 1: sign = 0 pass else: if sign == 1 or sign == 2: string += read else: save(string) string = "" sign = 0 elif read == '(': if sign == 1 or sign == 2: string += read else: save(string) string = "" save('(') elif read == ')': if sign == 1 or sign == 2: string += read else: save(string) string = "" save(')') elif read == '[': if sign == 1 or sign == 2: string += read else: save(string) string = "" save('[') elif read == ']': if sign == 1 or sign == 2: string += read else: save(string) string = "" save(']') elif read == '{': if sign == 1 or sign == 2: string += read else: save(string) string = "" save('{') elif read == '}': if sign == 1 or sign == 2: string += read else: save(string) string = "" save('}') elif read == '<': save(string) string = "" save('<') elif read == '>': save(string) string = "" save('>') elif read == ',': save(string) string = "" save(',') elif read == "'": string += read if sign == 1: sign = 0 save_const(string) string = "" else: if sign != 2: sign = 1 elif read == '"': string += read if sign == 2: sign = 0 save_const(string) string = "" else: if sign != 1: sign = 2 elif read == ':': if sign == 1 or sign == 2: string += read else: save(string) string = "" save(':') elif read == '+': save(string) string = "" save('+') elif read == '=': save(string) string = "" save('=') else: string += read except exception as e: print(e)def main(): if len(sys.argv) < 2: print("please input filename") else: pretreatment(sys.argv[1]) recognition('file.tmp') for i in signlist.keys(): print("(", signlist[i], ",", i, ")")if __name__ == '__main__': main() |
以上就是本文的全部内容，希望对大家的学习有所帮助，也希望大家多多支持开心学习网。
原文链接：https://blog.csdn.net/IT_DREAM_ER/article/details/53485583