import re TOKEN = re.compile(r"\s*(?:(\d+)|(.))") def tokenize(text): tokens = [] for number, symbol in TOKEN.findall(text): tokens.append(int(number) if number else symbol) return tokens