package be.jeffcheasey88.peeratcode.parser; import java.io.BufferedReader; import java.util.ArrayList; import java.util.List; import java.util.function.Consumer; public class Tokenizer { private List tokens; private boolean optionSpace; private Consumer> postProcessor; public Tokenizer(){ this.tokens = new ArrayList<>(); } public Tokenizer takeSpacement(){ this.optionSpace = true; return this; } public void postProcess(Consumer> postProcess){ this.postProcessor = postProcess; } public List getTokens(){ return this.tokens; } public void reset(){ this.tokens.clear(); } public void parse(BufferedReader reader) throws Exception{ int lineNumber = 0; String line; while((line = reader.readLine()) != null){ lineNumber++; for(int i = 0; i < line.length(); i++){ char c = line.charAt(i); Token token; if(Character.isAlphabetic(c) || Character.isDigit(c)){ String value = ""; int j = i; for(; j < line.length(); j++){ c = line.charAt(j); if(Character.isAlphabetic(c) || Character.isDigit(c)) value+=c; else break; } token = new Token(lineNumber, i+1, value, TokenType.NAME); i = j-1; }else if(Character.isWhitespace(c)){ if(!optionSpace) continue; String value = ""; int j = i; for(; j < line.length(); j++){ c = line.charAt(j); if(!Character.isWhitespace(c)) break; } token = new Token(lineNumber, i+1, value, TokenType.SPACE); i = j-1; }else{ token = new Token(lineNumber, i+1, ""+c, TokenType.DELIMITER); } this.tokens.add(token); } } if(this.postProcessor != null) this.postProcessor.accept(tokens); } public void parse(String line){ if(this.postProcessor != null) this.postProcessor.accept(tokens); for(int i = 0; i < line.length(); i++){ char c = line.charAt(i); Token token; if(Character.isAlphabetic(c) || Character.isDigit(c) || c == '_' || c == '$'){ String value = ""; int j = i; for(; j < line.length(); j++){ c = line.charAt(j); if(Character.isAlphabetic(c) || Character.isDigit(c) || c == '_' || c == '$') value+=c; else break; } token = new Token(1, i+1, value, TokenType.NAME); i = j-1; }else if(Character.isWhitespace(c)){ if(!optionSpace) continue; String value = ""; int j = i; for(; j < line.length(); j++){ c = line.charAt(j); if(!Character.isWhitespace(c)) break; } token = new Token(1, i+1, value, TokenType.SPACE); i = j-1; }else{ token = new Token(1, i+1, ""+c, TokenType.DELIMITER); } this.tokens.add(token); } } }