71 lines
No EOL
1.7 KiB
Java
71 lines
No EOL
1.7 KiB
Java
package be.jeffcheasey88.peeratcode.parser;
|
|
|
|
import java.io.BufferedReader;
|
|
import java.util.ArrayList;
|
|
import java.util.List;
|
|
import java.util.function.Consumer;
|
|
|
|
public class Tokenizer {
|
|
|
|
private List<Token> tokens;
|
|
|
|
private boolean optionSpace;
|
|
private Consumer<List<Token>> postProcessor;
|
|
|
|
public Tokenizer(){
|
|
this.tokens = new ArrayList<>();
|
|
}
|
|
|
|
public Tokenizer takeSpacement(){
|
|
this.optionSpace = true;
|
|
return this;
|
|
}
|
|
|
|
public void postProcess(Consumer<List<Token>> postProcess){
|
|
this.postProcessor = postProcess;
|
|
}
|
|
|
|
public List<Token> getTokens(){
|
|
return this.tokens;
|
|
}
|
|
|
|
public void parse(BufferedReader reader) throws Exception{
|
|
int lineNumber = 0;
|
|
String line;
|
|
while((line = reader.readLine()) != null){
|
|
lineNumber++;
|
|
|
|
for(int i = 0; i < line.length(); i++){
|
|
char c = line.charAt(i);
|
|
Token token;
|
|
if(Character.isAlphabetic(c) || Character.isDigit(c)){
|
|
String value = "";
|
|
int j = i;
|
|
for(; j < line.length(); j++){
|
|
c = line.charAt(j);
|
|
if(Character.isAlphabetic(c) || Character.isDigit(c)) value+=c;
|
|
else break;
|
|
}
|
|
token = new Token(lineNumber, i+1, value, TokenType.NAME);
|
|
i = j-1;
|
|
}else if(Character.isWhitespace(c)){
|
|
if(!optionSpace) continue;
|
|
String value = "";
|
|
int j = i;
|
|
for(; j < line.length(); j++){
|
|
c = line.charAt(j);
|
|
if(!Character.isWhitespace(c)) break;
|
|
}
|
|
token = new Token(lineNumber, i+1, value, TokenType.SPACE);
|
|
i = j-1;
|
|
}else{
|
|
token = new Token(lineNumber, i+1, ""+c, TokenType.DELIMITER);
|
|
}
|
|
this.tokens.add(token);
|
|
}
|
|
}
|
|
|
|
if(this.postProcessor != null) this.postProcessor.accept(tokens);
|
|
}
|
|
|
|
} |