Peer-at-Code/peer-at-code-parser-java
5
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Optimise TokenValidator & tmp re-build by tokens

Browse source
This commit is contained in:
jeffcheasey88 2023-07-11 11:52:42 +02:00
parent 1015bbadf2
commit 943fa54dac
5 changed files with 183 additions and 95 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
src/be/jeffcheasey88/peeratcode/parser/Parser.java
View file

@ -1,6 +1,7 @@
package be.jeffcheasey88.peeratcode.parser; package be.jeffcheasey88.peeratcode.parser;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.BufferedWriter;
import be.jeffcheasey88.peeratcode.parser.state.StateTree; import be.jeffcheasey88.peeratcode.parser.state.StateTree;
@ -24,4 +25,27 @@ public class Parser<E>{
this.state.seed(this.tokenizer, container); this.state.seed(this.tokenizer, container);
} }
//tmp
public void build(BufferedWriter writer) throws Exception{
Token[] confirmed = new Token[TokenValidator.MAX_VALIDATE];
System.arraycopy(tokenizer.getTokens().toArray(), 0, confirmed, 0, confirmed.length);
int line = 1;
int character = 1;
for(Token token : confirmed){
while(token.getLineNumber() != line){
writer.write("\n");
line++;
character = 1;
}
while(token.getCharacterNumber() != character){
writer.write(" ");
character++;
}
writer.write(token.getValue());
character+=token.getValue().length();
}
writer.flush();
writer.close();
}
} }

158
src/be/jeffcheasey88/peeratcode/parser/TokenValidator.java
View file

@ -1,10 +1,11 @@
package be.jeffcheasey88.peeratcode.parser; package be.jeffcheasey88.peeratcode.parser;
import java.util.Iterator;
import java.util.function.BiConsumer; import java.util.function.BiConsumer;
import java.util.function.Function; import java.util.function.Function;
public class TokenValidator implements Iterator<Token>{ public class TokenValidator{
public static int MAX_VALIDATE = 0;
private Token[] elements; private Token[] elements;
private int index; private int index;
@ -12,28 +13,25 @@ public class TokenValidator implements Iterator<Token>{
private Bag bag; private Bag bag;
public TokenValidator(Token[] tokens){ private TokenValidator(Bag bag){
this.elements = tokens; this.bag = bag;
this.validated = -1; }
public TokenValidator(Token[] elements){
this.elements = elements;
this.bag = new Bag(); this.bag = new Bag();
} }
@Override
public boolean hasNext(){ public boolean hasNext(){
return index < elements.length; return validated < elements.length;
}
@Override
public Token next(){
return elements[index++];
} }
public boolean validate(Function<Token, Boolean> action){ public boolean validate(Function<Token, Boolean> action){
if(index >= this.elements.length) return false; if(validated >= this.elements.length) return false;
if(action.apply(this.elements[index])){ if(action.apply(this.elements[validated])){
System.out.println("validate "+elements[index]); System.out.println("validate "+elements[validated]);
this.validated = index; if(validated > MAX_VALIDATE) MAX_VALIDATE = validated;
next(); this.validated++;
return true; return true;
} }
return false; return false;
@ -41,48 +39,23 @@ public class TokenValidator implements Iterator<Token>{
public boolean validate(Function<Token, Boolean> action, BiConsumer<Bag, Token> filler){ public boolean validate(Function<Token, Boolean> action, BiConsumer<Bag, Token> filler){
if(validate(action)){ if(validate(action)){
filler.accept(bag, elements[index-1]); filler.accept(bag, elements[validated-1]);
return true; return true;
} }
return false; return false;
} }
public void rollbackValidate(){
this.index = validated+1;
}
public TokenValidator pullValidated(){
Token[] validated = new Token[this.validated+1];
System.arraycopy(this.elements, 0, validated, 0, validated.length);
TokenValidator tk = new TokenValidator(validated);
tk.bag = bag;
return tk;
}
public void pushValidated(){
Token[] validated = new Token[this.elements.length-(this.validated+1)];
System.arraycopy(this.elements, this.validated+1, validated, 0, validated.length);
this.elements = validated;
this.index = 0;
this.validated = -1;
}
public TokenValidator branch(){ public TokenValidator branch(){
pushValidated(); TokenValidator branch = new TokenValidator(bag);
TokenValidator branch = new TokenValidator(this.elements); branch.elements = this.elements;
branch.bag = this.bag; branch.index = Math.max(0, this.validated-1);
branch.validated = this.validated;
return branch; return branch;
} }
public void merge(TokenValidator branch){ public void merge(TokenValidator branch){
branch.pushValidated(); this.index = branch.index;
this.elements = branch.elements; this.validated = branch.validated;
this.index = 0;
this.validated = -1;
}
public Token[] toArray(){
return this.elements;
} }
public void setBag(Bag bag){ public void setBag(Bag bag){
@ -93,4 +66,91 @@ public class TokenValidator implements Iterator<Token>{
return this.bag; return this.bag;
} }
// private Token[] elements;
// private int index;
// private int validated;
//
// private Bag bag;
//
// public TokenValidator(Token[] tokens){
// this.elements = tokens;
// this.validated = -1;
// this.bag = new Bag();
// }
//
// @Override
// public boolean hasNext(){
// return index < elements.length;
// }
//
// @Override
// public Token next(){
// return elements[index++];
// }
//
// public boolean validate(Function<Token, Boolean> action){
// if(index >= this.elements.length) return false;
// if(action.apply(this.elements[index])){
// System.out.println("validate "+elements[index]);
// this.validated = index;
// next();
// return true;
// }
// return false;
// }
//
// public boolean validate(Function<Token, Boolean> action, BiConsumer<Bag, Token> filler){
// if(validate(action)){
// filler.accept(bag, elements[index-1]);
// return true;
// }
// return false;
// }
//
// public void rollbackValidate(){
// this.index = validated+1;
// }
//
// public TokenValidator pullValidated(){
// Token[] validated = new Token[this.validated+1];
// System.arraycopy(this.elements, 0, validated, 0, validated.length);
// TokenValidator tk = new TokenValidator(validated);
// tk.bag = bag;
// return tk;
// }
//
// public void pushValidated(){
// Token[] validated = new Token[this.elements.length-(this.validated+1)];
// System.arraycopy(this.elements, this.validated+1, validated, 0, validated.length);
// this.elements = validated;
// this.index = 0;
// this.validated = -1;
// }
//
// public TokenValidator branch(){
// pushValidated();
// TokenValidator branch = new TokenValidator(this.elements);
// branch.bag = this.bag;
// return branch;
// }
//
// public void merge(TokenValidator branch){
// branch.pushValidated();
// this.elements = branch.elements;
// this.index = 0;
// this.validated = -1;
// }
//
// public Token[] toArray(){
// return this.elements;
// }
//
// public void setBag(Bag bag){
// this.bag = bag;
// }
//
// public Bag getBag(){
// return this.bag;
// }
//
} }

83
src/be/jeffcheasey88/peeratcode/parser/java/JavaParser.java
View file

@ -1,8 +1,10 @@
package be.jeffcheasey88.peeratcode.parser.java; package be.jeffcheasey88.peeratcode.parser.java;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File; import java.io.File;
import java.io.FileReader; import java.io.FileReader;
import java.io.FileWriter;
import java.lang.reflect.Modifier; import java.lang.reflect.Modifier;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@ -36,6 +38,8 @@ public class JavaParser extends Parser<JavaFile> {
parser.parse(reader, jFile); parser.parse(reader, jFile);
System.out.println((System.currentTimeMillis()-time)+"ms"); System.out.println((System.currentTimeMillis()-time)+"ms");
parser.build(new BufferedWriter(new FileWriter(new File("/home/ParserV2.txt"))));
} }
private static final private static final
@ -53,37 +57,37 @@ public class JavaParser extends Parser<JavaFile> {
if(modifier == null) modifier = 0; if(modifier == null) modifier = 0;
bag.set(modifier+JavaParser.getModifier(token.getValue())); bag.set(modifier+JavaParser.getModifier(token.getValue()));
}); });
BiFunction<JavaFile, TokenValidator, JavaFile> BiFunction<JavaFile, Bag, JavaFile>
END_PACKAGE = ((javafile, validator) -> javafile.setPackage(validator.getBag())), END_PACKAGE = ((javafile, bag) -> javafile.setPackage(bag)),
END_IMPORT = ((javafile, validator) -> javafile.addImport(validator.getBag())); END_IMPORT = ((javafile, bag) -> javafile.addImport(bag));
BiFunction<JavaFile, TokenValidator, Class> BiFunction<JavaFile, Bag, Class>
END_CLASS = ((javafile, validator) -> javafile.setClass(new Class(validator.getBag()))); END_CLASS = ((javafile, bag) -> javafile.setClass(new Class(bag)));
BiFunction<JavaElement, TokenValidator, Token> BiFunction<JavaElement, Bag, Token>
END_TYPE = ((javafile, validator) -> { END_TYPE = ((javafile, bag) -> {
System.out.println("return token "+validator.getBag().get()); System.out.println("return token "+bag.get());
return validator.getBag().get(); return bag.get();
}); });
BiFunction<JavaElement, TokenValidator, Variable> BiFunction<JavaElement, Bag, Variable>
END_VAR = ((javafile, validator) -> { END_VAR = ((javafile, bag) -> {
System.out.println("new var"); System.out.println("new var");
return new Variable(validator.getBag()); return new Variable(bag);
}); });
BiFunction<JavaElement, TokenValidator, JavaElement> BiFunction<JavaElement, Bag, JavaElement>
END_FUNC = ((javafile, validator) -> { END_FUNC = ((javafile, bag) -> {
System.out.println("new function"); System.out.println("new function");
return new be.jeffcheasey88.peeratcode.parser.java.Function(validator.getBag()); return new be.jeffcheasey88.peeratcode.parser.java.Function(bag);
}); });
BiFunction<JavaElement, TokenValidator, Integer> BiFunction<JavaElement, Bag, Integer>
END_MODIFIER = ((javafile, validator) ->{ END_MODIFIER = ((javafile, bag) ->{
return validator.getBag().get(); return bag.get();
}); });
BiFunction<JavaElement, TokenValidator, JavaElement> BiFunction<JavaElement, Bag, JavaElement>
END_OP = ((javafile, validator) -> { END_OP = ((javafile, bag) -> {
System.out.println("\top"); System.out.println("\top");
return javafile; return javafile;
}); });
BiFunction<JavaElement, TokenValidator, JavaElement> BiFunction<JavaElement, Bag, JavaElement>
END_ANNOTATION = ((javafile, validator) -> { END_ANNOTATION = ((javafile, bag) -> {
System.out.println("ANNOTATION"); System.out.println("ANNOTATION");
return javafile; return javafile;
}); });
@ -199,8 +203,8 @@ public class JavaParser extends Parser<JavaFile> {
StateTree<JavaElement> function_q18 = function_q17.then(LAMBDA_42); StateTree<JavaElement> function_q18 = function_q17.then(LAMBDA_42);
StateTree<JavaElement> function_q19 = function_q18.then(LAMBDA_43); StateTree<JavaElement> function_q19 = function_q18.then(LAMBDA_43);
function_q19.end((a,b) -> { System.out.println("\t\tolala"); return a;}); function_q19.end((a,b) -> { System.out.println("\t\tolala"); return a;});
StateTree<JavaElement> function_q21 = function_q18.then(new RedirectStateTree<>(variable_q0,(bag) -> "func_var")); StateTree<JavaElement> function_q21 = function_q18.then(new RedirectStateTree<>(variable_q0,(bag) -> "func_var")).loop();
StateTree<JavaElement> function_q22 = function_q18.then(new RedirectStateTree<>(operation_q0,(bag) -> "func_op")); StateTree<JavaElement> function_q22 = function_q18.then(new RedirectStateTree<>(operation_q0,(bag) -> "func_op")).loop();
function_q21.then(function_q19); function_q21.then(function_q19);
function_q22.then(function_q19); function_q22.then(function_q19);
function_q22.then(function_q21); function_q22.then(function_q21);
@ -342,44 +346,43 @@ public class JavaParser extends Parser<JavaFile> {
bag.set("?", sub); bag.set("?", sub);
}); });
BiFunction<JavaElement, TokenValidator, JavaElement> END_NATIVE_VALUE = (element, validator) -> { BiFunction<JavaElement, Bag, JavaElement> END_NATIVE_VALUE = (element, bag) -> {
Value result = validator.getBag().<Bag>get("?").get(); Value result = bag.<Bag>get("?").get();
if(result.get() != null){ if(result.get() != null){
validator.getBag().set(result.get()); bag.set(result.get());
return result.get(); return result.get();
} }
validator.getBag().set(result); bag.set(result);
return result; return result;
}; };
BiFunction<JavaElement, TokenValidator, JavaElement> END_VALUE = (element, validator) -> { BiFunction<JavaElement, Bag, JavaElement> END_VALUE = (element, bag) -> {
Value v = new Value(validator.getBag().get()); Value v = new Value(bag.get());
validator.getBag().set(v); bag.set(v);
return v; return v;
}; };
BiFunction<JavaElement, TokenValidator, JavaElement> END_BIVALUE = (element, validator) -> { BiFunction<JavaElement, Bag, JavaElement> END_BIVALUE = (element, bag) -> {
Value origin = (Value)element; Value origin = (Value)element;
Value right = validator.getBag().<Bag>get("?").get(); Value right = bag.<Bag>get("?").get();
if(right.get() != null) right = right.get(); if(right.get() != null) right = right.get();
if(right instanceof TriValue){ if(right instanceof TriValue){
TriValue last = (TriValue)right; TriValue last = (TriValue)right;
last.getCondition().switchInto(null); last.getCondition().switchInto(null);
BiValue condition = new BiValue(origin, last.getCondition(), validator.getBag().get("type")); BiValue condition = new BiValue(origin, last.getCondition(), bag.get("type"));
TriValue result = new TriValue(condition, last.success(), last.fail()); TriValue result = new TriValue(condition, last.success(), last.fail());
origin.switchInto(result); origin.switchInto(result);
validator.getBag().set(result); bag.set(result);
return result; return result;
} }
Value v = new BiValue(origin, right, validator.getBag().get("type")); Value v = new BiValue(origin, right, bag.get("type"));
origin.switchInto(v); origin.switchInto(v);
validator.getBag().set(v); bag.set(v);
return v; return v;
}; };
BiFunction<JavaElement, TokenValidator, JavaElement> END_TRIVALUE = (element, validator) -> { BiFunction<JavaElement, Bag, JavaElement> END_TRIVALUE = (element, bag) -> {
Bag bag = validator.getBag();
Value origin = (Value)element; Value origin = (Value)element;
Value v = new TriValue(origin, bag.<Bag>get("true").get(), bag.<Bag>get("false").get()); Value v = new TriValue(origin, bag.<Bag>get("true").get(), bag.<Bag>get("false").get());
origin.switchInto(v); origin.switchInto(v);
validator.getBag().set(v); bag.set(v);
return v; return v;
}; };
StateTree<JavaElement> value_q0 = new StateTree<>(); StateTree<JavaElement> value_q0 = new StateTree<>();

7
src/be/jeffcheasey88/peeratcode/parser/state/BuilderStateTree.java
View file

@ -7,16 +7,15 @@ import be.jeffcheasey88.peeratcode.parser.TokenValidator;
public class BuilderStateTree<E, B> extends StateTree<B>{ public class BuilderStateTree<E, B> extends StateTree<B>{
private BiFunction<E, TokenValidator, B> builder; private BiFunction<E, Bag, B> builder;
public BuilderStateTree(BiFunction<E, TokenValidator, B> builder){ public BuilderStateTree(BiFunction<E, Bag, B> builder){
super(); super();
this.builder = builder; this.builder = builder;
} }
B build(TokenValidator validator, E element){ B build(TokenValidator validator, E element){
B build = this.builder.apply(element, validator.pullValidated()); B build = this.builder.apply(element, validator.getBag());
validator.pushValidated();
validator.setBag(new Bag()); validator.setBag(new Bag());
super.internalSeed(validator, build); super.internalSeed(validator, build);

4
src/be/jeffcheasey88/peeratcode/parser/state/StateTree.java
View file

@ -5,6 +5,7 @@ import java.util.List;
import java.util.function.BiFunction; import java.util.function.BiFunction;
import java.util.function.Function; import java.util.function.Function;
import be.jeffcheasey88.peeratcode.parser.Bag;
import be.jeffcheasey88.peeratcode.parser.Token; import be.jeffcheasey88.peeratcode.parser.Token;
import be.jeffcheasey88.peeratcode.parser.TokenValidator; import be.jeffcheasey88.peeratcode.parser.TokenValidator;
import be.jeffcheasey88.peeratcode.parser.Tokenizer; import be.jeffcheasey88.peeratcode.parser.Tokenizer;
@ -27,6 +28,7 @@ public class StateTree<E>{
E build = internalSeed(validator, container); E build = internalSeed(validator, container);
if(build == null) break; if(build == null) break;
} }
System.out.println("Validate "+validator.MAX_VALIDATE+" tokens !");
} }
<B> B internalSeed(TokenValidator validator, E element){ <B> B internalSeed(TokenValidator validator, E element){
@ -69,7 +71,7 @@ public class StateTree<E>{
return this; return this;
} }
public <B> StateTree<B> end(BiFunction<E, TokenValidator, B> builder){ public <B> StateTree<B> end(BiFunction<E, Bag, B> builder){
BuilderStateTree<E, B> builderState = new BuilderStateTree<>(builder); BuilderStateTree<E, B> builderState = new BuilderStateTree<>(builder);
this.builder = builderState; this.builder = builderState;
return builderState; return builderState;