Peer-at-Code/peer-at-code-parser-java
5
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Optimise TokenValidator & tmp re-build by tokens

Browse source
This commit is contained in:
jeffcheasey88 2023-07-11 11:52:42 +02:00
parent 1015bbadf2
commit 943fa54dac
5 changed files with 183 additions and 95 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
src/be/jeffcheasey88/peeratcode/parser/Parser.java
View file

@ -1,6 +1,7 @@
package be.jeffcheasey88.peeratcode.parser;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import be.jeffcheasey88.peeratcode.parser.state.StateTree;
@ -24,4 +25,27 @@ public class Parser<E>{
this.state.seed(this.tokenizer, container);
}
//tmp
public void build(BufferedWriter writer) throws Exception{
Token[] confirmed = new Token[TokenValidator.MAX_VALIDATE];
System.arraycopy(tokenizer.getTokens().toArray(), 0, confirmed, 0, confirmed.length);
int line = 1;
int character = 1;
for(Token token : confirmed){
while(token.getLineNumber() != line){
writer.write("\n");
line++;
character = 1;
}
while(token.getCharacterNumber() != character){
writer.write(" ");
character++;
}
writer.write(token.getValue());
character+=token.getValue().length();
}
writer.flush();
writer.close();
}
}

158
src/be/jeffcheasey88/peeratcode/parser/TokenValidator.java
View file

@ -1,10 +1,11 @@
package be.jeffcheasey88.peeratcode.parser;
import java.util.Iterator;
import java.util.function.BiConsumer;
import java.util.function.Function;
public class TokenValidator implements Iterator<Token>{
public class TokenValidator{
public static int MAX_VALIDATE = 0;
private Token[] elements;
private int index;
@ -12,28 +13,25 @@ public class TokenValidator implements Iterator<Token>{
private Bag bag;
public TokenValidator(Token[] tokens){
this.elements = tokens;
this.validated = -1;
private TokenValidator(Bag bag){
this.bag = bag;
}
public TokenValidator(Token[] elements){
this.elements = elements;
this.bag = new Bag();
}
@Override
public boolean hasNext(){
return index < elements.length;
}
@Override
public Token next(){
return elements[index++];
return validated < elements.length;
}
public boolean validate(Function<Token, Boolean> action){
if(index >= this.elements.length) return false;
if(action.apply(this.elements[index])){
System.out.println("validate "+elements[index]);
this.validated = index;
next();
if(validated >= this.elements.length) return false;
if(action.apply(this.elements[validated])){
System.out.println("validate "+elements[validated]);
if(validated > MAX_VALIDATE) MAX_VALIDATE = validated;
this.validated++;
return true;
}
return false;
@ -41,48 +39,23 @@ public class TokenValidator implements Iterator<Token>{
public boolean validate(Function<Token, Boolean> action, BiConsumer<Bag, Token> filler){
if(validate(action)){
filler.accept(bag, elements[index-1]);
filler.accept(bag, elements[validated-1]);
return true;
}
return false;
}
public void rollbackValidate(){
this.index = validated+1;
}
public TokenValidator pullValidated(){
Token[] validated = new Token[this.validated+1];
System.arraycopy(this.elements, 0, validated, 0, validated.length);
TokenValidator tk = new TokenValidator(validated);
tk.bag = bag;
return tk;
}
public void pushValidated(){
Token[] validated = new Token[this.elements.length-(this.validated+1)];
System.arraycopy(this.elements, this.validated+1, validated, 0, validated.length);
this.elements = validated;
this.index = 0;
this.validated = -1;
}
public TokenValidator branch(){
pushValidated();
TokenValidator branch = new TokenValidator(this.elements);
branch.bag = this.bag;
TokenValidator branch = new TokenValidator(bag);
branch.elements = this.elements;
branch.index = Math.max(0, this.validated-1);
branch.validated = this.validated;
return branch;
}
public void merge(TokenValidator branch){
branch.pushValidated();
this.elements = branch.elements;
this.index = 0;
this.validated = -1;
}
public Token[] toArray(){
return this.elements;
this.index = branch.index;
this.validated = branch.validated;
}
public void setBag(Bag bag){
@ -93,4 +66,91 @@ public class TokenValidator implements Iterator<Token>{
return this.bag;
}
// private Token[] elements;
// private int index;
// private int validated;
//
// private Bag bag;
//
// public TokenValidator(Token[] tokens){
// this.elements = tokens;
// this.validated = -1;
// this.bag = new Bag();
// }
//
// @Override
// public boolean hasNext(){
// return index < elements.length;
// }
//
// @Override
// public Token next(){
// return elements[index++];
// }
//
// public boolean validate(Function<Token, Boolean> action){
// if(index >= this.elements.length) return false;
// if(action.apply(this.elements[index])){
// System.out.println("validate "+elements[index]);
// this.validated = index;
// next();
// return true;
// }
// return false;
// }
//
// public boolean validate(Function<Token, Boolean> action, BiConsumer<Bag, Token> filler){
// if(validate(action)){
// filler.accept(bag, elements[index-1]);
// return true;
// }
// return false;
// }
//
// public void rollbackValidate(){
// this.index = validated+1;
// }
//
// public TokenValidator pullValidated(){
// Token[] validated = new Token[this.validated+1];
// System.arraycopy(this.elements, 0, validated, 0, validated.length);
// TokenValidator tk = new TokenValidator(validated);
// tk.bag = bag;
// return tk;
// }
//
// public void pushValidated(){
// Token[] validated = new Token[this.elements.length-(this.validated+1)];
// System.arraycopy(this.elements, this.validated+1, validated, 0, validated.length);
// this.elements = validated;
// this.index = 0;
// this.validated = -1;
// }
//
// public TokenValidator branch(){
// pushValidated();
// TokenValidator branch = new TokenValidator(this.elements);
// branch.bag = this.bag;
// return branch;
// }
//
// public void merge(TokenValidator branch){
// branch.pushValidated();
// this.elements = branch.elements;
// this.index = 0;
// this.validated = -1;
// }
//
// public Token[] toArray(){
// return this.elements;
// }
//
// public void setBag(Bag bag){
// this.bag = bag;
// }
//
// public Bag getBag(){
// return this.bag;
// }
//
}

83
src/be/jeffcheasey88/peeratcode/parser/java/JavaParser.java
View file

@ -1,8 +1,10 @@
package be.jeffcheasey88.peeratcode.parser.java;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.lang.reflect.Modifier;
import java.util.ArrayList;
import java.util.List;
@ -36,6 +38,8 @@ public class JavaParser extends Parser<JavaFile> {
parser.parse(reader, jFile);
System.out.println((System.currentTimeMillis()-time)+"ms");
parser.build(new BufferedWriter(new FileWriter(new File("/home/ParserV2.txt"))));
}
private static final
@ -53,37 +57,37 @@ public class JavaParser extends Parser<JavaFile> {
if(modifier == null) modifier = 0;
bag.set(modifier+JavaParser.getModifier(token.getValue()));
});
BiFunction<JavaFile, TokenValidator, JavaFile>
END_PACKAGE = ((javafile, validator) -> javafile.setPackage(validator.getBag())),
END_IMPORT = ((javafile, validator) -> javafile.addImport(validator.getBag()));
BiFunction<JavaFile, TokenValidator, Class>
END_CLASS = ((javafile, validator) -> javafile.setClass(new Class(validator.getBag())));
BiFunction<JavaElement, TokenValidator, Token>
END_TYPE = ((javafile, validator) -> {
System.out.println("return token "+validator.getBag().get());
return validator.getBag().get();
BiFunction<JavaFile, Bag, JavaFile>
END_PACKAGE = ((javafile, bag) -> javafile.setPackage(bag)),
END_IMPORT = ((javafile, bag) -> javafile.addImport(bag));
BiFunction<JavaFile, Bag, Class>
END_CLASS = ((javafile, bag) -> javafile.setClass(new Class(bag)));
BiFunction<JavaElement, Bag, Token>
END_TYPE = ((javafile, bag) -> {
System.out.println("return token "+bag.get());
return bag.get();
});
BiFunction<JavaElement, TokenValidator, Variable>
END_VAR = ((javafile, validator) -> {
BiFunction<JavaElement, Bag, Variable>
END_VAR = ((javafile, bag) -> {
System.out.println("new var");
return new Variable(validator.getBag());
return new Variable(bag);
});
BiFunction<JavaElement, TokenValidator, JavaElement>
END_FUNC = ((javafile, validator) -> {
BiFunction<JavaElement, Bag, JavaElement>
END_FUNC = ((javafile, bag) -> {
System.out.println("new function");
return new be.jeffcheasey88.peeratcode.parser.java.Function(validator.getBag());
return new be.jeffcheasey88.peeratcode.parser.java.Function(bag);
});
BiFunction<JavaElement, TokenValidator, Integer>
END_MODIFIER = ((javafile, validator) ->{
return validator.getBag().get();
BiFunction<JavaElement, Bag, Integer>
END_MODIFIER = ((javafile, bag) ->{
return bag.get();
});
BiFunction<JavaElement, TokenValidator, JavaElement>
END_OP = ((javafile, validator) -> {
BiFunction<JavaElement, Bag, JavaElement>
END_OP = ((javafile, bag) -> {
System.out.println("\top");
return javafile;
});
BiFunction<JavaElement, TokenValidator, JavaElement>
END_ANNOTATION = ((javafile, validator) -> {
BiFunction<JavaElement, Bag, JavaElement>
END_ANNOTATION = ((javafile, bag) -> {
System.out.println("ANNOTATION");
return javafile;
});
@ -199,8 +203,8 @@ public class JavaParser extends Parser<JavaFile> {
StateTree<JavaElement> function_q18 = function_q17.then(LAMBDA_42);
StateTree<JavaElement> function_q19 = function_q18.then(LAMBDA_43);
function_q19.end((a,b) -> { System.out.println("\t\tolala"); return a;});
StateTree<JavaElement> function_q21 = function_q18.then(new RedirectStateTree<>(variable_q0,(bag) -> "func_var"));
StateTree<JavaElement> function_q22 = function_q18.then(new RedirectStateTree<>(operation_q0,(bag) -> "func_op"));
StateTree<JavaElement> function_q21 = function_q18.then(new RedirectStateTree<>(variable_q0,(bag) -> "func_var")).loop();
StateTree<JavaElement> function_q22 = function_q18.then(new RedirectStateTree<>(operation_q0,(bag) -> "func_op")).loop();
function_q21.then(function_q19);
function_q22.then(function_q19);
function_q22.then(function_q21);
@ -342,44 +346,43 @@ public class JavaParser extends Parser<JavaFile> {
bag.set("?", sub);
});
BiFunction<JavaElement, TokenValidator, JavaElement> END_NATIVE_VALUE = (element, validator) -> {
Value result = validator.getBag().<Bag>get("?").get();
BiFunction<JavaElement, Bag, JavaElement> END_NATIVE_VALUE = (element, bag) -> {
Value result = bag.<Bag>get("?").get();
if(result.get() != null){
validator.getBag().set(result.get());
bag.set(result.get());
return result.get();
}
validator.getBag().set(result);
bag.set(result);
return result;
};
BiFunction<JavaElement, TokenValidator, JavaElement> END_VALUE = (element, validator) -> {
Value v = new Value(validator.getBag().get());
validator.getBag().set(v);
BiFunction<JavaElement, Bag, JavaElement> END_VALUE = (element, bag) -> {
Value v = new Value(bag.get());
bag.set(v);
return v;
};
BiFunction<JavaElement, TokenValidator, JavaElement> END_BIVALUE = (element, validator) -> {
BiFunction<JavaElement, Bag, JavaElement> END_BIVALUE = (element, bag) -> {
Value origin = (Value)element;
Value right = validator.getBag().<Bag>get("?").get();
Value right = bag.<Bag>get("?").get();
if(right.get() != null) right = right.get();
if(right instanceof TriValue){
TriValue last = (TriValue)right;
last.getCondition().switchInto(null);
BiValue condition = new BiValue(origin, last.getCondition(), validator.getBag().get("type"));
BiValue condition = new BiValue(origin, last.getCondition(), bag.get("type"));
TriValue result = new TriValue(condition, last.success(), last.fail());
origin.switchInto(result);
validator.getBag().set(result);
bag.set(result);
return result;
}
Value v = new BiValue(origin, right, validator.getBag().get("type"));
Value v = new BiValue(origin, right, bag.get("type"));
origin.switchInto(v);
validator.getBag().set(v);
bag.set(v);
return v;
};
BiFunction<JavaElement, TokenValidator, JavaElement> END_TRIVALUE = (element, validator) -> {
Bag bag = validator.getBag();
BiFunction<JavaElement, Bag, JavaElement> END_TRIVALUE = (element, bag) -> {
Value origin = (Value)element;
Value v = new TriValue(origin, bag.<Bag>get("true").get(), bag.<Bag>get("false").get());
origin.switchInto(v);
validator.getBag().set(v);
bag.set(v);
return v;
};
StateTree<JavaElement> value_q0 = new StateTree<>();

7
src/be/jeffcheasey88/peeratcode/parser/state/BuilderStateTree.java
View file

@ -7,16 +7,15 @@ import be.jeffcheasey88.peeratcode.parser.TokenValidator;
public class BuilderStateTree<E, B> extends StateTree<B>{
private BiFunction<E, TokenValidator, B> builder;
private BiFunction<E, Bag, B> builder;
public BuilderStateTree(BiFunction<E, TokenValidator, B> builder){
public BuilderStateTree(BiFunction<E, Bag, B> builder){
super();
this.builder = builder;
}
B build(TokenValidator validator, E element){
B build = this.builder.apply(element, validator.pullValidated());
validator.pushValidated();
B build = this.builder.apply(element, validator.getBag());
validator.setBag(new Bag());
super.internalSeed(validator, build);

4
src/be/jeffcheasey88/peeratcode/parser/state/StateTree.java
View file

@ -5,6 +5,7 @@ import java.util.List;
import java.util.function.BiFunction;
import java.util.function.Function;
import be.jeffcheasey88.peeratcode.parser.Bag;
import be.jeffcheasey88.peeratcode.parser.Token;
import be.jeffcheasey88.peeratcode.parser.TokenValidator;
import be.jeffcheasey88.peeratcode.parser.Tokenizer;
@ -27,6 +28,7 @@ public class StateTree<E>{
E build = internalSeed(validator, container);
if(build == null) break;
}
System.out.println("Validate "+validator.MAX_VALIDATE+" tokens !");
}
<B> B internalSeed(TokenValidator validator, E element){
@ -69,7 +71,7 @@ public class StateTree<E>{
return this;
}
public <B> StateTree<B> end(BiFunction<E, TokenValidator, B> builder){
public <B> StateTree<B> end(BiFunction<E, Bag, B> builder){
BuilderStateTree<E, B> builderState = new BuilderStateTree<>(builder);
this.builder = builderState;
return builderState;