Skip to content

Commit 3969c8e

Browse files
ppaskappaska
committed
added import parser
1 parent 386784b commit 3969c8e

File tree

5 files changed

+105
-35
lines changed

5 files changed

+105
-35
lines changed

‎index.html‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -29,15 +29,7 @@
2929
<div class="container">
3030
<h4>JSPython development console</h4>
3131
<div id="editor">
32-
sum = 0
33-
i = 0
34-
35-
while i<1000:
36-
sum = sum + i
37-
i = i + 1
38-
39-
sum
40-
32+
from datapipe-js-array import sort, first as f, fullJoin
4133
</div>
4234
<button onclick="tokenize()">Tokenize</button>
4335
<button onclick="parse()">Parse</button>

‎src/common/ast-types.ts‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ export abstract class AstNode {
1010
| 'funcCall' | 'funcDef' | 'arrowFuncDef'
1111
| 'createObject' | 'createArray'
1212
| 'if' | 'for' | 'while'
13+
| 'import'
1314
| 'return' | 'continue' | 'break'
1415
) { }
1516
}
@@ -65,13 +66,13 @@ export class FunctionCallNode extends AstNode {
6566
}
6667

6768
export class FunctionDefNode extends AstNode {
68-
constructor(public name: string, public params: string[], public body: AstNode[] ) {
69+
constructor(public name: string, public params: string[], public body: AstNode[]) {
6970
super('funcDef');
7071
}
7172
}
7273

7374
export class ArrowFuncDefNode extends AstNode {
74-
constructor(public params: string[], public body: AstNode[] ) {
75+
constructor(public params: string[], public body: AstNode[]) {
7576
super('arrowFuncDef');
7677
}
7778
}
@@ -94,6 +95,17 @@ export class WhileNode extends AstNode {
9495
}
9596
}
9697

98+
export interface NameAlias {
99+
name: string,
100+
alias: string | undefined
101+
}
102+
103+
export class ImportNode extends AstNode {
104+
constructor(public module: NameAlias, public body: AstBlock, public parts: NameAlias[] | undefined = undefined) {
105+
super('import');
106+
}
107+
}
108+
97109
export class GetSingleVarNode extends AstNode {
98110
name: string;
99111
nullCoelsing: boolean | undefined = undefined;
@@ -124,7 +136,7 @@ export class CreateObjectNode extends AstNode {
124136
export class CreateArrayNode extends AstNode {
125137
constructor(
126138
public items: AstNode[]
127-
) {
139+
) {
128140
super('createArray');
129141
}
130142
}

‎src/evaluator/evaluator.ts‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,11 @@ export class Evaluator {
8585
}
8686

8787
private evalNode(node: AstNode, scope: Scope): unknown {
88+
if (node.type === 'import') {
89+
// skip this for now. As modules are implemented externally
90+
return null;
91+
}
92+
8893
if (node.type === 'if') {
8994
const ifNode = node as IfNode;
9095
const newScope = scope;
@@ -103,7 +108,7 @@ export class Evaluator {
103108

104109
const array = this.evalNode(forNode.sourceArray, newScope) as unknown[] | string;
105110

106-
for(let item of array){
111+
for (let item of array) {
107112
newScope.set(forNode.itemVarName, item);
108113
this.evalBlock({ body: forNode.body } as AstBlock, newScope);
109114
}
@@ -114,7 +119,7 @@ export class Evaluator {
114119
const forNode = node as WhileNode;
115120
const newScope = scope;
116121

117-
while(this.evalNode(forNode.condition, newScope)){
122+
while (this.evalNode(forNode.condition, newScope)) {
118123
this.evalBlock({ body: forNode.body } as AstBlock, newScope);
119124
}
120125

@@ -211,7 +216,7 @@ export class Evaluator {
211216
for (let i = 1; i < dotObject.nestedProps.length; i++) {
212217
const nestedProp = dotObject.nestedProps[i];
213218

214-
if((dotObject.nestedProps[i - 1] as any).nullCoelsing && !startObject) {
219+
if ((dotObject.nestedProps[i - 1] as any).nullCoelsing && !startObject) {
215220
startObject = {};
216221
}
217222

@@ -234,7 +239,7 @@ export class Evaluator {
234239
}
235240

236241
// no undefined values, make it rather null
237-
return (startObject === undefined)? null : startObject;
242+
return (startObject === undefined) ? null : startObject;
238243
}
239244

240245
if (node.type === 'createObject') {

‎src/parser/parser.spec.ts‎

Lines changed: 39 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
1-
import { AssignNode, BinOpNode, ConstNode } from "../common";
1+
import { AssignNode, BinOpNode, ConstNode, ImportNode } from "../common";
22
import { Tokenizer } from "../tokenizer";
33
import { Parser } from "./parser";
44

55
describe('Parser => ', () => {
66

77
it('1+2', async () => {
8-
let ast = new Parser().parse(new Tokenizer().tokenize("1+2") )
8+
let ast = new Parser().parse(new Tokenizer().tokenize("1+2"))
99
expect(ast.body.length).toBe(1);
1010
expect(ast.body[0].type).toBe("binOp");
1111
const binOp = ast.body[0] as BinOpNode
@@ -15,7 +15,7 @@ describe('Parser => ', () => {
1515
});
1616

1717
it('1+2-3', async () => {
18-
let ast = new Parser().parse(new Tokenizer().tokenize("1+2-3") )
18+
let ast = new Parser().parse(new Tokenizer().tokenize("1+2-3"))
1919
expect(ast.body.length).toBe(1);
2020
expect(ast.body[0].type).toBe("binOp");
2121
const binOp = ast.body[0] as BinOpNode
@@ -24,14 +24,40 @@ describe('Parser => ', () => {
2424
expect((binOp.right as ConstNode).value).toBe(3);
2525
});
2626

27-
// it('x=1+2', async () => {
28-
// let ast = new Parser().parse(new Tokenizer().tokenize("x=1+2"))
29-
// expect(ast.body.length).toBe(1);
30-
// expect(ast.body[0].type).toBe("assign");
31-
// const binOp = (ast.body[0] as AssignNode).source as BinOpNode
32-
// expect((binOp.left as ConstNode).value).toBe(1);
33-
// expect(binOp.op).toBe('+');
34-
// expect((binOp.right as ConstNode).value).toBe(2);
35-
// });
36-
27+
it('import datapipe-js-utils as utils', async () => {
28+
const script = `import datapipe-js-utils as utils`
29+
let ast = new Parser().parse(new Tokenizer().tokenize(script))
30+
expect(ast.body.length).toBe(1);
31+
expect(ast.body[0].type).toBe("import");
32+
const importNode = (ast.body[0] as ImportNode);
33+
expect(importNode.module.name).toBe("datapipe-js-utils");
34+
expect(importNode.module.alias).toBe("utils");
35+
});
36+
37+
it('import datapipe-js-utils', async () => {
38+
const script = `import datapipe-js-utils`
39+
let ast = new Parser().parse(new Tokenizer().tokenize(script))
40+
expect(ast.body.length).toBe(1);
41+
expect(ast.body[0].type).toBe("import");
42+
const importNode = (ast.body[0] as ImportNode);
43+
expect(importNode.module.name).toBe("datapipe-js-utils");
44+
expect(importNode.module.alias).toBe(undefined);
45+
});
46+
47+
it('from datapipe-js-array import sort, first as f, fullJoin', async () => {
48+
const script = `from datapipe-js-array import sort, first as f, fullJoin`
49+
let ast = new Parser().parse(new Tokenizer().tokenize(script))
50+
expect(ast.body.length).toBe(1);
51+
expect(ast.body[0].type).toBe("import");
52+
const importNode = (ast.body[0] as ImportNode);
53+
expect(importNode.module.name).toBe("datapipe-js-array");
54+
expect(importNode.module.alias).toBe(undefined);
55+
expect(importNode.parts).toBeDefined()
56+
if (importNode.parts) {
57+
expect(importNode.parts.length).toBe(3);
58+
expect(importNode.parts[1].name).toBe('first');
59+
expect(importNode.parts[1].alias).toBe('f');
60+
}
61+
});
62+
3763
});

‎src/parser/parser.ts‎

Lines changed: 41 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import {
22
BinOpNode, ConstNode, AstBlock, Token, ParserOptions, AstNode, Operators, AssignNode, TokenTypes,
33
GetSingleVarNode, FunctionCallNode, getTokenType, getTokenValue, isTokenTypeLiteral, getStartLine,
44
getStartColumn, getEndColumn, getEndLine, findOperators, splitTokens, DotObjectAccessNode, BracketObjectAccessNode,
5-
findTokenValueIndex, FunctionDefNode, CreateObjectNode, ObjectPropertyInfo, CreateArrayNode, ArrowFuncDefNode, ExpressionOperators, IfNode, ForNode, WhileNode
5+
findTokenValueIndex, FunctionDefNode, CreateObjectNode, ObjectPropertyInfo, CreateArrayNode, ArrowFuncDefNode, ExpressionOperators, IfNode, ForNode, WhileNode, ImportNode, NameAlias
66
} from '../common';
77

88
export class InstructionLine {
@@ -60,14 +60,15 @@ export class Parser {
6060
}
6161

6262
for (let i = 0; i < instructions.length; i++) {
63-
const instruction = instructions[i]
63+
const instruction = instructions[i];
64+
const firstToken = instruction.tokens[0];
6465

6566
if (!instruction.tokens.length) {
6667
continue;
6768
}
6869
const assignTokens = splitTokens(instruction.tokens, '=');
6970

70-
if (getTokenValue(instruction.tokens[0]) === 'def') {
71+
if (getTokenValue(firstToken) === 'def') {
7172
const funcName = getTokenValue(instruction.tokens[1]) as string;
7273
const paramsTokens = instruction.tokens.slice(
7374
instruction.tokens.findIndex(tkns => getTokenValue(tkns) === '(') + 1,
@@ -91,7 +92,7 @@ export class Parser {
9192

9293
ast.funcs.push(new FunctionDefNode(funcName, params, funcAst.body))
9394

94-
} else if (getTokenValue(instruction.tokens[0]) === 'if') {
95+
} else if (getTokenValue(firstToken) === 'if') {
9596

9697
const endDefOfDef = findTokenValueIndex(instruction.tokens, v => v === ':');
9798

@@ -111,7 +112,7 @@ export class Parser {
111112

112113
ast.body.push(new IfNode(conditionNode, ifBody, elseBody))
113114

114-
} else if (getTokenValue(instruction.tokens[0]) === 'for') {
115+
} else if (getTokenValue(firstToken) === 'for') {
115116

116117
const endDefOfDef = findTokenValueIndex(instruction.tokens, v => v === ':');
117118

@@ -125,7 +126,7 @@ export class Parser {
125126

126127
ast.body.push(new ForNode(sourceArray, itemVarName, forBody))
127128

128-
} else if (getTokenValue(instruction.tokens[0]) === 'while') {
129+
} else if (getTokenValue(firstToken) === 'while') {
129130

130131
const endDefOfDef = findTokenValueIndex(instruction.tokens, v => v === ':');
131132

@@ -138,6 +139,40 @@ export class Parser {
138139

139140
ast.body.push(new WhileNode(condition, body))
140141

142+
} else if (getTokenValue(firstToken) === 'import') {
143+
let asIndex = findTokenValueIndex(instruction.tokens, v => v === 'as');
144+
if (asIndex < 0) {
145+
asIndex = instruction.tokens.length;
146+
}
147+
148+
const module = {
149+
name: instruction.tokens.slice(1, asIndex).map(t => getTokenValue(t)).join(''),
150+
alias: instruction.tokens.slice(asIndex + 1).map(t => getTokenValue(t)).join('') || undefined
151+
} as NameAlias;
152+
153+
const body = {} as AstBlock; // empty for now
154+
ast.body.push(new ImportNode(module, body))
155+
} else if (getTokenValue(firstToken) === 'from') {
156+
let importIndex = findTokenValueIndex(instruction.tokens, v => v === 'import');
157+
if (importIndex < 0) {
158+
throw Error(`'import' must follow 'from'`);
159+
}
160+
161+
const module = {
162+
name: instruction.tokens.slice(1, importIndex).map(t => getTokenValue(t)).join('')
163+
} as NameAlias;
164+
165+
const parts = splitTokens(instruction.tokens.slice(importIndex + 1), ',')
166+
.map(t => {
167+
return {
168+
name: getTokenValue(t[0]),
169+
alias: (t.length === 3) ? getTokenValue(t[2]) : undefined
170+
} as NameAlias
171+
});
172+
173+
const body = {} as AstBlock; // empty for now
174+
175+
ast.body.push(new ImportNode(module, body, parts))
141176
} else if (assignTokens.length > 1) {
142177
const target = this.createExpressionNode(assignTokens[0]);
143178
const source = this.createExpressionNode(assignTokens[1]);

0 commit comments

Comments
 (0)