pug - a compiler pipeline

HTML Output<article> <h1>Title</h1> <p>Body</p></article>

Pug Sourcearticle h1 Title p Body

Pug Compiler

Lexer Parser Code Gen

Lexerarticle h1 Title p Body

• tag – "article"• indent• tag – "h1"• text – "Title"• new line• tag – "p"• text – "Body"• outdent

Lexerclass Lexer { constructor(str) { this.str = str; this.indent = 0; }}

Lexertag() { let match = /^[a-z]+/.exec(this.str); if (match) { this.str = this.str.substr(match[0].length); return {type: 'tag', val: match[0]}; }}

Lexertext() { let match = /^ .+/.exec(this.str); if (match) { this.str = this.str.substr(match[0].length); return {type: 'text', val: match[0].substr(1)}; }}

Lexernewline() { let match = /^\n( *)/.exec(this.str); if (match) { this.str = this.str.substr(match[0].length); let oldIndent = this.indent; this.indent = match[1].length; if (this.indent > oldIndent) return {type: 'indent'}; if (this.indent < oldIndent) return {type: 'outdent'}; return {type: 'newline'}; }}

Lexerfail() { throw new Error( 'Unexpected text' );}

getTokens() { let tokens = []; while (this.str.length) { tokens.push( this.newline() || this.tag() || this.text() || this.fail() ); } tokens.push({type: 'eos'}); return tokens;}

Parser

OutdentIndent NewLine

Text"Body"

Tag"h1"

Tag"article"

Text"Title"

Tag"p"

Parser

Text"Body"

Tag"h1"

Tag"article"

Text"Title"

Tag"p"

Parserclass Parser { constructor(tokens) { this.tokens = new TokenStream(tokens); }}

Token Stream• Next – get the next token and advance the stream• Peek – get the next token without advancing the stream• Expect(type) – get the next token and advance the stream

ParserparseFile() { let nodes = []; while (this.tokens.peek().type !== 'eos') { if (this.tokens.peek().type === 'tag') { nodes.push(this.parseTag()); } else { this.tokens.expect('newline'); } } this.tokens.expect('eos'); return {type: 'File', nodes};}

ParserparseTag() { let tok = this.tokens.expect('tag'); let body = null; switch (this.tokens.peek().type) { case 'text': body = this.parseText(); break; case 'indent': body = this.parseBlock(); break; } return {type: 'Tag', name: tok.val, body};}

ParserparseBlock() { this.tokens.expect('indent'); let nodes = []; while (this.tokens.peek().type !== 'outdent') { if (this.tokens.peek().type === 'tag') { nodes.push(this.parseTag()); } else { this.tokens.expect('newline'); } } this.tokens.expect('outdent'); return {type: 'Block', nodes};}

Code Gen

Text"Body"

Tag"h1"

Tag"article"

Text"Title"

Tag"p"

Code Gen

Text"Body"

Tag"h1"

Tag"article"

"Title"

Tag"p"

Code Gen

Text"Body"

"<h1>Title</h1>"

Tag"article"

Tag"p"

Code Gen

"Body"

"<h1>Title</h1>"

Tag"article"

Tag"p"

Code Gen

"<h1>Title</h1>"

Tag"article"

"<p>Body</p>"

Code Gen "<article><h1>Title</h1><p>Body</p></article>"

Code Genfunction render(node) { switch (node.type) { case 'Block': return renderBlock(node); case 'Tag': return renderTag(node); case 'Text': return renderText(node); }}

Code Genfunction renderBlock(node) { return node.nodes.map(render).join('\n');}

Code Genfunction renderTag(node) { if (!node.body) return '<' + node.name + '/>'; return ( '<' + node.name + '>' + render(node.body) + '</' + node.name + '>' );}

Code Genfunction renderText(node) { return node.val;}

Pug Compiler

Lexer Parser Code Gen

Includesarticle h1 Title include ./content.pug

LinkingFile A

IncludeFile B

FILE B

LinkingFile A

IncludeFile BFILE B

Pug Compiler Pipeline

Lexer Parser Loader Linker Code-Gen

String Tokens AST Collection of ASTs

AST String

Lexer Parser Loader Linker Code-Gen

Now it's your turn!

String Tokens AST Collection of ASTs

AST String