Merge pull request #1 from Chevrotain/dev

Initial version

.gitignore

@@ -102,3 +102,5 @@ dist
 
 # TernJS port file
 .tern-port
+
+.idea

README.md

@@ -1 +1,77 @@
-# quick-start
+# Overview
+
+This is the typescript based starter pack npm module to develop a parser/serializer for a language using [Chevortain](https://sap.github.io/chevrotain/docs/)
+parser toolkit.
+
+It bootstraps following constructs for a super simple sql dsl used in the [Chevrotain Tutorial](https://sap.github.io/chevrotain/docs/tutorial/step0_introduction.html)
+
+- Lexer
+- Parser
+- Visitor
+- Serializer
+- Models
+- Grammar Railroad diagram generation
+- Jest based unit tests
+
+## Usage
+
+- clone this repo
+
+```
+git clone https://github.com/Chevrotain/quick-start.git chevrotain-quick-start
+```
+
+- Change the package name and other things as appropriate in the `package.json`
+
+- Install dependencies
+
+```
+yarn install
+```
+
+- Start implementing your language.
+
+## Unit Tests
+
+Unit tests are a must when you are developing a new language. As you modify grammar, you want to make sure that new grammar is
+handling all use cases properly. This quick-starter comes with easy to use unit testing based on Jest.
+
+To add tests, edit the `dsl-texts.ts` file with new dsl text, model it supposed to parse into and text after
+serializing using couple of format options.
+
+To run a single test, just add `only: true` property to that entry. This allows you to debug to fix issues.
+
+## Resources
+
+Here are some resources to help with your language development
+
+- [Chevrotain Docs](https://sap.github.io/chevrotain/docs/)
+- [Chevrotain Chat](https://gitter.im/chevrotain-parser/Lobby)
+- [Chevrotain Apis](https://sap.github.io/chevrotain/documentation/7_0_1/globals.html)
+
+Here are some projects using Chevrotain in real world
+
+- [Soql Parser](https://github.com/paustint/soql-parser-js)
+- [Prettier Java Plugin Parser][sample_prettier_java]
+- [JHipster Domain Language][sample_jhipster]
+- [Metabase BI expression Parser][sample_metabase].
+- [Three.js VRML Parser][sample_threejs]
+- [Argdown Parser][sample_argdown]
+- [Stardog Union Parsers (GraphQL/SPARQL/and more...)][sample_stardog]
+- [Bombadil Toml Parser][sample_bombadil]
+- [Eve Interactive Programing Language Parser][sample_eve].
+- [BioModelAnalyzer's ChatBot Parser][sample_biomodel].
+
+[benchmark]: https://sap.github.io/chevrotain/performance/
+[sample_metabase]: https://github.com/metabase/metabase/blob/136dfb17954f4e4302b3bf2fee99ff7b7b12fd7c/frontend/src/metabase/lib/expressions/parser.js
+[sample_jhipster]: https://github.com/jhipster/jhipster-core/blob/master/lib/dsl/jdl_parser.js
+[sample_eve]: https://github.com/witheve/Eve/blob/master/src/parser/parser.ts
+[sample_biomodel]: https://github.com/Microsoft/BioModelAnalyzer/blob/master/ChatBot/src/NLParser/NLParser.ts
+[sample_bombadil]: https://github.com/sgarciac/bombadil/blob/master/src/parser.ts
+[sample_argdown]: https://github.com/christianvoigt/argdown/blob/master/packages/argdown-core/src/parser.ts
+[sample_threejs]: https://github.com/mrdoob/three.js/blob/dev/examples/js/loaders/VRMLLoader.js
+[sample_prettier_java]: https://github.com/jhipster/prettier-java/tree/master/packages/java-parser/src/productions
+[sample_stardog]: https://github.com/stardog-union/millan/tree/master/src
+[languages]: https://github.com/SAP/chevrotain/tree/master/examples/implementation_languages
+[backtracking]: https://github.com/SAP/chevrotain/blob/master/examples/parser/backtracking/backtracking.js
+[custom_apis]: https://sap.github.io/chevrotain/docs/guide/custom_apis.html

package.json

@@ -0,0 +1,61 @@
+{
+  "name": "@chevrotain/quick-start",
+  "description": "Your easy path to define a parser for your language based on Chevrotain DSL toolkit",
+  "version": "0.0.1",
+  "private": true,
+  "keywords": [
+    "chevrotain",
+    "starter pack",
+    "dsl",
+    "parser"
+  ],
+  "homepage": "https://github.com/Chevrotain/quick-start",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Chevrotain/quick-start"
+  },
+  "scripts": {
+    "clean": "rm -rf dist && rm -rf coverage && rm -rf yarn-error.log",
+    "compile": "tsc",
+    "compile:clean": "yarn clean && yarn compile",
+    "compile:watch": "tsc --watch",
+    "generate-diagram": "node scripts/generateDiagram.js",
+    "build": "rm -rf dist && tsc && yarn generate-diagram",
+    "prepublish": "yarn build",
+    "test": "ts-node -O '{\"module\":\"commonjs\"}' node_modules/jest/bin/jest.js ",
+    "test:watch": "ts-node -O '{\"module\":\"commonjs\"}' node_modules/jest/bin/jest.js --watch"
+  },
+  "dependencies": {
+    "chevrotain": "^7.0.1"
+  },
+  "devDependencies": {
+    "@types/jest": "^25.2.2",
+    "@types/node": "^12.12.31",
+    "jest": "^25.5.4",
+    "ts-jest": "^25.5.1",
+    "ts-node": "^8.10.1",
+    "typescript": "^3.9.3"
+  },
+  "jest": {
+    "collectCoverage": false,
+    "moduleFileExtensions": [
+      "ts",
+      "tsx",
+      "js",
+      "jsx",
+      "json",
+      "node"
+    ],
+    "roots": [
+      "test"
+    ],
+    "transform": {
+      "^.+\\.tsx?$": "ts-jest"
+    },
+    "testMatch": [
+      "**/*.spec.ts"
+    ],
+    "testTimeout": 120000
+  }
+}

scripts/generateDiagram.js

@@ -0,0 +1,17 @@
+const path = require('path');
+const fs = require('fs');
+const chevrotain = require('chevrotain');
+
+function generateDiagram() {
+  const {parser} = require('../dist/parser');
+  const serializedGrammar = parser.getSerializedGastProductions();
+  const htmlText = chevrotain.createSyntaxDiagramsCode(serializedGrammar);
+  const outPath = path.resolve(__dirname, '../dist');
+  fs.writeFileSync(outPath + '/diagram.html', htmlText);
+
+  return {
+    message: `Diagram is written to ${outPath}`,
+  };
+}
+
+generateDiagram();

src/index.ts

@@ -0,0 +1,4 @@
+export * from "./models";
+export { tokenize } from "./lexer";
+export { buildAst as parse } from "./visitor";
+export { serialize } from "./serializer";

src/lexer.ts

@@ -0,0 +1,70 @@
+import {createToken, Lexer} from 'chevrotain';
+
+export const Identifier = createToken({name: 'Identifier', pattern: /[a-zA-Z]\w*/});
+
+// We specify the "longer_alt" property to resolve keywords vs identifiers ambiguity.
+// See: https://github.com/SAP/chevrotain/blob/master/examples/lexer/keywords_vs_identifiers/keywords_vs_identifiers.js
+export const Select = createToken({
+  name: 'Select',
+  pattern: /SELECT/i,
+  longer_alt: Identifier,
+});
+
+export const From = createToken({
+  name: 'From',
+  pattern: /FROM/i,
+  longer_alt: Identifier,
+});
+
+export const Where = createToken({
+  name: 'Where',
+  pattern: /WHERE/i,
+  longer_alt: Identifier,
+});
+
+export const Comma = createToken({name: 'Comma', pattern: /,/});
+
+export const Integer = createToken({name: 'Integer', pattern: /0|[1-9]\d*/i});
+
+export const GreaterThan = createToken({name: 'GreaterThan', pattern: />/});
+
+export const Equals = createToken({name: 'Equals', pattern: /=/});
+
+export const LessThan = createToken({name: 'LessThan', pattern: /</});
+
+export const WhiteSpace = createToken({
+  name: 'WhiteSpace',
+  pattern: /\s+/,
+  group: Lexer.SKIPPED,
+});
+
+// note we are placing WhiteSpace first as it is very common thus it will speed up the lexer.
+export const allTokens = [
+  WhiteSpace,
+
+  // "keywords" appear before the Identifier
+  Select,
+  From,
+  Where,
+  Comma,
+
+  // The Identifier must appear after the keywords because all keywords are valid identifiers.
+  Identifier,
+  Integer,
+  GreaterThan,
+  Equals,
+  LessThan,
+];
+
+export const SqlLexer = new Lexer(allTokens);
+
+export function tokenize(text: string) {
+  const result = SqlLexer.tokenize(text);
+
+  if (result.errors.length > 0) {
+    const msg = result.errors.map((error) => `[${error.line}:${error.column}] ${error.message}`).join(', ');
+    throw new Error(`Error tokenizing the text. ${msg}`);
+  }
+
+  return result.tokens;
+}

src/models.ts

@@ -0,0 +1,63 @@
+export enum ModelType {
+  SelectStatement = 'SelectStatement',
+  Expression = 'Expression',
+  WhereClause = 'WhereClause',
+  FromClause = 'FromClause',
+  SelectClause = 'SelectClause',
+}
+
+export interface Model {
+  type: ModelType;
+}
+
+export interface SelectStatement extends Model {
+  type: ModelType.SelectStatement;
+  select: SelectClause;
+  from: FromClause;
+  where?: WhereClause;
+}
+
+export interface SelectClause extends Model {
+  type: ModelType.SelectClause;
+  columns: string[];
+}
+
+export interface FromClause extends Model {
+  type: ModelType.FromClause;
+  table: string;
+}
+
+export interface WhereClause extends Model {
+  type: ModelType.WhereClause;
+  condition: Expression;
+}
+
+export enum RelationalOperator {
+  '>' = '>',
+  '=' = '=',
+  '<' = '<',
+}
+
+export interface Expression extends Model {
+  type: ModelType.Expression;
+  lhs: string;
+  operator?: RelationalOperator;
+  rhs?: string;
+}
+
+export interface Expression {}
+
+export interface FormatOptions {
+  keywordsUpperCase?: boolean;
+  newLineForClause?: boolean;
+}
+
+export const FORMAT_CLEAN: FormatOptions = {
+  keywordsUpperCase: false,
+  newLineForClause: false,
+};
+
+export const FORMAT_DEFAULT: FormatOptions = {
+  keywordsUpperCase: true,
+  newLineForClause: true,
+};

src/nodes.ts

@@ -0,0 +1,36 @@
+import {CstNode, IToken} from 'chevrotain';
+
+export interface SelectStatementNode {
+  selectClause: CstNode[];
+  fromClause: CstNode[];
+  whereClause?: CstNode[];
+}
+
+export interface SelectClauseNode {
+  columns: IToken[];
+}
+
+export interface FromClauseNode {
+  table: IToken[];
+}
+
+export interface WhereClauseNode {
+  expression: CstNode[];
+}
+
+export interface ExpressionNode {
+  lhs: CstNode[];
+  relationalOperator: CstNode[];
+  rhs: CstNode[];
+}
+
+export interface AtomicExpressionNode {
+  Integer?: IToken[];
+  Identifier?: IToken[];
+}
+
+export interface RelationalOperatorNode {
+  GreaterThan?: IToken[];
+  LessThan?: IToken[];
+  Equals?: IToken[];
+}