feat: add separator tokenization #77

Sign in to view logs

Workflow file for this run

.github/workflows/CI.yml at ca65762

	name: CI
	on:
	push:
	branches:
	- master
	pull_request:
	jobs:
	test-oniguruma:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v2
	- name: Install dependencies
	run: npm ci
	- name: Run tests
	run: npm run test