pw_tokenizer: Fix undefined return values in TS

Updates the CSV parsing function to handle undefined values returned by
attempting to parse invalid data.

Change-Id: I3c13f37f6d09f6e87da3d6f1778f03c6bd770750
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/255964
Presubmit-Verified: CQ Bot Account <[email protected]>
Pigweed-Auto-Submit: Alexei Frolov <[email protected]>
Commit-Queue: Auto-Submit <[email protected]>
Reviewed-by: Chad Norvell <[email protected]>
Lint: Lint 🤖 <[email protected]>
Docs-Not-Needed: Alexei Frolov <[email protected]>

package.json

@@ -28,31 +28,54 @@
     "jest": "jest"
   },
   "devDependencies": {
+    "@codemirror/autocomplete": "^6.16.3",
+    "@codemirror/lang-python": "^6.1.6",
+    "@codemirror/state": "^6.4.1",
+    "@codemirror/theme-one-dark": "^6.1.2",
+    "@lit-labs/virtualizer": "^2.0.7",
     "@material-ui/core": "^4.12.1",
     "@material-ui/lab": "^4.0.0-alpha.60",
+    "@material/web": "^1.5.0",
+    "@open-wc/testing": "^3.0.0-next.5",
+    "@remcovaes/web-test-runner-vite-plugin": "^1.2.1",
     "@rollup/plugin-commonjs": "^19.0.0",
     "@rollup/plugin-node-resolve": "^13.3.0",
     "@rollup/plugin-terser": "^0.4.4",
     "@rollup/plugin-typescript": "^8.3.3",
+    "@shoelace-style/shoelace": "^2.15.0",
     "@types/crc": "^3.4.0",
     "@types/google-protobuf": "^3.15.5",
     "@types/jest": "^28.1.4",
+    "@types/mocha": "^10.0.6",
     "@types/node": "^22.8.4",
     "@types/papaparse": "^5.3.15",
     "@types/react": "^17.0.14",
     "@types/react-dom": "^17.0.9",
     "@typescript-eslint/eslint-plugin": "^5.59.7",
+    "@web/test-runner": "^0.18.0",
+    "@web/test-runner-commands": "^0.9.0",
+    "@web/test-runner-playwright": "^0.11.0",
     "ansi_up": "^5.1.0",
     "arg": "^5.0.2",
     "base64-js": "^1.5.1",
+    "codemirror": "^6.0.1",
     "debug": "^4.3.2",
     "eslint": "^8.47.0",
+    "eslint-config-prettier": "^8.8.0",
+    "eslint-config-standard-with-typescript": "^34.0.1",
+    "eslint-plugin-import": "^2.26.0",
+    "eslint-plugin-lit": "^1.8.3",
     "eslint-plugin-lit-a11y": "^4.1.0",
+    "eslint-plugin-n": "^15.7.0",
+    "eslint-plugin-promise": "^6.1.1",
     "html-react-parser": "^1.4.0",
     "http-server": "^13.0.2",
     "install-peers": "^1.0.3",
     "jest-environment-jsdom": "^28.1.3",
+    "lit": "^3.1.2",
+    "postcss": "^8.4.24",
     "prettier": "^3.1.1",
+    "prettier-plugin-jsdoc": "^0.4.2",
     "react": "^17.0.2",
     "react-dom": "^17.0.2",
     "requirejs": "^2.3.6",
@@ -61,40 +84,17 @@
     "rollup-plugin-node-builtins": "^2.1.2",
     "rollup-plugin-node-globals": "^1.4.0",
     "rollup-plugin-node-polyfills": "^0.2.1",
+    "rollup-plugin-postcss": "^4.0.2",
     "rollup-plugin-sourcemaps": "^0.6.3",
     "rxjs": "^7.2.0",
+    "sinon": "^17.0.1",
     "tmp": "0.2.1",
     "ts-jest": "^28.0.5",
     "ts-node": "^10.8.1",
     "tsc-watch": "^5.0.3",
     "tslib": "^2.4.0",
     "typescript": "^4.3.5",
-    "@codemirror/autocomplete": "^6.16.3",
-    "@codemirror/lang-python": "^6.1.6",
-    "@codemirror/state": "^6.4.1",
-    "@codemirror/theme-one-dark": "^6.1.2",
-    "@lit-labs/virtualizer": "^2.0.7",
-    "@material/web": "^1.5.0",
-    "@shoelace-style/shoelace": "^2.15.0",
-    "codemirror": "^6.0.1",
-    "lit": "^3.1.2",
-    "prettier-plugin-jsdoc": "^0.4.2",
-    "@open-wc/testing": "^3.0.0-next.5",
-    "@remcovaes/web-test-runner-vite-plugin": "^1.2.1",
-    "@types/mocha": "^10.0.6",
-    "@web/test-runner": "^0.18.0",
-    "@web/test-runner-commands": "^0.9.0",
-    "@web/test-runner-playwright": "^0.11.0",
-    "eslint-config-prettier": "^8.8.0",
-    "eslint-config-standard-with-typescript": "^34.0.1",
-    "eslint-plugin-import": "^2.26.0",
-    "eslint-plugin-lit": "^1.8.3",
-    "eslint-plugin-n": "^15.7.0",
-    "eslint-plugin-promise": "^6.1.1",
-    "sinon": "^17.0.1",
-    "vite": "^5.2.8",
-    "postcss": "^8.4.24",
-    "rollup-plugin-postcss": "^4.0.2"
+    "vite": "^5.2.8"
   },
   "dependencies": {
     "@protobuf-ts/protoc": "^2.7.0",

pw_tokenizer/ts/token_database.ts

@@ -23,7 +23,10 @@ interface TokenData {
   template: string;
 }
 
-function parseTokenNumber(num: string, lineNumber?: number): number {
+function parseTokenNumber(
+  num: string,
+  lineNumber?: number,
+): number | undefined {
   if (!/^[a-fA-F0-9]+$/.test(num)) {
     // Malformed number
     console.error(
@@ -33,6 +36,7 @@ function parseTokenNumber(num: string, lineNumber?: number): number {
           : '' + `is not a valid hex number`,
       ),
     );
+    return undefined;
   }
 
   try {
@@ -45,13 +49,14 @@ function parseTokenNumber(num: string, lineNumber?: number): number {
           : '' + `could not be parsed`,
       ),
     );
+    return undefined;
   }
 }
 
 function parseRemovalDate(
   dateString: string,
   lineNumber?: number,
-): Date | null {
+): Date | null | undefined {
   const dateContent = dateString.trim();
   if (dateContent === '') return null;
 
@@ -65,6 +70,7 @@ function parseRemovalDate(
           : '' + `could not be parsed`,
       ),
     );
+    return undefined;
   }
 }
 
@@ -85,16 +91,20 @@ export function parseCsvEntry(
   }
 
   // Column 0: Token
-  const token = parseTokenNumber(data.shift(), lineNumber);
+  const token = parseTokenNumber(data.shift()!, lineNumber);
 
   // Column 1: Removal date
-  const removalDate = parseRemovalDate(data.shift(), lineNumber);
+  const removalDate = parseRemovalDate(data.shift()!, lineNumber);
+
+  if (token === undefined || removalDate === undefined) {
+    return undefined;
+  }
 
   // Modern 4-column databases will have the domain in this position.
-  const domain = data.length > 1 ? data.shift() : '';
+  const domain = data.length > 1 ? data.shift()! : '';
 
   // Last column: Template strings
-  const template = data.shift();
+  const template = data.shift() ?? '';
 
   return {
     token,