-
Notifications
You must be signed in to change notification settings - Fork 6
/
isir-from-spreadsheet.html
13405 lines (12161 loc) · 489 KB
/
isir-from-spreadsheet.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
<!DOCTYPE html>
<html lang=en dir=ltr>
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/build/pure-min.css" crossorigin="anonymous">
<script src="https://cdn.jsdelivr.net/npm/[email protected]/umd/imm_dom.min.js" crossorigin="anonymous"></script>
<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/exceljs.min.js" crossorigin="anonymous"></script>
<style>
:not(:defined) { display:none }
body { padding: 2rem; }
h1 { margin-top: 0; padding-top: 0; }
header {
--header-color: #e8e8e8;
display: flex;
gap: 2rem;
position: sticky;
top: 0;
padding-bottom: 0.5rem;
background-color: white;
&>aside {
display: flex;
gap: 0.5rem;
border-bottom: 2px solid var(--header-color);
&>h3 {
margin: 0;
text-align: center;
writing-mode: vertical-rl;
text-orientation: sideways;
background-color: var(--header-color);
padding: 0.5rem;
transform: rotate(-0.5turn);
}
}
}
label { display: inline-block; margin-bottom: 0.5rem; }
.flex { display: flex; }
.flex-row { display: flex; flex-direction: row; }
.flex-col { display: flex; flex-direction: column; }
html.loaded-isir-frames .no-isir-frames { display: none!important; }
html:not(.loaded-isir-frames) .have-isir-frames { display: none!important; }
pre:has(#output_file_list) {
max-width: 60vw;
max-height: 10em;
overflow: auto;
}
.isir-validation {
.isir-field::before { content: '['}
.isir-field::after { content: ']'}
.isir-field { font-style: italic; color: lightcoral; }
.isir-field-idx { font-weight: 800; }
.isir-failed-value { font-weight: 500; color: red; }
}
.validation-details {
display: flex;
&>* {
padding-inline: 1em;
margin: 1em;
min-width: 20em;
background-color: #e8e8e8;
}
}
.isir-field-table tbody { white-space: pre; font-family: monospace; }
/* .field-idx { } */
/* .field-len { } */
/* .field-szvalue { } */
.field-result { color: blue; }
.field-result.field-invalid {
font-style: italic; color: red;
&>:first-child::before {
content: '[invalid] '
}
}
/* .field-path { } */
</style>
</head>
<body>
<h1>ISIR from Spreadsheet</h1>
<header>
<aside>
<h3>Input</h3>
<div>
<label>
<h4>Select <code>.xlsx</code>:</h4>
<input id=excel_file_src type=file onchange='on_use_excel_file(this.files[0], {isir_mode: document.getElementById("isir_header_trailer_opt").value})' accept=".xlsx, .xlsm, application/vnd.ms-excel, application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"/>
</label>
<label>
ISIR header / trailer
<select id=isir_header_trailer_opt>
<option selected value="saig">SAIG</option>
<option value="query">Query</option>
<option value="blank">Blank line</option>
</select>
</label>
</div>
<label class="have-isir-frames">
<h4>Select sample ISIR</h4>
<input type=number value=1 min=1 onchange='isir_viewer.show_isir(this.valueAsNumber-1)' />
<div>(of <span id=isir_sample_count>???</span> samples)</div>
</label>
</aside>
<aside>
<h3>Output</h3>
<pre><ul id=output_file_list></ul></pre>
</aside>
</header>
<output id=output_isirs></output>
<script type="module" data-name="./isir-from-spreadsheet.js">
//****************************
// Module `isir_from_spreadsheet` helps transform an Excel Spreadsheet of ISIR column data into ISIR frames
//
import {imm, imm_set, imm_html, imm_raf, imm_emit} from 'https://cdn.jsdelivr.net/npm/[email protected]/esm/index.min.js'
// For use in file HTMLInputElement selection event
// Transforms an Excel Spreadsheet of ISIR scenario column data into ISIR frames
export async function on_use_excel_file(file_src, {isir_mode}) {
imm_set(document.getElementById('output_file_list'))
let isir_workbook = new ExcelJS.Workbook()
await isir_workbook.xlsx.load(file_src)
console.group('Load ISIRs from Excel spreadsheet')
let isir_scenarios = []
try {
for (let each of iter_isirs_from_excel_workbook(isir_workbook)) {
isir_scenarios.push(each)
await imm_raf() // wait for next requestAnimationFrame
}
} finally { console.groupEnd() }
// Allow exploration of ISIR frames using `isir-viewer.html` interface
let isir_samples = isir_scenarios.map(ea => ea.isir_frame)
imm_emit(document, 'isir_samples_updated', {isir_samples})
await imm_raf() // wait for next requestAnimationFrame
// Create ISIR frame DAT file downloads
let filename_no_ext = file_src.name.replace(/\.[^.]*$/,'')
await _as_isir_downloads(isir_scenarios, {filename_no_ext, isir_mode})
}
const _as_text_file_blob_url = (mimetype='text/plain', text_content) =>
URL.createObjectURL( new Blob([text_content], {type: mimetype}) )
export function as_isir_dat_blob_url(isir_frames, opt={isir_mode}) {
// per _section 3.4_ of [Student Aid Internet Gateway (FTI-SAIG) TDClient Host Communication Guide](https://fsapartners.ed.gov/sites/default/files/2023-11/FTISAIGTDClientHostCommunicationsGuide.pdf)
const isir_record_len = 7704
const isir_mode = (opt.isir_mode ?? 'SAIG').toLowerCase()
let isir_headers = [], isir_footers = []
if (/query/.test(isir_mode)) {
isir_headers.push(`O*N05FTDF390 ,CLS=IGFT25OP,XXX,BAT=,`)
} else if (/saig/.test(isir_mode)) {
isir_headers.push(`O*N05TG99999 ,CLS=IDAP25OP,XXX,BAT=,`)
isir_headers.push('') // Note: 2023-24 year had an extra blank line after the header
isir_footers.push(`O*N95TG99999 ,CLS=IDAP25OP,XXX,BAT=,`)
} else if (!isir_mode || 'blank' == isir_mode) {
isir_headers.push('')
} else throw new Error(`Unknown ISIR mode ${JSON.stringify(isir_mode)}`)
return _as_text_file_blob_url('text/plain',
// using IDNR25OP for 2025ISIR Data - Daily State Non-Resident
[ isir_headers.map(e => e.padEnd(isir_record_len, ' '))
, isir_frames
, isir_footers.map(e => e.padEnd(isir_record_len, ' '))
, '' // end with newline
].flat().join('\r\n'))
}
// Helper: create ISIR frames as links to downloadable text file blobs
export async function _as_isir_downloads(isir_scenarios, {filename_no_ext, isir_mode}) {
let el_list = document.getElementById('output_file_list')
if (1) {
let isir_dat = { download: `isirs-for-${filename_no_ext}.dat.txt`,
href: as_isir_dat_blob_url(isir_scenarios.map(ea => ea.isir_frame), {isir_mode}) }
imm(el_list, imm_html.li(imm_html.a(isir_dat, isir_dat.download)))
await imm_raf() // wait for next requestAnimationFrame
}
if (1) {
let isir_ldjson = { download: `isirs-for-${filename_no_ext}.flat.ld-json`,
href: _as_text_file_blob_url('application/ld+json',
isir_scenarios
.map(ea => JSON.stringify(ea.isir_fields))
.concat([''])
.join('\r\n')
)}
imm(el_list, imm_html.li(imm_html.a(isir_ldjson, isir_ldjson.download)))
await imm_raf()
}
if (1) {
for (let each of isir_scenarios) {
let filename = each.scenario[0]
.replaceAll(/\W/g,' ')
.replaceAll(/[ ]+/g, ' ')
.trim()
let isir_dat = { download: `one-isir--${filename}.dat.txt`,
href: as_isir_dat_blob_url(each.isir_frame, {isir_mode}) }
imm(el_list, imm_html.li(imm_html.a(isir_dat, isir_dat.download)))
await imm_raf() // wait for next requestAnimationFrame
}
}
}
// load a list of ISIR scenarios from and Excel workbook
export function gen_isirs_from_excel_workbook(isir_workbook) {
return [... iter_isirs_from_excel_workbook(isir_workbook)] }
// iteratively load a list of ISIR scenarios from and Excel workbook
export function * iter_isirs_from_excel_workbook(isir_workbook) {
let ws_isir_mock = isir_workbook.getWorksheet('ISIR Records')
if (!ws_isir_mock)
throw new Error('Invalid Excel workbook: "ISIR Records" worksheet expected')
try {
console.groupCollapsed('Metadata columns')
var {field_column_mapping, isir_scenario_columns} = _load_worksheet_columns(ws_isir_mock)
} finally { console.groupEnd() }
for (let column of isir_scenario_columns)
yield _make_isir_from_scenario_column(column, field_column_mapping)
}
// Helper: Inspect the Excel worksheet columns for metadata versus scenario column data
function _load_worksheet_columns(ws_isir_mock) {
const _cell_as_text = cell =>
null == cell?.richText ? cell
: cell.richText.map(p => p.text).join('') // strip richText
const _array_from_xl_values = xl_col =>
xl_col.values.map(_cell_as_text)
const metadata_columns = {
__proto__: null,
'order': false,
'f#': false, // field index; not present in all the Excel spreadsheets
'section': false, // documentation; section name / path
'v&v': false, // documentation; valid data notes
'datatype': false, // documentation; data type hint
'len': 'col_lens',
'lens': 'col_lens',
'field': 'col_names',
}
let metadata = {}
let isir_scenario_columns = []
let header_row = _array_from_xl_values(ws_isir_mock.getRow(1))
for (let xl_col_idx=1; xl_col_idx <= ws_isir_mock.actualColumnCount; xl_col_idx++) {
let xl_col = ws_isir_mock.getColumn(xl_col_idx)
let hdr = (header_row[xl_col_idx] || '')
if (!hdr) {
console.log('Skip empty column %o', xl_col_idx)
} else if (xl_col.hidden) {
console.log('Skip HIDDEN column %o: %o', xl_col_idx, hdr)
} else if (/filter\s*$/i.test(hdr)) {
console.log('Skip filter column %o: %o', xl_col_idx, hdr)
} else if (Object.hasOwn(metadata_columns, hdr.toLowerCase())) {
console.log('Metadata column %o: %o', xl_col_idx, hdr)
let attr = metadata_columns[hdr.toLowerCase()]
if (attr) metadata[attr] = _array_from_xl_values(xl_col)
} else {
console.log('Scenario data column %o: %o', xl_col_idx, hdr)
isir_scenario_columns.push(
_array_from_xl_values(xl_col))
}
}
let field_column_mapping = _map_columns_to_isir_fields(metadata, ws_isir_mock)
return {field_column_mapping, isir_scenario_columns}
}
// Helper: Validate Len metadata column against the ISIR layout in isir_module.isir_record_fields
function _map_columns_to_isir_fields(metadata, ws_isir_mock) {
let field_column_mapping = new Map()
for (let xl_row_idx=2, field_idx=1; xl_row_idx <= ws_isir_mock.actualRowCount; xl_row_idx++, field_idx++) {
let xl_len = metadata.col_lens[xl_row_idx]
let field = isir_module.isir_record_fields[field_idx]
if (field.len != xl_len) {
console.error('Field length mismatch [row %o] len cell: %o field.len: %o %o', xl_row_idx, xl_len, field.len, field)
throw new Error('Field length mismatch')
}
field_column_mapping.set(xl_row_idx, field)
}
return field_column_mapping
}
// Helper: Given a ISIR column scenario and a field column mappting, return an isir_frame
function _make_isir_from_scenario_column(column, field_column_mapping) {
let scenario = column[1].split(/\r?\n/)
let isir_frame = isir_module.isir_blank(), isir_fields=[]
console.group('ISIR for:', scenario[0])
try {
let msg_invalid
for (let xl_row_idx=2; xl_row_idx<column.length; xl_row_idx++) {
let field = field_column_mapping.get(xl_row_idx)
if (!field) {
if (null == field)
console.error('Missing field %o', xl_row_idx)
continue // Skipping hidden field
}
let value = column[xl_row_idx]
if ('boolean' === typeof value)
value = value ? 'True' : 'False' // use capitalization from spec
if (null==value || ''===value) {
if (!/UUID/.test(field.name))
continue
else value = ''
}
value = 'string' === typeof value
? value.replace(/^_+|_+$/, '').trim()
: `${value.text ?? value ?? ''}`
try {
if (null != field.expect)
value = field.expect
else if (!value && field.empty)
value = field.empty
else { // Various FIXUPs for mock data
if (/UUID/.test(field.name)) {
if (!value) {
let uuid = crypto.getRandomValues(new Uint8Array((12+12+8)/2))
uuid = Array.from(uuid, v => v.toString(16).padStart(2,'0')).join('')
uuid = uuid.replace(
/([0-9a-fA-F]{8})([0-9a-fA-F]{4})([0-9a-fA-F]{4})([0-9a-fA-F]{4})([0-9a-fA-F]{12})/
, '$1-$2-$3-$4-$5')
value = uuid
}
} else if (field.name === 'Country') {
// fixup some misunderstandings about country codes
switch (value) {
case 'USA': value = 'US'; break
case 'UK': value = 'GB'; break
}
}
}
; [isir_frame, msg_invalid] = isir_module.isir_field_update(field, isir_frame, value, false)
if (msg_invalid) console.warn('%s (value: %o)', msg_invalid, value, field)
isir_fields.push({idx: field.idx, value, name: field.name, invalid: msg_invalid ? true : msg_invalid })
} catch (err) { console.warn([field.idx, field.name, value], err, err.info) }
}
isir_module.isir_load_report(isir_frame, {mode: 'warn'})
} finally { console.groupEnd() }
return {scenario, isir_frame, isir_fields}
}
export function isir_from_spreadsheet_init() {
window.on_use_excel_file = on_use_excel_file
return window.isir_from_spreadsheet = {
on_use_excel_file,
as_isir_dat_blob_url,
_as_isir_downloads,
gen_isirs_from_excel_workbook,
iter_isirs_from_excel_workbook,
}
}
; isir_from_spreadsheet_init()
</script>
<script type=module data-name='./isir-viewer.js'>
//****************************
// Module `isir_viewer` renders ISIR frames into user interface elements */
//
import {imm, imm_set, imm_html, imm_raf, imm_emit} from 'https://cdn.jsdelivr.net/npm/[email protected]/esm/index.min.js'
// Renders a given ISIR frame into validation report and all fields, grouped by section.
// Also accepts an index into `window.isir_samples`
export function show_isir(isir_frame=0) {
if (!window.isir_module) return;
document.documentElement.classList.add('loaded-isir-frames')
if (window.isir_samples)
document.getElementById('isir_sample_count').textContent = ''+window.isir_samples.length
if ('string' !== typeof isir_frame)
isir_frame = window.isir_samples[isir_frame]
if (!isir_frame)
return console.warn('No valid ISIR frame found')
// set active ISIR frame global
window.active_isir_frame = isir_frame
let isir_validation = new Map() // collect validation errors by field
let isir_report = isir_module.isir_load_report(isir_frame, {mode: isir_validation})
imm_set(document.getElementById('output_isirs'),
_render_validation_report(isir_validation),
_render_fields(isir_report))
// browsable ISIR object model, usable from the developer console
let isir_obj = isir_module.isir_model_from(isir_frame)
console.log('active_isir_obj:', window.active_isir_obj = isir_obj)
let evt_details = {isir_frame, isir_validation, isir_obj}
// notify other UI elements of the currently shown ISIR
imm_emit(document, 'isir_shown', evt_details)
return evt_details
}
// Renders ISIR validation results into a section, provding technical
function _render_validation_report(isir_validation) {
let el_report = imm_html.ul()
if (0 == isir_validation.size)
el_report = imm_html.em('No issues; clean validation')
else for (let [field, {value, result, issues}] of isir_validation.entries()) {
// leave out less useful fields
let {idx, name, path, pos_start, pos_end, validate, note, ... tech_detail} = field
let validate_detail = JSON.stringify({value, result}, null, 2).split(/\r?\n/).slice(1,-1)
tech_detail = JSON.stringify(tech_detail, null, 2).split(/\r?\n/).slice(1,-1)
imm(el_report, imm_html.li(
imm_html.details(
imm_html.summary(
imm_html.span({class:'isir-field'},
'Field ', imm_html.span({class:'isir-field-idx'}, `f_${field.idx}`),
' ', imm_html.span({class:'isir-field-name'}, field.name ? `${field.name}` : field.value ? '(expected value)' : '(filler)')),
' failed for "', imm_html.code({class:'isir-failed-value'}, `${value}`), '"'),
imm_html.div({class: 'validation-details'},
imm_html.div(
imm_html.h5('Valid content note:'),
imm_html.pre(' ', note.join('\r\n ')),
),
imm_html.div(
imm_html.h5('Field validation detail:'),
imm_html.ul(issues.map(msg => imm_html.li(`${msg}`))),
imm_html.pre(validate_detail.join('\r\n')),
),
imm_html.div(
imm_html.h5('Technical detail for ', imm_html.code(`isir_module.field_${field.idx}`),':'),
imm_html.pre(tech_detail.join('\r\n')),
))
)))
}
return imm_html.aside({class:'isir-validation'},
imm_html.h2('ISIR Field Validation'),
el_report)
}
// Renders ISIR fields into collapsable sections of tables, as grouped by isir_report sections
function _render_fields(isir_report) {
const _pos_to_cell = pos => (''+(1+pos)).padStart(3,' ')
const as_field_row = ({field, value, result, invalid}) => (
imm_html.tr(
imm_html.td({class: 'field-idx'}, 'f_'+field.idx),
imm_html.td({class: 'field-len'}, ''+field.len),
imm_html.td({class: 'field-pos'}, `${_pos_to_cell(field.pos_start)} to ${_pos_to_cell(field.pos_end-1)}`),
imm_html.td({class: 'field-szvalue'}, imm_html.code(`${value.trimEnd()}`)),
imm_html.td({class: `field-result ${invalid ? 'field-invalid' : ''}`},
imm_html.code(invalid ? ''+invalid : JSON.stringify(result) || '')),
imm_html.td({class: 'field-path'},
! field.name ? null
: imm_html.code(''+([].concat(field.path).join('.'))),
),
))
let all_sections = []
let sect_num = 0
for (let sect of isir_report) {
let {el_section, el_thead, el_tbody} = _isir_section_for(sect, {open: 0===sect_num++})
let field_headers = ['field', 'len', 'pos', 'value', 'result'].map(s => imm_html.th(s))
field_headers.push(
imm_html.th('path: ',
imm_html.code(''+([].concat(sect.path, ['']).join('.'))) ))
imm_set(el_thead, imm_html.tr(field_headers))
imm_set(el_tbody, sect.fields.map(as_field_row))
all_sections.push(el_section)
}
return imm_html.article({class:'isir-fields'},
imm_html.h2('ISIR Fields'),
all_sections)
}
// Cache the section details/summary elements so that it retains the
// open/closed UI interaction state between viewed ISIRs
let _show_elem_cache = new Map()
function _isir_section_for(sect, {open}) {
let res = _show_elem_cache.get(sect.section)
if (!res) {
_show_elem_cache.set(sect.section, res = {})
res.el_section = imm_html.section(
imm_html.details({open},
res.el_summary = imm_html.summary(),
imm_html.table({class: 'pure-table isir-field-table'},
res.el_thead = imm_html.thead(),
res.el_tbody = imm_html.tbody(),
)))
}
imm_set(res.el_summary,
imm_html.small(`[${sect.path.join('.')}] `),
sect.non_empty ? imm_html.b(sect.section) : imm_html.span(sect.section),
imm_html.small(` (non-empty: ${sect.non_empty})`),
)
return res
}
// Validates a list of ISIRs to compile unique validation error messages by field.
// Not currently accessible outside the Developer Console
export async function check_isirs_list(isir_list=window.isir_samples) {
let unique_warnings
for (unique_warnings of iter_check_isirs_list(isir_list)) {
console.log(unique_warnings)
await imm_raf()
}
return unique_warnings
}
export function * iter_check_isirs_list(isir_list=window.isir_samples) {
let unique_warnings = new Map()
let isir_validation = new Map() // collect validation errors by field
let n = 0
for (let isir of isir_list) {
if (0 == (++n % 100))
yield unique_warnings
isir_module.isir_load_report(isir, {mode: isir_validation})
if (0 != isir_validation.size) {
for (let [field, res] of isir_validation.entries()) {
let counts = unique_warnings.get(field) || new Map()
for (let key of res.issues)
counts.set(key, 1 + (counts.get(key) || 0))
unique_warnings.set(field, counts)
}
isir_validation.clear() // reset incremental map
}
}
return unique_warnings
}
// Install window.isir_viewer module, and hook into ISIR-Viewer specific events / globals for the user interface
export function isir_viewer_init() {
imm(document, {
isir_samples_updated(evt) {
let {isir_samples} = evt.detail
console.log('Loaded %o isir frames', isir_samples.length)
window.isir_samples = isir_samples
show_isir(isir_samples[0])
}})
if (window.isir_samples)
imm_dom.imm_emit(document, 'isir_samples_updated', {isir_samples: window.isir_samples})
return window.isir_viewer = {
show_isir,
check_isirs_list,
iter_check_isirs_list,
}
}
// export on globalThis
; isir_viewer_init()
</script>
<script type="module" data-name="./isir-module.js">
//****************************
// Module for ISIR record field read, validation, and update
//
// Definitions for sections and fields is generated (transpiled)
// from the [2024–25 Final ISIR Record Layout in Excel Format, 96KB](https://fsapartners.ed.gov/sites/default/files/2023-11/2024-25ISIRNov2023.xlsx)
// definition of ISIR Layout Volume 4 specification.
//
/**
* Extract raw string value from ISIR frame at field position
* @param {ISIRField} field
* @param {string} isir_frame
* @returns {string}
*/
export function isir_field_read_raw(field, isir_frame) {
return isir_frame.substring(field.pos_start, field.pos_end)
}
/**
* Extract field value from ISIR frame using field validation
* @param {ISIRField} field
* @param {string} isir_frame
* @param {*} mode - see {@link isir_field_validate}
* @returns {*} - string if mode is falsey, result of {@link isir_field_validate} otherwise
*/
export function isir_field_read(field, isir_frame, mode) {
let sz_value = isir_field_read_raw(field, isir_frame)
let res = isir_field_validate(field, sz_value, mode)
return false == mode ? res : res.value
}
/**
* Validate `value` using field specific logic.
* Used in field read and update operations.
*
* Upon failed validation, mode determines next action:
* if mode is false or 'ignore', no action and proceed
* if mode.set is a function, invoke to collect errors by field (Map protocol compatible)
* if mode == 'warn', use `console.warn` and proceed
* otherwise or mode == null, throw error
*
* @param {ISIRField} field
* @param {string} value -- for validation against field
* @param {*} mode -- options for handling validation errors
* @returns {value: string, field: ISIRField, result?:*, invalid?:bool|string}
*/
export function isir_field_validate(field, value, mode) {
let valid, res={__proto__: {field}, raw: value, value}, issues=[]
value = `${value}`.trimEnd()
// Detect left-padding spaces or zero issues. Spec change from prior years.
let m_padding = value.match(/^(?<pad_space>\s+)|^-?(?<pad_zero>\s*0\d+)/)
if (m_padding?.groups.pad_space) {
issues.push('not left justified')
value = value.trimStart()
}
valid = field.validate?.(value, field)
if (!valid && m_padding?.groups.pad_zero) {
// issues.push('zero padded') // currently too noisy for non-numeric fields (Street, SSNs, etc.)
value = value.replace(/^(-)?\s*0+([^0].*)/, '$1$2')
valid = field.validate?.(value, field)
}
res.value = value
if (valid) {
res.result = value
if ('object' == typeof valid) {
res.result = valid.result
valid = valid.valid ?? true
}
}
if (false == valid)
issues.push('invalid field value')
else if (0 < issues.length)
valid = false
else if (false != valid) {
if (null != valid)
res.invalid = false
return res
}
res.invalid = `f_${field.idx} raw: ${JSON.stringify(value)}`
res.issues = issues
if (false == mode || 'ignore' == mode) {
// passthrough
} else if (mode?.set) {
mode.set(field, res) // a map
} else {
let msg_invalid = `ISIR field f_${field.idx}[${field.name}] failed validation`
if (mode=='warn')
console.warn('%s (value: %o)', msg_invalid, value, field, issues)
else throw new ISIRValidationError(msg_invalid, res)
}
return res // return negative validation result
}
export class ISIRValidationError extends Error {
constructor(msg, info) { super(msg); this.info = info }
}
/**
* Pack string value into field length.
* Throws error when longer than available length.
* @param {ISIRField} field
* @param {string} value
* @returns {string} - packed to field length
*/
export function _isir_field_pack_value(field, value) {
let update = `${value}`.padEnd(field.len, ' ')
if (update.length !== field.len) {
let err = new Error('Invalid update length')
err.info = {field_len: field.len, update_len:update.length, update, field}
throw err
}
return update
}
/**
* Update ISIR frame at field position with `update` string.
* Throws error when `isir_frame` length invariant changes.
* @param {ISIRField} field
* @param {string} isir_frame
* @param {string} update - value packed to field length
* @returns {string} - of updated isir_frame
*/
export function _isir_field_raw_splice(field, isir_frame, update) {
let new_frame = isir_frame.substring(0, field.pos_start)
new_frame += update
new_frame += isir_frame.substring(field.pos_end)
if (new_frame.length != isir_frame.length)
throw new Error("Frame length change")
return new_frame
}
/**
* Update ISIR frame field using `value` *without* validation
* Throws error when `isir_frame` length invariant changes.
* @param {ISIRField} field
* @param {string} isir_frame
* @param {string} value
* @returns {string} - of updated isir_frame
*/
export function isir_field_update_raw(field, isir_frame, value) {
let sz_value = _isir_field_pack_value(field, value)
return _isir_field_raw_splice(field, isir_frame, sz_value)
}
/**
* Update ISIR frame field using `value` with validation
* Throws error when `isir_frame` length invariant changes.
* @param {ISIRField} field
* @param {string} isir_frame
* @param {string} value
* @returns {string} - of updated isir_frame
*/
export function isir_field_update(field, isir_frame, value, mode) {
let sz_value = _isir_field_pack_value(field, value)
let res = isir_field_validate(field, sz_value, mode)
let isir = _isir_field_raw_splice(field, isir_frame, sz_value)
return false == mode ? [isir, res.error, res] : isir
}
let _isir_blank // cache blank ISIR frame
/**
* Return a new blank ISIR frame using field definitions
* @returns string - isir_frame with spec defaults
*/
export function isir_blank() {
if (!_isir_blank) {
_isir_blank = ''
for (let field of isir_record_fields)
if (!field) ;
else if (field.empty)
_isir_blank += field.empty
else if (field.expect)
_isir_blank += `${field.expect}`.padEnd(field.len, ' ')
else _isir_blank += ' '.repeat(field?.len || 0)
}
return _isir_blank
}
/**
* Load all ISIR fields by section from an ISIR frame, performing validation
* @param {string} isir_frame
* @param {*} options
* @returns {*}
*/
export function isir_load_report(isir_frame, opt) {
return isir_record_sections.map(section =>
isir_section_report(section, isir_frame, opt))
}
/**
* Load all ISIR fields of a section from an ISIR frame, performing validation
* @param {ISIRSection} section
* @param {string} isir_frame
* @param {*} options
* @returns {*}
*/
export function isir_section_report(section, isir_frame, opt={}) {
let {mode, skip_empty} = opt.trim ? {mode: opt} : opt
let res_fields = [], non_empty=0, pos_start=isir_frame.length, pos_end=0
for (let field of section.field_list) {
//let {idx, name, alias, len} = field
let sz_value = isir_field_read_raw(field, isir_frame)
let res = isir_field_validate(field, sz_value, mode)
if (null != res.value && (! /^0*$/.test(res.value)) && !field.non_content) {
non_empty++
res_fields.push(res)
} else if (!skip_empty) {
res_fields.push(res)
}
}
return {__proto__: section, non_empty, fields: res_fields}
}
/**
* Load all ISIR fields into structured JSON from an ISIR frame, performing validation
* @param {string} isir_frame
* @param {*} options
* @returns {*}
*/
export function isir_load_json(isir_frame, opt) {
let isir_res = {__proto__: {isir: isir_frame}}
for (let section of isir_record_sections) {
let sect_res = isir_section_json(section, isir_frame, opt)
_isir_set_path(isir_res, section.path, sect_res)
}
return isir_res
}
/**
* Load all ISIR fields for a section into structured JSON from an ISIR frame, performing validation
* @param {ISIRSection} section
* @param {string} isir_frame
* @param {*} options - for {mode} option, see parameter from {@link isir_field_validate}
* @returns {*}
*/
export function isir_section_json(section, isir_frame, opt={}) {
let {mode, skip_empty} = opt.trim ? {mode: opt} : opt
let sect_res = {__proto__: {section}}
for (let field of section.field_list) {
if ( field.non_content ) continue; // then skip
let value = isir_field_read(field, isir_frame, mode)
if (!skip_empty || value || (null != value && '' != value))
_isir_set_path(sect_res, field.path, value)
}
return sect_res
}
const _absent_fill = (tgt, key, as_obj) => tgt[key] = ({}) // (as_obj ? {} : [])
/**
* (Advanced) Utility for creating ISIR structure from section paths and field paths.
* Cross reference with use in {@link isir_section_json} and {@link _init_isir_model}
*/
export function _isir_set_path(tip_obj, key_path, value, absent=_absent_fill) {
let key, last_obj=tip_obj, idx_last = key_path.length-1
for (let key_idx=0; key_idx<idx_last; key_idx++) {
key = key_path[key_idx]
tip_obj = (last_obj = tip_obj)[ key ]
if (undefined === tip_obj)
tip_obj = absent(last_obj, key, !isNaN(key_path[key_idx+1]), key_idx, key_path)
}
key = key_path[idx_last]
if (null != key)
tip_obj[ key ] = value
return tip_obj
}
let _isir_proto_
/**
* Load an ISIR structured object model from an ISIR frame
* @param {string} isir_frame
* @returns {*}
*/
export function isir_model_from(isir_frame) {
_isir_proto_ ??= _init_isir_model()
return Object.create(_isir_proto_, {$: {value: [isir_frame]}})
}
/**
* (Advanced) Utility for creating ISIR model prototypes from section paths and field paths.
*/
function _init_isir_model() {
let by_field_idx = {}, propByField = new Map()
for (let field of isir_record_fields)
if (null != field)
propByField.set(field,
by_field_idx['f_'+field.idx] = _isir_field_prop(field))
// structured object
let _fixup_structure = []
const _absent_structure = (tgt, key) => {
let grp = tgt[key] = {}
_fixup_structure.push([grp, [tgt, key]])
return grp }
let by_path = {}
for (let section of isir_record_sections) {
let sect_props = _isir_set_path(by_path, section.path.concat(null), void 0, _absent_structure)
for (let field of section.field_list)
if (field.path)
_isir_set_path(sect_props, field.path, {enumerable: true, ... propByField.get(field)}, _absent_structure)
}
for (let rec of _fixup_structure) {
let [tgt, key] = rec.pop()
// (Subtle) lazily create nested accessor objects using prototypes and shared mutable
tgt[key] = { get() { return Object.create(null, {$: {value: this.$}, ...rec[0]}) }, enumerable: true }
}
// (Subtle) create accessor object using prototypes and shared mutable
return Object.create(null, {...by_path, ...by_field_idx})
function _isir_field_prop(field, kw) {
let prop = { ... kw,
get() {
let sz_value = isir_field_read_raw(field, this.$[0])
let field_res = isir_field_validate(field, sz_value)
return field_res },
set(value) {
return this.$[0] = isir_field_update(field, this.$[0], value) },
}
return prop
}
}
//****************************
// ISIR field validator logic implementations
//
const _validate_expect = (sz_value, field) => (sz_value == field.expect) || (field.allow_blank ? sz_value == '' : false)
function _check_date(sz) {
if ('' === sz) return;
let sz_iso = sz.replace(/(\d{4})(\d{2})(\d{2})/, '$1-$2-$3')
let dt = new Date(sz_iso+'T00:00:00Z')
let [rt_sz] = isNaN(dt) ? '' : dt.toISOString().split('T',1)
let valid = sz_iso == rt_sz
return valid && {valid, result: sz_iso}
}
const _validate_date = (sz_value, field) => _validate_options(sz_value, field) && _check_date(sz_value)
const _validate_yearmonth = (sz_value, field) => _validate_options(sz_value, field) && _check_date(sz_value.length==6 ? sz_value+'01' : sz_value)
const _validate_fixed_decimal = (sz_value) => /\d*/.test(sz_value)
const _rx_correction_highlight_verify_flags = /^(?<correction>[012])(?<highlight>[01])(?<verify>[012])$/
const _validate_correction = (sz_value) => {
if ('000' == sz_value)
return true // valid, with default "no correction" result
let m = _rx_correction_highlight_verify_flags.exec(sz_value)
return m ? {valid: true, result: m.groups} : false
}
function _check_range(value, min, max) {
value = parseInt(value)
let valid = Number.isFinite(value) && parseInt(min) <= value && value <= parseInt(max)
return valid && {valid, result: value}
}
const _validate_by_op = {
__proto__: null,
uuid: (sz_value) => {