normalize.py

#!/usr/bin/env python3

import argparse
import io
import unicodedata

# Command line arguments.
arg_parser = argparse.ArgumentParser(
    description='Normalize all ground truth texts for the given text files.'
)
arg_parser.add_argument('filename', help='filename of text file', nargs='*')
arg_parser.add_argument(
    '-n',
    '--dry-run',
    help="show which files would be normalized but don't change them",
    action='store_true',
)
arg_parser.add_argument(
    '-v', '--verbose', help='show ignored files', action='store_true'
)
arg_parser.add_argument(
    '-f',
    '--form',
    help='normalization form (default: NFC)',
    choices=['NFC', 'NFKC', 'NFD', 'NFKD'],
    default='NFC',
)

args = arg_parser.parse_args()

# Read all files and overwrite them with normalized text if necessary.
for filename in args.filename:
    with io.open(filename, 'r', encoding='utf-8') as f:
        try:
            text = f.read()
        except UnicodeDecodeError:
            if args.verbose:
                print(filename + ' (ignored)')
            continue
        normalized_text = unicodedata.normalize(args.form, text)
        if text != normalized_text:
            print(filename)
            if not args.dry_run:
                with io.open(filename, 'w', encoding='utf-8') as out:
                    out.write(normalized_text)