Add basic file checks

.github/workflows/build.yml

@@ -13,8 +13,20 @@ on:
   workflow_dispatch: {}
 
 jobs:
+  check_files:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v3
+
+    - name: Check Hunspell encoding
+      run: bash ./scripts/check_encoding.sh
+
+    - name: Check tagger file newlines
+      run: bash ./scripts/check_newlines.sh
   build:
     runs-on: ubuntu-latest
+    needs: check_files
     strategy:
       matrix:
         python-version: ["3.11"]

scripts/check_encoding.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Shell script to check if all .dic and .aff files are encoded in ISO-8859-1
+
+get_encoding() {
+  for ext in dic aff; do
+    find ./data/spelling-dict/hunspell -type f -name "pt*.${ext}" -exec file {} \;
+  done
+}
+
+FILE_ENCODINGS=$(get_encoding)
+
+check_encoding() {
+  echo "${FILE_ENCODINGS}" | grep -v "ISO-8859 text"
+}
+
+
+if [[ -z $(check_encoding) ]]; then
+  echo "All .dic and .aff files are encoded in ISO-8859-1, we're good!"
+  exit 0
+else
+  echo "Some .dic and .aff files are not encoded in ISO-8859-1, please fix this."
+  echo "${FILE_ENCODINGS}"
+  exit 1
+fi

scripts/check_newlines.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+# Check for files that do not end with a newline
+
+check_newlines() {
+  find ./data/src-dict -name "*.txt" -type f -print0 | xargs -0 -n1 bash -c 'tail -c1 "$1" | read -r _ || echo "$1"' bash
+}
+
+NO_NEWLINE_FILES=$(check_newlines)
+
+if [[ -z "${NO_NEWLINE_FILES}" ]]; then
+  echo "All files end with a blank line, which is good."
+  exit 0
+else
+  echo "Some files do not end with a newline:"
+  echo "${NO_NEWLINE_FILES}"
+  exit 1
+fi