WIP

.github/workflows/region_maps.yml

@@ -52,7 +52,7 @@ jobs:
         env:
           RESOLUTION: 7
         run: |
-          lw-generator index generate extern/hplans/${{ matrix.region }}.geojson ${{ matrix.region }}.res${{ env.RESOLUTION }}.h3idz --resolution ${{ env.RESOLUTION }}
+          lw-generator regions generate extern/hplans/${{ matrix.region }}.geojson ${{ matrix.region }}.res${{ env.RESOLUTION }}.h3idz --resolution ${{ env.RESOLUTION }}
 
       - name: Release | Artifacts
         if: startsWith(github.ref, 'refs/tags/region_maps')

.gitignore

@@ -6,6 +6,7 @@ _*
 *.plt
 *.swp
 *.swo
+*.h3cdz
 .erlang.cookie
 ebin
 log
@@ -21,4 +22,4 @@ doc/
 *.geojson
 *.kml
 target/
-serialized/
+serialized/

Cargo.toml

@@ -1,7 +1,7 @@
 [workspace]
 resolver = "2"
 members = [
-    "generator",
+    "generator"
 ]
 
 [profile.release]

README.md

@@ -3,12 +3,12 @@
 LoRaWAN regions represented as serialized (little endian) [H3] indices which
 build a h3 map based on a geojson source file.
 
-`lw-generator` is used to generate compressed h3 index file (extension `.h3idz`)
-for a given region [upstream geojson]. Note that generating h3 index files can
-take a VERY long time.
+`lw-generator` is used to generate a compressed h3 region index file (extension
+`.h3idz`) for a given region [upstream geojson]. Note that generating h3 region
+index files can take a VERY long time.
 
-The h3 index files are built as [release assets] for this repository when tagged
-with a `region_maps_YYYY.MM.DD` tag
+The h3 region index files are built as [release assets] for this repository when
+tagged with a `region_maps_YYYY.MM.DD` tag
 
 `lw-generator` is also used to generate compressed region parameters for a given
 [region parameters json] file.
@@ -25,7 +25,7 @@ parameters json] files.
 Generate IN865.res7.h3irz fom IN865.geojson.
 
 ```
-$ target/release/lw-generator index generate extern/hplans/IN865.geojson IN865.res7.h3idz
+$ target/release/lw-generator regions generate extern/hplans/IN865.geojson IN865.res7.h3idz
 ```
 
 A `Makefile` is supplied to make manual generation easier, but note that index

generator/Cargo.toml

@@ -21,4 +21,8 @@ hextree = "0"
 byteorder = "1.4"
 rayon = "1.6"
 log = "0"
+isocountry = "0"
 simple_logger = "4"
+
+[target.'cfg(not(target_env = "msvc"))'.dependencies]
+tikv-jemallocator = "0.5"

generator/src/countries.rs

@@ -0,0 +1,165 @@
+use crate::{print_json, utils::*};
+use anyhow::Result;
+use byteorder::{ReadBytesExt, WriteBytesExt};
+use h3o::{CellIndex, Resolution};
+use hextree::{compaction::EqCompactor, Cell, HexTreeMap};
+use isocountry::CountryCode;
+use rayon::prelude::*;
+use std::{
+    collections::HashMap,
+    path,
+    sync::{Arc, Mutex},
+};
+
+/// Commands on region indexes
+#[derive(Debug, clap::Args)]
+pub struct Cmd {
+    #[command(subcommand)]
+    cmd: CountriesCmd,
+}
+
+impl Cmd {
+    pub fn run(&self) -> Result<()> {
+        self.cmd.run()
+    }
+}
+
+#[derive(Debug, clap::Subcommand)]
+pub enum CountriesCmd {
+    Generate(Generate),
+    Find(Find),
+}
+
+impl CountriesCmd {
+    pub fn run(&self) -> Result<()> {
+        match self {
+            Self::Generate(cmd) => cmd.run(),
+            Self::Find(cmd) => cmd.run(),
+        }
+    }
+}
+
+/// Generate a binary region index for geojson files in a given folder. The
+/// files are expected to have the three letter country code with `.geojson` as
+/// the extension
+#[derive(Debug, clap::Args)]
+pub struct Generate {
+    /// Path to a folder of GeoJson files to process
+    input: path::PathBuf,
+
+    /// Output file to write to
+    output: path::PathBuf,
+
+    /// Resolution to use for h3 cells
+    #[arg(default_value_t = Resolution::Seven, short, long)]
+    resolution: Resolution,
+}
+
+type CountryMap = HexTreeMap<CountryCode, EqCompactor>;
+
+impl Generate {
+    pub fn run(&self) -> Result<()> {
+        fn country_code_from_path(path: &path::Path) -> Result<CountryCode> {
+            path.file_stem()
+                .ok_or_else(|| anyhow::anyhow!("Missing filename"))
+                .and_then(|stem| {
+                    CountryCode::for_alpha3_caseless(&stem.to_string_lossy())
+                        .map_err(anyhow::Error::from)
+                })
+        }
+
+        fn insert_country_cells(
+            path: &path::Path,
+            resolution: Resolution,
+            country_code: CountryCode,
+            hextree: Arc<Mutex<CountryMap>>,
+        ) -> Result<()> {
+            let cells = read_geojson(path).and_then(|geojson| to_cells(geojson, resolution))?;
+
+            cells
+                .par_chunks(100_000)
+                .for_each_with(hextree, |hextree, chunk| {
+                    let mut hextree = hextree.lock().unwrap();
+                    chunk
+                        .iter()
+                        .filter_map(to_hextree_cell)
+                        .for_each(|hex_cell| hextree.insert(hex_cell, country_code));
+                });
+            log::info!("Inserted {}", country_code);
+            Ok(())
+        }
+
+        // Read filenames from folder
+        let paths = std::fs::read_dir(&self.input)?
+            .map(|entry| entry.map(|e| e.path()))
+            .collect::<Result<Vec<path::PathBuf>, std::io::Error>>()?;
+
+        // Filter out non-geojson files
+        let paths = paths
+            .into_iter()
+            .filter(|path| {
+                path.extension()
+                    .map(|ext| ext == "geojson")
+                    .unwrap_or(false)
+            })
+            .collect::<Vec<path::PathBuf>>();
+
+        let hexmap: Arc<Mutex<HexTreeMap<CountryCode, _>>> =
+            Arc::new(Mutex::new(HexTreeMap::with_compactor(EqCompactor)));
+
+        paths.into_par_iter().for_each(|path| {
+            let hexmap = hexmap.clone();
+            let _ = country_code_from_path(&path).and_then(|country_code| {
+                insert_country_cells(&path, self.resolution, country_code, hexmap)
+            });
+        });
+
+        let result = hexmap.lock().unwrap();
+        write_hexmap(&result, &self.output)?;
+        println!("size: {}", result.len());
+        Ok(())
+    }
+}
+
+/// Check membership of one or more h3 indexes in a country map file
+#[derive(Debug, clap::Args)]
+pub struct Find {
+    /// Country map file to search
+    input: path::PathBuf,
+    /// Indexes to look up countries for
+    cells: Vec<CellIndex>,
+}
+
+impl HexMapValueReader for CountryCode {
+    fn read_value(reader: &mut dyn std::io::Read) -> Result<Self> {
+        let decoded = reader.read_u32::<byteorder::LittleEndian>()?;
+        Ok(CountryCode::for_id(decoded)?)
+    }
+}
+
+impl HexMapValueWriter for CountryCode {
+    fn write_value(&self, writer: &mut dyn std::io::Write) -> Result<()> {
+        writer.write_u32::<byteorder::LittleEndian>(self.numeric_id())?;
+        Ok(())
+    }
+}
+
+impl Find {
+    pub fn run(&self) -> Result<()> {
+        log::info!("Reading hexmap");
+        let hex_map = read_hexmap::<_, CountryCode>(&self.input)?;
+        log::info!("Read hexmap: {}", hex_map.len());
+
+        let result: HashMap<String, CountryCode> = self
+            .cells
+            .clone()
+            .into_iter()
+            .filter_map(|cell| {
+                Cell::from_raw(u64::from(cell))
+                    .ok()
+                    .and_then(|cell| hex_map.get(cell).map(|(_, code)| (cell.to_string(), *code)))
+            })
+            .collect();
+        print_json(&result)
+    }
+}