Merge pull request #32 from micahflee/usao

Support US Attorneys' Office folders

.github/workflows/deploy.yml

@@ -5,7 +5,37 @@ on:
       - "main"
 
 jobs:
+  tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "20.x"
+      - name: Install pip and poetry
+        run: pip install --upgrade pip poetry
+      - uses: actions/cache@v3
+        with:
+          path: ~/.cache/pypoetry/virtualenvs
+          key: ${{ runner.os }}-poetry-${{ hashFiles('src/poetry.lock') }}
+      - name: Install Poetry dependencies
+        run: cd src && poetry install
+      - uses: actions/cache@v3
+        with:
+          path: src/frontend/node_modules
+          key: ${{ runner.os }}-node-${{ hashFiles('src/frontend/package-lock.json') }}
+      - name: Install Node dependencies
+        run: cd src/frontend && npm install
+      - name: Build frontend
+        run: cd src/frontend && npm run build
+      - name: Run tests
+        run: cd src && poetry run pytest -v
+
   docker:
+    needs: tests
     runs-on: ubuntu-latest
     steps:
       - name: Set up Docker Buildx

.github/workflows/tests.yml

@@ -12,7 +12,7 @@ jobs:
           python-version: "3.11"
       - uses: actions/setup-node@v3
         with:
-          node-version: "18.x"
+          node-version: "20.x"
       - name: Install pip and poetry
         run: pip install --upgrade pip poetry
       - uses: actions/cache@v3
@@ -21,6 +21,10 @@ jobs:
           key: ${{ runner.os }}-poetry-${{ hashFiles('src/poetry.lock') }}
       - name: Install Poetry dependencies
         run: cd src && poetry install
+      - uses: actions/cache@v3
+        with:
+          path: src/frontend/node_modules
+          key: ${{ runner.os }}-node-${{ hashFiles('src/frontend/package-lock.json') }}
       - name: Install Node dependencies
         run: cd src/frontend && npm install
       - name: Build frontend

src/app.py

@@ -237,7 +237,11 @@ def catch_all(path):
         )
     missing_dbs = []
     missing_sites = []
+
     sites = os.listdir(get_blueleaks_path())
+    if os.path.exists(os.path.join(get_blueleaks_path(), "usao")):
+        sites += os.listdir(os.path.join(get_blueleaks_path(), "usao"))
+
     for site in get_all_sites():
         if site not in sites:
             missing_sites.append(site)

src/build-structures.py

@@ -9,63 +9,71 @@
 
 
 def build_structure():
-    # Find all folders that have tables (CSV files)
+    # Make a list of folders that potentially have tables
+    potential_sites = {}
     for site in os.listdir(blueleaks_path):
         if os.path.isdir(os.path.join(blueleaks_path, site)):
-            structure = {}
+            potential_sites[site] = os.path.join(blueleaks_path, site)
+    for site in os.listdir(os.path.join(blueleaks_path, "usao")):
+        if os.path.isdir(os.path.join(blueleaks_path, "usao", site)):
+            potential_sites[site] = os.path.join(blueleaks_path, "usao", site)
+
+    # Find all folders that have tables (CSV files)
+    for site in potential_sites:
+        structure = {}
 
-            # Make a list of tables
-            tables = []
-            try:
-                for filename in os.listdir(os.path.join(blueleaks_path, site)):
-                    if filename.endswith(".csv"):
-                        tables.append(filename[0:-4])
-            except:
-                # lost+found folder throws a permission denied
-                pass
+        # Make a list of tables
+        tables = []
+        try:
+            for filename in os.listdir(os.path.join(potential_sites[site])):
+                if filename.endswith(".csv"):
+                    tables.append(filename[0:-4])
+        except:
+            # lost+found folder throws a permission denied
+            pass
 
-            # Skip if there aren't any tables
-            if len(tables) == 0:
-                click.echo(f"No tables found for {site}")
-                continue
+        # Skip if there aren't any tables
+        if len(tables) == 0:
+            click.echo(f"No tables found for {site}")
+            continue
 
-            # Start defining the structure
-            structure = {"name": site, "tables": {}}
+        # Start defining the structure
+        structure = {"name": site, "tables": {}}
 
-            for table in tables:
-                # Get a list of columns for this table
-                csv_filename = os.path.join(blueleaks_path, site, f"{table}.csv")
-                with open(csv_filename) as csv_file:
-                    reader = csv.DictReader(csv_file)
-                    fields = [
-                        {
-                            "name": sanitize_field_name(field),
-                            "show": True,
-                            "type": "text",
-                        }
-                        for field in reader.fieldnames
-                    ]
+        for table in tables:
+            # Get a list of columns for this table
+            csv_filename = os.path.join(potential_sites[site], f"{table}.csv")
+            with open(csv_filename) as csv_file:
+                reader = csv.DictReader(csv_file)
+                fields = [
+                    {
+                        "name": sanitize_field_name(field),
+                        "show": True,
+                        "type": "text",
+                    }
+                    for field in reader.fieldnames
+                ]
 
-                    count = 0
-                    for row in reader:
-                        count += 1
+                count = 0
+                for row in reader:
+                    count += 1
 
-                if count == 0:
-                    hidden = True
-                else:
-                    hidden = False
+            if count == 0:
+                hidden = True
+            else:
+                hidden = False
 
-                structure["tables"][table] = {
-                    "display": table,
-                    "hidden": hidden,
-                    "fields": fields,
-                    "joins": [],
-                }
+            structure["tables"][table] = {
+                "display": table,
+                "hidden": hidden,
+                "fields": fields,
+                "joins": [],
+            }
 
-            json_filename = os.path.join(get_default_structures_path(), f"{site}.json")
-            with open(json_filename, "w") as f:
-                f.write(json.dumps(structure, indent=4))
-            click.secho(f"Wrote {json_filename}", dim=True)
+        json_filename = os.path.join(get_default_structures_path(), f"{site}.json")
+        with open(json_filename, "w") as f:
+            f.write(json.dumps(structure, indent=4))
+        click.secho(f"Wrote {json_filename}", dim=True)
 
 
 if __name__ == "__main__":

src/frontend/src/Pages/Table/TableCell.vue

@@ -50,8 +50,11 @@ function preValue(value) {
 }
 
 function attachmentUrl(value) {
-  var url =
-    "/blueleaks-data/" + props.site + "/files/" + value.replace("\\", "/");
+  if (props.site.startsWith("usa")) {
+    var url = "/blueleaks-data/usao/" + props.site + "/files/" + value.replace("\\", "/");
+  } else {
+    var url = "/blueleaks-data/" + props.site + "/files/" + value.replace("\\", "/");
+  }
   return url;
 }
 

src/initialize.py

@@ -53,7 +53,11 @@ def load_file(path):
         for table in structure["tables"]:
             progress(site, table)
 
-            csv_filename = os.path.join(get_blueleaks_path(), site, f"{table}.csv")
+            if site.startswith("usa"):
+                csv_filename = os.path.join(get_blueleaks_path(), "usao", site, f"{table}.csv")
+            else:
+                csv_filename = os.path.join(get_blueleaks_path(), site, f"{table}.csv")
+
             with open(csv_filename) as csv_file:
                 reader = csv.DictReader(csv_file)
 

src/tests/conftest.py

@@ -175,6 +175,90 @@
     "vslea",
     "wifusion",
     "wsorca",
+    "usaeo",
+    "usaoalmtraining",
+    "usaoalntraining",
+    "usaoaretraining",
+    "usaoarwtraining",
+    "usaoaztraining",
+    "usaocaclea",
+    "usaocactraining",
+    "usaocaetraining",
+    "usaocantraining",
+    "usaocaslea",
+    "usaocastraining",
+    "usaocotraining",
+    "usaocrimevictims",
+    "usaoctlea",
+    "usaocttraining",
+    "usaodelea",
+    "usaodetraining",
+    "usaoflmtraining",
+    "usaoflntraining",
+    "usaogamtraining",
+    "usaogantraining",
+    "usaogastraining",
+    "usaogutraining",
+    "usaohitraining",
+    "usaoiantraining",
+    "usaoidtraining",
+    "usaoilctraining",
+    "usaoilntraining",
+    "usaoinntraining",
+    "usaokstraining",
+    "usaokyelea",
+    "usaokyetraining",
+    "usaokywtraining",
+    "usaolaetraining",
+    "usaolamtraining",
+    "usaolawtraining",
+    "usaomatraining",
+    "usaomdtraining",
+    "usaometraining",
+    "usaomietraining",
+    "usaomiwtraining",
+    "usaomnlea",
+    "usaomntraining",
+    "usaomoetraining",
+    "usaomsstraining",
+    "usaomttraining",
+    "usaoncetraining",
+    "usaoncmtraining",
+    "usaoncwtraining",
+    "usaonetraining",
+    "usaonjtraining",
+    "usaonmlea",
+    "usaonmtraining",
+    "usaonvtraining",
+    "usaonyetraining",
+    "usaonystraining",
+    "usaoohntraining",
+    "usaoohstraining",
+    "usaooketraining",
+    "usaookntraining",
+    "usaookwtraining",
+    "usaoortraining",
+    "usaopaetraining",
+    "usaoritraining",
+    "usaosctraining",
+    "usaoscvw",
+    "usaosdtraining",
+    "usaotnetraining",
+    "usaotnmtraining",
+    "usaotnwtraining",
+    "usaotxetraining",
+    "usaotxntraining",
+    "usaotxwtraining",
+    "usaouttraining",
+    "usaovawtraining",
+    "usaovttraining",
+    "usaowaetraining",
+    "usaowawtraining",
+    "usaowietraining",
+    "usaowitraining",
+    "usaowvntraining",
+    "usaowvstraining",
+    "usaowytraining",
 ]
 
 

src/tests/test_api.py

@@ -10,7 +10,7 @@ def test_api_structures(client, blueleaks_path, dbs_path, structures_paths):
         obj["implemented_sites"][0]["name"]
         == "Northern California Regional Intelligence Center"
     )
-    assert len(obj["unimplemented_sites"]) == 120
+    assert len(obj["unimplemented_sites"]) == 244
     assert "arictexas" in obj["unimplemented_sites"]
     assert "houstonhidta" in obj["unimplemented_sites"]
     assert "memiac" in obj["unimplemented_sites"]