Migrate to service template

.cruft.json

@@ -0,0 +1,23 @@
+{
+  "template": "[email protected]:CybercentreCanada/assemblyline-service-template.git",
+  "commit": "3a814ab822dc6f42558cb3727a2857afdcb50b2f",
+  "checkout": null,
+  "context": {
+    "cookiecutter": {
+      "service_name": "apivector",
+      "__svc_name": "apivector",
+      "__repository": "assemblyline-service-apivector",
+      "__pkg_name": "apivector",
+      "__class_name": "Apivector",
+      "short_description": "This service extracts library imports from windows PE files or memory dump to generate api vector classification.",
+      "short_description_fr": "Ce service extrait les importations de libraries des executables Windows ou des fichiers mémoire pour générer une classification vectorielle des api.",
+      "stage": "CORE",
+      "category": "Static Analysis",
+      "org_name_full": "CybercentreCanada",
+      "org_name_short": "cccs",
+      "license": "mit",
+      "_template": "[email protected]:CybercentreCanada/assemblyline-service-template.git"
+    }
+  },
+  "directory": null
+}

.dockerignore

@@ -0,0 +1,32 @@
+Dockerfile
+.idea
+.git
+.gitignore
+.vscode
+.dockerignore
+
+pipelines
+venv
+.venv
+env
+.env
+test
+tests
+examples
+docs
+
+build
+dist
+**/__pycache__
+**/*.pyc
+
+pip-log.txt
+pip-delete-this-directory.txt
+.tox
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.log

.gitignore

@@ -1,3 +1,7 @@
+# Created by https://www.toptal.com/developers/gitignore/api/python,vim,visualstudiocode
+# Edit at https://www.toptal.com/developers/gitignore?templates=python,vim,visualstudiocode
+
+### Python ###
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -6,11 +10,6 @@ __pycache__/
 # C extensions
 *.so
 
-# IDE files
-.pydevproject
-.python-version
-.idea
-
 # Distribution / packaging
 .Python
 build/
@@ -60,6 +59,22 @@ cover/
 *.mo
 *.pot
 
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
 # PyBuilder
 .pybuilder/
 target/
@@ -71,6 +86,43 @@ target/
 profile_default/
 ipython_config.py
 
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
 # Environments
 .env
 .venv
@@ -80,9 +132,85 @@ ENV/
 env.bak/
 venv.bak/
 
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
 # Cython debug symbols
 cython_debug/
 
-*malpedia_apivector_list.csv
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+### Python Patch ###
+# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration
+poetry.toml
+
+# ruff
+.ruff_cache/
+
+# LSP config files
+pyrightconfig.json
+
+### Vim ###
+# Swap
+[._]*.s[a-v][a-z]
+!*.svg  # comment out if you don't need vector files
+[._]*.sw[a-p]
+[._]s[a-rt-v][a-z]
+[._]ss[a-gi-z]
+[._]sw[a-p]
+
+# Session
+Session.vim
+Sessionx.vim
+
+# Temporary
+.netrwhist
+*~
+# Auto-generated tag files
+tags
+# Persistent undo
+[._]*.un~
+
+### VisualStudioCode ###
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+!.vscode/*.code-snippets
+
+# Local History for Visual Studio Code
+.history/
+
+# Built Visual Studio Code Extensions
+*.vsix
+
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+.ionide
 
-malpediaclient/config\.py
+# End of https://www.toptal.com/developers/gitignore/api/python,vim,visualstudiocode

.vscode/launch.json

@@ -0,0 +1,21 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "RunServiceOnce",
+            "type": "python",
+            "request": "launch",
+            "module": "assemblyline_v4_service.dev.run_service_once",
+            "cwd": "${workspaceFolder}",
+            "args": [
+                "-d",
+                "apivector.apivector.APIVector",
+                "${file}"
+            ],
+            "justMyCode": false,
+        },
+    ]
+}

CONTRIBUTING.md

@@ -2,24 +2,68 @@
 
 This guide covers the basics of how to contribute to the Assemblyline project.
 
-Python code should follow the PEP8 guidelines defined here: [PEP8 Guidelines](https://www.python.org/dev/peps/pep-0008/).
+Python code should follow the PEP8 guidelines defined here:
+[PEP8 Guidelines](https://www.python.org/dev/peps/pep-0008/).
 
 ## Tell us want you want to build/fix
-Before you start coding anything you should connect with the Assemblyline community via the [Assemblyline Discord server](https://discord.gg/GUAy9wErNu) and/or the [central Assemblyline GitHub project](https://github.com/CybercentreCanada/assemblyline/issues) to make sure no one else is working on the same thing and that whatever you are going to build still fits with the vision of the system.
+
+Before you start coding anything you should connect with the Assemblyline community via the
+[Assemblyline Discord server](https://discord.gg/GUAy9wErNu) and/or the
+[central Assemblyline GitHub project](https://github.com/CybercentreCanada/assemblyline/issues) to make sure no one
+else is working on the same thing and that whatever you are going to build still fits with the vision of the system.
 
 ## Git workflow
 
 - Clone the repo to your own account
 - Checkout and pull the latest commits from the master branch
 - Make a branch
-- Work in any way you like and make sure your changes actually work
-- When you're satisfied with your changes, create a pull requests to the main assemblyline repo
+- Work on your modifications and make sure your changes work as expected
+- When you're satisfied with your changes, create a pull requests to the Assemblyline repo
 
 #### Transfer your service repo
-If you've worked on a new service that you want to be included in the default service selection you'll have to transfer the repo into our control.
+
+If you've worked on a new service that you want to be included in the default service selection you'll have to transfer
+the associated repo into our control.
 
 #### You are not allow to merge:
 
-Even if you try to merge in your pull request, you will be denied. Only a few people in our team are allowed to merge code into our repositories.
+Even if you try to merge in your pull request, you will be denied. Only a few people in our team are allowed to merge
+code into our repositories.
 
 We check for new pull requests every day and will merge them in once they have been approved by someone in our team.
+
+# Guide de contribution d'Assemblyline
+
+Ce guide couvre les bases de la façon de contribuer au projet Assemblyline.
+
+Le code Python doit suivre les directives PEP8 définies ici:
+[Directives PEP8](https://www.python.org/dev/peps/pep-0008/).
+
+## Dites-nous que vous voulez construire / réparer
+
+Avant de commencer à coder quoi que ce soit, vous devriez vous connecter à la communauté Assemblyline via le
+[Serveur Discord Assemblyline](https://discord.gg/GUAy9wErNu) et/ou le
+[projet GitHub central Assemblyline](https://github.com/CybercentreCanada/assemblyline/issues) pour vous assurer que
+personne d'autre ne travaille sur la même chose et que tout ce que vous allez construire correspond toujours à la vision
+du système.
+
+## Flux de travail avec Git
+
+- Clonez le référentiel sur votre propre compte
+- Changez de branche pour la branche principale et la synchroniser avec le serveur de référence
+- Faire une nouvelle branche
+- Travaillez sur ce que vous souhaitez et assurez-vous que vos modifications fonctionnent comme prévu
+- Lorsque vous êtes satisfait de vos modifications, créez une demande de fusion sur le référentiel d'Assemblyline
+
+#### Transférer votre référentiel de service
+
+Si vous avez travaillé sur un nouveau service que vous souhaitez inclure dans la sélection de service par défaut, vous
+devrez transférer le référentiel associé sous notre contrôle.
+
+#### Vous n'êtes pas autorisé à compléter une fusion:
+
+Même si vous tentez de compléter une demande de fusion, vous serez refusé. Seules quelques personnes de notre équipe
+sont autorisées à fusionner dans nos référentiels.
+
+Nous vérifions les nouvelles demande de fusion tous les jours et les fusionnerons une fois qu'elles auront été approuvées
+par quelqu'un de notre équipe.

Dockerfile

@@ -1,19 +1,33 @@
 ARG branch=latest
 FROM cccs/assemblyline-v4-service-base:$branch
 
-# Set service to be run
-ENV SERVICE_PATH api_vector.api_vector.API_VECTOR
+# Python path to the service class from your service directory
+ENV SERVICE_PATH apivector.apivector.APIVector
 
-# Install python dependancies
+# Install apt dependencies
+USER root
+COPY pkglist.txt /tmp/setup/
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y --no-install-recommends \
+    $(grep -vE "^\s*(#|$)" /tmp/setup/pkglist.txt | tr "\n" " ") && \
+    rm -rf /tmp/setup/pkglist.txt /var/lib/apt/lists/*
+
+# Install python dependencies
+USER assemblyline
 COPY requirements.txt requirements.txt
-RUN pip install --no-cache-dir --user --requirement requirements.txt && rm -rf ~/.cache/pip
+RUN pip install \
+    --no-cache-dir \
+    --user \
+    --requirement requirements.txt && \
+    rm -rf ~/.cache/pip
 
 # Copy service code
 WORKDIR /opt/al_service
 COPY . .
 
 # Patch version in manifest
-ARG version=4.2.0.dev1
+ARG version=1.0.0.dev1
 USER root
 RUN sed -i -e "s/\$SERVICE_TAG/$version/g" service_manifest.yml