Merge pull request #1 from dyne/master

merge from upstream

Makefile

@@ -23,7 +23,7 @@ clean:
 install:
 	install -d ${DESTDIR}${PREFIX}
 	install -d ${DESTDIR}${PREFIX}/bin
-	install -p -m 644 dowse ${DESTDIR}${PREFIX}/.zshrc
+	install -p -m 644 dowse ${DESTDIR}${PREFIX}/zshrc
 	make -C src     install
 	make -C zlibs   install
 	make -C daemons install
@@ -35,9 +35,12 @@ install:
 	@modules/install.sh ${CONFDIR}
 	install -d ${DESTDIR}${PREFIX}/db
 	install    -p -m 644 build/*.zkv         ${DESTDIR}${PREFIX}/db
+	install    -p -m 644 build/*.idx         ${DESTDIR}${PREFIX}/db
 	install -s -p -m 755 build/dowse-to-osc    ${DESTDIR}${PREFIX}/bin
 	install -s -p -m 755 build/dowse-to-gource ${DESTDIR}${PREFIX}/bin
 	install -s -p -m 755 build/modprobe        ${DESTDIR}${PREFIX}/bin
+	install -s -p -m 755 build/netdiscover     ${DESTDIR}${PREFIX}/bin
+	install -s -p -m 755 build/netdata       ${DESTDIR}${PREFIX}/bin
 	install -s -p -m 755 build/dnscap        ${DESTDIR}${PREFIX}/bin
 	install -s -p -m 755 build/dowse.so      ${DESTDIR}${PREFIX}/bin
 	install -s -p -m 755 build/dnscrypt-proxy ${DESTDIR}${PREFIX}/bin

README.md

@@ -69,55 +69,43 @@ Installation and activation takes a few steps and needs root:
 2. Install all requirements, here below the list of packages:
 
 ```
-zsh iptables ebtables gettext-base procps net-tools autoconf
-libssl-dev libbind-dev libpcap-dev unzip wget gcc make liblo-dev
-libnetfilter-conntrack3 libnetfilter-queue-dev libjemalloc-dev
-libseccomp2 libsodium-dev libhiredis-dev libkmod-dev
+zsh iptables ebtables sqlite3 procps gettext-base procps net-tools
+autoconf libssl-dev libbind-dev libpcap-dev unzip wget gcc make
+liblo-dev libnetfilter-conntrack3 libnetfilter-queue-dev
+libjemalloc-dev libseccomp2 libsodium-dev libhiredis-dev libkmod-dev
 ```
 
-3. Run `make`
+3. Choose which user should be running dowse: your own is fine, or
+   eventually create one just for that to separate filesystem
+   permissions.
 
-4. Configure the files in the `conf/` folder: settings and network
-   The files are plain text and include documentation in comments.
+4. As the user of choice, run `make`
 
-5. Fire up the startup script **as root**: `sudo ./start.sh`
+5. As root, run `make install`
 
-5.1 Please note that if you are root and inside a ZSh shell, then an
-   interactive console is available: do `source dowse conf/settings`
-   (or another custom config file) and then proceed launching commands
-   prefixed with `dowse-` (tab completion available)
 
-6. Remember to deactivate the DHCP service (Automatic IP
-   configuration) on any other object on the network, typically your
-   ADSL router.
+7. As the dowse user of choice and inside the source, fire up the
+   startup script `./start.sh`
 
-If all went well now one should be able to connect any device to the
-internet as you did before, via Dowse.
-
-## Embedded ARM devices
+Dowse is now running, but there is no graphical interface at the
+moment. To interact and check the status there is only a console with
+commands prefixed with `dowse-` (tab completion available).
 
-If you are using an ARM device, for instance a RaspberryPi box, then
-you are probably running Raspian or a derivative, which lacks many of
-the packages needed to compile and run Dowse.
+To enter it run zsh without extensions and source the main script:
+first type `zsh -f` and press enter, then type `source
+/usr/local/dowse/zshrc` and press enter.
 
-In order to get these packages, one can add the Devuan.org repository
-to `/etc/apt/sources.list.d/devuan.list` with one line:
+If you like the dowse user to have an interactive console every time
+it logs in, then do `ln -s /usr/local/dowse/zshrc $HOME`.
 
-```
-deb http://packages.devuan.org/merged/ jessie main
-```
+If necessary edit the files in the `/etc/dowse` folder, maybe with the address for the local network you like to create or the wifi you like to connect to.
 
-Then import the Devuan developers keys and update the repositories:
-
-```
-apt-get install devuan-keyring
-apt-get update
-```
+If all went well now one should be able to connect any device to the
+internet as you did before, via Dowse.
 
-And then proceed with installing all the dependencies indicated above.
+## Embedded ARM devices
 
-When compilation is completed then it is possible to run `make` and
-build Dowse also on ARM devices, thanks to http://Devuan.org
+Using https://beta.devuan.org just compile and install Dowse following the procedure above. Images are available for a several popular ARM devices including RaspberryPI2 and 3, BananaPI, Cubieboard etc.
 
 # Visualization
 
@@ -126,16 +114,21 @@ easily processed by `gource`. This is the best way to "see dowse
 running": if you are running it locally, then install `gource` and do:
 
 ```
-./src/dowse-to-gource | gource --log-format custom -
+dowse-to-gource | gource --log-format custom -
+```
+
+or from remote:
+
+```
+ssh [email protected] -- dowse-to-gource | gource --log-format custom - 
 ```
 
-This will live render all the DNS activity occurring on your
-computer. Our utility subscribes to DNS events (reading from Redis,
-which listens only on localhost by default) and draws the sort of
-animation that is also showcased on our website.
+This will live render all the DNS activity occurring on your computer
+or local network, with the sort of animation that is also showcased on
+our website.
 
-One can also experiment with gource arguments and render all into a
-video file.
+One can also experiment with gource arguments and render all the
+output of dowse-to-gource into a video file.
 
 # Experimentation
 
@@ -144,7 +137,7 @@ low-latency devices that are running on the same network. To start it
 one must know the IP address of the device, then do:
 
 ```
-./src/dowse-to-osc osc.udp://10.0.0.2:999
+dowse-to-osc osc.udp://10.0.0.2:999
 ```
 
 This will start sending OSC messages over UDP to IP 10.0.0.2 port 999

TODO.org

@@ -32,6 +32,15 @@ implementations wishes for new daemons and modules
 *** nsd (advanced dns server + dnssec))
 *** credns (external zone verifier)
 
+* network object discovery
+
+  oui sanitized list: http://linuxnet.ca/ieee/oui/
+
+  arp-scan (most used, packaged): http://www.nta-monitor.com/wiki/index.php/Arp-scan_User_Guide
+
+  netdiscovery (passive listening): http://nixgeneration.com/~jaime/netdiscover/releases/
+
+
 * domain filtering and categorization
 
   make list importers based on redis, reachable from multiple programs
@@ -50,6 +59,8 @@ implementations wishes for new daemons and modules
   http://contentfilter.futuragts.com/wiki/doku.php?id=downloadable_blacklists
   http://www.urlblacklist.com/?sec=download
 
+  domain filtering hosts file: http://someonewhocares.org/hosts/zero/hosts
+
 * authentication
 
   as every node will store its own public configurations in the k/v

daemons/dnscap

@@ -26,6 +26,7 @@ dnscap-conf() { return 0; }
 
 dnscap-exec() {
     fn dnscap-exec $*
+    # not using launch here to access advanced sup functions
     exe=${1:-${execmap[dnscap]}}
     freq=($exe)
 

daemons/dnscrypt-proxy

@@ -2,14 +2,10 @@
 
 
 
-dnscrypt-exec() {
+dnscrypt-proxy-exec() {
     fn dnscrypt-exec
-    exe=${1:-${execmap[dnscrypt]}}
-    req=(exe)
-    freq=($exe)
-    ckreq || return 1
 
-    ${=exe} -a 127.0.0.1:53443 \
+    launch dnscrypt-proxy -a 127.0.0.1:53443 \
             -p $pid -l $H/log/dnscrypt.log \
             -r "176.56.237.171:443" \
             -k "67C0:0F2C:21C5:5481:45DD:7CB4:6A27:1AF2:EB96:9931:40A3:09B6:2B8D:1653:1185:9C66" \
@@ -18,7 +14,7 @@ dnscrypt-exec() {
 
     sleep 1 # race condition
 
-    savepid dnscrypt $pid
+    savepid dnscrypt-proxy $pid
 
     # TODO: offer a list of hosts from the csv list provided in dnscrypt source
 }

daemons/dnsmasq

@@ -111,13 +111,11 @@ EOF
 
 dnsmasq-exec() {
     fn dnsmasq-exec $*
-    exe=${1:-${execmap[dnsmasq]}}
-    freq=($conf $exe)
+    freq=($conf)
     ckreq || return $?
 
     ztmp
-    func "${execpath} -x $ztmpfile -C $conf"
-    $R/bin/sup ${exe} -x $ztmpfile -C $conf
+    launch  dnsmasq -x $ztmpfile -C $conf
     savepid dnsmasq $ztmpfile
 
     return $?

daemons/netdata

@@ -0,0 +1,38 @@
+#!/usr/bin/zsh
+
+netdata-conf() {
+    cat <<EOF
+[global]
+    # special port for dowse
+    port = 29999
+    ip version = ipv4
+    bind socket to ip = $address
+
+	run as user = $dowse_uid
+	web files owner = root
+    web files group = staff
+
+    error log  = $H/run/netdata-error.log
+    access log = $H/run/netdata-access.log
+    debug log  = $H/run/netdata-debug.log
+
+    memory deduplication = yes
+    # save - dump on exit
+    # map  - write to disk realtime
+    memory mode = ram
+
+EOF
+}
+
+netdata-exec() {
+    fn netdata-start $*
+    req=(conf)
+    freq=($conf)
+    ckreq || return $?
+
+    ztmp
+    launch netdata -pidfile $ztmpfile -c $conf -p 29999
+    savepid netdata $ztmpfile
+
+    return $?
+}

daemons/pgld

@@ -4,8 +4,8 @@ pgld-conf() {
     # required
     # [[ -f /proc/net/ip_tables_targets ]] && \
     #     grep -q NFQUEUE /proc/net/ip_tables_targets || {
-    $R/bin/sup ${execmap[modprobe]} -q xt_NFQUEUE
-    $R/bin/sup ${execmap[modprobe]} -q ipt_NFQUEUE
+    launch modprobe -q xt_NFQUEUE
+    launch modprobe -q ipt_NFQUEUE
 
     # below is all optional
     # [[ -f /proc/net/ip_tables_matches ]] && \
@@ -44,14 +44,12 @@ pgld-conf() {
 
 pgld-exec() {
     fn pgld-start $*
-    exe=${1:-${execmap[pgld]}}
-    req=(conf exe)
-    freq=($conf $exe)
+    req=(conf)
+    freq=($conf)
     ckreq || return 1
 
     ztmp
-    $R/bin/sup $R/bin/pgld -p $ztmpfile -r pgld_reject -a pgld_accept $conf
-
+    launch pgld -p $ztmpfile -r pgld_reject -a pgld_accept $conf
     savepid pgld $ztmpfile
 
     return $?

daemons/redis

@@ -100,7 +100,7 @@ redis-server-exec() {
     pidfile=`awk '/pidfile/ { print $2 }' $conf`
     func "redis $conf $* (pidfile: $pidfile)"
 
-    $exe $conf $*
+    launch redis-server $conf $*
 
     act "blank volatile databases"
     print "flushdb" | redis runtime > /dev/null

daemons/tor

@@ -25,15 +25,13 @@ EOF
 
 tor-exec() {
     fn tor-exec $*
-    execpath=${1:-${execmap[tor]}}
     conf=$1
-    req=(execpath)
     freq=($conf)
     ckreq || return 0
 
     act "launching tor"
 
-    $execpath -f $conf
+    launch tor -f $conf
 }
 
 tor-kill() {

doc/HACKING.md

@@ -57,6 +57,14 @@ Below is an updated list of existing types and their actions.
 
 Represents the event of a dns query by an object on the net.
 
+Sources:
+- dnscap (dowse plugin)
+
+Channel:
+- dns-query-channel
+
+Format:
+
 ```
 DNS,IP,{ACTION},EPOCH,DOMAIN,TLD[,GROUP]
 
@@ -72,8 +80,14 @@ Arguments:
 - `TLD`: is the last string of the domain, i.e: .org, .com, .net etc.
 - `GROUP`: optionally, a group the domain belongs (see domain-list)
 
-Sources:
-- dnscap (dowse plugin)
+Test from CLI:
+```shell
+db=`awk '/db_dynamic/ { print $3 }' src/database.h`
+cat <<EOF | redis-cli -n $db --raw
+SUBSCRIBE dns-query-channel
+EOF
+```
+
 
 ## OBJ
 
@@ -147,14 +161,6 @@ future. Here below some examples for commandline use:
 
 ## DNS (dns-query-channel)
 
-To subscribe to DNS events from CLI, do from the dowse source root:
-
-```shell
-db=`awk '/db_dynamic/ { print $3 }' src/database.h`
-cat <<EOF | redis-cli -n $db --raw
-SUBSCRIBE dns-query-channel
-EOF
-```
 
 ## PGL (pgl-info-channel)