wader · wader · Sep 20, 2024 · Sep 18, 2024 · Sep 19, 2024 · Sep 19, 2024
diff --git a/README.md b/README.md
@@ -124,6 +124,7 @@ mpeg_pes_packet,
 mpeg_spu,
 mpeg_ts,
 [msgpack](doc/formats.md#msgpack),
+[negentropy](doc/formats.md#negentropy),
 [nes](doc/formats.md#nes),
 ogg,
 ogg_page,

diff --git a/doc/formats.md b/doc/formats.md
@@ -96,6 +96,7 @@
 |`mpeg_spu`                                                      |Sub&nbsp;Picture&nbsp;Unit&nbsp;(DVD&nbsp;subtitle)                                                          |<sub></sub>|
 |`mpeg_ts`                                                       |MPEG&nbsp;Transport&nbsp;Stream                                                                              |<sub></sub>|
 |[`msgpack`](#msgpack)                                           |MessagePack                                                                                                  |<sub></sub>|
+|[`negentropy`](#negentropy)                                     |Negentropy&nbsp;message                                                                                      |<sub></sub>|
 |[`nes`](#nes)                                                   |iNES/NES&nbsp;2.0&nbsp;cartridge&nbsp;ROM&nbsp;format                                                        |<sub></sub>|
 |`ogg`                                                           |OGG&nbsp;file                                                                                                |<sub>`ogg_page` `vorbis_packet` `opus_packet` `flac_metadatablock` `flac_frame`</sub>|
 |`ogg_page`                                                      |OGG&nbsp;page                                                                                                |<sub></sub>|
@@ -140,7 +141,7 @@
 |`ip_packet`                                                     |Group                                                                                                        |<sub>`icmp` `icmpv6` `tcp_segment` `udp_datagram`</sub>|
 |`link_frame`                                                    |Group                                                                                                        |<sub>`bsd_loopback_frame` `ether8023_frame` `ipv4_packet` `ipv6_packet` `sll2_packet` `sll_packet`</sub>|
 |`mp3_frame_tags`                                                |Group                                                                                                        |<sub>`mp3_frame_vbri` `mp3_frame_xing`</sub>|
-|`probe`                                                         |Group                                                                                                        |<sub>`adts` `aiff` `apple_bookmark` `ar` `avi` `avro_ocf` `bitcoin_blkdat` `bplist` `bzip2` `caff` `elf` `fit` `flac` `gif` `gzip` `html` `jp2c` `jpeg` `json` `jsonl` `leveldb_table` `luajit` `macho` `macho_fat` `matroska` `moc3` `mp3` `mp4` `mpeg_ts` `nes` `ogg` `opentimestamps` `pcap` `pcapng` `png` `tar` `tiff` `toml` `tzif` `tzx` `wasm` `wav` `webp` `xml` `yaml` `zip`</sub>|
+|`probe`                                                         |Group                                                                                                        |<sub>`adts` `aiff` `apple_bookmark` `ar` `avi` `avro_ocf` `bitcoin_blkdat` `bplist` `bzip2` `caff` `elf` `fit` `flac` `gif` `gzip` `html` `jp2c` `jpeg` `json` `jsonl` `leveldb_table` `luajit` `macho` `macho_fat` `matroska` `midi` `moc3` `mp3` `mp4` `mpeg_ts` `nes` `ogg` `opentimestamps` `pcap` `pcapng` `png` `tar` `tiff` `toml` `tzif` `tzx` `wasm` `wav` `webp` `xml` `yaml` `zip`</sub>|
 |`tcp_stream`                                                    |Group                                                                                                        |<sub>`dns_tcp` `rtmp` `tls`</sub>|
 |`udp_payload`                                                   |Group                                                                                                        |<sub>`dns`</sub>|
 
@@ -873,37 +874,41 @@ Standard MIDI file.
 
 ### Notes
 
-1. Only supports the MIDI 1.0 specification.
-2. Does only basic validation on the MIDI data.
+1. Only supports the MIDI 1.0 MIDI file specification.
+2. Only supports _MThd_ and _MTrk_ chunks.
+3. Does only basic validation on the MIDI data.
 
 ### Sample queries
 
 1. Extract the track names from a MIDI file
 ```
-fq -d midi -d midi '.. | select(.event=="Track Name")? | "\(.name)"' twinkle.mid 
+fq -d midi -d midi '.. | select(.event=="track_name")? | "\(.track_name)"' midi/twinkle.mid 
 ```
 
 2. Extract the tempo changes from a MIDI file
 ```
-fq -d midi '.. | select(.event=="Tempo")?.tempo' twinkle.mid
+fq -d midi '.. | select(.event=="tempo")?.tempo' midi/twinkle.mid
 ```
 
 3. Extract the key changes from a MIDI file
 ```
-fq -d midi '.. | select(.event=="Key Signature")?.key' key-signatures.mid
+fq -d midi '.. | select(.event=="key_signature")?.key_signature' midi/twinkle.mid
 ```
 
-4. Extract NoteOn and NoteOff events:
+4. Extract NoteOn events:
 ```
-fq -d midi 'grep_by(.event=="Note On" or .event=="Note Off") | "\(.event)  \(.time.tick)  \(.note)"' twinkle.mid
+fq -d midi 'grep_by(.event=="note_on") | [.time.tick, .note_on.note] | join(" ")' midi/twinkle.mid
 ```
 
 ### Authors
-- transcriptaze.development@gmail.com
+- [transcriptaze](https://github.com/transcriptaze)
 
 ### References
 
 1. [The Complete MIDI 1.0 Detailed Specification](https://www.midi.org/specifications/item/the-midi-1-0-specification)
+2. [Standard MIDI Files](https://midi.org/standard-midi-files)
+3. [Standard MIDI File (SMF) Format](http://midi.teragonaudio.com/tech/midifile.htm)
+4. [MIDI Files Specification](http://www.somascape.org/midi/tech/mfile.html)
 
 ## moc3
 MOC3 file.
@@ -1012,6 +1017,39 @@ $ fq -d msgpack torepr file.msgpack
 ### References
 - https://github.com/msgpack/msgpack/blob/master/spec.md
 
+## negentropy
+Negentropy message.
+
+### View a full Negentropy message
+
+```
+$ fq -d negentropy dd file
+```
+
+### Or from hex
+
+```
+$ echo '6186b7abb47c0001108e4206828ee3bf34258465809a337c6c00019a68e37b177a50b3ae7164ccc628b962020114019c1381281c9e3849d5fbd514b7bb65ad0101e601fbf7451f5d22e7fa36ae3e910e9f5215020157014a1b26853e06e9c32eb41b1df4f9ab300201e6011840e273c84bb1344f1d4e15d9aa67920200016f12ee2340888653f10b0ec2d438ac9f0101840156d2d796f4dff004ab369b9bcfa4d81e020187013f1b3c8a019800d5764e2de6bdfd2785020114017caaf0acb5dfe249aa0f7f742402168a01018301e7b8c4decb1eae455ca5714281e3245302017a01409c22636b097362df125ddffb6d944302015b01f332208bee82acf8ed922853ee54057f020001fc3e51fdb0b92966e38017f7959903850101cc01428ce0c96d49f15b50143e4fb228cb9300000131712d30e5296a7a45d07bba452d61cd' | fq -R 'from_hex | negentropy | dd'
+```
+
+### Check how many ranges the message has and how many of those are of 'fingerprint' mode
+
+```
+$ fq -d negentropy '.bounds | length as $total | map(select(.mode == "fingerprint")) | length | {$total, fingerprint: .}' message
+```
+
+### Check get all ids in all idlists
+
+```
+$ fq -d negentropy '.bounds | map(select(.mode == "idlist") | .idlist | .ids) | flatten' message
+```
+
+### Authors
+- fiatjaf, https://fiatjaf.com
+
+### References
+- https://github.com/hoytech/negentropy
+
 ## nes
 iNES/NES 2.0 cartridge ROM format.
 

diff --git a/format/all/all.fqtest b/format/all/all.fqtest
@@ -141,6 +141,7 @@ mpeg_pes_packet      MPEG Packetized elementary stream packet
 mpeg_spu             Sub Picture Unit (DVD subtitle)
 mpeg_ts              MPEG Transport Stream
 msgpack              MessagePack
+negentropy           Negentropy message
 nes                  iNES/NES 2.0 cartridge ROM format
 ogg                  OGG file
 ogg_page             OGG page

diff --git a/format/all/all.go b/format/all/all.go
@@ -42,6 +42,7 @@ import (
 	_ "github.com/wader/fq/format/mp4"
 	_ "github.com/wader/fq/format/mpeg"
 	_ "github.com/wader/fq/format/msgpack"
+	_ "github.com/wader/fq/format/negentropy"
 	_ "github.com/wader/fq/format/nes"
 	_ "github.com/wader/fq/format/ogg"
 	_ "github.com/wader/fq/format/opentimestamps"

diff --git a/format/format.go b/format/format.go
@@ -148,6 +148,7 @@ var (
 	MPEG_TS             = &decode.Group{Name: "mpeg_ts"}
 	MPES_PES            = &decode.Group{Name: "mpeg_pes"}
 	MsgPack             = &decode.Group{Name: "msgpack"}
+	Negentropy          = &decode.Group{Name: "negentropy"}
 	NES                 = &decode.Group{Name: "nes"}
 	Ogg                 = &decode.Group{Name: "ogg"}
 	Ogg_Page            = &decode.Group{Name: "ogg_page"}

diff --git a/format/negentropy/negentropy.go b/format/negentropy/negentropy.go
@@ -0,0 +1,131 @@
+package negentropy
+
+// https://github.com/hoytech/negentropy
+
+import (
+	"embed"
+	"math"
+	"time"
+
+	"github.com/wader/fq/format"
+	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
+	"github.com/wader/fq/pkg/scalar"
+)
+
+//go:embed negentropy.md
+var negFS embed.FS
+
+func init() {
+	interp.RegisterFormat(
+		format.Negentropy,
+		&decode.Format{
+			Description: "Negentropy message",
+			DecodeFn:    decodeNegentropyMessage,
+			Groups:      []*decode.Group{},
+		})
+	interp.RegisterFS(negFS)
+}
+
+const (
+	version         = 0x61
+	fingerprintSize = 16
+
+	modeSkip        = 0
+	modeFingerprint = 1
+	modeIdlist      = 2
+)
+
+var modeMapper = scalar.SintMapSymStr{
+	modeSkip:        "skip",
+	modeFingerprint: "fingerprint",
+	modeIdlist:      "idlist",
+}
+
+type timestampDeltaTranslator struct{}
+
+func (tt *timestampDeltaTranslator) MapSint(s scalar.Sint) (scalar.Sint, error) {
+	if s.Actual == 0 {
+		s.Sym = -1
+		s.Description = "infinity"
+		return s, nil
+	} else {
+		s.Sym = s.Actual - 1
+		return s, nil
+	}
+}
+
+type timestampTranslator struct{ last time.Time }
+
+func (tt *timestampTranslator) MapSint(s scalar.Sint) (scalar.Sint, error) {
+	if s.Actual == 0 {
+		s.Description = "infinity"
+		tt.last = time.Unix(math.MaxInt64, 0)
+		return s, nil
+	} else {
+		timestamp := tt.last.Add(time.Second * time.Duration(s.Actual-1))
+		s.Description = timestamp.UTC().Format(time.RFC3339)
+		s.Actual = timestamp.Unix()
+		tt.last = timestamp
+		return s, nil
+	}
+}
+
+func decodeNegentropyMessage(d *decode.D) any {
+	tdt := &timestampDeltaTranslator{}
+	tt := &timestampTranslator{last: time.Unix(0, 0)}
+
+	d.Endian = decode.BigEndian
+
+	v := d.FieldU8("version")
+	if v != version {
+		d.Fatalf("unexpected version %d (expected %d), is this really a negentropy message?", v, version)
+	}
+
+	d.FieldStructArrayLoop("bounds", "bound", d.NotEnd, func(d *decode.D) {
+		delta := d.FieldSintFn("timestamp_delta", decodeVarInt, tdt)
+		d.FieldValueSint("timestamp", delta, tt)
+
+		size := d.FieldSintFn("id_prefix_size", decodeVarInt)
+		if size > 32 {
+			d.Fatalf("unexpected id prefix size bigger than 32: %d", size)
+		}
+		if size > 0 {
+			d.FieldRawLen("id_prefix", size*8, scalar.RawHex)
+		}
+
+		mode := d.FieldSintFn("mode", decodeVarInt, modeMapper)
+		switch mode {
+		case modeSkip:
+			return
+		case modeFingerprint:
+			d.FieldRawLen("fingerprint", fingerprintSize*8, scalar.RawHex)
+			return
+		case modeIdlist:
+			d.FieldStruct("idlist", func(d *decode.D) {
+				num := d.FieldSintFn("num", decodeVarInt)
+				d.FieldArray("ids", func(d *decode.D) {
+					for i := 0; i < int(num); i++ {
+						d.FieldRawLen("id", 32*8, scalar.RawHex)
+					}
+				})
+			})
+		default:
+			d.Fatalf("unexpected mode %d", mode)
+		}
+	})
+
+	return nil
+}
+
+func decodeVarInt(d *decode.D) int64 {
+	res := 0
+	for {
+		b := int(d.U8())
+		res = (res << 7) | (b & 127)
+		if (b & 128) == 0 {
+			break
+		}
+	}
+	return int64(res)
+}
diff --git a/format/negentropy/negentropy.md b/format/negentropy/negentropy.md
@@ -0,0 +1,29 @@
+### View a full Negentropy message
+
+```
+$ fq -d negentropy dd file
+```
+
+### Or from hex
+
+```
+$ echo '6186b7abb47c0001108e4206828ee3bf34258465809a337c6c00019a68e37b177a50b3ae7164ccc628b962020114019c1381281c9e3849d5fbd514b7bb65ad0101e601fbf7451f5d22e7fa36ae3e910e9f5215020157014a1b26853e06e9c32eb41b1df4f9ab300201e6011840e273c84bb1344f1d4e15d9aa67920200016f12ee2340888653f10b0ec2d438ac9f0101840156d2d796f4dff004ab369b9bcfa4d81e020187013f1b3c8a019800d5764e2de6bdfd2785020114017caaf0acb5dfe249aa0f7f742402168a01018301e7b8c4decb1eae455ca5714281e3245302017a01409c22636b097362df125ddffb6d944302015b01f332208bee82acf8ed922853ee54057f020001fc3e51fdb0b92966e38017f7959903850101cc01428ce0c96d49f15b50143e4fb228cb9300000131712d30e5296a7a45d07bba452d61cd' | fq -R 'from_hex | negentropy | dd'
+```
+
+### Check how many ranges the message has and how many of those are of 'fingerprint' mode
+
+```
+$ fq -d negentropy '.bounds | length as $total | map(select(.mode == "fingerprint")) | length | {$total, fingerprint: .}' message
+```
+
+### Check get all ids in all idlists
+
+```
+$ fq -d negentropy '.bounds | map(select(.mode == "idlist") | .idlist | .ids) | flatten' message
+```
+
+### Authors
+- fiatjaf, https://fiatjaf.com
+
+### References
+- https://github.com/hoytech/negentropy
diff --git a/format/negentropy/testdata/bigmessagewitheverything b/format/negentropy/testdata/bigmessagewitheverything