doorperson: add seed audio and write / read adaptor

MODULE.bazel

@@ -113,6 +113,7 @@ use_repo(
     "com_github_golang_protobuf",
     "com_github_gorilla_websocket",
     "com_github_itchyny_gojq",
+    "com_github_mewkiz_flac",
     "com_github_oapi_codegen_oapi_codegen_v2",
     "com_github_oapi_codegen_runtime",
     "com_github_sergi_go_diff",

go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/golang/protobuf v1.5.4
 	github.com/gorilla/websocket v1.5.3
 	github.com/itchyny/gojq v0.12.16
+	github.com/mewkiz/flac v1.0.12
 	github.com/oapi-codegen/oapi-codegen/v2 v2.4.1
 	github.com/oapi-codegen/runtime v1.1.1
 	github.com/sergi/go-diff v1.3.1
@@ -41,6 +42,7 @@ require (
 	github.com/google/go-dap v0.11.0 // indirect
 	github.com/google/uuid v1.5.0 // indirect
 	github.com/hashicorp/golang-lru v1.0.2 // indirect
+	github.com/icza/bitio v1.1.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/invopop/yaml v0.3.1 // indirect
 	github.com/itchyny/timefmt-go v0.1.6 // indirect
@@ -51,6 +53,7 @@ require (
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/mattn/go-shellwords v1.0.12 // indirect
+	github.com/mewkiz/pkg v0.0.0-20230226050401-4010bf0fec14 // indirect
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
 	github.com/perimeterx/marshmallow v1.1.5 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect

go/cmd/doorperson/BUILD.bazel

@@ -3,24 +3,33 @@ load("//go:rules.bzl", "go_binary", "go_library")
 
 go_library(
     name = "doorperson_lib",
-    srcs = ["main.go"],
+    srcs = [
+        "main.go",
+        "seed.go",
+    ],
+    data = [
+        "alice.flac",
+    ],
     importpath = "github.com/zemn-me/monorepo/go/cmd/doorperson",
     visibility = ["//visibility:private"],
     deps = [
+        "//go/ioutil",
         "//go/openai",
         "@com_github_gorilla_websocket//:websocket",
+        "@com_github_mewkiz_flac//:flac",
+        "@com_github_mewkiz_flac//frame",
         "@com_github_twilio_twilio_go//twiml",
         "@org_golang_x_sync//errgroup",
     ],
 )
 
+bazel_lint(
+    name = "bazel_lint",
+    srcs = ["BUILD.bazel"],
+)
+
 go_binary(
     name = "doorperson",
     embed = [":doorperson_lib"],
     visibility = ["//visibility:public"],
 )
-
-bazel_lint(
-    name = "bazel_lint",
-    srcs = ["BUILD.bazel"],
-)

go/cmd/doorperson/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"encoding/base64"
 	"encoding/json"
@@ -19,6 +20,7 @@ import (
 	"github.com/twilio/twilio-go/twiml"
 	"golang.org/x/sync/errgroup"
 
+	"github.com/zemn-me/monorepo/go/ioutil"
 	"github.com/zemn-me/monorepo/go/openai"
 )
 
@@ -213,6 +215,52 @@ func handleMediaStream(w http.ResponseWriter, r *http.Request) (err error) {
 	return group.Wait()
 }
 
+func initializeRealtimeSession(oaiConn *websocket.Conn) (err error) {
+	connWt, err := oaiConn.NextWriter(websocket.TextMessage)
+	defer connWt.Close()
+	if err != nil {
+		return
+	}
+
+	/*
+		if _, err = io.Copy(
+			connWt,
+			&ioutil.JSONReader{
+
+			}
+		)			*/
+
+	pcmBytes, err := pcmBytesFromFLACFile("go/cmd/doorperson/alice.flac")
+	if err != nil {
+		return err
+	}
+
+	var buf bytes.Buffer
+	if _, err = io.Copy(
+		&buf,
+		pcmBytes,
+	); err != nil {
+		return
+	}
+
+	pcmBytesB64 := base64.StdEncoding.EncodeToString(
+		buf.Bytes(),
+	)
+
+	_, err = io.Copy(
+		connWt,
+		&ioutil.JSONReader{
+			V: openai.RealtimeClientEventInputAudioBufferAppend{
+				Audio: pcmBytesB64,
+				Type:  "input_audio_buffer.append",
+			},
+		},
+	)
+	if err != nil {
+		return
+	}
+}
+
 // receiveFromTwilio receives audio data from Twilio and sends it to the OpenAI Realtime API.
 func receiveFromTwilio(ctx context.Context, twilioConn *websocket.Conn, openaiConn *websocket.Conn, streamSid *string) (err error) {
 	defer func() {
@@ -380,7 +428,6 @@ func sendToTwilio(ctx context.Context, wsConn *websocket.Conn, openaiConn *webso
 				}
 			case ToolCall:
 				// model called a function
-				fmt.Printf("%+v", v)
 				if v.Item.Name == "HangUp" {
 					cancel(fmt.Errorf("Model hung up the call: %v", io.EOF))
 				}
@@ -450,9 +497,19 @@ func sendSessionUpdate(ctx context.Context, openaiConn *websocket.Conn) error {
 			InputAudioFormat:  strPtr("g711_ulaw"),
 			OutputAudioFormat: strPtr("g711_ulaw"),
 			Voice:             strPtr(string(openai.CreateSpeechRequestVoiceAlloy)),
-			Instructions:      strPtr("Hi"),
-			Modalities:        &[]string{"text", "audio"},
-			Temperature:       &temperature,
+			Instructions: strPtr(`
+You are a helpful doorperson. You will shortly recieve an audio sample of
+the owner of the apartment's voice. You should imitate something like this
+voice for the conversation you are connected to, which will be someone
+who has dialed the call box you are speaking through that grants entry
+to the apartment.
+
+The owner of the apartment's name is Thomas.
+
+Opening the door has not yet been implemented.
+			`),
+			Modalities:  &[]string{"text", "audio"},
+			Temperature: &temperature,
 			Tools: &[]struct {
 				Description *string                 "json:\"description,omitempty\""
 				Name        *string                 "json:\"name,omitempty\""

go/cmd/doorperson/seed.go

@@ -0,0 +1,53 @@
+package main
+
+import (
+	"encoding/binary"
+	"errors"
+	"io"
+
+	"github.com/mewkiz/flac"
+	"github.com/mewkiz/flac/frame"
+
+	"github.com/zemn-me/monorepo/go/ioutil"
+)
+
+type PCMWriterTo struct {
+	*flac.Stream
+}
+
+func (p *PCMWriterTo) WriteTo(w io.Writer) (n int64, err error) {
+	var f *frame.Frame
+	f, err = p.ParseNext()
+	if err != nil {
+		return 0, err
+	}
+
+	for _, subframe := range f.Subframes {
+		for _, sample := range subframe.Samples {
+			sampleBytes := make([]byte, 2, 2)
+			binary.LittleEndian.PutUint16(sampleBytes, uint16(sample))
+			var delta int
+			delta, err = w.Write(sampleBytes)
+			n += int64(delta)
+			if err != nil {
+				return
+			}
+		}
+	}
+
+	return
+}
+
+// returns an io.ReadCloser of PCM bytes
+func pcmBytesFromFLACFile(path string) (rd io.ReadCloser, err error) {
+	stream, err := flac.ParseFile(path)
+	if err != nil {
+		return nil, errors.Join(err, stream.Close())
+	}
+
+	rd = ioutil.WriterToReader{
+		WriterTo: &PCMWriterTo{stream},
+	}
+
+	return
+}

go/ioutil/BUILD.bazel

@@ -0,0 +1,27 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
+load("//bzl:rules.bzl", "bazel_lint")
+load("//go:rules.bzl", "go_library")
+
+go_library(
+    name = "ioutil",
+    srcs = [
+        "chunked_writer_to.go",
+        "count_writer.go",
+        "json_writer_to.go",
+        "reader_from_writer_to.go",
+    ],
+    importpath = "github.com/zemn-me/monorepo/go/ioutil",
+    visibility = ["//visibility:public"],
+)
+
+bazel_lint(
+    name = "bazel_lint",
+    srcs = ["BUILD.bazel"],
+)
+
+go_test(
+    name = "ioutil_test",
+    size = "small",
+    srcs = ["reader_from_writer_test.go"],
+    embed = [":ioutil"],
+)

go/ioutil/chunked_writer_to.go

@@ -0,0 +1,10 @@
+package ioutil
+
+import "io"
+
+// A value that may be asked to write to something over and over
+// until exhausted.
+type ChunkedWriterTo interface {
+	io.WriterTo
+	Done() bool
+}