diff --git a/extensions/src/doodlebot/Doodlebot.ts b/extensions/src/doodlebot/Doodlebot.ts index 756c148df..c9a0c6404 100644 --- a/extensions/src/doodlebot/Doodlebot.ts +++ b/extensions/src/doodlebot/Doodlebot.ts @@ -632,6 +632,11 @@ export default class Doodlebot { }); } + getStoredIPAddress() { + if (!this.connection) { return "" } + return this.connection.ip; + } + parseWavHeader(uint8Array) { const dataView = new DataView(uint8Array.buffer); @@ -679,9 +684,6 @@ export default class Doodlebot { ws.onopen = () => { console.log('WebSocket connection opened'); - - let offset = 0; - let { sampleWidth, channels, rate } = this.parseWavHeader(uint8Array); let first = "(1," + String(sampleWidth) + "," + String(channels) + "," + String(rate) + ")"; console.log(first); @@ -695,21 +697,16 @@ export default class Doodlebot { return; } - // Calculate end position of the current chunk - //const end = Math.min(offset + CHUNK_SIZE, uint8Array.length); const chunk = chunks[i]; - console.log("sending"); const binaryString = Array.from(chunk).map((byte: any) => String.fromCharCode(byte)).join('');; const base64Data = btoa(binaryString); const jsonData = JSON.stringify({ audio_data: base64Data }); - console.log(jsonData); ws.send(jsonData); i = i + 1; sendNextChunk(); } - // Start sending chunks sendNextChunk(); }; @@ -717,11 +714,8 @@ export default class Doodlebot { console.error('WebSocket error:', error); }; - ws.onmessage = (event) => { - const response = JSON.parse(event.data); - if (response.type === 'ack' && response.status === 'received') { - console.log('Message successfully received by server'); - } + ws.onmessage = (message) => { + console.log(message); } ws.onclose = () => { @@ -837,6 +831,10 @@ export default class Doodlebot { await this.sendWebsocketCommand(command.display, value); } + async displayFile(file: string) { + await this.sendWebsocketCommand(command.display, file); + } + // Function to fetch and parse HTML template async fetchAndExtractList(endpoint) { try { diff --git a/extensions/src/doodlebot/index.ts b/extensions/src/doodlebot/index.ts index f9ef7209c..562e0e82b 100644 --- a/extensions/src/doodlebot/index.ts +++ b/extensions/src/doodlebot/index.ts @@ -3,6 +3,8 @@ import { DisplayKey, displayKeys, command, type Command, SensorKey, sensorKeys } import Doodlebot from "./Doodlebot"; import { splitArgsString } from "./utils"; import CustomArgument from './CustomArgument.svelte'; +import SoundArgument from './SoundArgument.svelte'; +import ImageArgument from './ImageArgument.svelte'; import EventEmitter from "events"; import { categoryByGesture, classes, emojiByGesture, gestureDetection, gestureMenuItems, gestures, objectDetection } from "./detection"; @@ -61,16 +63,27 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg imageStream: HTMLImageElement; videoDrawable: ReturnType; ip: string; + soundDictionary; init(env: Environment) { this.openUI("Connect"); this.setIndicator("disconnected"); + this.soundDictionary = {}; + + for (const target of env.runtime.targets) { + this.soundDictionary[target.id] = {}; + if (target.sprite) { + for (const sound of target.sprite.sounds) { + if (sound.asset.dataFormat == "wav") { + this.soundDictionary[target.id][sound.name] = sound.asset.data; + } + } + } + } - console.log(env); - - soundFiles = ["test"]; - imageFiles = ["test"]; + soundFiles = ["File"]; + imageFiles = ["File"]; // idea: set up polling mechanism to try and disable unused sensors // idea: set up polling mechanism to destroy gesture recognition loop @@ -81,6 +94,10 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg } + getCurrentSounds(id): string[] { + return Object.keys(this.soundDictionary[id]); + } + async setDoodlebot(doodlebot: Doodlebot) { this.doodlebot = doodlebot; this.setIndicator("connected"); @@ -90,6 +107,10 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg console.log(soundFiles); } + async getIPAddress() { + return this.doodlebot?.getStoredIPAddress(); + } + async setIndicator(status: "connected" | "disconnected") { if (this.indicator) (await this.indicator)?.close(); this.indicator = status == "connected" @@ -247,13 +268,18 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg await this.doodlebot?.disableSensor(sensor); } - @block({ + @block((self) => ({ type: "command", - text: (type: DisplayKey) => `display ${type}`, - arg: { type: "string", options: displayKeys.filter(key => key !== "clear"), defaultValue: "happy" } - }) - async setDisplay(display: DisplayKey) { - await this.doodlebot?.display(display); + text: (type: DisplayKey | string) => `display ${type}`, + arg: { type: "string", options: () => displayKeys.filter(key => key !== "clear").concat(imageFiles), defaultValue: "happy" } + })) + async setDisplay(display: DisplayKey | string) { + if (imageFiles.includes(display)) { + await this.doodlebot?.displayFile(display); + } else { + await this.doodlebot?.display(display as DisplayKey); + } + } @block({ @@ -286,42 +312,24 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg @block((self) => ({ type: "command", text: (sound) => `play sound file ${sound}`, - // arg: self.makeCustomArgument({ - // component: CustomArgument, - // initial: { value: "File", text: "File" } - // }) - arg: { type: "string", options: () => soundFiles } + arg: { + type: "string", options: () => soundFiles.concat(self.getCurrentSounds(self.runtime._editingTarget.id)) + } })) - // @(scratch.command( - // (self, tag) => tag`play sound file ${{ type: "string", options: self.soundFiles }}` - // )) async playSoundFile(sound: string, util: BlockUtilityWithID) { - await this.doodlebot?.sendWebsocketCommand("m", sound) - } - - @block((self) => ({ - type: "command", - text: (sound) => `play asset file ${sound}`, - // arg: self.makeCustomArgument({ - // component: CustomArgument, - // initial: { value: "File", text: "File" } - // }) - arg: { type: "string", options: () => soundFiles } - })) - // @(scratch.command( - // (self, tag) => tag`play sound file ${{ type: "string", options: self.soundFiles }}` - // )) - async playAssetFile(sound: string, util: BlockUtilityWithID) { - const { target } = util; - console.log(target); - if (target.sprite) { - console.log(target.sprite); - let soundArray = target.sprite.sounds[0].asset.data; + let currentId = this.runtime._editingTarget.id; + let costumeSounds = this.getCurrentSounds(currentId); + if (costumeSounds.includes(sound)) { + let soundArray = this.soundDictionary[currentId][sound]; console.log(soundArray); - this.doodlebot.sendAudioData(soundArray); + await this.doodlebot.sendAudioData(soundArray); + } else { + await this.doodlebot?.sendWebsocketCommand("m", sound) } + } + @block({ type: "command", text: (transparency) => `display video with ${transparency}% transparency`, @@ -405,45 +413,10 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg await new Promise((resolve) => setTimeout(resolve, audioDuration * 1000)); } - - async downloadAndUploadFile(fileUrl: string, uploadEndpoint: string): Promise { - try { - // Step 1: Download the file from the URL - const response = await fetch(fileUrl); - if (!response.ok) { - throw new Error(`Failed to fetch file from URL: ${response.statusText}`); - } - - // Convert response to a Blob - const fileBlob = await response.blob(); - - // Step 2: Prepare FormData for upload - const formData = new FormData(); - formData.append('file', fileBlob, 'uploaded-file'); // Optional: 'uploaded-file' is the filename - - // Step 3: Upload the file to the server - const uploadResponse = await fetch(uploadEndpoint, { - method: 'POST', - body: formData, - headers: { - // 'Content-Type': 'multipart/form-data', // No need to set Content-Type manually - }, - }); - - if (!uploadResponse.ok) { - throw new Error(`Failed to upload file: ${uploadResponse.statusText}`); - } - - const result = await uploadResponse.json(); - console.log('File uploaded successfully:', result); - } catch (error) { - console.error('Error:', error); - } - } - async setArrays() { imageFiles = await this.doodlebot.findImageFiles(); soundFiles = await this.doodlebot.findSoundFiles(); + console.log("SETTING"); } @@ -455,23 +428,33 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg return this.doodlebot?.getIPAddress(); } - @block({ - type: "command", - text: "Upload files" - }) - async uploadFiles() { - this.openUI("UI"); + // @block({ + // type: "command", + // text: "Upload files" + // }) + // async uploadFiles() { + // this.openUI("UI"); + // } + + @(scratch.command((self, $) => $`Upload sound file ${self.makeCustomArgument({ component: SoundArgument, initial: { value: "File", text: "File" } })}`)) + uploadSoundFile(test: string) { + } - @block({ - type: "command", - text: "Find files" - }) - async findFiles() { - let response = await this.doodlebot?.fetchAndExtractList("http://192.168.41.121:8080/sounds/"); - console.log(response); + @(scratch.command((self, $) => $`Upload image file ${self.makeCustomArgument({ component: ImageArgument, initial: { value: "File", text: "File" } })}`)) + uploadImageFile(test: string) { + } + // @block({ + // type: "command", + // text: "Find files" + // }) + // async findFiles() { + // let response = await this.doodlebot?.fetchAndExtractList("http://192.168.41.121:8080/sounds/"); + // console.log(response); + // } + @block({ type: "command", text: (_command, args, protocol) => `send (${_command}, ${args}) over ${protocol}`,