From 670bb8510535e158226a83542bcff84d673a5c8e Mon Sep 17 00:00:00 2001 From: Jose Vargas Date: Sun, 21 Jan 2024 22:59:04 +0100 Subject: [PATCH] Deploy website - based on 8376b6d23feb07dd389f8f97238fc683777e2f42 --- 404.html | 4 ++-- assets/js/{fe7d5959.a37c04e4.js => fe7d5959.6dabcb34.js} | 2 +- .../{runtime~main.b6f4f44f.js => runtime~main.f3d2b629.js} | 2 +- blog/archive.html | 4 ++-- docs.html | 4 ++-- docs/custom_task.html | 6 +++--- docs/deployment.html | 4 ++-- docs/development.html | 4 ++-- docs/getting_started.html | 4 ++-- docs/installation.html | 4 ++-- docs/output.html | 4 ++-- docs/roadmap.html | 4 ++-- index.html | 4 ++-- markdown-page.html | 4 ++-- 14 files changed, 27 insertions(+), 27 deletions(-) rename assets/js/{fe7d5959.a37c04e4.js => fe7d5959.6dabcb34.js} (72%) rename assets/js/{runtime~main.b6f4f44f.js => runtime~main.f3d2b629.js} (96%) diff --git a/404.html b/404.html index 89e071ef..80157cc0 100644 --- a/404.html +++ b/404.html @@ -7,13 +7,13 @@ Page Not Found | covfee: continuous video feedback tool - +
Skip to main content

Page Not Found

We could not find what you were looking for.

Please contact the owner of the site that linked you to the original URL and let them know their link is broken.

- + \ No newline at end of file diff --git a/assets/js/fe7d5959.a37c04e4.js b/assets/js/fe7d5959.6dabcb34.js similarity index 72% rename from assets/js/fe7d5959.a37c04e4.js rename to assets/js/fe7d5959.6dabcb34.js index 155894e5..fece4e3e 100644 --- a/assets/js/fe7d5959.a37c04e4.js +++ b/assets/js/fe7d5959.6dabcb34.js @@ -1 +1 @@ -"use strict";(self.webpackChunkdocs=self.webpackChunkdocs||[]).push([[90],{3905:(e,t,a)=>{a.d(t,{Zo:()=>c,kt:()=>m});var n=a(7294);function s(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function o(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function r(e){for(var t=1;t=0||(s[a]=e[a]);return s}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(s[a]=e[a])}return s}var l=n.createContext({}),p=function(e){var t=n.useContext(l),a=t;return e&&(a="function"==typeof e?e(t):r(r({},t),e)),a},c=function(e){var t=p(e.components);return n.createElement(l.Provider,{value:t},e.children)},d="mdxType",h={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},u=n.forwardRef((function(e,t){var a=e.components,s=e.mdxType,o=e.originalType,l=e.parentName,c=i(e,["components","mdxType","originalType","parentName"]),d=p(a),u=s,m=d["".concat(l,".").concat(u)]||d[u]||h[u]||o;return a?n.createElement(m,r(r({ref:t},c),{},{components:a})):n.createElement(m,r({ref:t},c))}));function m(e,t){var a=arguments,s=t&&t.mdxType;if("string"==typeof e||s){var o=a.length,r=new Array(o);r[0]=u;var i={};for(var l in t)hasOwnProperty.call(t,l)&&(i[l]=t[l]);i.originalType=e,i[d]="string"==typeof e?e:s,r[1]=i;for(var p=2;p{a.r(t),a.d(t,{contentTitle:()=>r,default:()=>c,frontMatter:()=>o,metadata:()=>i,toc:()=>l});var n=a(7462),s=(a(7294),a(3905));const o={title:"Developing custom tasks"},r=void 0,i={unversionedId:"custom_task",id:"custom_task",title:"Developing custom tasks",description:"New custom tasks or HIT types can be added by implementing a React component meeting a few conditions. For some tasks, a Python base class must be sub-classed too. To be valid, task components must meet these conditions:",source:"@site/docs/custom_task.mdx",sourceDirName:".",slug:"/custom_task",permalink:"/covfee/docs/custom_task",editUrl:"https://github.com/facebook/docusaurus/edit/master/website/docs/custom_task.mdx",tags:[],version:"current",frontMatter:{title:"Developing custom tasks"},sidebar:"docs",previous:{title:"Development install",permalink:"/covfee/docs/development"}},l=[{value:"Types of tasks",id:"types-of-tasks",children:[],level:2},{value:"Task state",id:"task-state",children:[],level:2},{value:"Tutorial: developing your own task",id:"tutorial-developing-your-own-task",children:[{value:"Background",id:"background",children:[],level:3},{value:"Installation",id:"installation",children:[],level:3},{value:"The tutorial",id:"the-tutorial",children:[],level:3},{value:"Task props",id:"task-props",children:[],level:3},{value:"State",id:"state",children:[],level:3},{value:"React Component",id:"react-component",children:[],level:3},{value:"Importing the task",id:"importing-the-task",children:[],level:3}],level:2},{value:"Running the task",id:"running-the-task",children:[],level:2}],p={toc:l};function c(e){let{components:t,...a}=e;return(0,s.kt)("wrapper",(0,n.Z)({},p,a,{components:t,mdxType:"MDXLayout"}),(0,s.kt)("p",null,"New custom tasks or HIT types can be added by implementing a React component meeting a few conditions. For some tasks, a Python base class must be sub-classed too. To be valid, task components must meet these conditions:"),(0,s.kt)("h2",{id:"types-of-tasks"},"Types of tasks"),(0,s.kt)("p",null,"There are two main types of tasks in covfee:"),(0,s.kt)("ul",null,(0,s.kt)("li",{parentName:"ul"},'Tasks with shared state, or "multiplayer" tasks are meant to be visited by multiple users at the same time. A chess game, or a collaborative live view form to be filled by multiple users at the same time is an example of a shared state task. Covfee internally manages state synchronization and persistence. A copy of the latest state is kept in the server.'),(0,s.kt)("li",{parentName:"ul"},'Tasks without shared state, or "single player" tasks are meant to be visited by a single user. A standard form is a good example.')),(0,s.kt)("h2",{id:"task-state"},"Task state"),(0,s.kt)("p",null,'Covfee tasks are, in their most basic form, simply React components. In other words, copy-pasting any React component as a Covfee task would "work". However, in any type of online experiment we normally want to store the results of the task (eg. the answers to a questionnaire). '),(0,s.kt)("p",null,"In covfee, task components do not explicitly store or commit these results to the server. Instead, the state of the task is stored automatically by Covfee. To make use of this, any task state that should be persisted by Covfee must be managed in a central Redux store. In other words, ",(0,s.kt)("strong",{parentName:"p"},"implementing Covfee tasks boils down to implementing React components with a Redux state store")," in all cases. State that does not need to be stored can be managed separately, for example using React's ",(0,s.kt)("inlineCode",{parentName:"p"},"useState"),"."),(0,s.kt)("p",null,"State persistance and loading will be managed internally:"),(0,s.kt)("ul",null,(0,s.kt)("li",{parentName:"ul"},(0,s.kt)("p",{parentName:"li"},"For shared state tasks, Covfee will keep the most up-to-date state in the server, which will always represent the latest state of the task.")),(0,s.kt)("li",{parentName:"ul"},(0,s.kt)("p",{parentName:"li"},"For non-shared state tasks, the latest state is kept in the browser and persisted when necessary. Covfee submits and persists state whenever:"),(0,s.kt)("ul",{parentName:"li"},(0,s.kt)("li",{parentName:"ul"},"A user closes the browser tab where the task is open, or otherwise leaves the task."),(0,s.kt)("li",{parentName:"ul"},'A user presses the "submit" button in the task.')))),(0,s.kt)("h2",{id:"tutorial-developing-your-own-task"},"Tutorial: developing your own task"),(0,s.kt)("h3",{id:"background"},"Background"),(0,s.kt)("p",null,"This tutorial requires a basic understanding of Typescript, HTML, React and Redux. For progressing to more complex custom tasks the following is recommended:"),(0,s.kt)("ul",null,(0,s.kt)("li",{parentName:"ul"},"Typescript, Javascript, HTML, CSS. For more complex tasks, a good level of CSS might be necessary."),(0,s.kt)("li",{parentName:"ul"},"React: functional components, hooks (useState, useEffect, useContext, useRef, useMemo), custom hooks, fetching data. It's important to understand the nuances behind hooks and dependencies: when do hooks run?"),(0,s.kt)("li",{parentName:"ul"},"Redux, Redux toolkit. Important to understand how to think about application state and how to design the state, actions and reducers.")),(0,s.kt)("p",null,"In general, ",(0,s.kt)("strong",{parentName:"p"},"you should know how to design and write a React application using Redux for state management"),"."),(0,s.kt)("h3",{id:"installation"},"Installation"),(0,s.kt)("p",null,"Please start by following the ",(0,s.kt)("a",{parentName:"p",href:"development"},"development install instructions")," to install Covfee."),(0,s.kt)("h3",{id:"the-tutorial"},"The tutorial"),(0,s.kt)("p",null,"In this tutorial, we will create a simple contact form asking users for their name and email. Additionally, the task will have one boolean prop (",(0,s.kt)("inlineCode",{parentName:"p"},"showPhoneField"),") that, if true, will add a phone number field to the form. The answers to the form will be preserved in task state. "),(0,s.kt)("p",null,"We will place Typescript files for the task in ",(0,s.kt)("inlineCode",{parentName:"p"},"covfee/client/tasks"),", in the following structure:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre"},"covfee\n client\n tasks\n - my_task\n - index.tsx # the main React component\n - slice.ts # the state specification\n - spec.ts # the props for the task\n ... # any extra files needed\n")),(0,s.kt)("h3",{id:"task-props"},"Task props"),(0,s.kt)("p",null,"First, we place the specification for the task props in the ",(0,s.kt)("inlineCode",{parentName:"p"},"spec.ts")," file:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre"},'import { BaseTaskSpec } from "@covfee-shared/spec/task"\n\n/**\n * @TJS-additionalProperties false\n */\nexport interface TutorialTaskSpec extends BaseTaskSpec {\n /**\n * @default "TutorialTask"\n */\n type: "TutorialTask"\n /**\n * Media file to be displayed.\n */\n showPhoneField?: boolean\n}\n')),(0,s.kt)("p",null,"These props will be parsed by Covfee so that they can be provided when the task is created. They allow the user to configure the way a task is rendered to the user. In this case, we only provide two props. One is the name of the task. This will be the name of the task's Python class and is a constant value. The other one is our only true prop ",(0,s.kt)("inlineCode",{parentName:"p"},"showPhoneField"),", which we define to be boolean. The comments provided here will be parsed and will be available in Python when creating task objects. The line ",(0,s.kt)("inlineCode",{parentName:"p"},"TJS-additionalProperties false")," specifies that no additional properties should be allowed when the task is validated."),(0,s.kt)("h3",{id:"state"},"State"),(0,s.kt)("p",null,"Tasks in covfee specify their state by creating a Redux Toolkit slice. For our purposes a slice is just an object that encapsulates the initial state and the reducers for the task. In this case, we need the state to hold the user's name, email and phone number. Our ",(0,s.kt)("inlineCode",{parentName:"p"},"slice.ts")," file looks like this:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre"},'import { createSlice } from "@reduxjs/toolkit"\n\nexport interface State {\n name: string\n email: string\n phone: string\n}\n\nexport const initialState: State = {\n name: "",\n email: "",\n phone: "",\n}\n\nexport const slice = createSlice({\n name: "form",\n initialState,\n reducers: {\n setName: (state, action) => {\n state.name = action.payload\n },\n setEmail: (state, action) => {\n state.email = action.payload\n },\n setPhone: (state, action) => {\n state.phone = action.payload\n },\n },\n})\n\nexport const { actions, reducer } = slice\n')),(0,s.kt)("p",null,"Here we specified the type for the state, the initial state, and wrote separate reducers to set name, email and phone number. "),(0,s.kt)("h3",{id:"react-component"},"React Component"),(0,s.kt)("p",null,"Following React + Redux design philosophy, the main job of the React component is to sync state with the DOM (view), by reading the state and by calling Redux actions in response to user events. In this case, because form input elements hold their own state, we are simply calling Redux actions whenever the content of the input elements changes. That's all!"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre"},'import React, { useState } from "react"\nimport { slice, actions } from "./slice"\nimport { TaskExport } from "../../types/node"\nimport { CovfeeTaskProps } from "../base"\nimport type { TutorialTaskSpec } from "./spec"\nimport { AllPropsRequired } from "../../types/utils"\n\ninterface Props extends CovfeeTaskProps {}\n\nconst TutorialTask: React.FC = (props) => {\n const args: AllPropsRequired = {\n ...props,\n spec: {\n ...props.spec,\n showPhoneField: true,\n },\n }\n\n return (\n
\n
\n \n actions.setName(e.target.value)}\n required\n />\n
\n
\n \n actions.setEmail(e.target.value)}\n required\n />\n
\n {args.spec.showPhoneField && (\n
\n \n actions.setPhone(e.target.value)}\n />\n
\n )}\n
\n )\n}\nexport type { TutorialTaskSpec }\nexport default {\n taskComponent: TutorialTask,\n taskSlice: slice,\n useSharedState: false,\n} as TaskExport\n\n')),(0,s.kt)("p",null,"Note the export statements at the end of the file. First we export the task specification, which should be available to Covfee (more on this later)"),(0,s.kt)("p",null,"Covfee expects every task to export a default object with these keys. ",(0,s.kt)("inlineCode",{parentName:"p"},"useSharedState"),' should only be set to true for "multiplayer" tasks.'),(0,s.kt)("h3",{id:"importing-the-task"},"Importing the task"),(0,s.kt)("p",null,"Now that the task is created, the task should be imported into Covfee source files. To do this, modify the file at ",(0,s.kt)("inlineCode",{parentName:"p"},"/covfee/client/tasks/index.ts")," to import your custom task. Simply follow the same pattern as for the rest of the imports in the file. The task props specification (",(0,s.kt)("em",{parentName:"p"},"spec"),") should also be imported in ",(0,s.kt)("inlineCode",{parentName:"p"},"/covfee/shared/spec/task.ts"),"."),(0,s.kt)("p",null,"Now we are ready to create the auto-generated code that Covfee is designed to create. Covfee auto-generates two files using the tasks' props specification:"),(0,s.kt)("ul",null,(0,s.kt)("li",{parentName:"ul"},"/covfee/shared/schemata.json - JSON schemata. Each task props specification (spec) is translated into a JSON schema, used for validation and for generating the dataclasses."),(0,s.kt)("li",{parentName:"ul"},"/covfee/shared/task_dataclasses.py contains the dataclasses that can be used to create task objects in Python. These are simple objects that take the task props as constructor arguments. Covfee internally translates them into database entries to initialize its database on startup.")),(0,s.kt)("p",null,"These files may be useful for debugging purposes to eg. make sure that the dataclasses have the right arguments, types and comments."),(0,s.kt)("p",null,"To (re-)create both of these auto-generated files run:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre"},"covfee-dev schemata\n")),(0,s.kt)("p",null,"After this step, it should be possible to manually verify that the dataclasses file contains a new class for our custom task."),(0,s.kt)("h2",{id:"running-the-task"},"Running the task"),(0,s.kt)("p",null,"The task is ready to be used. To do this in a dev environment, create a folder for the covfee project. This folder may be anywhere in the file system. The we will create the following structure:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre"},"my_folder\n - tutorial.py\n")),(0,s.kt)("p",null,"The naming is not relevant. Paste the following into the ",(0,s.kt)("inlineCode",{parentName:"p"},"tutorial.py"),":"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre"},'from covfee import tasks, HIT, Project\nfrom covfee.config import config\nfrom covfee.shared.dataclass import CovfeeApp\n\nconfig.load_environment("local")\n\nmy_task_1 = tasks.TutorialTaskSpec(name="My Task 1", showPhoneField=True)\nmy_task_2 = tasks.TutorialTaskSpec(name="My Task 2", showPhoneField=False)\n\nhit = HIT("Joint counter")\nj1 = hit.add_journey(nodes=[my_task_1, my_task_2])\n\nprojects = [Project("My Project", email="example@example.com", hits=[hit])]\napp = CovfeeApp(projects) # we must always create an app object of class CovfeeApp\n')),(0,s.kt)("p",null,"Here we import the necessary classes, create two task objects using the same (auto-generated) ",(0,s.kt)("inlineCode",{parentName:"p"},"TutorialTaskSpec")," class, and link them together in a journey. We create a HIT with a single journey, and a project with a single HIT. Finally, we create the ",(0,s.kt)("inlineCode",{parentName:"p"},"app")," object. Covfee internally looks for this object by name (important that it is called ",(0,s.kt)("inlineCode",{parentName:"p"},"app"),") and uses it as starting point to read the specification."),(0,s.kt)("p",null,"We can now start covfee with:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre"},"covfee make tutorial.py --force --dev\n")),(0,s.kt)("p",null,"This will parse the specification, create the database, and start the covfee server."),(0,s.kt)("p",null,"In development mode, we need to run the webpack server separately. To do it, in another terminal run:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre"},"covfee webpack\n")),(0,s.kt)("p",null,"After this the Covfee admin panel should be accessible in the URL displayed when running the Covfee server (http://localhost:5000/admin# by default). The links to the Journey (with the two tasks) should be available in the admin panel (under ",(0,s.kt)("em",{parentName:"p"},"Journeys"),")."),(0,s.kt)("p",null,"With this development setup, any changes to the client or server will be hot-reloaded. Therefore it should be possible and convenient to edit the task's React component or the backend while running the app."))}c.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkdocs=self.webpackChunkdocs||[]).push([[90],{3905:(e,t,a)=>{a.d(t,{Zo:()=>c,kt:()=>m});var n=a(7294);function s(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function o(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function r(e){for(var t=1;t=0||(s[a]=e[a]);return s}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(s[a]=e[a])}return s}var l=n.createContext({}),p=function(e){var t=n.useContext(l),a=t;return e&&(a="function"==typeof e?e(t):r(r({},t),e)),a},c=function(e){var t=p(e.components);return n.createElement(l.Provider,{value:t},e.children)},d="mdxType",h={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},u=n.forwardRef((function(e,t){var a=e.components,s=e.mdxType,o=e.originalType,l=e.parentName,c=i(e,["components","mdxType","originalType","parentName"]),d=p(a),u=s,m=d["".concat(l,".").concat(u)]||d[u]||h[u]||o;return a?n.createElement(m,r(r({ref:t},c),{},{components:a})):n.createElement(m,r({ref:t},c))}));function m(e,t){var a=arguments,s=t&&t.mdxType;if("string"==typeof e||s){var o=a.length,r=new Array(o);r[0]=u;var i={};for(var l in t)hasOwnProperty.call(t,l)&&(i[l]=t[l]);i.originalType=e,i[d]="string"==typeof e?e:s,r[1]=i;for(var p=2;p{a.r(t),a.d(t,{contentTitle:()=>r,default:()=>c,frontMatter:()=>o,metadata:()=>i,toc:()=>l});var n=a(7462),s=(a(7294),a(3905));const o={title:"Developing custom tasks"},r=void 0,i={unversionedId:"custom_task",id:"custom_task",title:"Developing custom tasks",description:"New custom tasks or HIT types can be added by implementing a React component meeting a few conditions. For some tasks, a Python base class must be sub-classed too. To be valid, task components must meet these conditions:",source:"@site/docs/custom_task.mdx",sourceDirName:".",slug:"/custom_task",permalink:"/covfee/docs/custom_task",editUrl:"https://github.com/facebook/docusaurus/edit/master/website/docs/custom_task.mdx",tags:[],version:"current",frontMatter:{title:"Developing custom tasks"},sidebar:"docs",previous:{title:"Development install",permalink:"/covfee/docs/development"}},l=[{value:"Types of tasks",id:"types-of-tasks",children:[],level:2},{value:"Task state",id:"task-state",children:[],level:2},{value:"Tutorial: developing your own task",id:"tutorial-developing-your-own-task",children:[{value:"Background",id:"background",children:[],level:3},{value:"Installation",id:"installation",children:[],level:3},{value:"The tutorial",id:"the-tutorial",children:[],level:3},{value:"Task props",id:"task-props",children:[],level:3},{value:"State",id:"state",children:[],level:3},{value:"React Component",id:"react-component",children:[],level:3},{value:"Importing the task",id:"importing-the-task",children:[],level:3}],level:2},{value:"Running the task",id:"running-the-task",children:[],level:2}],p={toc:l};function c(e){let{components:t,...a}=e;return(0,s.kt)("wrapper",(0,n.Z)({},p,a,{components:t,mdxType:"MDXLayout"}),(0,s.kt)("p",null,"New custom tasks or HIT types can be added by implementing a React component meeting a few conditions. For some tasks, a Python base class must be sub-classed too. To be valid, task components must meet these conditions:"),(0,s.kt)("h2",{id:"types-of-tasks"},"Types of tasks"),(0,s.kt)("p",null,"There are two main types of tasks in covfee:"),(0,s.kt)("ul",null,(0,s.kt)("li",{parentName:"ul"},'Tasks with shared state, or "multiplayer" tasks are meant to be visited by multiple users at the same time. A chess game, or a collaborative live view form to be filled by multiple users at the same time is an example of a shared state task. Covfee internally manages state synchronization and persistence. A copy of the latest state is kept in the server.'),(0,s.kt)("li",{parentName:"ul"},'Tasks without shared state, or "single player" tasks are meant to be visited by a single user. A standard form is a good example.')),(0,s.kt)("h2",{id:"task-state"},"Task state"),(0,s.kt)("p",null,'Covfee tasks are, in their most basic form, simply React components. In other words, copy-pasting any React component as a Covfee task would "work". However, in any type of online experiment we normally want to store the results of the task (eg. the answers to a questionnaire). '),(0,s.kt)("p",null,"In covfee, task components do not explicitly store or commit these results to the server. Instead, the state of the task is stored automatically by Covfee. To make use of this, any task state that should be persisted by Covfee must be managed in a central Redux store. In other words, ",(0,s.kt)("strong",{parentName:"p"},"implementing Covfee tasks boils down to implementing React components with a Redux state store")," in all cases. State that does not need to be stored can be managed separately, for example using React's ",(0,s.kt)("inlineCode",{parentName:"p"},"useState"),"."),(0,s.kt)("p",null,"State persistance and loading will be managed internally:"),(0,s.kt)("ul",null,(0,s.kt)("li",{parentName:"ul"},(0,s.kt)("p",{parentName:"li"},"For shared state tasks, Covfee will keep the most up-to-date state in the server, which will always represent the latest state of the task.")),(0,s.kt)("li",{parentName:"ul"},(0,s.kt)("p",{parentName:"li"},"For non-shared state tasks, the latest state is kept in the browser and persisted when necessary. Covfee submits and persists state whenever:"),(0,s.kt)("ul",{parentName:"li"},(0,s.kt)("li",{parentName:"ul"},"A user closes the browser tab where the task is open, or otherwise leaves the task."),(0,s.kt)("li",{parentName:"ul"},'A user presses the "submit" button in the task.')))),(0,s.kt)("h2",{id:"tutorial-developing-your-own-task"},"Tutorial: developing your own task"),(0,s.kt)("h3",{id:"background"},"Background"),(0,s.kt)("p",null,"This tutorial requires a basic understanding of Typescript, HTML, React and Redux. For progressing to more complex custom tasks the following is recommended:"),(0,s.kt)("ul",null,(0,s.kt)("li",{parentName:"ul"},"Typescript, Javascript, HTML, CSS. For more complex tasks, a good level of CSS might be necessary."),(0,s.kt)("li",{parentName:"ul"},"React: functional components, hooks (useState, useEffect, useContext, useRef, useMemo), custom hooks, fetching data. It's important to understand the nuances behind hooks and dependencies: when do hooks run?"),(0,s.kt)("li",{parentName:"ul"},"Redux, Redux toolkit. Important to understand how to think about application state and how to design the state, actions and reducers.")),(0,s.kt)("p",null,"In general, ",(0,s.kt)("strong",{parentName:"p"},"you should know how to design and write a React application using Redux for state management"),"."),(0,s.kt)("h3",{id:"installation"},"Installation"),(0,s.kt)("p",null,"Please start by following the ",(0,s.kt)("a",{parentName:"p",href:"development"},"development install instructions")," to install Covfee."),(0,s.kt)("h3",{id:"the-tutorial"},"The tutorial"),(0,s.kt)("p",null,"In this tutorial, we will create a simple contact form asking users for their name and email. Additionally, the task will have one boolean prop (",(0,s.kt)("inlineCode",{parentName:"p"},"showPhoneField"),") that, if true, will add a phone number field to the form. The answers to the form will be preserved in task state. "),(0,s.kt)("p",null,"We will place Typescript files for the task in ",(0,s.kt)("inlineCode",{parentName:"p"},"covfee/client/tasks"),", in the following structure:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre"},"covfee\n client\n tasks\n - my_task\n - index.tsx # the main React component\n - slice.ts # the state specification\n - spec.ts # the props for the task\n ... # any extra files needed\n")),(0,s.kt)("h3",{id:"task-props"},"Task props"),(0,s.kt)("p",null,"First, we place the specification for the task props in the ",(0,s.kt)("inlineCode",{parentName:"p"},"spec.ts")," file:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre"},'import { BaseTaskSpec } from "@covfee-shared/spec/task"\n\n/**\n * @TJS-additionalProperties false\n */\nexport interface TutorialTaskSpec extends BaseTaskSpec {\n /**\n * @default "TutorialTask"\n */\n type: "TutorialTask"\n /**\n * Media file to be displayed.\n */\n showPhoneField?: boolean\n}\n')),(0,s.kt)("p",null,"These props will be parsed by Covfee so that they can be provided when the task is created. They allow the user to configure the way a task is rendered to the user. In this case, we only provide two props. One is the name of the task. This will be the name of the task's Python class and is a constant value. The other one is our only true prop ",(0,s.kt)("inlineCode",{parentName:"p"},"showPhoneField"),", which we define to be boolean. The comments provided here will be parsed and will be available in Python when creating task objects. The line ",(0,s.kt)("inlineCode",{parentName:"p"},"TJS-additionalProperties false")," specifies that no additional properties should be allowed when the task is validated."),(0,s.kt)("h3",{id:"state"},"State"),(0,s.kt)("p",null,"Tasks in covfee specify their state by creating a Redux Toolkit slice. For our purposes a slice is just an object that encapsulates the initial state and the reducers for the task. In this case, we need the state to hold the user's name, email and phone number. Our ",(0,s.kt)("inlineCode",{parentName:"p"},"slice.ts")," file looks like this:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre"},'import { createSlice } from "@reduxjs/toolkit"\n\nexport interface State {\n name: string\n email: string\n phone: string\n}\n\nexport const initialState: State = {\n name: "",\n email: "",\n phone: "",\n}\n\nexport const slice = createSlice({\n name: "form",\n initialState,\n reducers: {\n setName: (state, action) => {\n state.name = action.payload\n },\n setEmail: (state, action) => {\n state.email = action.payload\n },\n setPhone: (state, action) => {\n state.phone = action.payload\n },\n },\n})\n\nexport const { actions, reducer } = slice\n')),(0,s.kt)("p",null,"Here we specified the type for the state, the initial state, and wrote separate reducers to set name, email and phone number. "),(0,s.kt)("h3",{id:"react-component"},"React Component"),(0,s.kt)("p",null,"Following React + Redux design philosophy, the main job of the React component is to sync state with the DOM (view), by reading the state and by calling Redux actions in response to user events. In this case, because form input elements hold their own state, we are simply calling Redux actions whenever the content of the input elements changes. That's all!"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre"},'import React, { useState } from "react"\nimport { slice, actions } from "./slice"\nimport { TaskExport } from "../../types/node"\nimport { CovfeeTaskProps } from "../base"\nimport type { TutorialTaskSpec } from "./spec"\nimport { AllPropsRequired } from "../../types/utils"\n\ninterface Props extends CovfeeTaskProps {}\n\nconst TutorialTask: React.FC = (props) => {\n const args: AllPropsRequired = {\n ...props,\n spec: {\n ...props.spec,\n showPhoneField: true,\n },\n }\n\n return (\n
\n
\n \n actions.setName(e.target.value)}\n required\n />\n
\n
\n \n actions.setEmail(e.target.value)}\n required\n />\n
\n {args.spec.showPhoneField && (\n
\n \n actions.setPhone(e.target.value)}\n />\n
\n )}\n
\n )\n}\nexport default {\n taskComponent: TutorialTask,\n taskSlice: slice,\n useSharedState: false,\n} as TaskExport\n\n')),(0,s.kt)("p",null,"Note the expor statement at the end of the file. Covfee expects every task to export a default object with these keys. ",(0,s.kt)("inlineCode",{parentName:"p"},"useSharedState"),' should only be set to true for "multiplayer" tasks.'),(0,s.kt)("h3",{id:"importing-the-task"},"Importing the task"),(0,s.kt)("p",null,"Now that the task is created, the task should be imported into Covfee source files. To do this, modify the file at ",(0,s.kt)("inlineCode",{parentName:"p"},"/covfee/client/tasks/index.ts")," to import your custom task. Simply follow the same pattern as for the rest of the imports in the file. The task props specification (",(0,s.kt)("em",{parentName:"p"},"spec"),") should also be imported in ",(0,s.kt)("inlineCode",{parentName:"p"},"/covfee/shared/spec/task.ts"),"."),(0,s.kt)("p",null,"Now we are ready to create the auto-generated code that Covfee is designed to create. Covfee auto-generates two files using the tasks' props specification:"),(0,s.kt)("ul",null,(0,s.kt)("li",{parentName:"ul"},"/covfee/shared/schemata.json - JSON schemata. Each task props specification (spec) is translated into a JSON schema, used for validation and for generating the dataclasses."),(0,s.kt)("li",{parentName:"ul"},"/covfee/shared/task_dataclasses.py contains the dataclasses that can be used to create task objects in Python. These are simple objects that take the task props as constructor arguments. Covfee internally translates them into database entries to initialize its database on startup.")),(0,s.kt)("p",null,"These files may be useful for debugging purposes to eg. make sure that the dataclasses have the right arguments, types and comments."),(0,s.kt)("p",null,"To (re-)create both of these auto-generated files run:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre"},"covfee-dev schemata\n")),(0,s.kt)("p",null,"After this step, it should be possible to manually verify that the dataclasses file contains a new class for our custom task."),(0,s.kt)("h2",{id:"running-the-task"},"Running the task"),(0,s.kt)("p",null,"The task is ready to be used. To do this in a dev environment, create a folder for the covfee project. This folder may be anywhere in the file system. The we will create the following structure:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre"},"my_folder\n - tutorial.py\n")),(0,s.kt)("p",null,"The naming is not relevant. Paste the following into the ",(0,s.kt)("inlineCode",{parentName:"p"},"tutorial.py"),":"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre"},'from covfee import tasks, HIT, Project\nfrom covfee.config import config\nfrom covfee.shared.dataclass import CovfeeApp\n\nconfig.load_environment("local")\n\nmy_task_1 = tasks.TutorialTaskSpec(name="My Task 1", showPhoneField=True)\nmy_task_2 = tasks.TutorialTaskSpec(name="My Task 2", showPhoneField=False)\n\nhit = HIT("Joint counter")\nj1 = hit.add_journey(nodes=[my_task_1, my_task_2])\n\nprojects = [Project("My Project", email="example@example.com", hits=[hit])]\napp = CovfeeApp(projects) # we must always create an app object of class CovfeeApp\n')),(0,s.kt)("p",null,"Here we import the necessary classes, create two task objects using the same (auto-generated) ",(0,s.kt)("inlineCode",{parentName:"p"},"TutorialTaskSpec")," class, and link them together in a journey. We create a HIT with a single journey, and a project with a single HIT. Finally, we create the ",(0,s.kt)("inlineCode",{parentName:"p"},"app")," object. Covfee internally looks for this object by name (important that it is called ",(0,s.kt)("inlineCode",{parentName:"p"},"app"),") and uses it as starting point to read the specification."),(0,s.kt)("p",null,"We can now start covfee with:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre"},"covfee make tutorial.py --force --dev\n")),(0,s.kt)("p",null,"This will parse the specification, create the database, and start the covfee server."),(0,s.kt)("p",null,"In development mode, we need to run the webpack server separately. To do it, in another terminal run:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre"},"covfee webpack\n")),(0,s.kt)("p",null,"After this the Covfee admin panel should be accessible in the URL displayed when running the Covfee server (http://localhost:5000/admin# by default). The links to the Journey (with the two tasks) should be available in the admin panel (under ",(0,s.kt)("em",{parentName:"p"},"Journeys"),")."),(0,s.kt)("p",null,"With this development setup, any changes to the client or server will be hot-reloaded. Therefore it should be possible and convenient to edit the task's React component or the backend while running the app."))}c.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/runtime~main.b6f4f44f.js b/assets/js/runtime~main.f3d2b629.js similarity index 96% rename from assets/js/runtime~main.b6f4f44f.js rename to assets/js/runtime~main.f3d2b629.js index f427a7d7..8be91493 100644 --- a/assets/js/runtime~main.b6f4f44f.js +++ b/assets/js/runtime~main.f3d2b629.js @@ -1 +1 @@ -(()=>{"use strict";var e,t,r,o,a,n={},f={};function c(e){var t=f[e];if(void 0!==t)return t.exports;var r=f[e]={id:e,loaded:!1,exports:{}};return n[e].call(r.exports,r,r.exports,c),r.loaded=!0,r.exports}c.m=n,c.c=f,e=[],c.O=(t,r,o,a)=>{if(!r){var n=1/0;for(b=0;b=a)&&Object.keys(c.O).every((e=>c.O[e](r[i])))?r.splice(i--,1):(f=!1,a0&&e[b-1][2]>a;b--)e[b]=e[b-1];e[b]=[r,o,a]},c.n=e=>{var t=e&&e.__esModule?()=>e.default:()=>e;return c.d(t,{a:t}),t},r=Object.getPrototypeOf?e=>Object.getPrototypeOf(e):e=>e.__proto__,c.t=function(e,o){if(1&o&&(e=this(e)),8&o)return e;if("object"==typeof e&&e){if(4&o&&e.__esModule)return e;if(16&o&&"function"==typeof e.then)return e}var a=Object.create(null);c.r(a);var n={};t=t||[null,r({}),r([]),r(r)];for(var f=2&o&&e;"object"==typeof f&&!~t.indexOf(f);f=r(f))Object.getOwnPropertyNames(f).forEach((t=>n[t]=()=>e[t]));return n.default=()=>e,c.d(a,n),a},c.d=(e,t)=>{for(var r in t)c.o(t,r)&&!c.o(e,r)&&Object.defineProperty(e,r,{enumerable:!0,get:t[r]})},c.f={},c.e=e=>Promise.all(Object.keys(c.f).reduce(((t,r)=>(c.f[r](e,t),t)),[])),c.u=e=>"assets/js/"+({7:"06c6ffc3",53:"935f2afb",81:"18891827",85:"1f391b9e",90:"fe7d5959",237:"1df93b7f",317:"b5e9cae4",414:"393be207",505:"a19b665e",514:"1be78505",533:"456dc1ab",572:"e0180904",608:"9e4087bc",714:"3643a9e5",918:"17896441",930:"fa4d91bf"}[e]||e)+"."+{7:"4b3d9224",53:"cffa800e",75:"73464b18",81:"0c0dcc3b",85:"16b86f47",90:"a37c04e4",237:"7208088c",317:"b0137541",414:"9c2fbb33",505:"0e22246f",514:"6d8274cf",533:"930459c2",572:"d88b69ec",608:"33324444",714:"89de50b7",767:"e55da0ba",918:"2aac1e29",930:"45991c3e"}[e]+".js",c.miniCssF=e=>"assets/css/styles.e47ef7b4.css",c.g=function(){if("object"==typeof globalThis)return globalThis;try{return this||new Function("return this")()}catch(e){if("object"==typeof window)return window}}(),c.o=(e,t)=>Object.prototype.hasOwnProperty.call(e,t),o={},a="docs:",c.l=(e,t,r,n)=>{if(o[e])o[e].push(t);else{var f,i;if(void 0!==r)for(var d=document.getElementsByTagName("script"),b=0;b{f.onerror=f.onload=null,clearTimeout(l);var a=o[e];if(delete o[e],f.parentNode&&f.parentNode.removeChild(f),a&&a.forEach((e=>e(r))),t)return t(r)},l=setTimeout(u.bind(null,void 0,{type:"timeout",target:f}),12e4);f.onerror=u.bind(null,f.onerror),f.onload=u.bind(null,f.onload),i&&document.head.appendChild(f)}},c.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},c.p="/covfee/",c.gca=function(e){return e={17896441:"918",18891827:"81","06c6ffc3":"7","935f2afb":"53","1f391b9e":"85",fe7d5959:"90","1df93b7f":"237",b5e9cae4:"317","393be207":"414",a19b665e:"505","1be78505":"514","456dc1ab":"533",e0180904:"572","9e4087bc":"608","3643a9e5":"714",fa4d91bf:"930"}[e]||e,c.p+c.u(e)},(()=>{var e={303:0,532:0};c.f.j=(t,r)=>{var o=c.o(e,t)?e[t]:void 0;if(0!==o)if(o)r.push(o[2]);else if(/^(303|532)$/.test(t))e[t]=0;else{var a=new Promise(((r,a)=>o=e[t]=[r,a]));r.push(o[2]=a);var n=c.p+c.u(t),f=new Error;c.l(n,(r=>{if(c.o(e,t)&&(0!==(o=e[t])&&(e[t]=void 0),o)){var a=r&&("load"===r.type?"missing":r.type),n=r&&r.target&&r.target.src;f.message="Loading chunk "+t+" failed.\n("+a+": "+n+")",f.name="ChunkLoadError",f.type=a,f.request=n,o[1](f)}}),"chunk-"+t,t)}},c.O.j=t=>0===e[t];var t=(t,r)=>{var o,a,n=r[0],f=r[1],i=r[2],d=0;if(n.some((t=>0!==e[t]))){for(o in f)c.o(f,o)&&(c.m[o]=f[o]);if(i)var b=i(c)}for(t&&t(r);d{"use strict";var e,t,r,o,a,n={},f={};function c(e){var t=f[e];if(void 0!==t)return t.exports;var r=f[e]={id:e,loaded:!1,exports:{}};return n[e].call(r.exports,r,r.exports,c),r.loaded=!0,r.exports}c.m=n,c.c=f,e=[],c.O=(t,r,o,a)=>{if(!r){var n=1/0;for(b=0;b=a)&&Object.keys(c.O).every((e=>c.O[e](r[i])))?r.splice(i--,1):(f=!1,a0&&e[b-1][2]>a;b--)e[b]=e[b-1];e[b]=[r,o,a]},c.n=e=>{var t=e&&e.__esModule?()=>e.default:()=>e;return c.d(t,{a:t}),t},r=Object.getPrototypeOf?e=>Object.getPrototypeOf(e):e=>e.__proto__,c.t=function(e,o){if(1&o&&(e=this(e)),8&o)return e;if("object"==typeof e&&e){if(4&o&&e.__esModule)return e;if(16&o&&"function"==typeof e.then)return e}var a=Object.create(null);c.r(a);var n={};t=t||[null,r({}),r([]),r(r)];for(var f=2&o&&e;"object"==typeof f&&!~t.indexOf(f);f=r(f))Object.getOwnPropertyNames(f).forEach((t=>n[t]=()=>e[t]));return n.default=()=>e,c.d(a,n),a},c.d=(e,t)=>{for(var r in t)c.o(t,r)&&!c.o(e,r)&&Object.defineProperty(e,r,{enumerable:!0,get:t[r]})},c.f={},c.e=e=>Promise.all(Object.keys(c.f).reduce(((t,r)=>(c.f[r](e,t),t)),[])),c.u=e=>"assets/js/"+({7:"06c6ffc3",53:"935f2afb",81:"18891827",85:"1f391b9e",90:"fe7d5959",237:"1df93b7f",317:"b5e9cae4",414:"393be207",505:"a19b665e",514:"1be78505",533:"456dc1ab",572:"e0180904",608:"9e4087bc",714:"3643a9e5",918:"17896441",930:"fa4d91bf"}[e]||e)+"."+{7:"4b3d9224",53:"cffa800e",75:"73464b18",81:"0c0dcc3b",85:"16b86f47",90:"6dabcb34",237:"7208088c",317:"b0137541",414:"9c2fbb33",505:"0e22246f",514:"6d8274cf",533:"930459c2",572:"d88b69ec",608:"33324444",714:"89de50b7",767:"e55da0ba",918:"2aac1e29",930:"45991c3e"}[e]+".js",c.miniCssF=e=>"assets/css/styles.e47ef7b4.css",c.g=function(){if("object"==typeof globalThis)return globalThis;try{return this||new Function("return this")()}catch(e){if("object"==typeof window)return window}}(),c.o=(e,t)=>Object.prototype.hasOwnProperty.call(e,t),o={},a="docs:",c.l=(e,t,r,n)=>{if(o[e])o[e].push(t);else{var f,i;if(void 0!==r)for(var d=document.getElementsByTagName("script"),b=0;b{f.onerror=f.onload=null,clearTimeout(l);var a=o[e];if(delete o[e],f.parentNode&&f.parentNode.removeChild(f),a&&a.forEach((e=>e(r))),t)return t(r)},l=setTimeout(u.bind(null,void 0,{type:"timeout",target:f}),12e4);f.onerror=u.bind(null,f.onerror),f.onload=u.bind(null,f.onload),i&&document.head.appendChild(f)}},c.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},c.p="/covfee/",c.gca=function(e){return e={17896441:"918",18891827:"81","06c6ffc3":"7","935f2afb":"53","1f391b9e":"85",fe7d5959:"90","1df93b7f":"237",b5e9cae4:"317","393be207":"414",a19b665e:"505","1be78505":"514","456dc1ab":"533",e0180904:"572","9e4087bc":"608","3643a9e5":"714",fa4d91bf:"930"}[e]||e,c.p+c.u(e)},(()=>{var e={303:0,532:0};c.f.j=(t,r)=>{var o=c.o(e,t)?e[t]:void 0;if(0!==o)if(o)r.push(o[2]);else if(/^(303|532)$/.test(t))e[t]=0;else{var a=new Promise(((r,a)=>o=e[t]=[r,a]));r.push(o[2]=a);var n=c.p+c.u(t),f=new Error;c.l(n,(r=>{if(c.o(e,t)&&(0!==(o=e[t])&&(e[t]=void 0),o)){var a=r&&("load"===r.type?"missing":r.type),n=r&&r.target&&r.target.src;f.message="Loading chunk "+t+" failed.\n("+a+": "+n+")",f.name="ChunkLoadError",f.type=a,f.request=n,o[1](f)}}),"chunk-"+t,t)}},c.O.j=t=>0===e[t];var t=(t,r)=>{var o,a,n=r[0],f=r[1],i=r[2],d=0;if(n.some((t=>0!==e[t]))){for(o in f)c.o(f,o)&&(c.m[o]=f[o]);if(i)var b=i(c)}for(t&&t(r);d Archive | covfee: continuous video feedback tool - +

Archive

Archive

- + \ No newline at end of file diff --git a/docs.html b/docs.html index 0cc7ce2c..483fe914 100644 --- a/docs.html +++ b/docs.html @@ -7,13 +7,13 @@ Overview | covfee: continuous video feedback tool - +

Overview

What is covfee?

Covfee is an extensible web framework for (continuous) annotation, built for crowd-sourcing and other online uses. Covfee has two main use cases:

  • A tool for online (continuous) annotation: Continuous annotation is when video files are annotated while you watch them. Audio files can also be annotated while you listen to them, and time series in general may be continuously annotated. Some audiovisual continuous annotation tasks making use of mouse and keyboard for user feedback (see Playground) are already coded, working, and tested in covfee. Preparing a continuous annotation process using these tasks amounts to preparing a JSON file specifying your HITs. Covfee also includes some non-continuous tasks for questionnaires and surveys that are provided for convenience. Using existing covfee tasks requires no coding.

  • A platform for custom online human interaction data collection and annotation: Covfee provides a framework for researchers with basic knowledge of web development in Javascript to prepare online (crowd-sourced) annotation processes and data collections. Implementing custom covfee tasks (continuous or not) has been boiled down to the writing of a single class (React component) which can make use of multiple helper classes for continuous data recording, input management and multiparty communication for tasks with multiple subjects. Client-server communication and access to the recorded data is abstracted away completely.

What covfee is not:

  • A complete media annotation platform. Covfee specializes on continuous media and does not offer many "basic" features of image annotation like bounding box or polygon annotation which are hard to do continuously.
  • Enterprise-level software. Covfee is meant to facilitate research and experimentation. It is not thoroughly tested and contains bugs.
  • Cross-browser compatible. For the time being, covfee is meant to be used in desktop or laptop computers, not on mobile phones or tablets. Also, continuous video annotation specifically currently works only in Google Chrome due to its reliance on requestVideoFrameCallback(). This will probably improve with more browser support. That being said, with this exception covfee should work correctly on all modern desktop browsers.
caution

Covfee is still in alpha stage and is not mature software.

The covfee workflow

Docusaurus with Keytar

Once covfee is installed, working with covfee as a requestor generally means following the three steps shown in the picture:

  1. The requestor creates a project specification in a .covfee.json file, which completely specifies the annotation interface. This documentation, paticularly the Playground is meant to help write the specification.

  2. The requestor runs covfee make, the script in charge of validating the specification and generating the covfee interface from it. The requestor can now enter the covfee admin panel and obtain anonymized links to each HIT in the specification. A CSV file with all the links can be downloaded to be uploaded to Amazon MTurk or otherwise shared with the annotators or study participants.

  3. The requestor may keep track of the annotation process using the admin panel. At any time it is possible to download the raw data in JSON and CSV files, which can then be processed locally.

For a step-by-step guide on how to work with covfee see Getting Started

Some of the main features of covfee are:

  • Projects are fully specified in the .covfee.json file. This makes it easy to reproduce annotation processes on other datasets if a covfee specification is available.
  • Secure link hashes Covfee automatically generates random hash links for each HIT. These hashes generated from a secret key are meant to offer protection against bots or scalping of the HIT links. Note that this form of protection is weaker under HTTP.
  • Forms support Questionnaire tasks can be used to request non-continuous feedback from participants via text boxes, buttons, sliders, and more .
  • Support for automatic qualification tasks. For continuous tasks, a HIT may be opened only if the annotator demonstrates certain level of ability in a qualification.
  • Admin panel The admin panel helps keep track of progress and download results.

Continuous annotation

Covfee supports continuous annotation tasks in modern desktop browsers. Browsers implement animation and video frame callbacks (via requestAnimationFrame() and requestVideoFrameCallback() which are designed to run on each screen refresh. On most modern displays and browsers this is a rate of 60fps. This is not guaranteed and may be lower depending on the capacity and load of the user's machine but in practice we have observed consistent annotation rates of close to 60fps on most user machines.

Extending covfee

Particular emphasis has ben put on allowing covfee to be easily extensible. Implementing new covfee tasks like the ones in the Playground is possible with only a basic knowledge of web development. Specifically, development of custom tasks requires at least a basic understanding of Javascript and React. Some useful resources to quickly get started:

The main advantages of implementing a custom task as part of covfee are:

  • Covfee takes care of the data recording. For continuous tasks, the available buffer class takes care of sending chunks of continuous data and logs to the server, and of reading them back for data visualization. All you need to do is call buffer.data() for every collected data point and buffer.log() to log any events of interest to you. For non-continuous tasks it's even more simple: calling the onSubmit() method with the result of your custom task will send the results to the server.
  • Covfee's socket.io module allows you to efficiently implement multiparty tasks, where multiple subjects are expected to take part in the task. The main use case for multiparty features is the recording of live online interactions (written, audio or audiovisual) with the ability to query subjects at any point or request their live feedback.
  • Covfee's key manager makes it easy to attach event handlers to keyboard and gamepad key presses. This is specially important for continuous annotation.
  • Access to covfee's admin panel will allow you to keep track of progress and download results easily.
  • Share your work Covfee tasks are modular and configurable and could be incorporated as part of covfee to be reused by others. Feel free to create a pull request or contact me if you have created a reusable covfee tasks.

See Custom Tasks for a step-by-step guide on how to write your custom tasks in covfee.

Contribute to covfee

Since there is an unlimited number of tasks we can teach algorithms to perform, annotation and data collection tasks often require specific implementations. Covfee has the long-term goal of dramatically improving the time and effort necessary to collect and annotate interaction data online. It was born out of the need for an annotation platform that better satisfies the characteristics of in-the-wild interaction data, where the behavior of single subjects is annotated for long periods of time, but always with the goal of supporting a broad array of experiments. To achieve its goal, covfee seeks to become a sufficient library of annotation task templates (covfee tasks), which researchers will be able to use as-is or to modify to satisfy their particular needs. You can contribute to our goal by contributing covfee tasks or contact me if you are interested in other forms of collaboration.

- + \ No newline at end of file diff --git a/docs/custom_task.html b/docs/custom_task.html index 03a3edd0..f7839441 100644 --- a/docs/custom_task.html +++ b/docs/custom_task.html @@ -7,13 +7,13 @@ Developing custom tasks | covfee: continuous video feedback tool - +
-

Developing custom tasks

New custom tasks or HIT types can be added by implementing a React component meeting a few conditions. For some tasks, a Python base class must be sub-classed too. To be valid, task components must meet these conditions:

Types of tasks

There are two main types of tasks in covfee:

  • Tasks with shared state, or "multiplayer" tasks are meant to be visited by multiple users at the same time. A chess game, or a collaborative live view form to be filled by multiple users at the same time is an example of a shared state task. Covfee internally manages state synchronization and persistence. A copy of the latest state is kept in the server.
  • Tasks without shared state, or "single player" tasks are meant to be visited by a single user. A standard form is a good example.

Task state

Covfee tasks are, in their most basic form, simply React components. In other words, copy-pasting any React component as a Covfee task would "work". However, in any type of online experiment we normally want to store the results of the task (eg. the answers to a questionnaire).

In covfee, task components do not explicitly store or commit these results to the server. Instead, the state of the task is stored automatically by Covfee. To make use of this, any task state that should be persisted by Covfee must be managed in a central Redux store. In other words, implementing Covfee tasks boils down to implementing React components with a Redux state store in all cases. State that does not need to be stored can be managed separately, for example using React's useState.

State persistance and loading will be managed internally:

  • For shared state tasks, Covfee will keep the most up-to-date state in the server, which will always represent the latest state of the task.

  • For non-shared state tasks, the latest state is kept in the browser and persisted when necessary. Covfee submits and persists state whenever:

    • A user closes the browser tab where the task is open, or otherwise leaves the task.
    • A user presses the "submit" button in the task.

Tutorial: developing your own task

Background

This tutorial requires a basic understanding of Typescript, HTML, React and Redux. For progressing to more complex custom tasks the following is recommended:

  • Typescript, Javascript, HTML, CSS. For more complex tasks, a good level of CSS might be necessary.
  • React: functional components, hooks (useState, useEffect, useContext, useRef, useMemo), custom hooks, fetching data. It's important to understand the nuances behind hooks and dependencies: when do hooks run?
  • Redux, Redux toolkit. Important to understand how to think about application state and how to design the state, actions and reducers.

In general, you should know how to design and write a React application using Redux for state management.

Installation

Please start by following the development install instructions to install Covfee.

The tutorial

In this tutorial, we will create a simple contact form asking users for their name and email. Additionally, the task will have one boolean prop (showPhoneField) that, if true, will add a phone number field to the form. The answers to the form will be preserved in task state.

We will place Typescript files for the task in covfee/client/tasks, in the following structure:

covfee
    client
        tasks
            - my_task
                - index.tsx       # the main React component
                - slice.ts        # the state specification
                - spec.ts         # the props for the task
                ...               # any extra files needed

Task props

First, we place the specification for the task props in the spec.ts file:

import { BaseTaskSpec } from "@covfee-shared/spec/task"

/**
 * @TJS-additionalProperties false
 */
export interface TutorialTaskSpec extends BaseTaskSpec {
  /**
   * @default "TutorialTask"
   */
  type: "TutorialTask"
  /**
   * Media file to be displayed.
   */
  showPhoneField?: boolean
}

These props will be parsed by Covfee so that they can be provided when the task is created. They allow the user to configure the way a task is rendered to the user. In this case, we only provide two props. One is the name of the task. This will be the name of the task's Python class and is a constant value. The other one is our only true prop showPhoneField, which we define to be boolean. The comments provided here will be parsed and will be available in Python when creating task objects. The line TJS-additionalProperties false specifies that no additional properties should be allowed when the task is validated.

State

Tasks in covfee specify their state by creating a Redux Toolkit slice. For our purposes a slice is just an object that encapsulates the initial state and the reducers for the task. In this case, we need the state to hold the user's name, email and phone number. Our slice.ts file looks like this:

import { createSlice } from "@reduxjs/toolkit"

export interface State {
  name: string
  email: string
  phone: string
}

export const initialState: State = {
  name: "",
  email: "",
  phone: "",
}

export const slice = createSlice({
  name: "form",
  initialState,
  reducers: {
    setName: (state, action) => {
      state.name = action.payload
    },
    setEmail: (state, action) => {
      state.email = action.payload
    },
    setPhone: (state, action) => {
      state.phone = action.payload
    },
  },
})

export const { actions, reducer } = slice

Here we specified the type for the state, the initial state, and wrote separate reducers to set name, email and phone number.

React Component

Following React + Redux design philosophy, the main job of the React component is to sync state with the DOM (view), by reading the state and by calling Redux actions in response to user events. In this case, because form input elements hold their own state, we are simply calling Redux actions whenever the content of the input elements changes. That's all!

import React, { useState } from "react"
import { slice, actions } from "./slice"
import { TaskExport } from "../../types/node"
import { CovfeeTaskProps } from "../base"
import type { TutorialTaskSpec } from "./spec"
import { AllPropsRequired } from "../../types/utils"

interface Props extends CovfeeTaskProps<TutorialTaskSpec> {}

const TutorialTask: React.FC<Props> = (props) => {
  const args: AllPropsRequired<Props> = {
    ...props,
    spec: {
      ...props.spec,
      showPhoneField: true,
    },
  }

  return (
    <form>
      <div>
        <label htmlFor="name">Name:</label>
        <input
          type="text"
          id="name"
          onChange={(e) => actions.setName(e.target.value)}
          required
        />
      </div>
      <div>
        <label htmlFor="email">Email:</label>
        <input
          type="email"
          id="email"
          onChange={(e) => actions.setEmail(e.target.value)}
          required
        />
      </div>
      {args.spec.showPhoneField && (
        <div>
          <label htmlFor="phone">Phone:</label>
          <input
            type="tel"
            id="phone"
            onChange={(e) => actions.setPhone(e.target.value)}
          />
        </div>
      )}
    </form>
  )
}
export type { TutorialTaskSpec }
export default {
  taskComponent: TutorialTask,
  taskSlice: slice,
  useSharedState: false,
} as TaskExport

Note the export statements at the end of the file. First we export the task specification, which should be available to Covfee (more on this later)

Covfee expects every task to export a default object with these keys. useSharedState should only be set to true for "multiplayer" tasks.

Importing the task

Now that the task is created, the task should be imported into Covfee source files. To do this, modify the file at /covfee/client/tasks/index.ts to import your custom task. Simply follow the same pattern as for the rest of the imports in the file. The task props specification (spec) should also be imported in /covfee/shared/spec/task.ts.

Now we are ready to create the auto-generated code that Covfee is designed to create. Covfee auto-generates two files using the tasks' props specification:

  • /covfee/shared/schemata.json - JSON schemata. Each task props specification (spec) is translated into a JSON schema, used for validation and for generating the dataclasses.
  • /covfee/shared/task_dataclasses.py contains the dataclasses that can be used to create task objects in Python. These are simple objects that take the task props as constructor arguments. Covfee internally translates them into database entries to initialize its database on startup.

These files may be useful for debugging purposes to eg. make sure that the dataclasses have the right arguments, types and comments.

To (re-)create both of these auto-generated files run:

covfee-dev schemata

After this step, it should be possible to manually verify that the dataclasses file contains a new class for our custom task.

Running the task

The task is ready to be used. To do this in a dev environment, create a folder for the covfee project. This folder may be anywhere in the file system. The we will create the following structure:

my_folder
    - tutorial.py

The naming is not relevant. Paste the following into the tutorial.py:

from covfee import tasks, HIT, Project
from covfee.config import config
from covfee.shared.dataclass import CovfeeApp

config.load_environment("local")

my_task_1 = tasks.TutorialTaskSpec(name="My Task 1", showPhoneField=True)
my_task_2 = tasks.TutorialTaskSpec(name="My Task 2", showPhoneField=False)

hit = HIT("Joint counter")
j1 = hit.add_journey(nodes=[my_task_1, my_task_2])

projects = [Project("My Project", email="example@example.com", hits=[hit])]
app = CovfeeApp(projects)   # we must always create an app object of class CovfeeApp

Here we import the necessary classes, create two task objects using the same (auto-generated) TutorialTaskSpec class, and link them together in a journey. We create a HIT with a single journey, and a project with a single HIT. Finally, we create the app object. Covfee internally looks for this object by name (important that it is called app) and uses it as starting point to read the specification.

We can now start covfee with:

covfee make tutorial.py --force --dev

This will parse the specification, create the database, and start the covfee server.

In development mode, we need to run the webpack server separately. To do it, in another terminal run:

covfee webpack

After this the Covfee admin panel should be accessible in the URL displayed when running the Covfee server (http://localhost:5000/admin# by default). The links to the Journey (with the two tasks) should be available in the admin panel (under Journeys).

With this development setup, any changes to the client or server will be hot-reloaded. Therefore it should be possible and convenient to edit the task's React component or the backend while running the app.

- +

Developing custom tasks

New custom tasks or HIT types can be added by implementing a React component meeting a few conditions. For some tasks, a Python base class must be sub-classed too. To be valid, task components must meet these conditions:

Types of tasks

There are two main types of tasks in covfee:

  • Tasks with shared state, or "multiplayer" tasks are meant to be visited by multiple users at the same time. A chess game, or a collaborative live view form to be filled by multiple users at the same time is an example of a shared state task. Covfee internally manages state synchronization and persistence. A copy of the latest state is kept in the server.
  • Tasks without shared state, or "single player" tasks are meant to be visited by a single user. A standard form is a good example.

Task state

Covfee tasks are, in their most basic form, simply React components. In other words, copy-pasting any React component as a Covfee task would "work". However, in any type of online experiment we normally want to store the results of the task (eg. the answers to a questionnaire).

In covfee, task components do not explicitly store or commit these results to the server. Instead, the state of the task is stored automatically by Covfee. To make use of this, any task state that should be persisted by Covfee must be managed in a central Redux store. In other words, implementing Covfee tasks boils down to implementing React components with a Redux state store in all cases. State that does not need to be stored can be managed separately, for example using React's useState.

State persistance and loading will be managed internally:

  • For shared state tasks, Covfee will keep the most up-to-date state in the server, which will always represent the latest state of the task.

  • For non-shared state tasks, the latest state is kept in the browser and persisted when necessary. Covfee submits and persists state whenever:

    • A user closes the browser tab where the task is open, or otherwise leaves the task.
    • A user presses the "submit" button in the task.

Tutorial: developing your own task

Background

This tutorial requires a basic understanding of Typescript, HTML, React and Redux. For progressing to more complex custom tasks the following is recommended:

  • Typescript, Javascript, HTML, CSS. For more complex tasks, a good level of CSS might be necessary.
  • React: functional components, hooks (useState, useEffect, useContext, useRef, useMemo), custom hooks, fetching data. It's important to understand the nuances behind hooks and dependencies: when do hooks run?
  • Redux, Redux toolkit. Important to understand how to think about application state and how to design the state, actions and reducers.

In general, you should know how to design and write a React application using Redux for state management.

Installation

Please start by following the development install instructions to install Covfee.

The tutorial

In this tutorial, we will create a simple contact form asking users for their name and email. Additionally, the task will have one boolean prop (showPhoneField) that, if true, will add a phone number field to the form. The answers to the form will be preserved in task state.

We will place Typescript files for the task in covfee/client/tasks, in the following structure:

covfee
    client
        tasks
            - my_task
                - index.tsx       # the main React component
                - slice.ts        # the state specification
                - spec.ts         # the props for the task
                ...               # any extra files needed

Task props

First, we place the specification for the task props in the spec.ts file:

import { BaseTaskSpec } from "@covfee-shared/spec/task"

/**
 * @TJS-additionalProperties false
 */
export interface TutorialTaskSpec extends BaseTaskSpec {
  /**
   * @default "TutorialTask"
   */
  type: "TutorialTask"
  /**
   * Media file to be displayed.
   */
  showPhoneField?: boolean
}

These props will be parsed by Covfee so that they can be provided when the task is created. They allow the user to configure the way a task is rendered to the user. In this case, we only provide two props. One is the name of the task. This will be the name of the task's Python class and is a constant value. The other one is our only true prop showPhoneField, which we define to be boolean. The comments provided here will be parsed and will be available in Python when creating task objects. The line TJS-additionalProperties false specifies that no additional properties should be allowed when the task is validated.

State

Tasks in covfee specify their state by creating a Redux Toolkit slice. For our purposes a slice is just an object that encapsulates the initial state and the reducers for the task. In this case, we need the state to hold the user's name, email and phone number. Our slice.ts file looks like this:

import { createSlice } from "@reduxjs/toolkit"

export interface State {
  name: string
  email: string
  phone: string
}

export const initialState: State = {
  name: "",
  email: "",
  phone: "",
}

export const slice = createSlice({
  name: "form",
  initialState,
  reducers: {
    setName: (state, action) => {
      state.name = action.payload
    },
    setEmail: (state, action) => {
      state.email = action.payload
    },
    setPhone: (state, action) => {
      state.phone = action.payload
    },
  },
})

export const { actions, reducer } = slice

Here we specified the type for the state, the initial state, and wrote separate reducers to set name, email and phone number.

React Component

Following React + Redux design philosophy, the main job of the React component is to sync state with the DOM (view), by reading the state and by calling Redux actions in response to user events. In this case, because form input elements hold their own state, we are simply calling Redux actions whenever the content of the input elements changes. That's all!

import React, { useState } from "react"
import { slice, actions } from "./slice"
import { TaskExport } from "../../types/node"
import { CovfeeTaskProps } from "../base"
import type { TutorialTaskSpec } from "./spec"
import { AllPropsRequired } from "../../types/utils"

interface Props extends CovfeeTaskProps<TutorialTaskSpec> {}

const TutorialTask: React.FC<Props> = (props) => {
  const args: AllPropsRequired<Props> = {
    ...props,
    spec: {
      ...props.spec,
      showPhoneField: true,
    },
  }

  return (
    <form>
      <div>
        <label htmlFor="name">Name:</label>
        <input
          type="text"
          id="name"
          onChange={(e) => actions.setName(e.target.value)}
          required
        />
      </div>
      <div>
        <label htmlFor="email">Email:</label>
        <input
          type="email"
          id="email"
          onChange={(e) => actions.setEmail(e.target.value)}
          required
        />
      </div>
      {args.spec.showPhoneField && (
        <div>
          <label htmlFor="phone">Phone:</label>
          <input
            type="tel"
            id="phone"
            onChange={(e) => actions.setPhone(e.target.value)}
          />
        </div>
      )}
    </form>
  )
}
export default {
  taskComponent: TutorialTask,
  taskSlice: slice,
  useSharedState: false,
} as TaskExport

Note the expor statement at the end of the file. Covfee expects every task to export a default object with these keys. useSharedState should only be set to true for "multiplayer" tasks.

Importing the task

Now that the task is created, the task should be imported into Covfee source files. To do this, modify the file at /covfee/client/tasks/index.ts to import your custom task. Simply follow the same pattern as for the rest of the imports in the file. The task props specification (spec) should also be imported in /covfee/shared/spec/task.ts.

Now we are ready to create the auto-generated code that Covfee is designed to create. Covfee auto-generates two files using the tasks' props specification:

  • /covfee/shared/schemata.json - JSON schemata. Each task props specification (spec) is translated into a JSON schema, used for validation and for generating the dataclasses.
  • /covfee/shared/task_dataclasses.py contains the dataclasses that can be used to create task objects in Python. These are simple objects that take the task props as constructor arguments. Covfee internally translates them into database entries to initialize its database on startup.

These files may be useful for debugging purposes to eg. make sure that the dataclasses have the right arguments, types and comments.

To (re-)create both of these auto-generated files run:

covfee-dev schemata

After this step, it should be possible to manually verify that the dataclasses file contains a new class for our custom task.

Running the task

The task is ready to be used. To do this in a dev environment, create a folder for the covfee project. This folder may be anywhere in the file system. The we will create the following structure:

my_folder
    - tutorial.py

The naming is not relevant. Paste the following into the tutorial.py:

from covfee import tasks, HIT, Project
from covfee.config import config
from covfee.shared.dataclass import CovfeeApp

config.load_environment("local")

my_task_1 = tasks.TutorialTaskSpec(name="My Task 1", showPhoneField=True)
my_task_2 = tasks.TutorialTaskSpec(name="My Task 2", showPhoneField=False)

hit = HIT("Joint counter")
j1 = hit.add_journey(nodes=[my_task_1, my_task_2])

projects = [Project("My Project", email="example@example.com", hits=[hit])]
app = CovfeeApp(projects)   # we must always create an app object of class CovfeeApp

Here we import the necessary classes, create two task objects using the same (auto-generated) TutorialTaskSpec class, and link them together in a journey. We create a HIT with a single journey, and a project with a single HIT. Finally, we create the app object. Covfee internally looks for this object by name (important that it is called app) and uses it as starting point to read the specification.

We can now start covfee with:

covfee make tutorial.py --force --dev

This will parse the specification, create the database, and start the covfee server.

In development mode, we need to run the webpack server separately. To do it, in another terminal run:

covfee webpack

After this the Covfee admin panel should be accessible in the URL displayed when running the Covfee server (http://localhost:5000/admin# by default). The links to the Journey (with the two tasks) should be available in the admin panel (under Journeys).

With this development setup, any changes to the client or server will be hot-reloaded. Therefore it should be possible and convenient to edit the task's React component or the backend while running the app.

+ \ No newline at end of file diff --git a/docs/deployment.html b/docs/deployment.html index 9f577455..bb20707b 100644 --- a/docs/deployment.html +++ b/docs/deployment.html @@ -7,13 +7,13 @@ Deploying to a server | covfee: continuous video feedback tool - +

Deploying to a server

Deploying covfee is necessary when you want to make your HITs available to others over the internet. We do not recommend developing your tasks in deployment mode. It is normally more convenient to create your HIT specifications locally, deploy covfee, and then run covfee make on the deployed instance to initialize your database.

Deployment configuration

covfee reads its configuration from the folder in which it is run (ie. the project folder). When ran in deployment mode, covfee will look for the file covfee.deployment.config.py for its configuration.

Basic configuration

If you are deploying covfee in a typical server, add a configuration file like the following to your project folder:

BASE_URL = 'http://example.com/covfee'
APP_PORT = 80

The BASE_URL option should point to the public URL of your covfee instance.

Hosting media files externally

To host media files externally, add the following to your covfee.deployment.config.py:

MEDIA_URL = 'http://example.com/covfee-media'
MEDIA_SERVER = False

The MEDIA_URL option allows you to customize the location of your media files. By default, covfee will serve media files from a folder called media in your project folder. Anything in this folder will be made public. If you want to host your media files externally, you can disable this behavior by setting MEDIA_SERVER to False and pointing MEDIA_URL to your files' location.

Deployment options

gunicorn

gunicorn is the most common deployment option for Flask applications. To run a production copy of covfee on your covfee project directory:

  1. Install covfee per the Getting Started instructions. It is recommended to install covfee in a virtual environment.

  2. Build covfee. This will build the Javascript bundle used by the covfee app.

cd /path/to/covfee-project
covfee-build
  1. Run the covfee server:
covfee-gunicorn

Apache mod_wsgi

covfee can be run under Apache by using mod_wsgi. mod_wsgi must be installed for Python 3 for covfee to work. To ensure this install:

sudo apt-get install libapache2-mod-wsgi-py3

Next, add a file covfee.wsgi with the following to your project directory:

import sys
import os
sys.stdout = sys.stderr
activate_this = '/path/to/virtualenv/bin/activate_this.py'
with open(activate_this) as file_:
    exec(file_.read(), dict(__file__=activate_this))
os.chdir('/path/to/covfee-project')
from covfee.start import create_app
application = create_app()

This code will allow Apache to create your app instance. /path/to/virtualenv is the path to your virtual environment where covfee is installed.

Finally, Apache must know about your site. For this add lines like the following to your apache site configuration:

WSGIDaemonProcess covfee user=me group=me threads=3
WSGIProcessGroup covfee
WSGIScriptAlias /covfee /path/to/covfee-project/covfee.wsgi

WSGIScriptAlias must point to the covfee.wsgi file you created in the previous step. The first parameter /covfee is the url path under which covfee will be accesible.

More deployment options

Covfee is built with Flask and is therefore easy to deploy using any of the options supported by Flask. Take a look at the Flask deployment options for more.

- + \ No newline at end of file diff --git a/docs/development.html b/docs/development.html index 4fe9f5d1..0337eee7 100644 --- a/docs/development.html +++ b/docs/development.html @@ -7,13 +7,13 @@ Development install | covfee: continuous video feedback tool - +

Development install

This guide explains how to install covfee for development. This type of install is recommended for modifying covfee's Javascript code, including the implementation of custom tasks, covfee documentation, or the Pyton backend. A good understanding of Javascript and/or Python is recommended.

Setup

caution

Covfee runs on Linux, Mac OS X and Windows, but you are more likely to encounter bugs on Windows. We recommend that you use WSL on Windows.

Covfee's frontend is built using webpack, which has a convenient hot-reloading development server. The Flask server used for the backend also supports reloading on changes. This guide will get you to run backend and frontend development servers:

  1. Install the latest version of node.js. Make sure that the npm command is available in your terminal.
tip

We strongly recommend that you install covfee in a Python virtual environment. To create and activate one using venv:

python3 -m venv ./env
source ./env/bin/activate
  1. Install the covfee Python package in editable mode:
git clone git@github.com:josedvq/covfee.git
cd covfee
python3 -m pip install -e .

The covfee command should now be available in the terminal. Type covfee --help to make sure.

  1. Install Javascript dependencies:
yarn install
  1. Generate schemata (for validation) and build webpack bundles:
covfee-dev schemata
covfee-dev build

The development install is ready.

Running a covfee project in dev mode

Covfee requires a project folder with at least one .covfee.json file to run. Covfee provides sample projects, or see Getting Started for instructions on how to create your own project folder and specification.

To run one of the covfee samples:

  1. Move to the sample project folder:
cd samples/basic
covfee make . --no-launch
  1. Start the webpack server from the same folder:
covfee webpack

This will start a Webpack development server with hot-reloading.

  1. In another terminal, at the same folder, start the flask server in dev mode. This will start the Python backend server with hot-reloading:
covfee start --dev

You should now be able to access the admin panel and open the covfee HITs. Any changes made to Python or Javascript code should be immediately hot-reloaded.

Schemata updates

The covfee schemata file created previously (covfee-dev schemata) is used by covfee make for validating covfee specifications. Therefore, if you make changes to covfee's Typescript interfaces in covfee/shared during development (by, for, example, adding a new task specification), the schemata file must be re-generated for covfee validate your new interfaces correctly. To regenerate covfee schemata run:

covfee-dev schemata

This command may also need to be called when switching branches if the Typescript specification differs between branches.

- + \ No newline at end of file diff --git a/docs/getting_started.html b/docs/getting_started.html index a19867f4..9f5889bd 100644 --- a/docs/getting_started.html +++ b/docs/getting_started.html @@ -7,13 +7,13 @@ Getting Started | covfee: continuous video feedback tool - +

Getting Started

This guide takes you through the process of creating a simple covfee interface which gathers continuous data from subjects.

caution

Covfee makes use of Dict[str, Any] as input format. In this guide we assume that you are familiar with the JSON syntax.

1. Install covfee

Follow the installation instructions if you have not done so already.

2. Specify your HITs

In this example project, we will assume that we have three annotators that will be performing the same HIT (separately). The HIT consists in four tasks: 1) filling a questionnaire where we ask for demographic information, 2) reading the instructions for affect annotation, 3) annotating affect in a video and 4) filling a questionnaire where we ask them for feedback on their experience during the annotation process.

This can be set up by creating a covfee project containing the HIT as follows:

Covfee will produce multiple URLs per HIT, which can be sent to the three annotators. Covfee projects may contain multiple HITs (like HIT2 in the picture) but this project only requires a single HIT. Covfee tasks can also be very diverse, and you can even implement your own if you know Javascript. You can see live examples of the tasks available in covfee in the Playground section.

You make use of covfee by first creating a JSON file specifying your HITs. First create a folder for your covfee project and a file example.covfee.json with the following content:

{
  "id": "getting_started",
  "name": "My Covfee Project",
  "email": "example@example.com",
  "hits": [
    {
      "id": "hit1",
      "name": "HIT1",
      "tasks": [
        ...
      ],
      "interface": {
        "type": "timeline"
      }
    }
  ]
}

This is the skeleton of a covfee project and example.covfee.json will contain the full specification of it. It starts with the name and email of the contact person in charge of the study, which will be available to participants. The hits section contains an object per HIT, in this case a single one. We will specify our tasks next.

Task 1. Questionnaire

We will start by specifying the form / questionnaire that we want annotators to fill in in Task 1. We will ask them for their age (number input), sex (select input) and nationality (free text input):

{
  "name": "Demographics",
  "type": "QuestionnaireTask",
  "form": {
    "fields": [
      {
        "name": "age",
        "label": "Your age:",
        "input": {
          "inputType": "InputNumber"
        }
      },
      {
        "name": "sex",
        "label": "Your sex:",
        "input": {
          "inputType": "Select",
          "options": [
            {"label": "Male", "value": "m"},
            {"label": "Female", "value": "f"}
          ]
        }
      },
      {
        "name": "nationality",
        "label": "Your nationality:",
        "input": {
          "inputType": "Input"
        }
      }
    ]
  }
}

Go ahead and paste the text above into the tasks section of the project specification.

Task 2. Instructions

Next we will specify the instructions task. Covfee's InstructionTask supports linking to a Markdown file with instructions. A Markdown file may contain rich text, images, audio and even videos. We will make use of it for our instructions:

{
  "name": "Instructions",
  "type": "InstructionsTask",
  "content": {
    "type": "link",
    "url": "$$www$$/instructions.md"
  }
}

Now we only need to create our instructions.md file with the following content:

# Instructions

You will be shown a video of a person. Please rate their arousal using your keyboard up/down arrows.

An important matter is where to place this file. Covfee is able to read local files placed in a specific location: a folder named www located directly under your covfee project folder. Go ahead and create this folder and place your instructions.md file inside. After, this your file structure should look like:

myproject
    example.covfee.json
    www
        instructions.md
tip

The $$www$$ special variable above is used to point to the special www folder. Note that this is the only way to work with local files unless you set up a file server yourself.

Task 3. Annotation

Next, we will add the continuous task in which subjects must annotate a one dimensional variable: the arousal of the subject, in a video of a person reading:

{
  "name": "Arousal annotation",
  "type": "Continuous1DTask",
  "media": {
    "type": "video",
    "url": "$$www$$/person_reading.mp4"
  },
  "intensityInput": {
    "mode": "ranktrace"
  }
}

Here again we want to provide a local video to covfee by placing it in the www folder. You can download this video to use as a sample.

Alternatively, you can provide the URL of a publicly hosted video like:

      ...
      "url": "https://cdn.jsdelivr.net/gh/josedvq/covfee@master/samples/keypoints/www/person_reading.mp4"
      ...

Task 4. Questionnaire

Finally, we will ask annotators to give their feedback about the annotation process, to know if we can improve something.

{
  "name": "Feedback",
  "type": "QuestionnaireTask",
  "form": {
    "fields": [
        {
            "name": "rating",
            "label": "How would you rate your experience in completing this experiment?",
            "required": true,
            "input": {
                "inputType": "Rate",
                "half": true
            }
        },{
            "name": "feedback",
            "label": "Do you have any comments that can help us improve the experience?",
            "input": {
                "inputType": "Input.TextArea"
            }
        }
    ]
  }
}
info

By now your covfee specification (example.covfee.json file) should contain four tasks whithin its tasks section and look exactly like this file. Your project folder should look like this:

myproject
    example.covfee.json
    www
        instructions.md
        myvideo.mp4

2. Run covfee

Covfee uses the previous JSON specification to create a database that will store the responses to the tasks.

All you need to do now is to run covfee make . from the project folder:

cd myproject
covfee make .

covfee make . will read all the .covfee.json files in the folder, initialize a database for covfee, run covfee, and launch a browser window where you can access the admin panel. You should now see your web browser open and take you to covfee's admin panel.

3. Annotate!

You can get URLs for your HITs from covfee's admin panel, as well as download completed annotations.

More

For more samples, please check the samples folder in Github

- + \ No newline at end of file diff --git a/docs/installation.html b/docs/installation.html index b716e407..688ac836 100644 --- a/docs/installation.html +++ b/docs/installation.html @@ -7,13 +7,13 @@ Installation | covfee: continuous video feedback tool - +

Installation

caution

A new version of covfee is coming. At the moment, this version is only installable using the development install instructions.

I'll update these instructions when the new version is ready to be installed from PyPI.

- + \ No newline at end of file diff --git a/docs/output.html b/docs/output.html index 40c614c4..bf58dd66 100644 --- a/docs/output.html +++ b/docs/output.html @@ -7,13 +7,13 @@ Reading the output | covfee: continuous video feedback tool - +

Reading the output

Downloading covfee annotations

Covfee annotations can be downloaded from the admin panel, either for the complete project ("Download results") or for a specific HIT using the buttons on the HIT's row.

About continuous annotations

Annotation resolution

Covfee records continuous annotations in a fixed-size array with a configurable resolution via an fpsfps variable (frames or annotations per second). Every index of this array is associated to a particular media time (eg. video time) via:

i=round(mediaTime∗fps)i = round(mediaTime * fps)

The process to record one data point in covfee is the following:

  1. Covfee reads the current mediaTimemediaTime (eg. video time) of the media being annotated.
  2. Covfee reads the annotated value. This may be the position of the cursor or whether a key is being pressed.
  3. Covfee calculates the index ii of the array corresponding to this mediaTimemediaTime (see above) and stores the data point in this position.

The fpsfps variable can be set separately for each continuous task.

  • For video tasks, we recommend to set fpsfps equal to the frame rate of the video, as covfee will attempt to record one annotation for every frame of the video (see next section). This also means that no rounding will be done by covfee as every mediaTimemediaTime will correspond exactly to a frame number.
tip

If fpsfps is not set for a video task, covfee will default to 60fps. This is because covfee cannot read the frame rate of the video on the fly. Make sure to set the fpsfps property of the task for optimal results.

  • For other continuous tasks (eg. audio), fps can be set to any desired value. In practice, however, most browsers will not trigger events faster than 60fps. If fpsfps is not set, covfee defaults to 60fps.

  • In adition to the annotated value, covfee stores the value of mediaTimemediaTime for every data point. This may be used to test or eliminate the effect of rounding post-hoc.

Skipped data points

  • For continuous annotation of videos in the browser, covfee tries to record a new data point on every frame of the video (via requestVideoFrameCallback()). Therefore, in principle, continuous video annotations in covfee may have the same resolution as the input video. In practice, however, browsers may skip frames when playing a video, depending on the user machine's speed and load.
info

In our experience it is rare to see more than one or two frames skipped in a row with 60fps video in most modern machines.

  • For other types of media covfee tries to record a data point on every screen refresh (via requestAnimationFrame()), which is every 60Hz on most machines. In practice however, the browser may also skip calls depending on the computer's load.

Therefore, it is possible that covfee skips continuous data points regardless of the value of fpsfps. It is possible however to know when a data point has been skipped as this will correspond to a row of zeroes in covfee's output:

It is your choice whether to ignore or interpolate missing data points.

info

The way in which videos are processed in the browser may be hard to understand if you are used processing video in Python, C++ or other traditional environments. In the browser, there is no guarantee that a video can be played frame by frame, although it is possible to change a video's playbackRate to speed it up or slow it down. The browser takes care of playing the video as close as possible to the desired playbackRate but depending on the speed and load of the user's machine, the browser may skip frames of the video to meet the playbackRate.

- + \ No newline at end of file diff --git a/docs/roadmap.html b/docs/roadmap.html index a9455c2a..52e26e5d 100644 --- a/docs/roadmap.html +++ b/docs/roadmap.html @@ -7,13 +7,13 @@ About covfee and roadmap | covfee: continuous video feedback tool - +
- + \ No newline at end of file diff --git a/index.html b/index.html index 606ba3cb..a39a97a2 100644 --- a/index.html +++ b/index.html @@ -7,13 +7,13 @@ Hello from covfee: continuous video feedback tool | covfee: continuous video feedback tool - +

covfee: continuous video feedback tool

extensible web framework for continuous annotation

Get Started
Annotate affect, actions and keypoints

Annotate affect, actions and keypoints

Use covfee's continuous annotation techniques for affect (Ranktrace, Gtrace) action localization (via keyboard press) and keyboard (via mouse cursor with automatic speed adjustment).

Script your annotation flow

Script your annotation flow

Use your favorite programming language to generate a covfee specification file, containing the details of your HITs, no matter how complex.

Qualification tasks and consent forms

Qualification tasks and consent forms

Add consent forms and qualification tasks to your HITs.

Made for crowd-sourcing

Made for crowd-sourcing

Easily get batch URLs from Covfee's admin panel, track and download annotations in bulk.

Modern and extensible

Modern and extensible

Implement your annotation techniques by subclassing a Typescript class and using Covfee's helper classes, with only a basic knowledge of Javascript.

Jose Vargas Quiros

Jose Vargas Quiros

Jose is on a mission to reduce the time and effort necessary to obtain high-quality human behavior annotations and datasets.

- + \ No newline at end of file diff --git a/markdown-page.html b/markdown-page.html index d9d3a4be..883698f2 100644 --- a/markdown-page.html +++ b/markdown-page.html @@ -7,13 +7,13 @@ Markdown page example | covfee: continuous video feedback tool - +

Markdown page example

You don't need React to write simple standalone pages.

- + \ No newline at end of file