Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Features #38 #30 #31: keep objects always visible #327

Open
wants to merge 11 commits into
base: master
Choose a base branch
from
Open

Features #38 #30 #31: keep objects always visible #327

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
b2e6099
stayVisible
maxfyk Dec 30, 2022
031d545
fix target events
maxfyk Dec 30, 2022
e992270
Update controller.js
maxfyk Dec 31, 2022
b3f6608
fix bug
maxfyk Jan 1, 2023
de15703
Update three.js
maxfyk Jan 1, 2023
685902c
targetPresentPrevious -> targetPresentBefore
maxfyk Jan 1, 2023
762d3e2
add comments
maxfyk Jan 1, 2023
b830f1d
Update controller.js
maxfyk Jan 1, 2023
ddd11eb
Update controller.js
maxfyk Jan 1, 2023
232c36e
Update controller.js
maxfyk Jan 1, 2023
6f25475
remove console.logs
maxfyk Jan 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 24 additions & 12 deletions src/image-target/aframe.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,13 @@ AFRAME.registerSystem('mindar-image-system', {
tick: function() {
},

setup: function({imageTargetSrc, maxTrack, showStats, uiLoading, uiScanning, uiError, missTolerance, warmupTolerance, filterMinCF, filterBeta}) {
setup: function({imageTargetSrc, maxTrack, showStats, uiLoading, uiScanning, uiError, missTolerance, warmupTolerance, filterMinCF, filterBeta, stayVisible, stayVisibleScale}) {
this.imageTargetSrc = imageTargetSrc;
this.maxTrack = maxTrack;
this.filterMinCF = filterMinCF;
this.filterBeta = filterBeta;
this.stayVisible = stayVisible;
this.stayVisibleScale = stayVisibleScale;
this.missTolerance = missTolerance;
this.warmupTolerance = warmupTolerance;
this.showStats = showStats;
Expand Down Expand Up @@ -110,23 +112,25 @@ AFRAME.registerSystem('mindar-image-system', {
this.controller = new Controller({
inputWidth: video.videoWidth,
inputHeight: video.videoHeight,
maxTrack: this.maxTrack,
maxTrack: this.maxTrack,
filterMinCF: this.filterMinCF,
filterBeta: this.filterBeta,
stayVisible: this.stayVisible,
stayVisibleScale: this.stayVisibleScale,
missTolerance: this.missTolerance,
warmupTolerance: this.warmupTolerance,
onUpdate: (data) => {
if (data.type === 'processDone') {
if (this.mainStats) this.mainStats.update();
}
else if (data.type === 'updateMatrix') {
const {targetIndex, worldMatrix} = data;
const {targetIndex, worldMatrix, targetPresent} = data;

for (let i = 0; i < this.anchorEntities.length; i++) {
if (this.anchorEntities[i].targetIndex === targetIndex) {
this.anchorEntities[i].el.updateWorldMatrix(worldMatrix, );
if (worldMatrix) {
this.ui.hideScanning();
this.anchorEntities[i].el.updateWorldMatrix(worldMatrix, targetPresent);
if (worldMatrix && !this.stayVisible) {
this.ui.hideScanning();
}
}
}
Expand All @@ -149,7 +153,7 @@ AFRAME.registerSystem('mindar-image-system', {
await this.controller.dummyRun(this.video);
this.el.emit("arReady");
this.ui.hideLoading();
this.ui.showScanning();
if (!this.stayVisible) this.ui.showScanning();

this.controller.processVideo(this.video);
},
Expand Down Expand Up @@ -203,6 +207,8 @@ AFRAME.registerComponent('mindar-image', {
filterBeta: {type: 'number', default: -1},
missTolerance: {type: 'int', default: -1},
warmupTolerance: {type: 'int', default: -1},
stayVisible: {type: 'boolean', default: false},
stayVisibleScale: {type: 'int', default: 50},
showStats: {type: 'boolean', default: false},
autoStart: {type: 'boolean', default: true},
uiLoading: {type: 'string', default: 'yes'},
Expand All @@ -214,12 +220,14 @@ AFRAME.registerComponent('mindar-image', {
const arSystem = this.el.sceneEl.systems['mindar-image-system'];

arSystem.setup({
imageTargetSrc: this.data.imageTargetSrc,
imageTargetSrc: this.data.imageTargetSrc,
maxTrack: this.data.maxTrack,
filterMinCF: this.data.filterMinCF === -1? null: this.data.filterMinCF,
filterBeta: this.data.filterBeta === -1? null: this.data.filterBeta,
missTolerance: this.data.missTolerance === -1? null: this.data.missTolerance,
warmupTolerance: this.data.warmupTolerance === -1? null: this.data.warmupTolerance,
stayVisible: this.data.stayVisible,
stayVisibleScale: this.data.stayVisibleScale,
showStats: this.data.showStats,
uiLoading: this.data.uiLoading,
uiScanning: this.data.uiScanning,
Expand All @@ -230,7 +238,7 @@ AFRAME.registerComponent('mindar-image', {
arSystem.start();
});
}
},
},
remove: function () {
const arSystem = this.el.sceneEl.systems['mindar-image-system'];
arSystem.stop();
Expand All @@ -253,6 +261,8 @@ AFRAME.registerComponent('mindar-image-target', {
const root = this.el.object3D;
root.visible = false;
root.matrixAutoUpdate = false;

this.targetPresentBefore = false;
},

setupMarker([markerWidth, markerHeight]) {
Expand All @@ -268,11 +278,13 @@ AFRAME.registerComponent('mindar-image-target', {
this.postMatrix.compose(position, quaternion, scale);
},

updateWorldMatrix(worldMatrix) {
if (!this.el.object3D.visible && worldMatrix !== null) {
updateWorldMatrix(worldMatrix, targetPresent) {
if (this.targetPresentBefore === false && targetPresent === true) {
this.el.emit("targetFound");
} else if (this.el.object3D.visible && worldMatrix === null) {
this.targetPresentBefore = true;
} else if (this.targetPresentBefore === true && targetPresent === false) {
this.el.emit("targetLost");
this.targetPresentBefore = false;
}

this.el.object3D.visible = worldMatrix !== null;
Expand Down
47 changes: 29 additions & 18 deletions src/image-target/controller.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ const DEFAULT_WARMUP_TOLERANCE = 5;
const DEFAULT_MISS_TOLERANCE = 5;

class Controller {
constructor({inputWidth, inputHeight, onUpdate=null, debugMode=false, maxTrack=1,
constructor({inputWidth, inputHeight, onUpdate=null, debugMode=false, stayVisible=false, stayVisibleScale=50, maxTrack=1,
warmupTolerance=null, missTolerance=null, filterMinCF=null, filterBeta=null}) {

this.inputWidth = inputWidth;
Expand All @@ -28,6 +28,8 @@ class Controller {
this.markerDimensions = null;
this.onUpdate = onUpdate;
this.debugMode = debugMode;
this.stayVisible = stayVisible;
this.stayVisibleScale = stayVisibleScale;
this.processingVideo = false;
this.interestedTargetIndex = -1;
this.trackingStates = [];
Expand Down Expand Up @@ -173,7 +175,7 @@ class Controller {
return acc + (!!s.isTracking? 1: 0);
}, 0);

// detect and match only if less then maxTrack
// detect and match only if less than maxTrack
if (nTracking < this.maxTrack) {

const matchingIndexes = [];
Expand All @@ -196,19 +198,17 @@ class Controller {
// tracking update
for (let i = 0; i < this.trackingStates.length; i++) {
const trackingState = this.trackingStates[i];

if (trackingState.isTracking) {
if (trackingState.isTracking && trackingState.currentModelViewTransform) {
let modelViewTransform = await this._trackAndUpdate(inputT, trackingState.currentModelViewTransform, i);
if (modelViewTransform === null) {
trackingState.isTracking = false;
trackingState.isTracking = false;
} else {
trackingState.currentModelViewTransform = modelViewTransform;
}
}

// if not showing, then show it once it reaches warmup number of frames
if (!trackingState.showing) {
if (trackingState.isTracking) {
if (trackingState.isTracking || this.stayVisible) {
trackingState.trackMiss = 0;
trackingState.trackCount += 1;
if (trackingState.trackCount > this.warmupTolerance) {
Expand All @@ -218,33 +218,44 @@ class Controller {
}
}
}

// if showing, then count miss, and hide it when reaches tolerance
if (trackingState.showing) {
if (!trackingState.isTracking) {
trackingState.trackCount = 0;
trackingState.trackMiss += 1;

if (trackingState.trackMiss > this.missTolerance) {
trackingState.showing = false;
trackingState.trackingMatrix = null;
this.onUpdate && this.onUpdate({type: 'updateMatrix', targetIndex: i, worldMatrix: null});
if (trackingState.trackMiss > this.missTolerance && !this.stayVisible) {
trackingState.showing = false;
trackingState.trackingMatrix = null;
this.onUpdate && this.onUpdate({type: 'updateMatrix', targetIndex: i, worldMatrix: null, targetPresent: false});
}
} else {
trackingState.trackMiss = 0;
}
}

// if showing, then call onUpdate, with world matrix
if (trackingState.showing) {
const worldMatrix = this._glModelViewMatrix(trackingState.currentModelViewTransform, i);
let worldMatrix = [], targetPresent;
if (trackingState.trackMiss <= this.missTolerance && trackingState.currentModelViewTransform) {
worldMatrix = this._glModelViewMatrix(trackingState.currentModelViewTransform, i);
targetPresent = true;
}
else if (this.stayVisible) {
// if target is not present and stayVisible is true, use "fake" worldMatrix that centers objects on the screen
const dimensions = this.markerDimensions[i];
worldMatrix = [
1, 0, 0, 0, 0, 1, 0, 0, -0, -0, 1, 0, -dimensions[0] / 2, -dimensions[1] / 2,
-(dimensions[0] * dimensions[1]) / (100 + this.stayVisibleScale), 1
];
targetPresent = false;
}
trackingState.trackingMatrix = trackingState.filter.filter(Date.now(), worldMatrix);

const clone = [];
for (let j = 0; j < trackingState.trackingMatrix.length; j++) {
clone[j] = trackingState.trackingMatrix[j];
}
this.onUpdate && this.onUpdate({type: 'updateMatrix', targetIndex: i, worldMatrix: clone});
this.onUpdate && this.onUpdate({type: 'updateMatrix', targetIndex: i, worldMatrix: clone, targetPresent: targetPresent});
}
}

Expand Down Expand Up @@ -307,7 +318,7 @@ class Controller {
_glModelViewMatrix(modelViewTransform, targetIndex) {
const height = this.markerDimensions[targetIndex][1];

// Question: can someone verify this interpreation is correct?
// Question: can someone verify this interpreation is correct?
// I'm not very convinced, but more like trial and error and works......
//
// First, opengl has y coordinate system go from bottom to top, while the marker corrdinate goes from top to bottom,
Expand All @@ -318,7 +329,7 @@ class Controller {
// [0 -1 0 h]
// [0 0 -1 0]
// [0 0 0 1]
//
//
// This is tested that if we reverse marker coordinate from bottom to top and estimate the modelViewTransform,
// then the above matrix is not necessary.
//
Expand Down
20 changes: 14 additions & 6 deletions src/image-target/three.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,16 @@ cssScaleDownMatrix.compose(new Vector3(), new Quaternion(), new Vector3(0.001, 0
export class MindARThree {
constructor({
container, imageTargetSrc, maxTrack, uiLoading = "yes", uiScanning = "yes", uiError = "yes",
filterMinCF = null, filterBeta = null, warmupTolerance = null, missTolerance = null
filterMinCF = null, filterBeta = null, warmupTolerance = null, missTolerance = null,
stayVisible = false, stayVisibleScale = 50,
}) {
this.container = container;
this.imageTargetSrc = imageTargetSrc;
this.maxTrack = maxTrack;
this.filterMinCF = filterMinCF;
this.filterBeta = filterBeta;
this.stayVisible = stayVisible;
this.stayVisibleScale = stayVisibleScale;
this.warmupTolerance = warmupTolerance;
this.missTolerance = missTolerance;
this.ui = new UI({ uiLoading, uiScanning, uiError });
Expand All @@ -29,6 +32,7 @@ export class MindARThree {
this.renderer.outputEncoding = sRGBEncoding;
this.renderer.setPixelRatio(window.devicePixelRatio);
this.camera = new PerspectiveCamera();
this.targetPresentBefore = false;
this.anchors = [];

this.renderer.domElement.style.position = 'absolute';
Expand Down Expand Up @@ -121,12 +125,14 @@ export class MindARThree {
inputHeight: video.videoHeight,
filterMinCF: this.filterMinCF,
filterBeta: this.filterBeta,
stayVisible: this.stayVisible,
stayVisibleScale: this.stayVisibleScale,
warmupTolerance: this.warmupTolerance,
missTolerance: this.missTolerance,
maxTrack: this.maxTrack,
onUpdate: (data) => {
if (data.type === 'updateMatrix') {
const { targetIndex, worldMatrix } = data;
const { targetIndex, worldMatrix, targetPresent } = data;

for (let i = 0; i < this.anchors.length; i++) {
if (this.anchors[i].targetIndex === targetIndex) {
Expand All @@ -148,21 +154,23 @@ export class MindARThree {
this.anchors[i].group.matrix = m;
}

if (this.anchors[i].visible && worldMatrix === null) {
if (this.targetPresentBefore === true && targetPresent === false) {
this.anchors[i].visible = false;
if (this.anchors[i].onTargetLost) {
this.anchors[i].onTargetLost();
this.targetPresentBefore = false;
}
}

if (!this.anchors[i].visible && worldMatrix !== null) {
if (!this.targetPresentBefore === false && targetPresent === true) {
this.anchors[i].visible = true;
if (this.anchors[i].onTargetFound) {
this.anchors[i].onTargetFound();
this.targetPresentBefore = true;
}
}

if (worldMatrix !== null) {
if (worldMatrix !== null && !this.stayVisible) {
this.ui.hideScanning();
}
}
Expand Down Expand Up @@ -193,7 +201,7 @@ export class MindARThree {

await this.controller.dummyRun(this.video);
this.ui.hideLoading();
this.ui.showScanning();
if (!this.stayVisible) this.ui.showScanning();

this.controller.processVideo(this.video);
resolve();
Expand Down