import * as THREE from './lib/three.module.js'; import {globalObjectCategory} from './obj_cfg.js'; import {saveWorldList} from "./save.js" import { intersect } from './util.js'; function Annotation(sceneMeta, world, frameInfo){ this.world = world; this.data = this.world.data; //this.coordinatesOffset = this.world.coordinatesOffset; this.boxes_load_time = 0; this.frameInfo = frameInfo; this.modified = false; this.setModified = function(){ this.modified=true; if (pointsGlobalConfig.autoSave) { saveWorldList([this.world]); } }; this.resetModified = function(){this.modified=false;}; this.sort_boxes = function(){ this.boxes = this.boxes.sort(function(x,y){ return x.position.y - y.position.y; }); }; this.findBoxByTrackId = function(id){ if (this.boxes){ let box = this.boxes.find(function(x){ return x.obj_track_id == id; }); return box; } return null; }; this.findIntersectedBoxes = function(box){ return this.boxes.filter(b=>b!=box).filter(b=>intersect(box, b)); }; this.preload = function(on_preload_finished){ this.on_preload_finished = on_preload_finished; this.load_annotation((boxes)=>this.proc_annotation(boxes)); }; this.go_cmd_received = false; this.webglScene = null; this.on_go_finished = null; this.go = function(webglScene, on_go_finished){ this.webglScene = webglScene; if (this.preloaded){ //this.boxes.forEach(b=>this.webglScene.add(b)); if (this.data.cfg.color_obj != "no"){ this.color_boxes(); } if (on_go_finished) on_go_finished(); } else { this.go_cmd_received = true; this.on_go_finished = on_go_finished; } }; // internal funcs below this._afterPreload = function(){ this.preloaded = true; console.log("annotation preloaded"); if (this.on_preload_finished){ this.on_preload_finished(); } if (this.go_cmd_received){ this.go(this.webglScene, this.on_go_finished); } }; this.unload = function(){ if (this.boxes){ this.boxes.forEach((b)=>{ //this.webglGroup.remove(b); if (b.boxEditor) b.boxEditor.detach(); }); } }; this.deleteAll = function(){ this.remove_all_boxes(); }; this.boxToAnn = function(box){ let ann = { psr: { position:{ x: box.position.x, y: box.position.y, z: box.position.z, }, scale:{ x: box.scale.x, y: box.scale.y, z: box.scale.z, }, rotation:{ x:box.rotation.x, y:box.rotation.y, z:box.rotation.z, }, }, obj_type: box.obj_type, obj_id: String(box.obj_track_id), obj_attr: box.obj_attr, //vertices: vertices, }; return ann; }; this.toBoxAnnotations = function(){ let anns = this.boxes.map((b)=>{ //var vertices = psr_to_xyz(b.position, b.scale, b.rotation); let ann = this.boxToAnn(b); if (b.annotator) ann.annotator = b.annotator; if (b.follows) ann.follows = b.follows; return ann; }); anns.sort((a,b)=>a.obj_id- b.obj_id); return anns; }; // to real-world position (no offset) this.ann_to_vector_global = function(box) { let posG = this.world.lidarPosToScene(box.position); let rotG = this.world.lidarRotToScene(box.rotation); return [ posG.x - this.world.coordinatesOffset[0], posG.y-this.world.coordinatesOffset[1], posG.z-this.world.coordinatesOffset[2], rotG.x, rotG.y, rotG.z, box.scale.x, box.scale.y, box.scale.z, ]; }; // real-world position to ann this.vector_global_to_ann = function(v) { let posG = new THREE.Vector3(v[0]+this.world.coordinatesOffset[0], v[1]+this.world.coordinatesOffset[1], v[2]+this.world.coordinatesOffset[2]); let rotG = new THREE.Euler(v[3],v[4],v[5]); let rotL = this.world.sceneRotToLidar(rotG); let posL = this.world.scenePosToLidar(posG); return { position: {x: posL.x, y: posL.y, z: posL.z}, rotation: {x: rotL.x, y: rotL.y, z: rotL.z}, scale: {x: v[6], y: v[7], z: v[8]} }; }; // this.vector_to_ann = function(v){ // return { // position:{ // x:v[0],// + this.coordinatesOffset[0], // y:v[1],// + this.coordinatesOffset[1], // z:v[2],// + this.coordinatesOffset[2], // }, // rotation:{ // x:v[3], // y:v[4], // z:v[5], // }, // scale:{ // x:v[6], // y:v[7], // z:v[8], // }, // }; // }; this.remove_all_boxes = function(){ if (this.boxes){ this.boxes.forEach((b)=>{ this.webglGroup.remove(b); this.world.data.dbg.free(); b.geometry.dispose(); b.material.dispose(); b.world = null; b.boxEditor = null; }); this.boxes = []; } else{ console.error("destroy empty world!") } }; this.new_bbox_cube=function(color){ var h = 0.5; var body = [ //top -h,h,h, h,h,h, h,h,h, h,-h,h, h,-h,h, -h,-h,h, -h,-h,h, -h, h, h, //botom -h,h,-h, h,h,-h, h,h,-h, h,-h,-h, h,-h,-h, -h,-h,-h, -h,-h,-h, -h, h, -h, // vertical lines -h,h,h, -h,h,-h, h,h,h, h,h,-h, h,-h,h, h,-h,-h, -h,-h,h, -h,-h,-h, //direction h, 0, h, 1.5*h, 0, h, //h/2, -h, h+0.1, h, 0, h+0.1, //h/2, h, h+0.1, h, 0, h+0.1, //side direction // h, h/2, h, h, h, 0, // h, h/2, -h, h, h, 0, // h, 0, 0, h, h, 0, ]; this.world.data.dbg.alloc(); var bbox = new THREE.BufferGeometry(); bbox.setAttribute( 'position', new THREE.Float32BufferAttribute(body, 3 ) ); if (!color){ color = 0x00ff00; } /* https://threejs.org/docs/index.html#api/en/materials/LineBasicMaterial linewidth is 1, regardless of set value. */ var material = new THREE.LineBasicMaterial( { color: color, linewidth: 1, opacity: this.data.cfg.box_opacity, transparent: true } ); var box = new THREE.LineSegments( bbox, material ); box.scale.x=1.8; box.scale.y=4.5; box.scale.z=1.5; box.name="bbox"; box.obj_type="car"; //box.computeLineDistances(); return box; }; this.createCuboid = function(pos, scale, rotation, obj_type, track_id, obj_attr){ let mesh = this.new_bbox_cube(parseInt("0x"+globalObjectCategory.get_obj_cfg_by_type(obj_type).color.slice(1))); mesh.position.x = pos.x; mesh.position.y = pos.y; mesh.position.z = pos.z; mesh.scale.x = scale.x; mesh.scale.y = scale.y; mesh.scale.z = scale.z; mesh.rotation.x = rotation.x; mesh.rotation.y = rotation.y; mesh.rotation.z = rotation.z; mesh.obj_track_id = track_id; //tracking id mesh.obj_type = obj_type; mesh.obj_attr = obj_attr; mesh.obj_local_id = this.get_new_box_local_id(); mesh.world = this.world; return mesh; }; /* pos: offset position, after transformed */ this.add_box=function(pos, scale, rotation, obj_type, track_id, obj_attr){ let mesh = this.createCuboid(pos, scale, rotation, obj_type, track_id, obj_attr) this.boxes.push(mesh); this.sort_boxes(); this.webglGroup.add(mesh); return mesh; }; this.load_box = function(box){ this.webglGroup.add(box); }; this.unload_box = function(box){ this.webglGroup.remove(box); }; this.remove_box=function(box){ this.world.data.dbg.free(); box.geometry.dispose(); box.material.dispose(); //selected_box.dispose(); this.boxes = this.boxes.filter(function(x){return x !=box;}); }; this.set_box_opacity=function(box_opacity){ this.boxes.forEach(function(x){ x.material.opacity = box_opacity; }); }; this.translate_box_position=function(pos, theta, axis, delta){ switch (axis){ case 'x': pos.x += delta*Math.cos(theta); pos.y += delta*Math.sin(theta); break; case 'y': pos.x += delta*Math.cos(Math.PI/2 + theta); pos.y += delta*Math.sin(Math.PI/2 + theta); break; case 'z': pos.z += delta; break; } }; this.find_boxes_inside_rect = function(x,y,w,h, camera){ let selected_boxes_by_rect = []; if (!this.boxes) return selected_boxes_by_rect; var p = new THREE.Vector3(); for (var i=0; i< this.boxes.length; i++){ let box_center = this.boxes[i].position; let pw = this.world.lidarPosToScene(box_center); p.set(pw.x, pw.y, pw.z); p.project(camera); p.x = p.x/p.z; p.y = p.y/p.z; //console.log(p); if ((p.x > x) && (p.x < x+w) && (p.y>y) && (p.ythis.webglGroup.add(b)); this.world.webglGroup.add(this.webglGroup); this.boxes_load_time = new Date().getTime(); console.log(this.boxes_load_time, this.frameInfo.scene, this.frameInfo.frame, "loaded boxes ", this.boxes_load_time - this.create_time, "ms"); this.sort_boxes(); this._afterPreload(); }; this.load_annotation=function(on_load){ if (this.data.cfg.disableLabels){ on_load([]); }else { var xhr = new XMLHttpRequest(); // we defined the xhr var _self = this; xhr.onreadystatechange = function () { if (this.readyState != 4) return; if (this.status == 200) { let ann = _self.frameInfo.anno_to_boxes(this.responseText); on_load(ann); } // end of state change: it can be after some time (async) }; xhr.open('GET', "/load_annotation"+"?scene="+this.frameInfo.scene+"&frame="+this.frameInfo.frame, true); xhr.send(); } }; this.reloadAnnotation=function(done){ this.load_annotation(ann=>{ this.reapplyAnnotation(ann, done); }); }; this.reapplyAnnotation = function(boxes, done){ // these boxes haven't attached a world //boxes = this.transformBoxesByOffset(boxes); // mark all old boxes this.boxes.forEach(b=>{b.delete=true;}); let pendingBoxList=[]; boxes.forEach(nb=>{ // nb is annotation format, not a true box let old_box = this.boxes.find(function(x){ return x.obj_track_id == nb.obj_id && x.obj_track_id != "" && nb.obj_id != "" && x.obj_type == nb.obj_type;; }); if (old_box){ // found // update psr delete old_box.delete; // unmark delete flag old_box.position.set(nb.psr.position.x, nb.psr.position.y, nb.psr.position.z); old_box.scale.set(nb.psr.scale.x, nb.psr.scale.y, nb.psr.scale.z); old_box.rotation.set(nb.psr.rotation.x, nb.psr.rotation.y, nb.psr.rotation.z); old_box.obj_attr = nb.obj_attr; old_box.annotator = nb.annotator; old_box.changed=false; // clear changed flag. }else{ // not found let box=this.createOneBoxByAnn(nb); pendingBoxList.push(box); } }); // delete removed let toBeDelBoxes = this.boxes.filter(b=>b.delete); toBeDelBoxes.forEach(b=>{ if (b.boxEditor){ b.boxEditor.detach("donthide"); } this.webglGroup.remove(b); this.remove_box(b); }) pendingBoxList.forEach(b=>{ this.boxes.push(b); }) //todo, restore point color //todo, update imagecontext, selected box, ... //refer to normal delete operation // re-color again this.world.lidar.recolor_all_points(); this.color_boxes(); // add new boxes pendingBoxList.forEach(b=>{ this.webglGroup.add(b); }) this.resetModified(); if (done) done(); } this.createOneBoxByAnn = function(annotation){ let b = annotation; let mesh = this.createCuboid(b.psr.position, b.psr.scale, b.psr.rotation, b.obj_type, b.obj_id, b.obj_attr); if (b.annotator){ mesh.annotator = b.annotator; } if (b.follows) mesh.follows = b.follows; return mesh; }; this.createBoxes = function(annotations){ return annotations.map((b)=>{ return this.createOneBoxByAnn(b); }); }; this.box_local_id = 0; this.get_new_box_local_id=function(){ var ret = this.box_local_id; this.box_local_id+=1; return ret; }; this.color_box = function(box) { if (this.data.cfg.color_obj == "category" || this.data.cfg.color_obj == "no") { let color = globalObjectCategory.get_color_by_category(box.obj_type); box.material.color.r=color.x; box.material.color.g=color.y; box.material.color.b=color.z; } else { let color = globalObjectCategory.get_color_by_id(box.obj_track_id); box.material.color.r=color.x; box.material.color.g=color.y; box.material.color.b=color.z; } } this.color_boxes = function() { this.boxes.forEach(box=>{ this.color_box(box); }) } } export{Annotation}