diff --git a/blender/multi-cam-object-extended.blend b/blender/multi-cam-object-extended.blend index 6366db2..23b655c 100644 Binary files a/blender/multi-cam-object-extended.blend and b/blender/multi-cam-object-extended.blend differ diff --git a/blender/multi-cam-object-extended.blend1 b/blender/multi-cam-object-extended.blend1 index f95feac..da9d735 100644 Binary files a/blender/multi-cam-object-extended.blend1 and b/blender/multi-cam-object-extended.blend1 differ diff --git a/images/references/bunny_multi_cam/bunny_s0.png b/images/references/bunny_multi_cam/bunny_s0.png new file mode 100644 index 0000000..e7b95d3 Binary files /dev/null and b/images/references/bunny_multi_cam/bunny_s0.png differ diff --git a/images/references/bunny_multi_cam/bunny_s1.png b/images/references/bunny_multi_cam/bunny_s1.png new file mode 100644 index 0000000..a3be544 Binary files /dev/null and b/images/references/bunny_multi_cam/bunny_s1.png differ diff --git a/images/references/bunny_multi_cam/bunny_s2.png b/images/references/bunny_multi_cam/bunny_s2.png new file mode 100644 index 0000000..dc96ea4 Binary files /dev/null and b/images/references/bunny_multi_cam/bunny_s2.png differ diff --git a/images/references/bunny_multi_cam/bunny_s3.png b/images/references/bunny_multi_cam/bunny_s3.png new file mode 100644 index 0000000..f02604f Binary files /dev/null and b/images/references/bunny_multi_cam/bunny_s3.png differ diff --git a/images/references/bunny_multi_cam/bunny_s4.png b/images/references/bunny_multi_cam/bunny_s4.png new file mode 100644 index 0000000..0cdf728 Binary files /dev/null and b/images/references/bunny_multi_cam/bunny_s4.png differ diff --git a/images/references/bunny_multi_cam/bunny_s5.png b/images/references/bunny_multi_cam/bunny_s5.png new file mode 100644 index 0000000..a396f83 Binary files /dev/null and b/images/references/bunny_multi_cam/bunny_s5.png differ diff --git a/images/references/bunny_multi_cam/bunny_s6.png b/images/references/bunny_multi_cam/bunny_s6.png new file mode 100644 index 0000000..166014e Binary files /dev/null and b/images/references/bunny_multi_cam/bunny_s6.png differ diff --git a/images/references/bunny_multi_cam/bunny_s7.png b/images/references/bunny_multi_cam/bunny_s7.png new file mode 100644 index 0000000..63e5a58 Binary files /dev/null and b/images/references/bunny_multi_cam/bunny_s7.png differ diff --git a/images/references/bunny_multi_cam/constant_emitter/smoothed/bunny_s0.png b/images/references/bunny_multi_cam/constant_emitter/smoothed/bunny_s0.png new file mode 100644 index 0000000..283a771 Binary files /dev/null and b/images/references/bunny_multi_cam/constant_emitter/smoothed/bunny_s0.png differ diff --git a/images/references/bunny_multi_cam/constant_emitter/smoothed/bunny_s1.png b/images/references/bunny_multi_cam/constant_emitter/smoothed/bunny_s1.png new file mode 100644 index 0000000..46a57ca Binary files /dev/null and b/images/references/bunny_multi_cam/constant_emitter/smoothed/bunny_s1.png differ diff --git a/images/references/bunny_multi_cam/constant_emitter/smoothed/bunny_s2.png b/images/references/bunny_multi_cam/constant_emitter/smoothed/bunny_s2.png new file mode 100644 index 0000000..00f84e1 Binary files /dev/null and b/images/references/bunny_multi_cam/constant_emitter/smoothed/bunny_s2.png differ diff --git a/images/references/bunny_multi_cam/constant_emitter/smoothed/bunny_s3.png b/images/references/bunny_multi_cam/constant_emitter/smoothed/bunny_s3.png new file mode 100644 index 0000000..e7e82b5 Binary files /dev/null and b/images/references/bunny_multi_cam/constant_emitter/smoothed/bunny_s3.png differ diff --git a/images/references/bunny_multi_cam/constant_emitter/smoothed/bunny_s4.png b/images/references/bunny_multi_cam/constant_emitter/smoothed/bunny_s4.png new file mode 100644 index 0000000..64bde5e Binary files /dev/null and b/images/references/bunny_multi_cam/constant_emitter/smoothed/bunny_s4.png differ diff --git a/images/references/bunny_multi_cam/constant_emitter/smoothed/bunny_s5.png b/images/references/bunny_multi_cam/constant_emitter/smoothed/bunny_s5.png new file mode 100644 index 0000000..af9bba0 Binary files /dev/null and b/images/references/bunny_multi_cam/constant_emitter/smoothed/bunny_s5.png differ diff --git a/images/references/bunny_multi_cam/constant_emitter/smoothed/bunny_s6.png b/images/references/bunny_multi_cam/constant_emitter/smoothed/bunny_s6.png new file mode 100644 index 0000000..ac55ab7 Binary files /dev/null and b/images/references/bunny_multi_cam/constant_emitter/smoothed/bunny_s6.png differ diff --git a/images/references/bunny_multi_cam/constant_emitter/smoothed/bunny_s7.png b/images/references/bunny_multi_cam/constant_emitter/smoothed/bunny_s7.png new file mode 100644 index 0000000..e7b9247 Binary files /dev/null and b/images/references/bunny_multi_cam/constant_emitter/smoothed/bunny_s7.png differ diff --git a/images/references/bunny_multi_cam/volume_case/test-0.png b/images/references/bunny_multi_cam/constant_emitter/test-0.png similarity index 100% rename from images/references/bunny_multi_cam/volume_case/test-0.png rename to images/references/bunny_multi_cam/constant_emitter/test-0.png diff --git a/images/references/bunny_multi_cam/volume_case/test-1.png b/images/references/bunny_multi_cam/constant_emitter/test-1.png similarity index 100% rename from images/references/bunny_multi_cam/volume_case/test-1.png rename to images/references/bunny_multi_cam/constant_emitter/test-1.png diff --git a/images/references/bunny_multi_cam/volume_case/test-2.png b/images/references/bunny_multi_cam/constant_emitter/test-2.png similarity index 100% rename from images/references/bunny_multi_cam/volume_case/test-2.png rename to images/references/bunny_multi_cam/constant_emitter/test-2.png diff --git a/images/references/bunny_multi_cam/volume_case/test-3.png b/images/references/bunny_multi_cam/constant_emitter/test-3.png similarity index 100% rename from images/references/bunny_multi_cam/volume_case/test-3.png rename to images/references/bunny_multi_cam/constant_emitter/test-3.png diff --git a/images/references/bunny_multi_cam/volume_case/test-4.png b/images/references/bunny_multi_cam/constant_emitter/test-4.png similarity index 100% rename from images/references/bunny_multi_cam/volume_case/test-4.png rename to images/references/bunny_multi_cam/constant_emitter/test-4.png diff --git a/images/references/bunny_multi_cam/volume_case/test-5.png b/images/references/bunny_multi_cam/constant_emitter/test-5.png similarity index 100% rename from images/references/bunny_multi_cam/volume_case/test-5.png rename to images/references/bunny_multi_cam/constant_emitter/test-5.png diff --git a/images/references/bunny_multi_cam/volume_case/test-6.png b/images/references/bunny_multi_cam/constant_emitter/test-6.png similarity index 100% rename from images/references/bunny_multi_cam/volume_case/test-6.png rename to images/references/bunny_multi_cam/constant_emitter/test-6.png diff --git a/images/references/bunny_multi_cam/volume_case/test-7.png b/images/references/bunny_multi_cam/constant_emitter/test-7.png similarity index 100% rename from images/references/bunny_multi_cam/volume_case/test-7.png rename to images/references/bunny_multi_cam/constant_emitter/test-7.png diff --git a/images/references/bunny_multi_cam/bunny_sensor_0.png b/images/references/bunny_multi_cam/with_envmap/bunny_sensor_0.png similarity index 100% rename from images/references/bunny_multi_cam/bunny_sensor_0.png rename to images/references/bunny_multi_cam/with_envmap/bunny_sensor_0.png diff --git a/images/references/bunny_multi_cam/bunny_sensor_1.png b/images/references/bunny_multi_cam/with_envmap/bunny_sensor_1.png similarity index 100% rename from images/references/bunny_multi_cam/bunny_sensor_1.png rename to images/references/bunny_multi_cam/with_envmap/bunny_sensor_1.png diff --git a/images/references/bunny_multi_cam/bunny_sensor_2.png b/images/references/bunny_multi_cam/with_envmap/bunny_sensor_2.png similarity index 100% rename from images/references/bunny_multi_cam/bunny_sensor_2.png rename to images/references/bunny_multi_cam/with_envmap/bunny_sensor_2.png diff --git a/images/references/bunny_multi_cam/bunny_sensor_3.png b/images/references/bunny_multi_cam/with_envmap/bunny_sensor_3.png similarity index 100% rename from images/references/bunny_multi_cam/bunny_sensor_3.png rename to images/references/bunny_multi_cam/with_envmap/bunny_sensor_3.png diff --git a/images/references/bunny_multi_cam/bunny_sensor_4.png b/images/references/bunny_multi_cam/with_envmap/bunny_sensor_4.png similarity index 100% rename from images/references/bunny_multi_cam/bunny_sensor_4.png rename to images/references/bunny_multi_cam/with_envmap/bunny_sensor_4.png diff --git a/images/references/bunny_multi_cam/bunny_sensor_5.png b/images/references/bunny_multi_cam/with_envmap/bunny_sensor_5.png similarity index 100% rename from images/references/bunny_multi_cam/bunny_sensor_5.png rename to images/references/bunny_multi_cam/with_envmap/bunny_sensor_5.png diff --git a/images/references/bunny_multi_cam/bunny_sensor_6.png b/images/references/bunny_multi_cam/with_envmap/bunny_sensor_6.png similarity index 100% rename from images/references/bunny_multi_cam/bunny_sensor_6.png rename to images/references/bunny_multi_cam/with_envmap/bunny_sensor_6.png diff --git a/images/references/bunny_multi_cam/bunny_sensor_7.png b/images/references/bunny_multi_cam/with_envmap/bunny_sensor_7.png similarity index 100% rename from images/references/bunny_multi_cam/bunny_sensor_7.png rename to images/references/bunny_multi_cam/with_envmap/bunny_sensor_7.png diff --git a/images/references/bunny_multi_cam/with_envmap/bunny_smoothed/bunny_s0.png b/images/references/bunny_multi_cam/with_envmap/bunny_smoothed/bunny_s0.png new file mode 100644 index 0000000..b538ed2 Binary files /dev/null and b/images/references/bunny_multi_cam/with_envmap/bunny_smoothed/bunny_s0.png differ diff --git a/images/references/bunny_multi_cam/with_envmap/bunny_smoothed/bunny_s1.png b/images/references/bunny_multi_cam/with_envmap/bunny_smoothed/bunny_s1.png new file mode 100644 index 0000000..5d84b9c Binary files /dev/null and b/images/references/bunny_multi_cam/with_envmap/bunny_smoothed/bunny_s1.png differ diff --git a/images/references/bunny_multi_cam/with_envmap/bunny_smoothed/bunny_s2.png b/images/references/bunny_multi_cam/with_envmap/bunny_smoothed/bunny_s2.png new file mode 100644 index 0000000..458ec34 Binary files /dev/null and b/images/references/bunny_multi_cam/with_envmap/bunny_smoothed/bunny_s2.png differ diff --git a/images/references/bunny_multi_cam/with_envmap/bunny_smoothed/bunny_s3.png b/images/references/bunny_multi_cam/with_envmap/bunny_smoothed/bunny_s3.png new file mode 100644 index 0000000..d3cd15e Binary files /dev/null and b/images/references/bunny_multi_cam/with_envmap/bunny_smoothed/bunny_s3.png differ diff --git a/images/references/bunny_multi_cam/with_envmap/bunny_smoothed/bunny_s4.png b/images/references/bunny_multi_cam/with_envmap/bunny_smoothed/bunny_s4.png new file mode 100644 index 0000000..429f8f4 Binary files /dev/null and b/images/references/bunny_multi_cam/with_envmap/bunny_smoothed/bunny_s4.png differ diff --git a/images/references/bunny_multi_cam/with_envmap/bunny_smoothed/bunny_s5.png b/images/references/bunny_multi_cam/with_envmap/bunny_smoothed/bunny_s5.png new file mode 100644 index 0000000..d75272e Binary files /dev/null and b/images/references/bunny_multi_cam/with_envmap/bunny_smoothed/bunny_s5.png differ diff --git a/images/references/bunny_multi_cam/with_envmap/bunny_smoothed/bunny_s6.png b/images/references/bunny_multi_cam/with_envmap/bunny_smoothed/bunny_s6.png new file mode 100644 index 0000000..aec9027 Binary files /dev/null and b/images/references/bunny_multi_cam/with_envmap/bunny_smoothed/bunny_s6.png differ diff --git a/images/references/bunny_multi_cam/with_envmap/bunny_smoothed/bunny_s7.png b/images/references/bunny_multi_cam/with_envmap/bunny_smoothed/bunny_s7.png new file mode 100644 index 0000000..63bcf04 Binary files /dev/null and b/images/references/bunny_multi_cam/with_envmap/bunny_smoothed/bunny_s7.png differ diff --git a/scenes/material-preview/bunny-multi-sensor/bunny_rec/bunny_vol_rec_128.vol b/scenes/material-preview/bunny-multi-sensor/bunny_rec/bunny_vol_rec_128.vol new file mode 100644 index 0000000..ed9fc0a Binary files /dev/null and b/scenes/material-preview/bunny-multi-sensor/bunny_rec/bunny_vol_rec_128.vol differ diff --git a/scenes/material-preview/bunny-multi-sensor/bunny_rec/end-result/meshes/Plane.ply b/scenes/material-preview/bunny-multi-sensor/bunny_rec/end-result/meshes/Plane.ply new file mode 100644 index 0000000..7aa22da Binary files /dev/null and b/scenes/material-preview/bunny-multi-sensor/bunny_rec/end-result/meshes/Plane.ply differ diff --git a/scenes/material-preview/bunny-multi-sensor/bunny_rec/end-result/meshes/bunny_rec_446.ply b/scenes/material-preview/bunny-multi-sensor/bunny_rec/end-result/meshes/bunny_rec_446.ply new file mode 100644 index 0000000..fd26e13 Binary files /dev/null and b/scenes/material-preview/bunny-multi-sensor/bunny_rec/end-result/meshes/bunny_rec_446.ply differ diff --git a/scenes/material-preview/bunny-multi-sensor/bunny_rec/end-result/multi-cam-object-extended.xml b/scenes/material-preview/bunny-multi-sensor/bunny_rec/end-result/multi-cam-object-extended.xml new file mode 100644 index 0000000..7074490 --- /dev/null +++ b/scenes/material-preview/bunny-multi-sensor/bunny_rec/end-result/multi-cam-object-extended.xml @@ -0,0 +1,239 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/scenes/material-preview/bunny-multi-sensor/bunny_rec/meshes/bunny_rec.ply b/scenes/material-preview/bunny-multi-sensor/bunny_rec/meshes/bunny_rec.ply deleted file mode 100644 index 376c43e..0000000 Binary files a/scenes/material-preview/bunny-multi-sensor/bunny_rec/meshes/bunny_rec.ply and /dev/null differ diff --git a/scenes/material-preview/bunny-multi-sensor/bunny_rec/meshes/bunny_rec_128.ply b/scenes/material-preview/bunny-multi-sensor/bunny_rec/meshes/bunny_rec_128.ply new file mode 100644 index 0000000..7f5a926 Binary files /dev/null and b/scenes/material-preview/bunny-multi-sensor/bunny_rec/meshes/bunny_rec_128.ply differ diff --git a/scenes/material-preview/bunny-multi-sensor/bunny_rec/meshes/bunny_rec_2.ply b/scenes/material-preview/bunny-multi-sensor/bunny_rec/meshes/bunny_rec_2.ply deleted file mode 100644 index e8e7c5e..0000000 Binary files a/scenes/material-preview/bunny-multi-sensor/bunny_rec/meshes/bunny_rec_2.ply and /dev/null differ diff --git a/scenes/material-preview/bunny-multi-sensor/bunny_rec/multi-cam-object-extended.xml b/scenes/material-preview/bunny-multi-sensor/bunny_rec/multi-cam-object-extended.xml index 057538e..9cd8aec 100644 --- a/scenes/material-preview/bunny-multi-sensor/bunny_rec/multi-cam-object-extended.xml +++ b/scenes/material-preview/bunny-multi-sensor/bunny_rec/multi-cam-object-extended.xml @@ -36,7 +36,7 @@ - + @@ -61,7 +61,7 @@ - + @@ -85,7 +85,7 @@ - + @@ -109,7 +109,7 @@ - + @@ -134,7 +134,7 @@ - + @@ -158,6 +158,9 @@ + + + @@ -181,7 +184,7 @@ - + @@ -206,7 +209,7 @@ - + @@ -219,9 +222,7 @@ - - - + @@ -247,7 +248,7 @@ - + diff --git a/scripts/grid_volume_generator.py b/scripts/grid_volume_generator.py index 2a64920..5b387b4 100644 --- a/scripts/grid_volume_generator.py +++ b/scripts/grid_volume_generator.py @@ -3,8 +3,8 @@ mi.set_variant("cuda_ad_rgb") -grid_res = 256 +grid_res = 4 mi.VolumeGrid( - dr.full(mi.TensorXf, 1, (grid_res, grid_res, grid_res, 1)) -).write("full_volume.vol") + dr.full(mi.TensorXf, 0.002, (grid_res, grid_res, grid_res, 1)) +).write("empty.vol") diff --git a/src/constants.py b/src/constants.py index 76b1a75..ec1bd20 100644 --- a/src/constants.py +++ b/src/constants.py @@ -71,7 +71,7 @@ CLOSE_STATUS_STR = "CLOSE" INITIAL_STATUS_STR = "INITIAL" RENDER_STATUS_STR = "RENDER" -MARGIN_PERCENTAGE_LABEL = "Margin per update" +MARGIN_PERCENTAGE_LABEL = "Margin per penalty" MARGIN_PENALTY_LABEL = "Margin Penalty" NONE_STR = "None" EXPONENTIAL_DECAY_STR = "Exponential Decay" @@ -119,7 +119,10 @@ CLEARCOAT_GLOSS_PATTERN: re.Pattern = re.compile(r".*\.clearcoat_gloss") # special pattern: activates custom optimizer -GRID_VOLUME_TO_OPTIMIZER_PATTERN: re.Pattern = re.compile(r"grid_volume_data.sigma_t.data") +GRID_VOLUME_DATA_SIGMA_T_STR: str = "grid_volume_data.sigma_t.data" +GRID_VOLUME_TO_OPTIMIZER_PATTERN: re.Pattern = re.compile( + GRID_VOLUME_DATA_SIGMA_T_STR +) # max Index of refraction value is taken from # https://en.wikipedia.org/wiki/List_of_refractive_indices @@ -196,8 +199,9 @@ See also: https://mitsuba.readthedocs.io/en/stable/src/generated/plugins_shapes.html# """ VERTEX_POSITIONS_PATTERN: re.Pattern = re.compile(r".*\.vertex_positions") -# TODO: what can be the min/max vertex position value? MAX_VERTEX_POSITION_VALUE: mi.Point3f = mi.Point3f(100.0, 100.0, 100.0) +VERTEX_NORMALS_PATTERN: re.Pattern = re.compile(r".*\.vertex_normals") +MAX_VERTEX_NORMALS_VALUE: mi.Point3f = mi.Point3f(100.0, 100.0, 100.0) """ Combine Patterns """ SUPPORTED_MITSUBA_PARAMETER_PATTERNS: list = [ @@ -236,6 +240,7 @@ VERTEX_COLOR_PATTERN, ALBEDO_DATA_PATTERN, VERTEX_POSITIONS_PATTERN, + VERTEX_NORMALS_PATTERN, ] PATTERNS_INTRODUCE_DISCONTINUITIES: list = [ # see also parameter 'D' flags https://mitsuba.readthedocs.io/en/stable/src/generated/plugins_bsdfs.html#technical-details @@ -252,10 +257,11 @@ CLEARCOAT_GLOSS_PATTERN, PHASE_G_PATTERN, VERTEX_POSITIONS_PATTERN, + VERTEX_NORMALS_PATTERN, ] PATTERNS_REQUIRE_VOLUMETRIC_INTEGRATOR = [ALBEDO_PATTERN, SIGMA_T_PATTERN] -OPTIMIZER_PATTERNS: list = [GRID_VOLUME_TO_OPTIMIZER_PATTERN] +OPTIMIZER_PATTERNS: list = [GRID_VOLUME_TO_OPTIMIZER_PATTERN] ### Constant dictionary: key: Pattern, value: default min and max clamp value def getDefaultLegalValues(pattern: re.Pattern) -> tuple([int, int]): @@ -277,7 +283,10 @@ def getDefaultLegalValues(pattern: re.Pattern) -> tuple([int, int]): result = (DEFAULT_MIN_CLAMP_VALUE, MAX_SCALE_VALUE) elif pattern is RADIANCE_PATTERN: result = (DEFAULT_MIN_CLAMP_VALUE, MAX_RADIANCE_VALUE) - elif pattern is VERTEX_POSITIONS_PATTERN: + elif ( + pattern is VERTEX_POSITIONS_PATTERN + or pattern is VERTEX_NORMALS_PATTERN + ): # currently arbitrary: more or less user responsibility result = (-MAX_VERTEX_POSITION_VALUE, MAX_VERTEX_POSITION_VALUE) diff --git a/src/material_optimizer_controller.py b/src/material_optimizer_controller.py index 8f5c7c3..25332f7 100644 --- a/src/material_optimizer_controller.py +++ b/src/material_optimizer_controller.py @@ -451,8 +451,10 @@ def onOptimizationParamChanged(self, row, col): return False, None, None, None try: - # special case: vertex positions - if VERTEX_POSITIONS_PATTERN.search(paramRow): + # special case: vertex_ + if VERTEX_POSITIONS_PATTERN.search( + paramRow + ) or VERTEX_NORMALS_PATTERN.search(paramRow): if ( paramCol == COLUMN_LABEL_MIN_CLAMP_LABEL or paramCol == COLUMN_LABEL_MAX_CLAMP_LABEL @@ -650,7 +652,13 @@ def onOutputBtnPressed( mi.util.write_bitmap(outputTextureFileName, v) elif type(v) is mi.Float: floatArray = [f for f in v] - if len(v) > 1: + if VERTEX_POSITIONS_PATTERN.search(k): + if any( + k in checkedRow + for checkedRow in self.getCheckedRows() + ): + self.outputPlyMesh(outputFileDir, k) + elif len(v) > 1: # special output: multi dimensional mi.Float as numpy array outputNDimArrayFileName = ( outputFileDir @@ -701,6 +709,30 @@ def onOutputBtnPressed( f"The output can be found at: '{absPath}'" ) + def outputPlyMesh(self, outputFileDir, k): + if ".vertex_positions" not in k: + return + + parentStr = k.replace(".vertex_positions", "") + mesh = mi.Mesh( + "optimized_mesh", + vertex_count=self.model.sceneParams[parentStr + ".vertex_count"], + face_count=self.model.sceneParams[parentStr + ".face_count"], + has_vertex_normals=True, + has_vertex_texcoords=False, + ) + mesh_params = mi.traverse(mesh) + mesh_params["vertex_positions"] = dr.ravel(self.model.sceneParams[k]) + mesh_params["vertex_normals"] = dr.ravel( + self.model.sceneParams[parentStr + ".vertex_normals"] + ) + mesh_params["faces"] = dr.ravel( + self.model.sceneParams[parentStr + ".faces"] + ) + print(mesh_params.update()) + outputMeshName = outputFileDir + f"//optimized_mesh_{k}.ply" + mesh.write_ply(outputMeshName) + def prepareFigureAndOutputEachElem( self, lossHist, diff --git a/src/material_optimizer_model.py b/src/material_optimizer_model.py index 898c145..ab55ef8 100644 --- a/src/material_optimizer_model.py +++ b/src/material_optimizer_model.py @@ -200,25 +200,34 @@ def copyMitsubaTypeIfSupported(self, result, k, v): elif vType is mi.TensorXf: result[k] = mi.TensorXf(v) - def updateAfterStep(self, opts, params): + def updateAfterStep(self, opts, params, it=0): for opt in opts: # Optimizer: take a gradient descent step opt.step() for key in opt.keys(): - self.ensureLegalParamValues(opt, key) + self.ensureLegalParamValues(opt, key, it) # Update the scene state to the new optimized values params.update(opt) - def ensureLegalParamValues(self, opt, key): + def ensureLegalParamValues(self, opt, key, it): # Post-process the optimized parameters to ensure legal values - if VERTEX_POSITIONS_PATTERN.search(key): + if VERTEX_POSITIONS_PATTERN.search( + key + ) or VERTEX_NORMALS_PATTERN.search(key): self.ensureLegalVertexPositions(opt, key) elif GRID_VOLUME_TO_OPTIMIZER_PATTERN.search(key): - opt[key] = dr.clamp( - opt[key], - 0.0, - self.optimizationParams[key][COLUMN_LABEL_MAX_CLAMP_LABEL], - ) + if it > 32: + opt[key] = dr.select( + opt[key] <= dr.min(opt[key])[0] + 0.3 * np.std(opt[key]), + 0.0, + 1.0, + ) + else: + opt[key] = dr.clamp( + opt[key], + 0.0, + self.optimizationParams[key][COLUMN_LABEL_MAX_CLAMP_LABEL], + ) else: opt[key] = dr.clamp( opt[key], @@ -389,12 +398,12 @@ def minIdxInDrList(lis: list): minValue = dr.min(lis) return lis.index(minValue) - def computeLoss(self, sensor: mi.Sensor, seed: int = 0): + def computeLoss(self, sensor: mi.Sensor, spp: int, seed: int = 0): # Perform a (noisy) differentiable rendering of the scene image = self.render( self.scene, sensor=sensor, - spp=self.samplesPerPixel, + spp=spp, params=self.sceneParams, seed=seed, ) @@ -411,7 +420,7 @@ def computeLoss(self, sensor: mi.Sensor, seed: int = 0): image2 = self.render( self.scene, sensor=sensor, - spp=self.samplesPerPixel, + spp=spp, params=self.sceneParams, seed=seed + 1, ) @@ -496,14 +505,6 @@ def computeMargin(self, sensorLossOnPriorIt): return self.marginPercentage return sensorLossOnPriorIt * self.marginPercentage - def increaseFailAndResetOptIfNecessary(self, opts, tmpFailTracker): - tmpFailTracker += 1 - if tmpFailTracker % 5 == 0: - for opt in opts: - for param in opt.variables.keys(): - logging.info(f"Reset optimization for {param}") - opt.reset(param) - def penalizeLearningRates(self, opts, it): if self.marginPenalty == EXPONENTIAL_DECAY_STR: for opt in opts: @@ -647,33 +648,27 @@ def optimizationLoop( totalLoss = 0.0 for sensorIdx, sensor in enumerate(sensors): currentLoss, diffRender = self.model.computeLoss( - sensor=sensor, seed=it + sensor=sensor, spp=self.model.samplesPerPixel, seed=it ) totalLoss += currentLoss[0] - if it == 0: - dr.backward(currentLoss) - tmpLossTracker[sensorIdx].append(currentLoss[0]) - self.model.updateAfterStep(opts, self.model.sceneParams) - else: + dr.backward(currentLoss) + self.model.updateAfterStep(opts, self.model.sceneParams) + if it > 0: sensorLossOnPriorIt = tmpLossTracker[sensorIdx][-1] margin = self.model.computeMargin(sensorLossOnPriorIt) - if currentLoss[0] < sensorLossOnPriorIt + margin: - dr.backward(currentLoss) - self.model.updateAfterStep( - opts, self.model.sceneParams + if currentLoss[0] > sensorLossOnPriorIt + margin: + msg = f"Current loss ({currentLoss[0]:.3f}) is larger" + msg += f" than loss on prior iteration plus some" + msg += ( + f" margin ({(sensorLossOnPriorIt + margin):.3f})" ) - tmpLossTracker[sensorIdx].append(currentLoss[0]) - else: - # no changes if total loss exceeds the prior loss + margin - self.model.increaseFailAndResetOptIfNecessary( - opts, tmpFailTracker - ) - self.model.penalizeLearningRates(opts, it) - totalLoss = lossHist[-1] - msg = "Skipped backpropopagation and scene update for" - msg += f" sensor index {sensorIdx} at iteration {it}." + msg += f" on sensor index {sensorIdx}." logging.info(msg) + tmpFailTracker += 1 + if tmpFailTracker % 3 == 0: + self.model.penalizeLearningRates(opts, it) + tmpLossTracker[sensorIdx].append(currentLoss[0]) self.model.updatePlotProgress( showDiffRender, it, itPercent, diffRender, totalLoss @@ -688,9 +683,7 @@ def optimizationLoop( break if showDiffRender: - showDiffRender( - diffRender=None, plotStatus=CLOSE_STATUS_STR - ) + showDiffRender(diffRender=None, plotStatus=CLOSE_STATUS_STR) optLog = self.model.endOptimizationLog( sceneParamsHist, startTime, optLog ) @@ -718,6 +711,7 @@ def optimizationLoop( self.model.initPlotProgress(showDiffRender) step_size = 16 it = 0 + spp = self.model.samplesPerPixel while it < self.model.iterationCount: for step in range(step_size): @@ -728,34 +722,27 @@ def optimizationLoop( totalLoss = 0.0 for sensorIdx, sensor in enumerate(sensors): currentLoss, diffRender = self.model.computeLoss( - sensor=sensor, seed=it + sensor=sensor, spp=spp, seed=it ) totalLoss += currentLoss[0] - if it == 0: - dr.backward(currentLoss) - tmpLossTracker[sensorIdx].append(currentLoss[0]) - self.model.updateAfterStep( - opts, self.model.sceneParams - ) - else: + dr.backward(currentLoss) + self.model.updateAfterStep( + opts, self.model.sceneParams, it + ) + if it > 0: sensorLossOnPriorIt = tmpLossTracker[sensorIdx][-1] margin = self.model.computeMargin(sensorLossOnPriorIt) - if currentLoss[0] < sensorLossOnPriorIt + margin: - dr.backward(currentLoss) - self.model.updateAfterStep( - opts, self.model.sceneParams - ) - tmpLossTracker[sensorIdx].append(currentLoss[0]) - else: - dr.backward(currentLoss) - self.model.updateAfterStep( - opts, self.model.sceneParams - ) - tmpLossTracker[sensorIdx].append(currentLoss[0]) + if currentLoss[0] > sensorLossOnPriorIt + margin: + msg = f"Current loss ({currentLoss[0]}) is larger" + msg += f" than loss on prior iteration plus some" + msg += f" margin ({sensorLossOnPriorIt + margin})" + msg += f" on sensor index {sensorIdx}." + logging.info(msg) tmpFailTracker += 1 if tmpFailTracker % 3 == 0: self.model.penalizeLearningRates(opts, it) + tmpLossTracker[sensorIdx].append(currentLoss[0]) self.model.updatePlotProgress( showDiffRender, it, itPercent, diffRender, totalLoss @@ -781,20 +768,25 @@ def optimizationLoop( ) return lossHist, sceneParamsHist, optLog - key = "grid_volume_data.sigma_t.data" - grid_res = min(256, self.model.sceneParams[key].shape[0] * 2) + grid_res = min( + 128, + self.model.sceneParams[GRID_VOLUME_DATA_SIGMA_T_STR].shape[0] + * 2, + ) step_size = min(64, step_size * 2) - logging.info(f"New configuration: ic={step_size}, res={grid_res}") + spp = min(2, spp * 2) + logging.info( + f"New configuration: ic={step_size}, res={grid_res}, spp={spp}" + ) for opt in opts: - opt[key] = dr.upsample( - opt[key], shape=(grid_res, grid_res, grid_res) + opt[GRID_VOLUME_DATA_SIGMA_T_STR] = dr.upsample( + opt[GRID_VOLUME_DATA_SIGMA_T_STR], + shape=(grid_res, grid_res, grid_res), ) self.model.sceneParams.update(opt) if showDiffRender: - showDiffRender( - diffRender=None, plotStatus=CLOSE_STATUS_STR - ) + showDiffRender(diffRender=None, plotStatus=CLOSE_STATUS_STR) optLog = self.model.endOptimizationLog( sceneParamsHist, startTime, optLog ) diff --git a/src/material_optimizer_view.py b/src/material_optimizer_view.py index 0338baf..15396dc 100644 --- a/src/material_optimizer_view.py +++ b/src/material_optimizer_view.py @@ -228,8 +228,11 @@ def initTable(self, sceneParams: dict): else: item = QTableWidgetItem(NOT_IMPLEMENTED_STRING) else: - # special case: mi.Point3f for max/min clamp value of vertex pos. - if VERTEX_POSITIONS_PATTERN.search(param): + # special case: mi.Point3f for max/min clamp value + isVertexPosOrNormal = VERTEX_POSITIONS_PATTERN.search( + param + ) or VERTEX_NORMALS_PATTERN.search(param) + if isVertexPosOrNormal: if ( label == COLUMN_LABEL_MIN_CLAMP_LABEL or label == COLUMN_LABEL_MAX_CLAMP_LABEL