Displacement-MicroMap-Toolkit squash

commit d139556f7b1171e8c3c0a8024d1b79412be9f4c4

Co-authored-by: Brian Barran <[email protected]>
Co-authored-by: Christoph Kubisch <[email protected]>
Co-authored-by: David Akeley <[email protected]>
Co-authored-by: Martin-Karl Lefrancois <[email protected]>
Co-authored-by: Mathias Heyer <[email protected]>
Co-authored-by: Neil Bickford <[email protected]>
Co-authored-by: Pascal Gautron <[email protected]>
Co-authored-by: Pyarelal Knowles <[email protected]>
Co-authored-by: Tristan Lorach <[email protected]>

Author: 9 people

.gitmodules

@@ -10,6 +10,9 @@
 	path = external/libpng
 	url = https://git.code.sf.net/p/libpng/code
 	shallow = true
+[submodule "external/heightmap_rtx"]
+       path = external/heightmap_rtx
+       url = https://github.com/NVIDIAGameWorks/heightmap_rtx.git
 [submodule "external/Displacement-MicroMap-SDK"]
 	path = external/Displacement-MicroMap-SDK
 	url = ../Displacement-MicroMap-SDK.git

CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.13 FATAL_ERROR)
 cmake_policy(SET CMP0079 NEW) # Allow linking with targets in other directories
 project(
     micromesh_toolkit
-    VERSION 1.1
+    VERSION 1.2
     DESCRIPTION "Micromesh Tools"
     LANGUAGES C CXX)
 
@@ -58,8 +58,7 @@ endif()
 #--------------------------------------------------------------------------------------------------
 # Package shared by all projects
 
-# set(SUPPORT_AFTERMATH on)
-# _add_package_NsightAftermath()
+_add_package_NsightAftermath()
 _add_package_ZLIB()
 _add_package_VulkanSDK()
 _add_package_ShaderC()
@@ -141,6 +140,7 @@ target_link_libraries(png_static zlibstatic)
 target_include_directories(png_static PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/external/libpng> ${CMAKE_CURRENT_LIST_DIR}/external/libpng)
 ###
 add_subdirectory(external/tinyexr)
+add_subdirectory(external/heightmap_rtx)
 add_subdirectory(meshops_core)
 add_subdirectory(meshops_bake)
 add_subdirectory(meshops_remesher)

README.md

@@ -204,18 +204,14 @@ tools. It relies on `VK_NV_mesh_shader` to allow rasterized display of
 micromeshes. `VK_KHR_acceleration_structure` is required for baking micromaps.
 If available, `VK_NV_displacement_micromap` is used to render micromeshes with
 raytracing when choosing *Rendering -> RTX*. `VK_NV_displacement_micromap` was
-introduced with the Ada Lovelace architecture based GPUs. If you see a message
-about the missing extension, update to the latest driver (note: beta drivers are
-available at https://developer.nvidia.com/vulkan-driver).
-
-> The rasterization of micromeshes, especially compressed is a bit of
-  a more complex topic on its own. Therefore there will be a future
-  dedicated sample that goes into the details of how it works
-  and showcases more features, such as dynamic level-of-detail.
-  We recommend to wait for this, rather than attempt to
-  embed the code from the toolbox. The future sample will also
-  provide more performant solutions and address compute-based
-  rasterization as well.
+introduced with the RTX 40 Series Ada Lovelace architecture based GPUs. Previous
+RTX cards have support, but performance will be better with Ada. If you see a
+message about the missing extension, update to the latest driver (note: beta
+drivers are available at https://developer.nvidia.com/vulkan-driver).
+
+> The rasterization of micromeshes, especially compressed, is a bit of
+  a more complex topic on its own. A dedicated sample is available at
+  https://github.com/nvpro-samples/vk_displacement_micromaps.
 
 > At the time of writing `VK_NV_displacement_micromap` is in beta still, meaning
   the extension is subject to changes. Do not use its headers here in production
@@ -266,7 +262,23 @@ from an original input mesh in the [Micro-Mesh Asset Pipeline
 slide-deck](https://developer.download.nvidia.com/ProGraphics/nvpro-samples/slides/Micro-Mesh_Asset_Pipeline.pdf)
 as well as in [docs/asset_pipeline.md](docs/asset_pipeline.md)
 
-## About the Latest Release (1.1)
+## About the Latest Release
+
+Version 1.2
+
+- Add eR11_unorm_packed_align32 support in meshopsOpCompressDisplacementMicromaps()
+- Add micromesh_tool --normal-textures-stem to generate normal maps
+- The micromesh_tool --resample-resolution argument now takes a 2D resolution
+- Faceted shading option when raytracing, or when missing normal maps
+- Update to the latest nvpro_core
+- Use heightmap_rtx to preview heightmap displacement when raytracing
+- Write textures in parallel when saving scenes
+- Fix baking heightmaps from base mesh materials - now using reference mesh
+- Fix support for loading \*.exr heightmaps
+- Filter outliers for dynamic range when using --height-textures-stem (experimental)
+- SW texture interpolation to preview heightmap displacement with rasterization
+
+Version 1.1
 
 - Heightmap rendering in micromesh_toolkit with dynamic LOD
 - Memory estimate for baking high resolution heightmaps in batches

automation/link_heightmaps.py

@@ -30,12 +30,19 @@
 parser.add_argument('--filter-out', nargs='+', help='Regex for material names to ignore')
 parser.add_argument('--copy-from', type=pathlib.Path, help='Source gltf file to use as a template. Takes precedence over the image filename search.')
 parser.add_argument('--match-one-image', action='store_true', help='A heightmap will be matched to an image and only added to materials sharing that image. Reduces false positives.')
+parser.add_argument('--match-one-material', action='store_true', help='Match a heightmap to at most one material, even if a color texture is shared by multiple materials.')
+parser.add_argument('--heightmap-regex', default=r'height|disp', help='Override the default regex for detecting heightmap-ish filenames')
+parser.add_argument('--image-name-weight', type=float, default=1.0, help='Weight given to the similarity of other images in a material')
+parser.add_argument('--material-name-weight', type=float, default=0.1, help='Weight given to the similarity of the material name')
+parser.add_argument('--match-materials-only', action='store_true', help='Match material names even if they have no other textures.')
 #parser.add_argument('--heightmaps', nargs='+', type=pathlib.Path, help='Provide heightmap names explicitly in the order of materials')
 args = parser.parse_args()
 
+imageSuffixes = set(['.png', '.jpg', '.exr', '.bmp'])
+
 # Return true if the filename could be a heightmap
 def heightmapishName(name):
-    return re.search(r'height|disp', name, re.IGNORECASE)
+    return re.search(args.heightmap_regex, name, re.IGNORECASE)
 
 def find_in_object(obj, target_name, depth):
     if target_name in obj:
@@ -65,13 +72,15 @@ def find_ids(data, object_name, target_names, target_id):
 def heightmapMatchMetric(candidate, image, material_names):
     candidate_name = str(candidate.relative_to(filepath.parent))
     # Helps when height or displacement is a common term for all textures
-    heightmap_terms = re.findall(r'height|disp', candidate_name, re.IGNORECASE)
+    heightmap_terms = re.findall(args.heightmap_regex, candidate_name, re.IGNORECASE)
+    # Remove the heightmap terms so they don't interfere with matching
+    candidate_name = re.sub(args.heightmap_regex, '_', candidate_name)
     image_name = str(image.relative_to(filepath.parent))
     image_seq = SequenceMatcher(None, candidate_name, image_name)
     image_similarity = image_seq.ratio()
     match_seq = SequenceMatcher(None, candidate_name.lower(), '$'.join(material_names).lower())
     material_similarity = match_seq.ratio()
-    score = image_similarity + material_similarity * 0.1 + len(heightmap_terms)
+    score = image_similarity * args.image_name_weight + material_similarity * args.material_name_weight + len(heightmap_terms)
     if args.verbose:
         print('Image similarity score {} (filename {}, material name {}, heightmap terms {}):'.format(score, image_similarity, material_similarity * 0.1, len(heightmap_terms)))
         print('  {}'.format(candidate_name))
@@ -81,6 +90,14 @@ def heightmapMatchMetric(candidate, image, material_names):
         print()
     return score
 
+# Returns material_ids, in descending order of their material names' similarity
+# to the candidate file name
+def orderMaterialSimilarity(candidate, materials, material_ids):
+    candidate_name = str(candidate.relative_to(filepath.parent))
+    candidate_name = re.sub(args.heightmap_regex, '_', candidate_name)
+    sorted_pairs = sorted((SequenceMatcher(None, candidate_name, materials[material_id]['name']).ratio(), material_id) for material_id in material_ids)
+    return [pair[1] for pair in reversed(sorted_pairs)]
+
 def filterMaterialName(material_name):
     if args.filter and not any(re.search(pattern, material_name) for patterrn in args.filter):
         if args.verbose:
@@ -111,6 +128,7 @@ def filterMaterialName(material_name):
         path = filepath.parent / urllib.parse.unquote(uri)
         #if args.verbose: print('Existing image {}'.format(path))
         images.add(path)
+        imageSuffixes.add(path.suffix)
         image_ids[path] = id
 
     image_types = set()
@@ -147,7 +165,7 @@ def filterMaterialName(material_name):
         if args.verbose: print('Searching "{}"...'.format(str(dir)))
         for file in os.listdir(dir):
             candidate = dir / file
-            if image.suffix == candidate.suffix and candidate not in images and heightmapishName(file):
+            if candidate.suffix in imageSuffixes and candidate not in images and heightmapishName(file):
                 if args.verbose: print('Found candidate {}'.format(str(candidate)))
                 candidates.add(candidate)
 
@@ -161,6 +179,18 @@ def filterMaterialName(material_name):
             material_names = list(data['materials'][material_id]['name'] for material_id in material_ids)
             matches += [(heightmapMatchMetric(candidate, image, material_names), candidate, image, material_ids)]
 
+        # HACK: this script started with only trying to match other texture
+        # names. It's entirely possible there are no textures but material (and
+        # even mesh) names would be fine matches. This quick workaround adds
+        # "null" images so that material names can still be matched. This script
+        # should really be rewritten more hollistically.
+        if args.match_materials_only:
+            all_material_ids = range(len(data['materials']))
+            fake_image = filepath.parent / "null"
+            for material_id in all_material_ids:
+                material_names = [data['materials'][material_id]['name']]
+                matches += [(heightmapMatchMetric(candidate, fake_image, material_names), candidate, fake_image, [material_id])]
+
     # Assume the most similar heightmap names belong to the same materials as existing images in a greedy fashion
     unlinked_heightmaps = []
     unlinked_material_ids = set()
@@ -174,7 +204,7 @@ def filterMaterialName(material_name):
             elif not warn_multiple_matches:
                 warn_multiple_matches = True
                 print("Warning: adding the same heightmap multiple times consider enabling '--match-one-image'", file=sys.stderr)
-        for material_id in set(material_ids) - unlinked_material_ids:
+        for material_id in orderMaterialSimilarity(heightmap, data['materials'], set(material_ids) - unlinked_material_ids):
             unlinked_material_ids.add(material_id)
 
             # Filter out materials with existing displacement
@@ -201,6 +231,9 @@ def filterMaterialName(material_name):
             unlinked_heightmaps += [(material_id, material_name, heightmap, image, sampler_id)]
             best_image[heightmap.name] = image
 
+            if args.match_one_image:
+                break
+
     had_changes = False
 
     if not unlinked_heightmaps:

micromesh_tool/src/debug_util.h common/debug_util.h

@@ -306,8 +306,9 @@ MESHOPS_API micromesh::Result MESHOPS_CALL meshopsOpBake(Context context, BakerO
       MESHOPS_LOGE(context, "Non-zero OpBake_input::ResamplerInput::texCoordIndex (%u) is not supported", texture.textureCoord);
       return micromesh::Result::eInvalidValue;
     }
-    bool generatedTextureType = texture.textureType == TextureType::eQuaternionMap || texture.textureType == TextureType::eOffsetMap
-                                || texture.textureType == TextureType::eHeightMap;
+    bool generatedTextureType = texture.textureType == TextureType::eQuaternionMap
+                                || texture.textureType == TextureType::eOffsetMap || texture.textureType == TextureType::eHeightMap
+                                || texture.textureType == TextureType::eNewNormalMap;
     if(texture.texture)
     {
       if(generatedTextureType)

external/Displacement-MicroMap-SDK

@@ -37,7 +37,7 @@
 
 namespace meshops {
 
-using namespace glsl_shared;
+using namespace shaders;
 
 static bool getGlobalMinMax(ArrayView<const nvmath::vec2f> minMaxs, nvmath::vec2f& globalMinMax, bool filterZeroToOne, const uint32_t maxFilterWarnings);
 
@@ -215,7 +215,7 @@ uint64_t BakerVK::estimateBaseGpuMemory(uint64_t distances, uint64_t triangles,
   // Persistent gpu memory used between batches
   uint64_t result = 0;
   result += nvh::align_up(sizeof(float) * distances, alignment);                  // m_distanceBuf
-  result += nvh::align_up(sizeof(glsl_shared::Triangle) * triangles, alignment);  // m_trianglesBuf
+  result += nvh::align_up(sizeof(shaders::Triangle) * triangles, alignment);      // m_trianglesBuf
   result += nvh::align_up(sizeof(nvmath::vec2f) * triangles, alignment);          // m_triangleMinMaxBuf
   result += BakerMeshVK::estimateGpuMemory(triangles, vertices, requireDirectionBounds);
 
@@ -820,9 +820,9 @@ uint64_t BakerMeshVK::estimateGpuMemory(uint64_t triangles, uint64_t vertices, b
 
   // Buffers allocated in create() and createTessellated()
   uint64_t result = 0;
-  result += nvh::align_up(sizeof(glsl_shared::CompressedVertex) * vertices, alignment);  // vertices - raw position and compressed attributes
+  result += nvh::align_up(sizeof(shaders::CompressedVertex) * vertices, alignment);  // vertices - raw position and compressed attributes
   result += nvh::align_up(sizeof(nvmath::vec3ui) * triangles, alignment);  // indices
-  result += nvh::align_up(sizeof(glsl_shared::BakerMeshInfo), alignment);
+  result += nvh::align_up(sizeof(shaders::BakerMeshInfo), alignment);
   if(requireDirectionBounds)
   {
     result += nvh::align_up(sizeof(nvmath::vec2f) * vertices, alignment);  // directionBoundsBuf
@@ -983,7 +983,7 @@ void ResamplerPipeline::destroy(VkDevice device)
 //--------------------------------------------------------------------------------------------------
 //
 //
-void BakerPipeline::run(meshops::ContextVK& vk, const meshops::OpBake_input& input, glsl_shared::BakerPushConstants& pushConstants, bool finalBatch)
+void BakerPipeline::run(meshops::ContextVK& vk, const meshops::OpBake_input& input, shaders::BakerPushConstants& pushConstants, bool finalBatch)
 {
   nvh::ScopedTimer t("Run Compute Pass");
 
@@ -1023,11 +1023,11 @@ void BakerPipeline::run(meshops::ContextVK& vk, const meshops::OpBake_input& inp
   vk.resAllocator->finalizeAndReleaseStaging();
 }
 
-void ResamplerPipeline::run(meshops::ContextVK&              vk,
-                            const meshops::OpBake_input&     input,
-                            ArrayView<meshops::Texture>      outputTextures,
-                            glsl_shared::BakerPushConstants& pushConstants,
-                            nvvk::Buffer&                    triangleMinMaxBuf)
+void ResamplerPipeline::run(meshops::ContextVK&          vk,
+                            const meshops::OpBake_input& input,
+                            ArrayView<meshops::Texture>  outputTextures,
+                            shaders::BakerPushConstants& pushConstants,
+                            nvvk::Buffer&                triangleMinMaxBuf)
 {
   nvh::ScopedTimer t("Run Resampler");
   assert(outputTextures.size());
@@ -1062,21 +1062,38 @@ void ResamplerPipeline::run(meshops::ContextVK&              vk,
     generatingHeightmap = generatingHeightmap || input.resamplerInput[i].textureType == TextureType::eHeightMap;
   }
 
-  // If we're generating a heightmap during resampling, we should scale the
-  // output by the bounds. This would need a second pass to first compute the
-  // values we would write. Instead, we can approximate the bounds by using the
-  // already-computed micromesh displacement values. These do get computed in
-  // meshops_bake.cpp, but this operation is not that common anyway.
+  // Heightmap generation is a byproduct and intended for use with the input
+  // base mesh (i.e. the micromap displaced output would be discarded). If we're
+  // generating a heightmap during resampling, we should scale the output by the
+  // bounds. This would need a second pass to first compute the values we would
+  // write. Instead, we can approximate the bounds by using the already-computed
+  // micromesh displacement values. These do get computed in meshops_bake.cpp,
+  // but this operation is not that common anyway.
   pushConstants.globalMinMax = nvmath::vec2f(0.0f, 1.0f);
   if(generatingHeightmap)
   {
+    // Ignore the top and bottom 1% heights when choosing a heightmap scale.
     auto minMaxs =
         ArrayView(static_cast<nvmath::vec2f*>(vk.resAllocator->map(triangleMinMaxBuf)), input.baseMeshView.triangleCount());
-    if(!getGlobalMinMax(minMaxs, pushConstants.globalMinMax, false, 0))
+    auto               minMaxFloatsView = ArrayView<float>(minMaxs);
+    std::vector<float> minMaxFloats(minMaxFloatsView.begin(), minMaxFloatsView.end());
+    std::sort(minMaxFloats.begin(), minMaxFloats.end());
+    size_t ignoreOutliers        = minMaxFloats.size() / 100;
+    pushConstants.globalMinMax.x = minMaxFloats[ignoreOutliers];
+    pushConstants.globalMinMax.y = minMaxFloats[minMaxFloats.size() - 1 - ignoreOutliers];
+    vk.resAllocator->unmap(triangleMinMaxBuf);
+
+    // Print the scale - currently this is the only output the user has
+    auto globalBiasScale = BiasScalef::minmax_unit(pushConstants.globalMinMax);
+    LOGW("\nHeightmap range: [%f, %f] (bias %f, scale %f)\n", pushConstants.globalMinMax.x,
+         pushConstants.globalMinMax.y, globalBiasScale.bias, globalBiasScale.scale);
+    if(input.settings.fitDirectionBounds)
     {
-      pushConstants.globalMinMax = {0.0f, 1.0f};
+      // When direction bounds are non-uniform, the direction vectors change.
+      // Even if the height values were rescaled, they will not work with the
+      // original mesh.
+      LOGW("Warning: heightmap will not work with the original base mesh due to --fit-direction-bounds\n");
     }
-    vk.resAllocator->unmap(triangleMinMaxBuf);
   }
 
   // Create a list of resolutions. Each geometry instance will be rasterized at