Type description improvements (AcademySoftwareFoundation#2176)

This change list is an overhaul of the type system in shader generation.

Improvements include:
- Removing the singleton pattern for TypeDesc storage and access, making the type system thread safe.
- Improved handling of TypeDesc internal data, supporting any amount of data for the future, while keeping the class size minimal.
- Handling of re-registration of type descriptions.
- Support for keeping type descriptions alive after the lifetime of ShaderGenerator and GenContext. For applications that holds/uses the ShadePort reflection data that is generated by shader generation, even after the ShaderGenerator itself is destroyed. (To do this an application can create and pass in an external TypeSystem).

Co-work with @ld-kerley

javascript/MaterialXTest/browser/shaderGenerator.spec.js

@@ -34,17 +34,17 @@ describe('Generate Shaders', function ()
 
         const generators = []
         if (typeof mx.EsslShaderGenerator != 'undefined')
-            generators.push(new mx.EsslShaderGenerator());
+            generators.push(mx.EsslShaderGenerator.create());
         if (typeof mx.GlslShaderGenerator != 'undefined')
-            generators.push(new mx.GlslShaderGenerator());
+            generators.push(mx.GlslShaderGenerator.create());
         if (typeof mx.MslShaderGenerator != 'undefined')
-            generators.push(new mx.MslShaderGenerator());
+            generators.push(mx.MslShaderGenerator.create());
         if (typeof mx.OslShaderGenerator != 'undefined')
-            generators.push(new mx.OslShaderGenerator());
+            generators.push(mx.OslShaderGenerator.create());
         if (typeof mx.VkShaderGenerator != 'undefined')
-            generators.push(new mx.VkShaderGenerator());
+            generators.push(mx.VkShaderGenerator.create());
         if (typeof mx.MdlShaderGenerator != 'undefined')
-            generators.push(new mx.MdlShaderGenerator());
+            generators.push(mx.MdlShaderGenerator.create());
 
         const elem = mx.findRenderableElement(doc);
         for (let gen of generators)

javascript/MaterialXView/source/viewer.js

@@ -1486,7 +1486,7 @@ export class Viewer
         this.mx = mtlxIn;
 
         // Initialize base document
-        this.generator = new this.mx.EsslShaderGenerator();
+        this.generator = this.mx.EsslShaderGenerator.create();
         this.genContext = new this.mx.GenContext(this.generator);
 
         this.document = this.mx.createDocument();

javascript/README.md

@@ -179,7 +179,7 @@ Make sure to consume `JsMaterialXGenShader.js` instead of `JsMaterialXCore.js` a
 #### Generating Essl Shader Code & Compiling with WebGL
 To generate WebGL 2 compatible shader code a generator context and an instance of the `EsslShaderGenerator` class is required. 
 ```javascript
-const gen = new mx.EsslShaderGenerator();
+const gen = mx.EsslShaderGenerator.create();
 const genContext = new mx.GenContext(gen);
 ```
 The standard libraries need to be loaded and imported into the document. This step is required as the standard libraries contain all the definitions and snippets needed for assembly of the shader code.

python/MaterialXTest/genshader.py

@@ -54,6 +54,7 @@ def test_ShaderInterface(self):
         self.assertTrue(foundTarget)
         context = mx_gen_shader.GenContext(shadergen)
         context.registerSourceCodeSearchPath(searchPath)
+        shadergen.registerTypeDefs(doc);
 
         # Test generator with complete mode
         context.getOptions().shaderInterfaceType = mx_gen_shader.ShaderInterfaceType.SHADER_INTERFACE_COMPLETE;

python/Scripts/generateshader.py

@@ -107,6 +107,7 @@ def main():
         codeSearchPath.append(os.path.dirname(inputFilename))
         context = mx_gen_shader.GenContext(shadergen)
         context.registerSourceCodeSearchPath(codeSearchPath)
+        shadergen.registerTypeDefs(doc);
 
         # If we're generating Vulkan-compliant GLSL then set the binding context
         if opts.vulkanCompliantGlsl:

resources/Materials/TestSuite/stdlib/structs/struct_texcoord.mtlx

@@ -0,0 +1,30 @@
+<?xml version="1.0"?>
+<materialx version="1.39">
+
+  <typedef name="texcoord_struct">
+    <member name="ss" type="float" value="0.5"/>
+    <member name="tt" type="float" value="0.5"/>
+  </typedef>
+
+  <nodedef name="ND_extract_s_texcoord" node="extract_s" nodegroup="shader" >
+    <input name="in" type="texcoord_struct" value="{0.1;0.1}" />
+    <output name="out" type="float" value="0.0" />
+  </nodedef>
+
+  <implementation name="IM_extract_s_texcoord_genglsl" nodedef="ND_extract_s_texcoord" target="genglsl" sourcecode="{{in}}.ss" />
+  <implementation name="IM_extract_s_texcoord_genmsl" nodedef="ND_extract_s_texcoord" target="genmsl" sourcecode="{{in}}.ss" />
+  <implementation name="IM_extract_s_texcoord_genosl" nodedef="ND_extract_s_texcoord" target="genosl" sourcecode="{{in}}.ss" />
+  <implementation name="IM_extract_s_texcoord_genmdl" nodedef="ND_extract_s_texcoord" target="genmdl" sourcecode="{{in}}.ss" />
+
+  <extract_s name="extract" type="float">
+    <input name="in" type="texcoord_struct" value="{0.01;0.5}"/>
+  </extract_s>
+  <surface_unlit name="unlit_surface1" type="surfaceshader">
+    <input name="emission" type="float" nodename="extract" />
+    <input name="opacity" type="float" value="1.0" />
+  </surface_unlit>
+  <surfacematerial name="test_struct_texcoord" type="material">
+    <input name="surfaceshader" type="surfaceshader" nodename="unlit_surface1" />
+  </surfacematerial>
+
+</materialx>

resources/Materials/TestSuite/stdlib/structs/struct_texcoordGroup.mtlx

@@ -0,0 +1,43 @@
+<?xml version="1.0"?>
+<materialx version="1.38">
+
+  <typedef name="texcoord_struct">
+    <member name="ss" type="float" value="0.5"/>
+    <member name="tt" type="float" value="0.5"/>
+  </typedef>
+  <typedef name="texcoordGroup_struct">
+    <member name="st_0" type="texcoord_struct" value="{0.1;0.1}"/>
+    <member name="st_1" type="texcoord_struct" value="{0.5;0.5}"/>
+    <member name="st_2" type="texcoord_struct" value="{0.9;0.9}"/>
+  </typedef>
+
+  <nodedef name="ND_extract_first_s_texcoordGroup" node="extract_first_s_group" nodegroup="shader" >
+    <input name="in" type="texcoordGroup_struct" value="{{0.1;0.2};{0.3;0.4};{0.5;0.6}}"/>
+    <output name="out" type="float" value="0.0" />
+  </nodedef>
+
+  <implementation name="IM_extract_first_s_texcoordGroup_genglsl" nodedef="ND_extract_first_s_texcoordGroup" target="genglsl" sourcecode="{{in}}.st_0.ss">
+    <input name="in" type="texcoordGroup_struct" implname="data" />
+  </implementation>
+  <implementation name="IM_extract_first_s_texcoordGroup_genmdl" nodedef="ND_extract_first_s_texcoordGroup" target="genmdl" sourcecode="{{in}}.st_0.ss">
+    <input name="in" type="texcoordGroup_struct" implname="data" />
+  </implementation>
+  <implementation name="IM_extract_first_s_texcoordGroup_genmsl" nodedef="ND_extract_first_s_texcoordGroup" target="genmsl" sourcecode="{{in}}.st_0.ss">
+    <input name="in" type="texcoordGroup_struct" implname="data" />
+  </implementation>
+  <implementation name="IM_extract_first_s_texcoordGroup_genosl" nodedef="ND_extract_first_s_texcoordGroup" target="genosl" sourcecode="{{in}}.st_0.ss">
+    <input name="in" type="texcoordGroup_struct" implname="data" />
+  </implementation>
+
+  <extract_first_s_group name="extractfirstsgroup" type="float">
+    <input name="in" type="texcoordGroup_struct" value="{{0.1;0.2};{0.3;0.4};{0.5;0.6}}"/>
+  </extract_first_s_group>
+  <surface_unlit name="unlit_surface1" type="surfaceshader">
+    <input name="emission" type="float" nodename="extractfirstsgroup" />
+    <input name="opacity" type="float" value="1.0" />
+  </surface_unlit>
+  <surfacematerial name="test_struct_texcoordGroup" type="material">
+    <input name="surfaceshader" type="surfaceshader" nodename="unlit_surface1" />
+  </surfacematerial>
+
+</materialx>

source/JsMaterialX/JsMaterialXGenEssl/JsEsslShaderGenerator.cpp

@@ -11,9 +11,17 @@
 namespace ems = emscripten;
 namespace mx = MaterialX;
 
+namespace
+{
+    // Creator wrapper to avoid having to expose the TypeSystem class in JavaScript
+    mx::ShaderGeneratorPtr EsslShaderGenerator_create()
+    {
+        return mx::EsslShaderGenerator::create();
+    }
+}
+
 EMSCRIPTEN_BINDINGS(EsslShaderGenerator)
 {
-    ems::class_<mx::EsslShaderGenerator, ems::base<mx::HwShaderGenerator>>("EsslShaderGenerator")
-        .smart_ptr_constructor("EsslShaderGenerator", &std::make_shared<mx::EsslShaderGenerator>)
-        ;
+    ems::class_<mx::EsslShaderGenerator, ems::base<mx::GlslShaderGenerator>>("EsslShaderGenerator")
+        .class_function("create", &EsslShaderGenerator_create);
 }

source/JsMaterialX/JsMaterialXGenGlsl/JsGlslShaderGenerator.cpp

@@ -11,9 +11,17 @@
 namespace ems = emscripten;
 namespace mx = MaterialX;
 
+namespace
+{
+    // Creator wrapper to avoid having to expose the TypeSystem class in JavaScript
+    mx::ShaderGeneratorPtr GlslShaderGenerator_create()
+    {
+        return mx::GlslShaderGenerator::create();
+    }
+}
+
 EMSCRIPTEN_BINDINGS(GlslShaderGenerator)
 {
-    ems::class_<mx::GlslShaderGenerator, ems::base<mx::ShaderGenerator>>("GlslShaderGenerator")
-        .smart_ptr_constructor("GlslShaderGenerator", &std::make_shared<mx::GlslShaderGenerator>)
-        ;
+    ems::class_<mx::GlslShaderGenerator, ems::base<mx::HwShaderGenerator>>("GlslShaderGenerator")
+        .class_function("create", &GlslShaderGenerator_create);
 }

source/JsMaterialX/JsMaterialXGenMdl/JsMdlShaderGenerator.cpp

@@ -11,9 +11,17 @@
 namespace ems = emscripten;
 namespace mx = MaterialX;
 
+namespace
+{
+    // Creator wrapper to avoid having to expose the TypeSystem class in JavaScript
+    mx::ShaderGeneratorPtr MdlShaderGenerator_create()
+    {
+        return mx::MdlShaderGenerator::create();
+    }
+}
+
 EMSCRIPTEN_BINDINGS(MdlShaderGenerator)
 {
     ems::class_<mx::MdlShaderGenerator, ems::base<mx::ShaderGenerator>>("MdlShaderGenerator")
-        .smart_ptr_constructor("MdlShaderGenerator", &std::make_shared<mx::MdlShaderGenerator>)
-        ;
+        .class_function("create", &MdlShaderGenerator_create);
 }

source/JsMaterialX/JsMaterialXGenMsl/JsMslShaderGenerator.cpp

@@ -11,9 +11,17 @@
 namespace ems = emscripten;
 namespace mx = MaterialX;
 
+namespace
+{
+    // Creator wrapper to avoid having to expose the TypeSystem class in JavaScript
+    mx::ShaderGeneratorPtr MslShaderGenerator_create()
+    {
+        return mx::MslShaderGenerator::create();
+    }
+}
+
 EMSCRIPTEN_BINDINGS(MslShaderGenerator)
 {
-    ems::class_<mx::MslShaderGenerator, ems::base<mx::ShaderGenerator>>("MslShaderGenerator")
-        .smart_ptr_constructor("MslShaderGenerator", &std::make_shared<mx::MslShaderGenerator>)
-        ;
+    ems::class_<mx::MslShaderGenerator, ems::base<mx::HwShaderGenerator>>("MslShaderGenerator")
+        .class_function("create", &MslShaderGenerator_create);
 }

source/JsMaterialX/JsMaterialXGenOsl/JsOslShaderGenerator.cpp

@@ -11,9 +11,17 @@
 namespace ems = emscripten;
 namespace mx = MaterialX;
 
+namespace
+{
+    // Creator wrapper to avoid having to expose the TypeSystem class in JavaScript
+    mx::ShaderGeneratorPtr OslShaderGenerator_create()
+    {
+        return mx::OslShaderGenerator::create();
+    }
+}
+
 EMSCRIPTEN_BINDINGS(OslShaderGenerator)
 {
     ems::class_<mx::OslShaderGenerator, ems::base<mx::ShaderGenerator>>("OslShaderGenerator")
-        .smart_ptr_constructor("OslShaderGenerator", &std::make_shared<mx::OslShaderGenerator>)
-        ;
+        .class_function("create", &OslShaderGenerator_create);
 }

source/JsMaterialX/JsMaterialXGenVk/JsVkShaderGenerator.cpp

@@ -11,9 +11,17 @@
 namespace ems = emscripten;
 namespace mx = MaterialX;
 
+namespace
+{
+    // Creator wrapper to avoid having to expose the TypeSystem class in JavaScript
+    mx::ShaderGeneratorPtr VkShaderGenerator_create()
+    {
+        return mx::VkShaderGenerator::create();
+    }
+}
+
 EMSCRIPTEN_BINDINGS(VkShaderGenerator)
 {
-    ems::class_<mx::VkShaderGenerator, ems::base<mx::ShaderGenerator>>("VkShaderGenerator")
-        .smart_ptr_constructor("VkShaderGenerator", &std::make_shared<mx::VkShaderGenerator>)
-        ;
+    ems::class_<mx::VkShaderGenerator, ems::base<mx::GlslShaderGenerator>>("VkShaderGenerator")
+        .class_function("create", &VkShaderGenerator_create);
 }

source/MaterialXGenGlsl/EsslShaderGenerator.cpp

@@ -13,10 +13,10 @@ MATERIALX_NAMESPACE_BEGIN
 const string EsslShaderGenerator::TARGET = "essl";
 const string EsslShaderGenerator::VERSION = "300 es"; // Current target is WebGL 2.0
 
-EsslShaderGenerator::EsslShaderGenerator() :
-    GlslShaderGenerator()
+EsslShaderGenerator::EsslShaderGenerator(TypeSystemPtr typeSystem) :
+    GlslShaderGenerator(typeSystem)
 {
-    _syntax = EsslSyntax::create();
+    _syntax = EsslSyntax::create(typeSystem);
     // Add in ESSL specific keywords
     const StringSet reservedWords = { "precision", "highp", "mediump", "lowp" };
     _syntax->registerReservedWords(reservedWords);

source/MaterialXGenGlsl/EsslShaderGenerator.h

@@ -20,9 +20,18 @@ using EsslShaderGeneratorPtr = shared_ptr<class EsslShaderGenerator>;
 class MX_GENGLSL_API EsslShaderGenerator : public GlslShaderGenerator
 {
   public:
-    EsslShaderGenerator();
-
-    static ShaderGeneratorPtr create() { return std::make_shared<EsslShaderGenerator>(); }
+    /// Constructor.
+    EsslShaderGenerator(TypeSystemPtr typeSystem);
+
+    /// Creator function.
+    /// If a TypeSystem is not provided it will be created internally.
+    /// Optionally pass in an externally created TypeSystem here, 
+    /// if you want to keep type descriptions alive after the lifetime
+    /// of the shader generator. 
+    static ShaderGeneratorPtr create(TypeSystemPtr typeSystem = nullptr)
+    {
+        return std::make_shared<EsslShaderGenerator>(typeSystem ? typeSystem : TypeSystem::create());
+    }
 
     /// Return a unique identifier for the target this generator is for
     const string& getTarget() const override { return TARGET; }

source/MaterialXGenGlsl/EsslSyntax.cpp

@@ -12,7 +12,7 @@
 
 MATERIALX_NAMESPACE_BEGIN
 
-EsslSyntax::EsslSyntax()
+EsslSyntax::EsslSyntax(TypeSystemPtr typeSystem) : GlslSyntax(typeSystem)
 {
 }
 

source/MaterialXGenGlsl/EsslSyntax.h

@@ -17,9 +17,9 @@ MATERIALX_NAMESPACE_BEGIN
 class MX_GENGLSL_API EsslSyntax : public GlslSyntax
 {
   public:
-    EsslSyntax();
+    EsslSyntax(TypeSystemPtr typeSystem);
 
-    static SyntaxPtr create() { return std::make_shared<EsslSyntax>(); }
+    static SyntaxPtr create(TypeSystemPtr typeSystem) { return std::make_shared<EsslSyntax>(typeSystem); }
 };
 
 MATERIALX_NAMESPACE_END

source/MaterialXGenGlsl/GlslShaderGenerator.cpp

@@ -41,8 +41,8 @@ const string GlslSamplingIncludeFilename = "stdlib/genglsl/lib/mx_sampling.glsl"
 // GlslShaderGenerator methods
 //
 
-GlslShaderGenerator::GlslShaderGenerator() :
-    HwShaderGenerator(GlslSyntax::create())
+GlslShaderGenerator::GlslShaderGenerator(TypeSystemPtr typeSystem) :
+    HwShaderGenerator(typeSystem, GlslSyntax::create(typeSystem))
 {
     //
     // Register all custom node implementation classes
@@ -734,7 +734,7 @@ ShaderNodeImplPtr GlslShaderGenerator::getImplementation(const NodeDef& nodedef,
         throw ExceptionShaderGenError("NodeDef '" + nodedef.getName() + "' has no outputs defined");
     }
 
-    const TypeDesc outputType = TypeDesc::get(outputs[0]->getType());
+    const TypeDesc outputType = context.getTypeDesc(outputs[0]->getType());
 
     if (implElement->isA<NodeGraph>())
     {

source/MaterialXGenGlsl/GlslShaderGenerator.h

@@ -22,9 +22,18 @@ using GlslShaderGeneratorPtr = shared_ptr<class GlslShaderGenerator>;
 class MX_GENGLSL_API GlslShaderGenerator : public HwShaderGenerator
 {
   public:
-    GlslShaderGenerator();
-
-    static ShaderGeneratorPtr create() { return std::make_shared<GlslShaderGenerator>(); }
+    /// Constructor.
+    GlslShaderGenerator(TypeSystemPtr typeSystem);
+
+    /// Creator function.
+    /// If a TypeSystem is not provided it will be created internally.
+    /// Optionally pass in an externally created TypeSystem here, 
+    /// if you want to keep type descriptions alive after the lifetime
+    /// of the shader generator. 
+    static ShaderGeneratorPtr create(TypeSystemPtr typeSystem = nullptr)
+    {
+        return std::make_shared<GlslShaderGenerator>(typeSystem ? typeSystem : TypeSystem::create());
+    }
 
     /// Generate a shader starting from the given element, translating
     /// the element and all dependencies upstream into shader code.