DX11 HLSL Tesselation issue. 1BILLION POLYS (or not as it happens)

dnc

So I have started writing my own DX11 Tesselation shader, at a hope of looking at how they could be used for VFX. I started trying to get some basic tesselation working in Maya 2014 with the DX11 shader with viewport 2.0 and I am having some trouble with the domain shader.

I think I am passing the right data sets through the Vertex, Hull, Patcher, Tesselator and Domain shaders to the Pixel shader but I get a compile error:
Fun Projects\DX11_Displacement_and_Tesselation.fx(148,1): error X3507: 'domainShader': Not all control paths return a value
Fun Projects\DX11_Displacement_and_Tesselation.fx(188,19): There was an error compiling expression


I'm not sure what is the problem as I am fairly sure I am outputting float4 position data for the vertices.


Any help please, I'm stumped here.

Here's my code.

/***********************************************************

Created by O Cullen    

DX11
HLSL Simple Tesselation Shader - This shader is used to apply simple tesselation methods in Maya.

// Maya DX 11 Notes - Taken from AutodeskUberShader
//------------------------------------
// Shader uses 'pre-multiplied alpha' as its render state and this Uber Shader is build to work in unison with that.
// Alternatively, in Maya, the dx11Shader node allows you to set your own render states by supplying the 'overridesDrawState' annotation in the technique
// You may find it harder to get proper transparency sorting if you choose to do so.

// The technique annotation 'isTransparent' is used to tell Maya how treat the technique with respect to transparency.
//    - If set to 0 the technique is always considered opaque
//    - If set to 1 the technique is always considered transparent
//    - If set to 2 the plugin will check if the parameter marked with the OPACITY semantic is less than 1.0
//    - If set to 3 the plugin will use the transparencyTest annotation to create a MEL procedure to perform the desired test.
// Maya will then render the object twice. Front faces follow by back faces.

// For some objects you may need to switch the Transparency Algorithm to 'Depth Peeling' to avoid transparency issues.
// Models that require this usually have internal faces.

**************************************************************/

//////////////////////////////////////////////////////////////
// Matrices
float4x4 WorldViewProjection     : WorldViewProjection   < string UIWidget = "None"; >;
float4x4 WorldInverseTranspose   : WorldInverseTranspose < string UIWidget = "None"; >;
float4x4 ViewInverse             : ViewInverse           < string UIWidget = "None"; >;
float4x4 World                   : World                 < string UIWidget = "None"; >;
float4x4 View                    : View                  < string UIWidget = "None"; >;
float4x4 Projection              : Projection            < string UIWidget = "None"; >;


//////////////////////////////////////////////////////////////
// Parameters

/*****possibly enclose this section in cbuffer to remove Maya warning*** */

float4 diffuseColour
<
    //string UIWidget = "Color";
    string UIName = "Diffuse Colour";
    int UIOrder = 100;
> = {0.6f, 0.6f, 1.0f, 1.0f};

float tessellationAmount
<
    string UIName = "Tesselation Amount";
    int UIOrder = 101;
> = 4.0f;

float3 padding
<
    string UIName = "Tesselation Padding";
    int UIOrder = 102;
> = {1.0f, 1.0f, 1.0f};


//////////////////////////////////////////////////////////////
// Data Structs
// input from application
struct application2vertex {
    float4 position      : POSITION;
};

// Vertex shader to Hull shader
struct vertex2hull {
    float4 position        : SV_POSITION;
};

// Hull to Tesselator
struct hull2tesselator {
    float4 position             : SV_POSITION; //could just be POSITION
};

//Patch Function 
struct hullPatcher
{
    float edges[3]        : SV_TessFactor;        // tessellation amount for each edge of patch
    float inside        : SV_InsideTessFactor;    // tessellation amount within a patch surface (would be float2 for quads)
};

//Domain to Pixel Shader
struct domain2pixel {
    float4 position          : SV_POSITION;
};
    

//////////////////////////////////////////////////////////////
// Functions

//Patch Function 
hullPatcher patchFunction(InputPatch<vertex2hull, 3> inputPatch, uint patchId : SV_PrimitiveID)
{
    hullPatcher output;
    // Set the tessellation factors for the three edges of the triangle.
    output.edges[0] = tessellationAmount;
    output.edges[1] = tessellationAmount;
    output.edges[2] = tessellationAmount;

    // Set the tessellation factor for tessallating inside the triangle.
    output.inside = tessellationAmount;

    return output;
}


//////////////////////////////////////////////////////////////
// Vertex Shader 
vertex2hull vertexShader(application2vertex In)
{
    vertex2hull Out;
    Out.position = In.position;

    return Out;
}


//////////////////////////////////////////////////////////////
// Hull Shader 
[domain("tri")]
[partitioning("integer")]
[outputtopology("triangle_cw")]
[outputcontrolpoints(3)]
[patchconstantfunc("patchFunction")]

hull2tesselator hullShader(
InputPatch <vertex2hull, 3> patch, 
uint pointId : SV_OutputControlPointID, 
uint patchId : SV_PrimitiveID
)
{
    hull2tesselator output;

    // Set the position for this control point as the output position.
    output.position = patch[pointId].position;

    return output;
}


//////////////////////////////////////////////////////////////
// Domain Shader
[domain("tri")]

domain2pixel domainShader(hullPatcher input, 
float3 uvwCoord : SV_DomainLocation, 
const OutputPatch<hull2tesselator, 3> patch) 
{
    domain2pixel output;
    float3 vertexPosition;
    
    //Determine the position of the new vertex
    vertexPosition = uvwCoord.x * patch[0].position.xyz + 
                     uvwCoord.y * patch[1].position.xyz + 
                     uvwCoord.z * patch[2].position.xyz;

                     //Transform the new pixels into Homogenous clip space.
    output.position = float4(vertexPosition.xyz, 1.0f);         //Convert it into a float 4 position data

    output.position = mul(output.position, WorldViewProjection);  
}


//////////////////////////////////////////////////////////////
// Pixel Shader 
float4 pixelShader(domain2pixel In) : COLOR
{
    float4 final = diffuseColour;
    return final;
}


//////////////////////////////////////////////////////////////
// Techniques

technique11 Base11 <
    bool overridesDrawState = false;
    int isTransparent = 0;
> {
    pass p0    <> 
    {
        SetVertexShader(CompileShader(vs_5_0, vertexShader()));
        SetHullShader(CompileShader(hs_5_0, hullShader()));
        SetDomainShader(CompileShader(ds_5_0, domainShader()));
        SetGeometryShader(NULL);
        SetPixelShader(CompileShader(ps_5_0, pixelShader()));
    }
}
    

Drew++

In your domain shader, just return output....

//////////////////////////////////////////////////////////////
// Domain Shader
[domain("tri")]

domain2pixel domainShader(hullPatcher input, float3 uvwCoord : SV_DomainLocation, const OutputPatch<hull2tesselator, 3> patch) 
{
    domain2pixel output;
    float3 vertexPosition;
    
    //Determine the position of the new vertex
    vertexPosition = uvwCoord.x * patch[0].position.xyz + 
    uvwCoord.y * patch[1].position.xyz + 
    uvwCoord.z * patch[2].position.xyz;

    //Transform the new pixels into Homogenous clip space.
    output.position = float4(vertexPosition.xyz, 1.0f);         //Convert it into a float 4 position data

    output.position = mul(output.position, WorldViewProjection);  

// you forgot this :-P
    return output;
}

dnc

Thanks Drew, that was very silly of me.