Shaders HLSL9 Anti-Aliasing Head Ache

[TheStolenBattenberg]

Hey there,

I post here way too often, but here's a new problem from the hell that is shader development & implementation...

I'm trying (again) and failing (again) to implement a rather nice post screen AA type called 'NFAA' (Normal Filter Anti Aliasing). The edge detection (generation of the screen space normal map) is working fine, but the actual blurring of edges isn't. I've had no problem implementing FXAA, I just hate FXAA, it blurs everything.

Can someone take a look at this code and tell me what's up?

Spoiler: Vertex Shader

//

// HLSL Vertex Passthough Shader
//
struct a2v {
    float3 Position  : POSITION;
    float2 Texcoord  : TEXCOORD0;
};

struct v2p {
    float4 Position : POSITION;
    float2 Texcoord : TEXCOORD0;
};

void main(in a2v IN, out v2p OUT) {
    float4 object_space_pos = float4(IN.Position.x, IN.Position.y, IN.Position.z, 1.0);
    
    OUT.Position = mul(gm_Matrices[MATRIX_WORLD_VIEW_PROJECTION], object_space_pos);
    OUT.Texcoord = IN.Texcoord;
}

Spoiler: Pixel Shader

//
// NFAA 3.0
//
struct v2p {
    float4 Position : POSITION;
    float2 Texcoord : TEXCOORD0;
};
 
struct p2s {
    float4 Color0    : COLOR0;
};

uniform float fScreenResolutionW;
uniform float fScreenResolutionH;

float3 RGBtoHCV(float3 RGB) {
    float4 P = (RGB.g < RGB.b) ? float4(float2(RGB.b, RGB.g), -1.0, 2.0/3.0) : float4(float2(RGB.g, RGB.b), 0.0, -1.0/3.0);
    float4 Q = (RGB.r < P.x)   ? float4(float3(P.x, P.y, P.w), RGB.r)        : float4(RGB.r, float3(P.y, P.z, P.x));
    float  C = Q.x - min(Q.w, Q.y);
    float  H = abs((Q.w - Q.y) / (6.0 * C + 0.0000000001) + Q.z);
    return float3(H, C, Q.x);
}

float lumRGB(float3 RGB) {
    float3 ABC = RGBtoHCV(RGB);
    return dot(1.0 - ABC, 1.0 - ABC) / 3.0;
}
    
void main(in v2p IN, out p2s OUT) {
    float2 vPixelViewport = float2(1.0 / fScreenResolutionW, 1.0 / fScreenResolutionH);
    float  fScale = 1.0;
    float  fBlur  = 0.5;
    
    float2 upOffset = float2(0.0, vPixelViewport.y) * fScale;
    float2 rtOffset = float2(vPixelViewport.x, 0.0) * fScale;
    
    // Blur Coordinates
    float topHeight         = lumRGB(tex2D(gm_BaseTexture, IN.Texcoord.xy + upOffset).rgb);
    float bottomHeight      = lumRGB(tex2D(gm_BaseTexture, IN.Texcoord.xy - upOffset).rgb);
    float rightHeight       = lumRGB(tex2D(gm_BaseTexture, IN.Texcoord.xy + rtOffset).rgb);
    float leftHeight        = lumRGB(tex2D(gm_BaseTexture, IN.Texcoord.xy - rtOffset).rgb);
    float leftTopHeight     = lumRGB(tex2D(gm_BaseTexture, IN.Texcoord.xy - rtOffset + upOffset).rgb);
    float leftBottomHeight  = lumRGB(tex2D(gm_BaseTexture, IN.Texcoord.xy - rtOffset - upOffset).rgb);
    float rightBottomHeight = lumRGB(tex2D(gm_BaseTexture, IN.Texcoord.xy + rtOffset - upOffset).rgb);
    float rightTopHeight    = lumRGB(tex2D(gm_BaseTexture, IN.Texcoord.xy + rtOffset + upOffset).rgb);
        
    // Normal Map Generation
    float sum0  = rightTopHeight   + topHeight    + rightBottomHeight;
    float sum1  = leftTopHeight    + bottomHeight + leftBottomHeight;
    float sum2  = leftTopHeight    + leftHeight   + rightTopHeight;
    float sum3  = leftBottomHeight + rightHeight  + rightBottomHeight;   
    float sumx = (sum1 - sum0);
    float sumy = (sum2 - sum3);
    sumx = sign(sumx) * pow(abs(sumx), fBlur);
    sumy = sign(sumy) * pow(abs(sumy), fBlur);
    
    // Combine & Scale
    float2 Normal = (normalize(float3(sumx, sumy, 1.0))).xy * vPixelViewport * fScale;
    Normal.xy *= 1.0;
    
    // Colour
    float4 Scene0 = tex2D(gm_BaseTexture, IN.Texcoord.xy) * 4.0;
    float4 Scene1 = tex2D(gm_BaseTexture, IN.Texcoord.xy + Normal.xy) * 2.0;
    float4 Scene2 = tex2D(gm_BaseTexture, IN.Texcoord.xy - Normal.xy) * 2.0;
    float4 Scene3 = tex2D(gm_BaseTexture, IN.Texcoord.xy + float2(Normal.x, -Normal.y));
    float4 Scene4 = tex2D(gm_BaseTexture, IN.Texcoord.xy - float2(Normal.x, -Normal.y));
    
    // Final Colour
    OUT.Color0.rgb = (Scene0.rgb + Scene1.rgb + Scene2.rgb + Scene3.rgb + Scene4.rgb) * 0.1;
    //OUT.Color0.rgb = normalize(float3(sumx, sumy , 1) * 0.5 + 0.5);
    OUT.Color0.a = 1.0;
}