[gap]

Originally Posted by Jeff-Groves

Try a normals dds in grey.
Like the original one in detaile_nm.dds

That file is more a height map then a normal map.

Okay, after several tests I am reasonably sure that the one terrain texture having an effect on the "glaring shore glitch", is Tex02 (in terrain .ini files), the one commonly set up to a sandy beach texture. The darker this texture, the more obvious the problem.

Despite the fact that the glare is only visible in step segments of the coast, where the sand texture gets blended with the rock texture, neither Tex01 nor Tex05 (respectively: the texture applied to cliffs and its normal map) have any significant effect on the problem.

Now looking into terrain shaders:


TerrainSurfacePS.fx

#include <data/Shaders/Shadowmap/ShadowMapUtils.inc>

float3 SunLightColor 		: register(c0);
float4 AmbLightColor 		: register(c1);
float3 FogColor		 		: register(c2);
float3 CamPosition   		: register(c3);
float4 Params	     		: register(c4);
float  MaxDepth	     		: register(c5);
float3 UnderWaterColor 		: register(c6);
float  g_drawShadows		: register(c7);
float4 sixColorAmbient[6]	: register(c8);

#ifdef EDITOR_MODE
float2 g_edTexOffset		: register(c15);
float2 g_edTexRot			: register(c16);
#endif

#ifdef OUTPUTDEPTH
float2 biasParams:register(c30);
#endif

struct PS_IN                                 
{     
#ifdef OUTPUTDEPTH
	float depth		:TEXCOORD0;
#else
	float4 Tex0      : TEXCOORD0;
	float3 WSPos	 : TEXCOORD1;// world space pos
#ifdef CLOUDS    
    float2 Tex2      : TEXCOORD2;
#endif    
    float3 SunDir    : TEXCOORD3;
#ifdef SPECULAR    
    float3 EyeVertex : TEXCOORD4;
#endif    
    float3 VPos		 : TEXCOORD5;
    float3 VNormal	 : TEXCOORD6;
    float3 ProjPos	 : TEXCOORD7;
    float4 Color0    : COLOR0;     
    float4 Color1    : COLOR1; 
#endif
};                                               

sampler HardSlope 		: register(s0);
sampler Sand      		: register(s1);
sampler Grass     		: register(s2);
sampler	Detail	  		: register(s3);
sampler Normal    		: register(s4);
sampler	Clouds	  		: register(s5);
sampler	MaskMaterials	: register(s6);
sampler Mask      		: register(s12);//register(s7);// s7 is for shadowmap

#ifdef EDITOR_MODE
sampler EditorMask		: register(s8);
#endif

#define MASK_TEXTURE_TILE_SIZE		float2(512, 512)// size of a tile inside the masks texture (which is 2048x512)
#define MASK_TEXTURE_TILE_MIPS		8 // maximum mip levels of a sub-texture

#define MASK_NUM_TILES				4 // number of texture masks inside the texture pack
#define MASK_INV_NUM_TILES			(1.0 / MASK_NUM_TILES)

float CalculateMipmapLevel(float2 uv, float2 textureSize)
{
    float2 dx = ddx(uv * textureSize.x);
    float2 dy = ddy(uv * textureSize.y);
    float d = max(dot(dx, dx), dot(dy, dy));
    return 0.5 * log2(d);
}

float3 GetMaskedTextures(in float3 Color, in sampler MaskTexture, const in float2 Tex0, const in float2 Tex1)
{
	//float maskRock = saturate(In.Color0.r + In.Color0.g);
	float4 MaskC = tex2D(MaskTexture, Tex1);// * maskRock;
	
	float2 TexMask = Tex0 * 2;
	float lod = CalculateMipmapLevel(TexMask, MASK_TEXTURE_TILE_SIZE);
	lod = clamp(lod, 0.0, MASK_TEXTURE_TILE_MIPS);	
	float texSize = pow(2.0, MASK_TEXTURE_TILE_MIPS - lod);
	float2 size = float2(texSize * MASK_NUM_TILES, texSize);
	float2 TexMat = frac(TexMask);
	TexMat.x = TexMat.x * ((size.x * MASK_INV_NUM_TILES - 1.0) / size.x) + 0.5 / size.x;
	TexMat.y = TexMat.y * ((size.y					    - 1.0) / size.y) + 0.5 / size.y;	
	
	// mat 0
	float4 TexMatOff = float4(TexMat, 0, lod);
	Color = lerp(Color,	tex2Dlod(MaskMaterials, TexMatOff), MaskC.r);  
	
	// mat 2
	TexMatOff = float4(TexMat.x + MASK_INV_NUM_TILES * 2, TexMat.y, 0, lod);
	Color = lerp(Color,	tex2Dlod(MaskMaterials, TexMatOff), MaskC.b);  
	
	// mat 3
	TexMatOff = float4(TexMat.x + MASK_INV_NUM_TILES * 3, TexMat.y, 0, lod);
	Color = lerp(Color,	tex2Dlod(MaskMaterials, TexMatOff), MaskC.a);  
	
	// mat 1 is shadow, we don't need to sample again, just use mask
	Color *= (1 - MaskC.g);
    	
	/*	
	// the material texture has 4 textures inside it
	// we need to convert the texture coords to those 4 (meaning tileable textures from 0 to 0.5 and from 0.5 to 1 on x/y)
	float2 matTc	= float2(Tex0.x, Tex0.y * 4);// tile
	float quadTc	= matTc.x * 4;
	float floorQTc	= 0.25 * floor(quadTc);
	float ceilQTc	= 0.25 * ceil(quadTc);
	float2 tc;
	tc.y = matTc.y;

	// mat 0
	tc.x = matTc.x - floorQTc;
	Color = lerp(Color,	tex2D(MaskMaterials, tc), MaskC.r);  

	// mat 2
	tc.x = matTc.x - ceilQTc - 0.25;
	Color = lerp(Color,	tex2D(MaskMaterials, tc), MaskC.b);
	
	// mat 3
	tc.x = matTc.x - ceilQTc;
	Color = lerp(Color,	tex2D(MaskMaterials, tc), MaskC.a);
	
	// mat 1 : shadow, must always be last
	//tc.x = matTc.x - floorQTc + 0.25;
	Color *= (1 - MaskC.g);
	*/	
	
	
	// mask details as well, but let some detail come out so "some" normal mapping is applied
	//maskRockGrass *= saturate(1.4 - min(MaskC.r + MaskC.g + MaskC.b + MaskC.a, 1));	
	
	return Color;
}

float4 main( PS_IN In ) : COLOR              
{	
#ifdef OUTPUTDEPTH
	return In.depth;
	
	/*float d = In.depth;
	float z_slope = abs(ddx(d)) + abs(ddy(d));
	float K0=1E-5;
	float K1=1E-1;
	float zBias0=10*K0;
	float zBias1=10*K1;
	zBias0=biasParams.x;
	zBias1=biasParams.y;
	float od = d+zBias0+z_slope*zBias1;	
	return od;
	*/
#else
#ifdef REFLECTION
	clip(In.VPos.y);
#else
	clip(In.VPos.y + MaxDepth);
#endif

	float3 Color;
	float3 HardSlopeC = tex2D(HardSlope, In.Tex0.xy);	
	float3 SandC	  = tex2D(Sand,      In.Tex0.xy);
	float3 GrassC	  = tex2D(Grass,     In.Tex0.xy);
	
#ifdef CLOUDS
	float3 CloudsC	  = tex2D(Clouds,	 In.Tex2);
#endif

	// smoothen the texture transition
	// this can be done in the vertex shader with acceptable quality loss, if there is vertex bandwidth to spare
	float fDelta1 = saturate(lerp(In.VPos.y - 0.5, In.Color0.g, 0.1));
	float fDelta2 = saturate(lerp(In.Color0.r, In.Color0.g, 0.53));
	In.Color0.r   = 1 - fDelta1;	
	In.Color0.g   = min(1, fDelta2 * 2) - In.Color0.r;

	// blend textures
	Color  = lerp(HardSlopeC, SandC,	In.Color0.r); 
	Color  = lerp(Color,      GrassC,	In.Color0.g);

	float3 DetailC;
#ifdef MICRO_DETAIL		// - needs an overlay ready detail texture, doesn't work with the current one
	// add micro detail for all terrain
	//DetailC = tex2D(Detail,	 In.Tex0.xy * 8);
	//Color	=  1.7 * DetailC * Color;	// calculate overlay factor
#endif

	//float maskRockGrass = (1.0 - In.Color0.r);

	// add macro detail for the entire terrain
	//DetailC = 1.1 - (tex2D(Detail, In.Tex0.xy * 0.05) - 0.5); // for b & w
	//Color  *= DetailC;
	DetailC = tex2D(Detail, In.Tex0.xy * 0.08);
	Color *= DetailC;
	
#ifdef EXTRA_TEXTURE_MAPS
	Color = GetMaskedTextures(Color, Mask, In.Tex0.xy, In.Tex0.zw);
#endif

#ifdef EDITOR_MODE

	// rotate and translate
	float2 texEd;
	float2 te = In.Tex0.zw - float2(0.5, 0.5);
	texEd.x = te.x * g_edTexRot.x + te.y * g_edTexRot.y;
	texEd.y = te.y * g_edTexRot.x - te.x * g_edTexRot.y;
	texEd += float2(0.5, 0.5);
	
	float2 trans;
	trans.x = g_edTexOffset.x * g_edTexRot.x + g_edTexOffset.y * g_edTexRot.y;
	trans.y = g_edTexOffset.y * g_edTexRot.x - g_edTexOffset.x * g_edTexRot.y;		
	texEd += trans;
	Color = GetMaskedTextures(Color, EditorMask, In.Tex0.xy, texEd);
//return float4(Color, 1);	
#endif

	// use clamped sunlight for clouds and specular
	float3 clampedSunColor = min(1, SunLightColor);

#ifdef CLOUDS		
	Color *= 1 - CloudsC * AmbLightColor.w * float4(clampedSunColor, 1);
#endif

	// normal map for micro detail
	float3 NormalC = tex2D(Normal,	 In.Tex0.xy).xyz * 2 - 1;
	float maskRock = 1 - saturate(In.Color0.r + In.Color0.g);	
	float3 normal  = float3(NormalC.rg * maskRock, NormalC.b);// only apply nm to rock
	//float maskGrass = In.Color0.g;
	//normal.rg *= saturate(1 - maskGrass + 0.5);

  	float dist = distance(CamPosition, In.VPos);

  	// fade tree shadows in the distance
  	float treeDist = saturate((dist - Params.z) * Params.w); 

	// add normal map for macro detail
	NormalC = tex2D(Normal,	 In.Tex0.xy / 5.0).xyz * 2 - 1;
	
	normal.xy += (NormalC.xy * maskRock) * 1.5 ; // multiply by how much we want the macro normal map to stand out	
	//normal.z = sqrt( 1 - normal.x * normal.x - normal.y * normal.y );	
	normal = normalize(normal);

	// add diffuse		
	float3 lightDir		= normalize(In.SunDir);
	float3 lightDiff	= SunLightColor * saturate((dot(normal, lightDir) + 0.5) / 1.5);

	// add shadows, not for reflection
#ifndef REFLECTION
	if ( g_drawShadows > 0.0 )
		lightDiff *= GetJitteredCascadeShadowMapValue_16(In.ProjPos, float4(In.WSPos, 1)).r;
#endif	

#ifdef SIX_COLOR_AMBIENT	
	// 6-color ambient, use vertex normal "slightly" combined with the per pixel normal, since it gives a more uniform look
	float3 vnormal = In.VNormal;
	vnormal.xz += normal.xz;
	vnormal = normalize(vnormal);
	
	float3 squaredNormal = vnormal * vnormal;
	int3 isNegative;
	isNegative.x = step(0, -vnormal.x);// must use a function that returns integer values
	isNegative.y = step(0, -vnormal.y);
	isNegative.z = step(0, -vnormal.z);

	float3 ambient = 
		squaredNormal.x * sixColorAmbient[isNegative.x]   +
		squaredNormal.y * sixColorAmbient[isNegative.y+2] +
		squaredNormal.z * sixColorAmbient[isNegative.z+4];		

	ambient *= saturate((dot(normalize(In.VNormal.rbg), -lightDir) + 1.5) / 2.5);
	Color *= lightDiff + ambient;
#else
	Color *= lightDiff + AmbLightColor;	
#endif

	Color *= lerp(In.Color1.xyz, 1, treeDist);// fade out tree shadows where there are no trees

#ifdef SPECULAR
	float specPerturb	= max(Color.r, max(Color.g, Color.b));
	float3 eyeDir       = normalize(In.EyeVertex);
	float3 halfAngle    = normalize(lightDir + eyeDir);
	float specPower		= maskRock * 32 + In.Color0.g * 4;
	float specMask		= maskRock * 0.6 + In.Color0.g * 0.3;
	float specFactor    = pow(saturate(dot(normal, halfAngle)), specPower) * specMask * specPerturb;
	Color			   += clampedSunColor * specFactor;
#endif

	// add fog
    dist				= saturate((dist - Params.x) * Params.y);
   	Color				= lerp(Color, FogColor, dist);

#ifndef REFLECTION
	// add underwater color
	//Color				= lerp(Color, UnderWaterColor, saturate(-In.VPos.y/MaxDepth));
#endif
   	
//    	if ( In.VPos.y < 0 )
//    	{
// 		float ratio = -In.VPos.y / 3;
// 		float3 blendColor = {0.3, 0.7, 0.7};
// 		Color = lerp(Color, Color * blendColor, ratio);
//    	}
   		
#ifdef REFLECTION
	Color = lerp(Color, FogColor, 0.2);
	return float4(Color, 1);
	//return float4(Color,(1-saturate(In.VPos.y/40))*0.65);
	//return float4(lerp(float3(0,0,0),Color,(1-saturate(In.VPos.y/40))*0.65), 1);
#endif
	return float4(Color, 1);
#endif
}

TerrainSurfaceVS.fx

float4x4 matVP    : register(c0);
float3 SunLightDir : register(c4);
float3 CamPosition : register(c5);
float4 Clouds	   : register(c6);
float4 Params	   : register(c7);
float4 NormalParams: register(c8);
float3 matW		   : register(c9);// for shadows, xyz = translation
//#undef SPECULAR

// NormalParams.x is the sTangent X and Z.
// NormalParams.y is the normal Y 

// Params.x = fogZmin
// Params.y = 1.0 / ( fogZmax - fogZmin )
// Params.z = texture tile
// Params.w = detail texture tile

// Clouds.x, Clouds.y = clouds texture offset
// Clouds.z = clouds coverage

struct VS_IN                                   
{                                             
     float4 Pos       : POSITION;  
     float3 Normal    : NORMAL;// Y holds the sTangent Y, build the normal with Normal.x, NormalParams.y, Normal.z
     float4 Color0    : COLOR0;
     float4 Color1    : COLOR1;
 };       
 
 struct VS_OUT                                 
 {           
     float4 ProjPos   : POSITION;
#ifdef OUTPUTDEPTH
	 float	depth		:TEXCOORD0;
#else
     float4 Tex0      : TEXCOORD0;
	 float3 WSPos	  : TEXCOORD1;
#ifdef CLOUDS     
     float2 Tex2	  : TEXCOORD2;  
#endif     
     float3 SunDir    : TEXCOORD3;
#ifdef SPECULAR 
     float3 EyeVertex : TEXCOORD4;
#endif     
     float3 VPos	  : TEXCOORD5;  
     float3 VNormal	  : TEXCOORD6;  
     float3 OProjPos  : TEXCOORD7;// needed for shadow jittering
     float4 Color0    : COLOR0;
     float4 Color1    : COLOR1;     
#endif
 };                                             
 
 VS_OUT main( VS_IN In )                        
 {     
    VS_OUT Out;
    
    //float dist = distance(CamPosition, In.Pos);
    //In.Pos.y *= min(2000, 2000 - max(0, dist - 1000)) / 2000;
	
#ifdef OUTPUTDEPTH
    In.Pos.xyz += matW;
	Out.ProjPos = mul(In.Pos, matVP); 
	Out.depth=Out.ProjPos.z;
	return Out;
#else

	Out.ProjPos = mul(In.Pos, matVP); 

	float2 TexCoord;
	TexCoord.x = ( In.Pos.x + 250 ) / 500;
	TexCoord.y = ( 250 - In.Pos.z ) / 500;
	
    Out.Tex0.xy	= TexCoord * Params.z;    
    Out.Tex0.zw	= float2(TexCoord.x, 1 - TexCoord.y);// tex1 outputs the tex coords in the 0..1 range for each tile
    
#ifdef CLOUDS    
    Out.Tex2	= (TexCoord + Clouds.xy) * Clouds.z;
#endif    

	// decompress normal	
	In.Normal = (In.Normal - 0.5) * 2;
	In.Normal.xz *= 255;// divided in code
	
	// calc bitangent & normalize tangent vectors		
	float3 normal    = normalize(float3(In.Normal.x, NormalParams.y, In.Normal.z));
	float3 sTangent  = normalize(float3(NormalParams.x, In.Normal.y, NormalParams.x));
	float3 bitangent = normalize(cross(sTangent, normal));
	
	// put light dir into tangent space
	Out.SunDir.x	= dot(sTangent,  SunLightDir);
	Out.SunDir.y	= dot(bitangent, SunLightDir);
	Out.SunDir.z	= dot(normal,	 SunLightDir);

#ifdef SPECULAR
	// put eye direction into tangent space
	float3 eyeDir   = normalize(CamPosition - In.Pos.xyz);
	Out.EyeVertex.x = dot(sTangent,  eyeDir);
	Out.EyeVertex.y = dot(bitangent, eyeDir);
	Out.EyeVertex.z = dot(normal,    eyeDir);
#endif	
	
    Out.Color0  	= In.Color0;
    Out.Color1  	= In.Color1;      
    Out.VPos    	= In.Pos.xyz;  
    Out.VNormal 	= float3(In.Normal.x, NormalParams.y, In.Normal.z);      
    Out.WSPos		= In.Pos.xyz + matW;
    Out.OProjPos	= Out.ProjPos.xyz;
    
    return Out;  
#endif
};

TerrainUnderwaterPS.fx

#include <data/Shaders/Shadowmap/ShadowMapUtils.inc>

float3 SunLightColor : register(c0);
float4 AmbLightColor : register(c1);
float3 FogColor		 : register(c2);
float3 CausticsParam : register(c5);

// Caustics.x = caustics color factor
// Caustics.y = 1.0 / animation frames
// Caustics.z = blend factor between frames( 0 to 1 )

 struct PS_IN                                 
 {                                             
     float4 Tex0		: TEXCOORD0;
     float4 Tex2	    : TEXCOORD1; 
     float3 SunDir		: TEXCOORD2;
     float3 TexParams   : TEXCOORD3;        
     float4  Depth		: TEXCOORD4;

#ifdef SHADOWS     
     float3 WSPos		: TEXCOORD5;
#endif     
};                                               

sampler HardSlope   : register(s0);
sampler Sand		: register(s1);
sampler Normal		: register(s4);
sampler Caustics	: register(s12);// s7 is for shadows

float4 main( PS_IN In, float2 vPos : VPOS ) : COLOR              
{
	clip(-In.Depth.w);
	
	float3 Color;

	float3 HardSlopeC	= tex2D(HardSlope,	In.Tex0.xy);
	float3 SandC		= tex2D(Sand,		In.Tex0.xy);
	float3 NormalC		= tex2D(Normal,		In.Tex0.zw).xyz;
	//NormalC.rg *= 0.5;
	NormalC = normalize(NormalC);

	Color = lerp(HardSlopeC, SandC,  In.TexParams.y);
	
#ifdef SHADOWS    
	Color *= GetJitteredCascadeShadowMapValue_16(In.Depth.xyz, float4(In.WSPos.xyz, 1)).r;
#endif	

	// blend between 2 frames of animation, using the mipmap calculated in the vertex shader
	// the lod level is in TexCoord.w, gets smaller with depth
	float3 CausticsC1 = tex2Dlod(Caustics, In.Tex2);
	In.Tex2.x += CausticsParam.y;
	float3 CausticsC2 = tex2Dlod(Caustics, In.Tex2);
	
	float3 lightDir = normalize(In.SunDir);
	float lightCoef = saturate(dot(NormalC, lightDir));

	//float3 colorFactor = min(1, SunLightColor * In.TexParams.x * CausticsParam.x);
	Color += lerp(CausticsC1, CausticsC2, CausticsParam.z) * CausticsParam.x;// * colorFactor;
	Color *= SunLightColor * lightCoef + AmbLightColor;
	
	Color = lerp(Color, FogColor, In.TexParams.z);

	return float4(Color, 1);
}

TerrainUnderwaterVS.fx

float4x4 matWVP    : register(c0);
float3 SunLightDir : register(c4);
float3 CamPosition : register(c5);
float4 Caustics	   : register(c6);
float4 Params	   : register(c7);
float4 NormalParams: register(c8);
float3 matW		   : register(c9);

// NormalParams.x is the sTangent X and Z.
// NormalParams.y is the normal Y

// Caustics.x = texture offset
// Caustics.y = tile factor on X
// Caustics.z = tile factor on Y

struct VS_IN                                   
{                                             
     float4 Pos       : POSITION;  
     float3 Normal    : NORMAL;// Y holds the sTangent Y, build the normal with Normal.x, NormalParams.y, Normal.z
     float4 Color0    : COLOR0;
     float4 Color1    : COLOR1;
 };       
 
 struct VS_OUT                                 
 {                                             
     float4 ProjPos   : POSITION;
     float4 Tex0      : TEXCOORD0;
     float4 Tex2	  : TEXCOORD1;  
     float3 SunDir    : TEXCOORD2;
     float3 TexParams : TEXCOORD3;// put fog, color0.w and color1.w in a single texture param
     float4  Depth	  : TEXCOORD4;// proj pos in xyz, height in w
     
#ifdef SHADOWS     
     float3 WSPos	  : TEXCOORD5;
#endif     
};                                             
 
VS_OUT main( VS_IN In )                        
{     
	VS_OUT Out;
	Out.ProjPos = mul(In.Pos, matWVP); 
	
	float2 TexCoord;
	TexCoord.x = ( In.Pos.x + 250 ) / 500;
	TexCoord.y = ( 250 - In.Pos.z ) / 500;
		
	Out.Tex0.xy	= TexCoord * Params.z;
	Out.Tex0.zw = TexCoord * (Params.w * 5);

	Out.Depth.xyz   = Out.ProjPos.xyz;
		
#ifdef SHADOWS   	
	Out.WSPos.xyz = In.Pos.xyz + matW;
#endif	
	
	// offset the caustics texture
	Out.Tex2.x  = TexCoord.x * Caustics.y + Caustics.x;
	Out.Tex2.y  = TexCoord.y * Caustics.z;
	Out.Tex2.z  = 0;	
	
	// in Tex2.w put the mipmap level to lookup, it's smaller the bigger the depth
	float depthVal  = -min(0, In.Pos.y + 1.5);		// from 0 to 15 meters clip to mipmap 0
	depthVal        = max(0, min(1, depthVal / 20));// depthVal has to be in the [0,1] range
	Out.Tex2.w      = round(9 * depthVal);			// choose mimap level
	Out.TexParams.x = min(5, Out.Tex2.w + 2);		// put color modifier in texparams.w, make the caustics brighter with depth

	// calc bitangent & normalize tangent vectors
	float3 normal    = normalize(float3(In.Normal.x, NormalParams.y, In.Normal.z));
	float3 sTangent  = normalize(float3(NormalParams.x, In.Normal.y, NormalParams.x));
	float3 bitangent = normalize(cross(sTangent, normal));
		
	// put light dir into tangent space
	Out.SunDir.x	= dot(sTangent,  SunLightDir);
	Out.SunDir.y	= dot(bitangent, SunLightDir);
	Out.SunDir.z	= dot(normal,	 SunLightDir);

	// fog distance
	float dist = distance(CamPosition, In.Pos.xyz);
	Out.TexParams.z = saturate((dist - Params.x) * Params.y);
	
	Out.TexParams.y = In.Color0.r;
	Out.Depth.w     = In.Pos.y;
	return Out;  
};

Before I start banging my head against that big pile of code, is there anyone who is familiar with SH shaders and who knows where to look?