*** a/doc/src/sgml/gist.sgml
--- b/doc/src/sgml/gist.sgml
***************
*** 377,383 **** my_decompress(PG_FUNCTION_ARGS)
        <para>
         Returns a value indicating the <quote>cost</quote> of inserting the new
         entry into a particular branch of the tree.  Items will be inserted
!        down the path of least <function>penalty</function> in the tree.
        </para>
  
        <para>
--- 377,385 ----
        <para>
         Returns a value indicating the <quote>cost</quote> of inserting the new
         entry into a particular branch of the tree.  Items will be inserted
!        down the path of least <function>penalty</function> in the tree. Value
!        returned by <function>penalty</function> should be non-negative. Negative
!        values returned by <function>penalty</function> are treating as zero.
        </para>
  
        <para>
*** a/src/backend/access/gist/gistutil.c
--- b/src/backend/access/gist/gistutil.c
***************
*** 526,536 **** gistpenalty(GISTSTATE *giststate, int attno,
--- 526,540 ----
  
  	if (giststate->penaltyFn[attno].fn_strict == FALSE ||
  		(isNullOrig == FALSE && isNullAdd == FALSE))
+ 	{
  		FunctionCall3Coll(&giststate->penaltyFn[attno],
  						  giststate->supportCollation[attno],
  						  PointerGetDatum(orig),
  						  PointerGetDatum(add),
  						  PointerGetDatum(&penalty));
+ 		if (penalty < 0.0)
+ 			penalty = 0.0;
+ 	}
  	else if (isNullOrig && isNullAdd)
  		penalty = 0.0;
  	else