Add strong stack protector mode for the original propolice in GCC3.

This includes additional functions to be protected --- those that
have local array definitions, or have references to local frame
addresses.

Miod verified that this works on real hardware, and not just on the
cross-compiled monster I tested this on.

6 files changed, 51 insertions, 12 deletions

diff --git a/gnu/usr.bin/gcc/gcc/doc/invoke.texi b/gnu/usr.bin/gcc/gcc/doc/invoke.texi
index 5b3c2096131..b2b61f6758a 100644
--- a/gnu/usr.bin/gcc/gcc/doc/invoke.texi
+++ b/gnu/usr.bin/gcc/gcc/doc/invoke.texi
@@ -221,7 +221,7 @@ in the following sections.
 -Wno-import  -Wnonnull  -Wpacked  -Wpadded @gol
 -Wparentheses  -Wpointer-arith  -Wredundant-decls @gol
 -Wreturn-type  -Wsequence-point  -Wshadow @gol
--Wsign-compare  -Wstrict-aliasing @gol
+-Wsign-compare  -Wstack-protector  -Wstrict-aliasing @gol
 -Wswitch  -Wswitch-default  -Wswitch-enum @gol
 -Wsystem-headers  -Wtrigraphs  -Wundef  -Wuninitialized @gol
 -Wunknown-pragmas  -Wunreachable-code @gol
@@ -279,6 +279,7 @@ in the following sections.
 -fno-sched-interblock  -fno-sched-spec  -fsched-spec-load @gol
 -fsched-spec-load-dangerous  -fsignaling-nans @gol
 -fsingle-precision-constant  -fssa  -fssa-ccp  -fssa-dce @gol
+-fstack-protector  -fstack-protector-all  -fstack-protector-strong @gol
 -fstrength-reduce  -fstrict-aliasing  -ftracer  -fthread-jumps @gol
 -funroll-all-loops  -funroll-loops  @gol
 --param @var{name}=@var{value} @gol
@@ -2765,6 +2766,11 @@ itself is likely to take inordinate amounts of time.
 @item -Werror
 @opindex Werror
 Make all warnings into errors.
+
+@item -Wstack-protector
+@opindex Wstack-protector
+This option is only active when @option{-fstack-protector} is active.  It
+warns about functions that will not be protected against stack smashing.
 @end table
 
 @node Debugging Options
@@ -4256,6 +4262,21 @@ Perform Sparse Conditional Constant Propagation in SSA form.  Requires
 Perform aggressive dead-code elimination in SSA form.  Requires @option{-fssa}.
 Like @option{-fssa}, this is an experimental feature.
 
+@item -fstack-protector
+Emit extra code to check for buffer overflows, such as stack smashing
+attacks.  This is done by adding a guard variable to functions with
+vulnerable objects.  This includes functions with buffers larger
+than 8 bytes.  The guards are initialized when a function is entered
+and then checked when the function exits.  If a guard check fails,
+an error message is printed and the program exits.
+
+@item -fstack-protector-all
+Like @option{-fstack-protector} except that all functions are protected.
+
+@item -fstack-protector-strong
+Like @option{-fstack-protector} but includes additional functions to
+be protected --- those that have local array definitions, or have
+references to local frame addresses.
 
 @item --param @var{name}=@var{value}
 @opindex param
diff --git a/gnu/usr.bin/gcc/gcc/flags.h b/gnu/usr.bin/gcc/gcc/flags.h
index 59a496a73bf..5cb5afd9422 100644
--- a/gnu/usr.bin/gcc/gcc/flags.h
+++ b/gnu/usr.bin/gcc/gcc/flags.h
@@ -715,6 +715,7 @@ extern const char *flag_random_seed;
 
 extern int flag_propolice_protection;
 extern int flag_stack_protection;
+extern int flag_strong_protection;
 
 /* Warn when not issuing stack smashing protection for some reason */
 
diff --git a/gnu/usr.bin/gcc/gcc/function.c b/gnu/usr.bin/gcc/gcc/function.c
index 6585afc2bbb..a542606fa05 100644
--- a/gnu/usr.bin/gcc/gcc/function.c
+++ b/gnu/usr.bin/gcc/gcc/function.c
@@ -656,7 +656,7 @@ assign_stack_temp_for_type (mode, size, keep, type)
   struct temp_slot *p, *best_p = 0;
   rtx slot;
   int char_array = (flag_propolice_protection
-		    && keep == 1 && search_string_def (type));
+		    && keep == 1 && search_string_def (type, TRUE));
 
   /* If SIZE is -1 it means that somebody tried to allocate a temporary
      of a variable size.  */
diff --git a/gnu/usr.bin/gcc/gcc/protector.c b/gnu/usr.bin/gcc/gcc/protector.c
index 9d50c47fdfd..bc135a6c988 100644
--- a/gnu/usr.bin/gcc/gcc/protector.c
+++ b/gnu/usr.bin/gcc/gcc/protector.c
@@ -293,7 +293,8 @@ search_string_from_argsandvars (caller)
       {
 	if (PARM_PASSED_IN_MEMORY (parms) && DECL_NAME (parms))
 	  {
-	    string_p = search_string_def (TREE_TYPE(parms));
+	    string_p = search_string_def (
+		TREE_TYPE (parms), TREE_ADDRESSABLE (parms));
 	    if (string_p) return TRUE;
 	  }
       }
@@ -325,7 +326,8 @@ search_string_from_local_vars (block)
 	      && DECL_RTL_SET_P (types)
 	      && GET_CODE (DECL_RTL (types)) == MEM
 
-	      && search_string_def (TREE_TYPE (types)))
+	      && search_string_def (
+		 TREE_TYPE (types), TREE_ADDRESSABLE (types)))
 	    {
 	      rtx home = DECL_RTL (types);
 
@@ -374,14 +376,20 @@ search_string_from_local_vars (block)
  * search a character array from the specified type tree
  */
 int
-search_string_def (type)
+search_string_def (type, addressable)
      tree type;
+     int addressable;
 {
   tree tem;
     
   if (! type)
     return FALSE;
 
+  if (flag_strong_protection
+      && (TREE_CODE (type) == ARRAY_TYPE
+          || addressable))
+    return TRUE;
+
   switch (TREE_CODE (type))
     {
     case ARRAY_TYPE:
@@ -420,7 +428,7 @@ search_string_def (type)
       /* to protect every functions, sweep any arrays to the frame top */
       is_array = TRUE;
 
-      return search_string_def(TREE_TYPE(type));
+      return search_string_def(TREE_TYPE (type), FALSE);
 	
     case UNION_TYPE:
     case QUAL_UNION_TYPE:
@@ -440,7 +448,7 @@ search_string_def (type)
 		  || (TREE_CODE (tem) == VAR_DECL && TREE_STATIC (tem)))
 	        continue;
 
-	      if (search_string_def(TREE_TYPE(tem)))
+	      if (search_string_def(TREE_TYPE (tem), FALSE))
 		{
 		  TREE_VISITED (type) = 0;
 		  return TRUE;
@@ -978,7 +986,8 @@ arrange_var_order (block)
 	      && GET_CODE (DECL_RTL (types)) == MEM
 	      && GET_MODE (DECL_RTL (types)) == BLKmode
 
-	      && (is_array=0, search_string_def (TREE_TYPE (types))
+	      && (is_array=0, search_string_def (
+		  TREE_TYPE (types), TREE_ADDRESSABLE (types))
 		  || (! current_function_defines_vulnerable_string
 		      && is_array)))
 	    {
@@ -1057,7 +1066,8 @@ copy_args_for_protection ()
 	      }
 	    */
 
-	    string_p = search_string_def (TREE_TYPE(parms));
+	    string_p = search_string_def (
+		TREE_TYPE (parms), TREE_ADDRESSABLE (parms));
 
 	    /* check if it is a candidate to move */
 	    if (string_p || search_pointer_def (TREE_TYPE (parms)))
diff --git a/gnu/usr.bin/gcc/gcc/protector.h b/gnu/usr.bin/gcc/gcc/protector.h
index 7b47789503c..de3a4176e09 100644
--- a/gnu/usr.bin/gcc/gcc/protector.h
+++ b/gnu/usr.bin/gcc/gcc/protector.h
@@ -33,7 +33,7 @@ extern void prepare_stack_protection  PARAMS ((int inlinable));
 #ifdef TREE_CODE
 /* search a character array from the specified type tree */
 
-extern int search_string_def PARAMS ((tree names));
+extern int search_string_def PARAMS ((tree names, int addressable));
 #endif
 
 /* examine whether the input contains frame pointer addressing */
diff --git a/gnu/usr.bin/gcc/gcc/toplev.c b/gnu/usr.bin/gcc/gcc/toplev.c
index 23301a13fcf..cfcacd3450a 100644
--- a/gnu/usr.bin/gcc/gcc/toplev.c
+++ b/gnu/usr.bin/gcc/gcc/toplev.c
@@ -928,9 +928,11 @@ int force_align_functions_log;
 /* Nonzero means use propolice as a stack protection method */
 int flag_propolice_protection = 1;
 int flag_stack_protection = 0;
+int flag_strong_protection = 0;
 #else
 int flag_propolice_protection = 0;
 int flag_stack_protection = 0;
+int flag_strong_protection = 0;
 #endif
 
 int flag_trampolines = 0;
@@ -1226,7 +1228,9 @@ static const lang_independent_options f_options[] =
   {"stack-protector", &flag_propolice_protection, 1,
    N_("Enables stack protection") },
   {"stack-protector-all", &flag_stack_protection, 1,
-   N_("Enables stack protection of every function") } ,
+   N_("Enables stack protection of every function") },
+  {"stack-protector-strong", &flag_strong_protection, 1,
+   N_("Enables smart stack protection of certain functions") },
   {"trampolines", &flag_trampolines, 1,
    N_("Allows trampolines") },
 };
@@ -5311,7 +5315,10 @@ process_options ()
     /* This combination makes optimized frame addressings and causes
        a internal compilation error at prepare_stack_protection.
        so don't allow it.  */
-    if (flag_stack_protection && !flag_propolice_protection)
+    if (flag_strong_protection && flag_stack_protection)
+      flag_strong_protection = FALSE;
+    if ((flag_stack_protection || flag_strong_protection)
+	&& !flag_propolice_protection)
       flag_propolice_protection = TRUE;
 }