Handling out-of-memory

Heikki Linnakangas <[email protected]>
Newsgroups gmane.comp.db.unixodbc.devel
Organization VMware
Message-ID <[email protected]> 
Hi,

While working on the PostgreSQL driver, I bumped into two little bugs in 
handling out-of-memory situation. Patch attached. It's probably 
self-evident from the patch what the bugs are, but I'll explain anyway:

1. In __post_internal_error_ex, check for NULL return from malloc, and 
avoid some unnecessary allocations so that we don't need to handle the 
case that they fail. (In my test case, the reason that 
__post_internal_error_ex got called in the first place was that dlopen() 
failed with Out-of-Memory while loading the driver, so it's not 
surprising that those allocations failed too)

2. In __alloc_desc, there is a check for calloc returning NULL, but some 
of the initialization code was misplaced, and was being called on the 
NULL pointer anyway.

There are a lot more places where we don't check for malloc returning 
NULL, but fixing those two made my test case work. In the test case, I 
opened a large number of connections, and to induce the OOM condition, I 
ran it with a small "ulimit -v".

- Heikki

_______________________________________________
unixODBC-dev mailing list
[email protected]
http://mailman.unixodbc.org/mailman/listinfo/unixodbc-dev
fix-oom-crash.patch (text/x-diff, 9 KB) 
*** DriverManager/__info.c	(revision 72)
--- DriverManager/__info.c	(working copy)
***************
*** 3832,3838 ****
  
      SQLCHAR msg[ SQL_MAX_MESSAGE_LENGTH + 32 ];
      ERROR *e1, *e2;
-     SQLWCHAR *tmp;
  
      /*
       * add our prefix
--- 3832,3837 ----
***************
*** 3842,3862 ****
      strcat((char*) msg, (char*) message_text );
  
      e1 = malloc( sizeof( ERROR ));
      e2 = malloc( sizeof( ERROR ));
  
      memset( e1, 0, sizeof( *e1 ));
      memset( e2, 0, sizeof( *e2 ));
  
      e1 -> native_error = native_error;
      e2 -> native_error = native_error;
!     tmp = ansi_to_unicode_alloc( sqlstate, SQL_NTS, __get_connection( error_header ));
!     wide_strcpy( e1 -> sqlstate, tmp );
!     wide_strcpy( e2 -> sqlstate, tmp );
!     free( tmp );
!     tmp = ansi_to_unicode_alloc( msg, SQL_NTS, __get_connection( error_header ) );
!     e1 -> msg = wide_strdup( tmp );
!     e2 -> msg = wide_strdup( tmp );
!     free( tmp );
      e1 -> return_val = SQL_ERROR;
      e2 -> return_val = SQL_ERROR;
  
--- 3841,3880 ----
      strcat((char*) msg, (char*) message_text );
  
      e1 = malloc( sizeof( ERROR ));
+     if (e1 == NULL)
+         return;
      e2 = malloc( sizeof( ERROR ));
+     if (e2 == NULL)
+     {
+         free(e1);
+         return;
+     }
  
      memset( e1, 0, sizeof( *e1 ));
      memset( e2, 0, sizeof( *e2 ));
  
      e1 -> native_error = native_error;
      e2 -> native_error = native_error;
!     ansi_to_unicode_copy(e1 -> sqlstate,
!                          sqlstate, SQL_NTS, __get_connection( error_header ));
!     wide_strcpy( e2 -> sqlstate, e1 -> sqlstate );
! 
!     e1 -> msg = ansi_to_unicode_alloc( msg, SQL_NTS, __get_connection( error_header ) );
!     if ( !e1 -> msg )
!     {
!         free( e1 );
!         free( e2 );
!         return;
!     }
!     e2 -> msg = wide_strdup( e1 -> msg );
!     if ( !e2 -> msg )
!     {
!         free( e1 -> msg);
!         free( e1 );
!         free( e2 );
!         return;
!     }
! 
      e1 -> return_val = SQL_ERROR;
      e2 -> return_val = SQL_ERROR;
  
***************
*** 3879,3908 ****
      e2 -> diag_row_number = 0;
  
      if ( class_origin == SUBCLASS_ODBC )
!         tmp = ansi_to_unicode_alloc((SQLCHAR*) "ODBC 3.0", SQL_NTS, __get_connection( error_header ) );
      else
!         tmp = ansi_to_unicode_alloc((SQLCHAR*) "ISO 9075", SQL_NTS, __get_connection( error_header ) );
!     wide_strcpy( e1 -> diag_class_origin, tmp );
!     wide_strcpy( e2 -> diag_class_origin, tmp );
!     free( tmp );
  
      if ( subclass_origin == SUBCLASS_ODBC )
!         tmp = ansi_to_unicode_alloc((SQLCHAR*) "ODBC 3.0", SQL_NTS, __get_connection( error_header ) );
      else
!         tmp = ansi_to_unicode_alloc((SQLCHAR*) "ISO 9075", SQL_NTS, __get_connection( error_header ) );
!     wide_strcpy( e1 -> diag_subclass_origin, tmp );
!     wide_strcpy( e2 -> diag_subclass_origin, tmp );
!     free( tmp );
  
!     tmp = ansi_to_unicode_alloc((SQLCHAR*) "", SQL_NTS, __get_connection( error_header ) );
!     wide_strcpy( e1 -> diag_connection_name, tmp );
!     wide_strcpy( e2 -> diag_connection_name, tmp );
!     free( tmp );
  
!     tmp = ansi_to_unicode_alloc((SQLCHAR*) "", SQL_NTS, __get_connection( error_header ) );
!     wide_strcpy( e1 -> diag_server_name, tmp );
!     wide_strcpy( e2 -> diag_server_name, tmp );
!     free( tmp );
  
      /*
       * the list for SQLError puts both local and driver 
--- 3897,3924 ----
      e2 -> diag_row_number = 0;
  
      if ( class_origin == SUBCLASS_ODBC )
!     	ansi_to_unicode_copy( e1 -> diag_class_origin, (SQLCHAR*) "ODBC 3.0",
! 			      SQL_NTS, __get_connection( error_header ) );
      else
!     	ansi_to_unicode_copy( e1 -> diag_class_origin, (SQLCHAR*) "ISO 9075",
! 			      SQL_NTS, __get_connection( error_header ) );
!     wide_strcpy( e2 -> diag_class_origin, e1 -> diag_class_origin );
  
      if ( subclass_origin == SUBCLASS_ODBC )
!     	ansi_to_unicode_copy( e1 -> diag_subclass_origin, (SQLCHAR*) "ODBC 3.0",
! 			      SQL_NTS, __get_connection( error_header ) );
      else
!     	ansi_to_unicode_copy( e1 -> diag_subclass_origin, (SQLCHAR*) "ISO 9075",
! 			      SQL_NTS, __get_connection( error_header ) );
!     wide_strcpy( e2 -> diag_subclass_origin, e1 -> diag_subclass_origin );
  
!     ansi_to_unicode_copy( e1 -> diag_connection_name, (SQLCHAR*) "", SQL_NTS,
! 			  __get_connection( error_header ) );
!     wide_strcpy( e2 -> diag_connection_name, e1 -> diag_connection_name );
  
!     ansi_to_unicode_copy( e1 -> diag_server_name, (SQLCHAR*) "", SQL_NTS,
! 			  __get_connection( error_header ) );
!     wide_strcpy( e2 -> diag_server_name, e1 -> diag_server_name );
  
      /*
       * the list for SQLError puts both local and driver 
***************
*** 3925,3944 ****
       * leave space for the error prefix
       */
  
!     SQLWCHAR msg[ SQL_MAX_MESSAGE_LENGTH + 32 ], *tmp;
      ERROR *e1, *e2;
  
      /*
       * add our prefix
       */
  
!     tmp = ansi_to_unicode_alloc((SQLCHAR*) ERROR_PREFIX, SQL_NTS, __get_connection( error_header ));
!     wide_strcpy( msg, tmp );
!     free( tmp );
      wide_strcat( msg, message_text );
  
      e1 = malloc( sizeof( ERROR ));
      e2 = malloc( sizeof( ERROR ));
  
      memset( e1, 0, sizeof( *e1 ));
      memset( e2, 0, sizeof( *e2 ));
--- 3941,3966 ----
       * leave space for the error prefix
       */
  
!     SQLWCHAR msg[ SQL_MAX_MESSAGE_LENGTH + 32 ];
      ERROR *e1, *e2;
  
      /*
       * add our prefix
       */
  
!     ansi_to_unicode_copy(msg, (SQLCHAR*) ERROR_PREFIX, SQL_NTS,
! 			 __get_connection( error_header ));
      wide_strcat( msg, message_text );
  
      e1 = malloc( sizeof( ERROR ));
+     if ( !e1 )
+         return;
      e2 = malloc( sizeof( ERROR ));
+     if ( !e2 )
+     {
+         free(e1);
+         return;
+     }
  
      memset( e1, 0, sizeof( *e1 ));
      memset( e2, 0, sizeof( *e2 ));
***************
*** 3971,3990 ****
      e2 -> diag_row_number = 0;
  
      if ( class_origin == SUBCLASS_ODBC )
!         tmp = ansi_to_unicode_alloc((SQLCHAR*) "ODBC 3.0", SQL_NTS, __get_connection( error_header ) );
      else
!         tmp = ansi_to_unicode_alloc((SQLCHAR*) "ISO 9075", SQL_NTS, __get_connection( error_header ) );
!     wide_strcpy( e1 -> diag_class_origin, tmp );
!     wide_strcpy( e2 -> diag_class_origin, tmp );
!     free( tmp );
  
      if ( subclass_origin == SUBCLASS_ODBC )
!         tmp = ansi_to_unicode_alloc((SQLCHAR*) "ODBC 3.0", SQL_NTS, __get_connection( error_header ) );
      else
!         tmp = ansi_to_unicode_alloc((SQLCHAR*) "ISO 9075", SQL_NTS, __get_connection( error_header ) );
!     wide_strcpy( e1 -> diag_subclass_origin, tmp );
!     wide_strcpy( e2 -> diag_subclass_origin, tmp );
!     free( tmp );
  
      e1 -> diag_connection_name[ 0 ] = 0;
      e2 -> diag_connection_name[ 0 ] = 0;
--- 3993,4012 ----
      e2 -> diag_row_number = 0;
  
      if ( class_origin == SUBCLASS_ODBC )
!         ansi_to_unicode_copy( e1 -> diag_class_origin, (SQLCHAR*) "ODBC 3.0",
! 							  SQL_NTS, __get_connection( error_header ) );
      else
!         ansi_to_unicode_copy( e1 -> diag_class_origin, (SQLCHAR*) "ISO 9075",
! 							  SQL_NTS, __get_connection( error_header ) );
!     wide_strcpy( e2 -> diag_class_origin, e1 -> diag_class_origin );
  
      if ( subclass_origin == SUBCLASS_ODBC )
!         ansi_to_unicode_copy( e1 -> diag_subclass_origin, (SQLCHAR*) "ODBC 3.0",
! 							  SQL_NTS, __get_connection( error_header ) );
      else
!         ansi_to_unicode_copy( e1 ->diag_subclass_origin, (SQLCHAR*) "ISO 9075",
! 							  SQL_NTS, __get_connection( error_header ) );
!     wide_strcpy( e2 -> diag_subclass_origin, e1 -> diag_subclass_origin );
  
      e1 -> diag_connection_name[ 0 ] = 0;
      e2 -> diag_connection_name[ 0 ] = 0;
*** DriverManager/__handles.c	(revision 72)
--- DriverManager/__handles.c	(working copy)
***************
*** 1354,1360 ****
  
  DMHDESC __alloc_desc( void )
  {
!     DMHDESC descriptor = NULL;
  
      mutex_entry( &mutex_lists );
  
--- 1354,1360 ----
  
  DMHDESC __alloc_desc( void )
  {
!     DMHDESC descriptor;
  
      mutex_entry( &mutex_lists );
  
***************
*** 1375,1392 ****
  #endif    
          descriptor_root = descriptor;
          descriptor -> type = HDESC_MAGIC;
-     }
  
!     setup_error_head( &descriptor -> error, descriptor,
!             SQL_HANDLE_DESC );
  
  #ifdef HAVE_LIBPTH
!     pth_mutex_init( &descriptor -> mutex );
  #elif HAVE_LIBPTHREAD
!     pthread_mutex_init( &descriptor -> mutex, NULL );
  #elif HAVE_LIBTHREAD
!     mutex_init( &descriptor -> mutex, USYNC_THREAD, NULL );
  #endif
  
      mutex_exit( &mutex_lists );
  
--- 1375,1391 ----
  #endif    
          descriptor_root = descriptor;
          descriptor -> type = HDESC_MAGIC;
  
!         setup_error_head( &descriptor -> error, descriptor, SQL_HANDLE_DESC );
  
  #ifdef HAVE_LIBPTH
!         pth_mutex_init( &descriptor -> mutex );
  #elif HAVE_LIBPTHREAD
!         pthread_mutex_init( &descriptor -> mutex, NULL );
  #elif HAVE_LIBTHREAD
!         mutex_init( &descriptor -> mutex, USYNC_THREAD, NULL );
  #endif
+     }
  
      mutex_exit( &mutex_lists );
lmpx.com only provides a reader for public news (NNTP) servers. It is not affiliated with the servers or forums shown here and is not responsible for the content of articles, which is written by their respective authors.