RE: About AT modules and buffering through SIM memory again

--- kannel-dev/gateway/gw/smsc_at2.c	Fri Feb  8 19:30:18 2002
+++ gateway/gw/smsc_at2.c	Tue Feb 12 18:24:19 2002
@@ -51,6 +51,10 @@
     int		skip_smsc_addr;
     int		prepend_zero_smsc;
     double	send_line_sleep;	/* wait seconds after having sent a line */
+#ifdef USE_SIM_BUFFERING
+	char *cpms_sm; /* set message storage to SM */
+	char *cpms_me; /* set message storage to ME */
+#endif
 };
 
 /* indexes into table below. Has to match ! */
@@ -67,6 +71,17 @@
 #define	MAX_MODEM_TYPES		9
 struct modem_def ModemTypes[MAX_MODEM_TYPES] =
 {
+#ifdef USE_SIM_BUFFERING
+    { "autodetect",    "AT+IFC=2,2"    , 0     , "AT+CNMI=1,2,0,0,0",  NULL,           NULL,   0, 0, 1, 0, 0, 0.1, "AT+CPMS=\"SM\"","AT+CPMS=\"ME\"" },
+    { "wavecom",       "AT+IFC=2,2"    , 9600  , "AT+CNMI=1,2,0,0,0",  "WAVECOM",      NULL,   0, 0, 1, 1, 1, 0.1, "AT+CPMS=\"SM\"","AT+CPMS=\"ME\"" },
+    { "premicell",     "AT+IFC=2,2"    , 9600  , "AT+CNMI=1,2,0,0,0",  "PREMICEL",     NULL,   0, 0, 0, 0, 0, 0.1, "AT+CPMS=\"SM\"","AT+CPMS=\"ME\"" },
+    { "siemens-tc35",  "AT\\Q3"        , 19200 , "AT+CNMI=1,2,0,0,1",  "SIEMENS",      "TC35", 0, 1, 1, 1, 1, 0.1, "AT+CPMS=\"SM\"","AT+CPMS=\"ME\"" },
+    { "siemens",       "AT\\Q3"        , 19200 , "AT+CNMI=1,2,0,0,0",  "SIEMENS",      "M20",  0, 1, 1, 1, 1, 0.1, "AT+CPMS=\"SM\"","AT+CPMS=\"ME\"" },
+    { "nokiaphone",    "AT+IFC=2,2"    , 9600  , "AT+CNMI=1,2,0,0,0",  "NokiaPhone" ,  NULL,   0, 1, 1, 1, 1, 0.1, "AT+CPMS=\"SM\"","AT+CPMS=\"ME\"" },
+    { "falcom",		"AT+IFC=2,2"    , 9600  , "AT+CNMI=1,2,0,0,0",  "Falcom",       NULL,   0, 0, 1, 0, 0, 0.1, "AT+CPMS=\"SM\"","AT+CPMS=\"ME\"" },
+    { "ericsson",	"AT+IFC=2,2"    , 9600  , "AT+CNMI=3,2,0,0",    "R520m",        NULL,   0, 0, 1, 1, 1, 0.1, "AT+CPMS=\"SM\",\"SM\",\"SM\"","AT+CPMS=\"ME\",\"ME\",\"ME\"" },
+    { "alcatel",	"AT+IFC=2,2"    , 9600  , "AT+CNMI=1,3,0,0,0",  "Alcatel",  	NULL,   0, 0, 1, 1, 1, 0.1, "AT+CPMS=\"SM\"","AT+CPMS=\"ME\"" } 
+#else
     { "autodetect", 	"AT+IFC=2,2" 	, 0     , "AT+CNMI=1,2,0,0,0",	NULL,		NULL, 	0, 0, 1, 0, 0, 0.1 },
     { "wavecom", 	"AT+IFC=2,2" 	, 9600  , "AT+CNMI=1,2,0,0,0",	"WAVECOM",	NULL,	0, 0, 1, 1, 1, 0.1 },
     { "premicell", 	"AT+IFC=2,2"	, 9600  , "AT+CNMI=1,2,0,0,0",	"PREMICEL",	NULL,	0, 0, 0, 0, 0, 0.1 },
@@ -76,6 +91,7 @@
     { "falcom",		"AT+IFC=2,2"	, 9600  , "AT+CNMI=1,2,0,0,0",	"Falcom",	NULL,	0, 0, 1, 0, 0, 0.1 },
     { "ericsson",	"AT+IFC=2,2"	, 9600  , "AT+CNMI=3,2,0,0",	"R520m",	NULL,	0, 0, 1, 1, 1, 0.1 },
     { "alcatel",    "AT+IFC=2,2"    , 9600  , "AT+CNMI=1,3,0,0,0",  "Alcatel",  NULL,   0, 0, 1, 1, 1, 0.1 } 
+#endif
 };
 
 /* maximum data to attempt to read in one go */
@@ -99,6 +115,27 @@
 /* The number of times to attempt to send a message should sending fail */
 #define RETRY_SEND 3
  
+#ifdef USE_SIM_BUFFERING
+/* define USE_SIM_BUFFERING to poll SIM memory at regular interval to catch messages in memory
+   this was tested to be somewhat problematic with Siemens M20 modem (for details see
+   MsgID <[email protected]> in Kannel-devel mailing list)
+   it was tested to work on Wavecom WM02B */
+
+/* once in how many listener loop to check the memory of the card */
+#define POLL_INTERVAL 20
+
+/* Define this to use SM memory (generally slower then ME memory) */
+#define USE_SM_MEM
+#undef USE_ME_MEM
+
+#ifdef USE_SM_MEM
+#define MEM_SELECT_COMMAND ModemTypes[privdata->modemid].cpms_sm
+#else 
+#define MEM_SELECT_COMMAND ModemTypes[privdata->modemid].cpms_me
+#endif
+
+#endif
+
 typedef struct PrivAT2data
 {
     List	*outgoing_queue;
@@ -116,6 +153,12 @@
     int 	phase2plus;
     Octstr	*validityperiod;    
     int		alt_dcs;
+#ifdef USE_SIM_BUFFERING
+	int input_mem_sms_capacity;
+	int input_mem_sms_used;
+	int output_mem_sms_capacity;
+	int output_mem_sms_used;
+#endif
 } PrivAT2data;
 
 
@@ -130,7 +173,7 @@
 void	at2_flush_buffer(PrivAT2data *privdata);
 int	at2_init_device(PrivAT2data *privdata);
 int	at2_send_modem_command(PrivAT2data *privdata,char *cmd, time_t timeout, int greaterflag);
-int	at2_wait_modem_command(PrivAT2data *privdata, time_t timeout, int greaterflag);
+int	at2_wait_modem_command(PrivAT2data *privdata, time_t timeout, int greaterflag, int* messages_collected);
 void	at2_set_speed(PrivAT2data *privdata, int bps);
 void	at2_device_thread(void *arg);
 int	at2_shutdown_cb(SMSCConn *conn, int finish_sending);
@@ -153,6 +196,11 @@
 void	at2_detect_speed(PrivAT2data *privdata);
 int	at2_detect_modem_type(PrivAT2data *privdata);
 int	at2_modem2id(char *name);
+int	swap_nibbles(char byte);
+#ifdef USE_SIM_BUFFERING
+void at2_check_messages_in_memory(PrivAT2data *privdata);
+int at2_memory_check(PrivAT2data* privdata);
+#endif
 
 /******************************************************************************
 ** For debugging purposes octstr_destroy might be a macro,
@@ -555,7 +603,7 @@
     	the next command. 10 sec should be suficient */
     	if(!privdata->pin_ready)
     	{
-   	   ret = at2_wait_modem_command(privdata,10, 0);
+   	   ret = at2_wait_modem_command(privdata,10, 0,NULL);
     	   if(ret == -1) /* timeout */
     	       return -1;
     	}
@@ -611,6 +659,12 @@
      ret = at2_send_modem_command(privdata, ModemTypes[privdata->modemid].init1, 0, 0);
     	if(ret != 0)
     	    return -1;
+#ifdef USE_SIM_BUFFERING
+		/* set message storage location for SIM buffering using the CPMS command */
+		ret = at2_send_modem_command(privdata, MEM_SELECT_COMMAND,0, 0);
+		if (ret != 0)
+			return -1;
+#endif
     info(0, "AT SMSC successfully opened.");
     return 0;
 }
@@ -632,7 +686,7 @@
 int at2_send_modem_command(PrivAT2data *privdata,char *cmd, time_t timeout, int gt_flag)
 {
     at2_write_line(privdata,cmd);
-    return at2_wait_modem_command(privdata, timeout, gt_flag);
+    return at2_wait_modem_command(privdata, timeout, gt_flag, NULL);
 }
 
 
@@ -641,7 +695,7 @@
 ** waits for the modem to send us something
 */
 
-int at2_wait_modem_command(PrivAT2data *privdata, time_t timeout, int gt_flag)
+int at2_wait_modem_command(PrivAT2data *privdata, time_t timeout, int gt_flag, int* messages_collected)
 {
     Octstr *line = NULL;
     Octstr *line2 = NULL;
@@ -651,6 +705,7 @@
     time_t cur_time;
     Msg	*msg;
     int len;
+	int cmgr_flag = 0;
  
     time(&end_time);
     if(timeout == 0)
@@ -705,7 +760,11 @@
 	   	ret = 1;
 	   	goto end;
 	   }
-           if (-1 != octstr_search(line, octstr_imm("+CMT"), 0))
+           if (-1 != octstr_search(line, octstr_imm("+CMT"), 0) 
+#ifdef USE_SIM_BUFFERING
+			   || ((-1 != octstr_search(line, octstr_imm("+CMGR:"), 0)) && (cmgr_flag = 1)) 
+#endif
+			   )
            {
            	line2 = at2_wait_line(privdata,1,0);
  
@@ -731,9 +790,17 @@
                     	{
 			    msg->sms.smsc_id = octstr_duplicate(privdata->conn->id);
                     	    bb_smscconn_receive(privdata->conn, msg);
+							if (messages_collected)
+								++(*messages_collected);
                     	}
+#ifdef USE_SIM_BUFFERING
+						if (!cmgr_flag) {
+#endif
                     	if(privdata->phase2plus)
 		    	    at2_write_line(privdata,"AT+CNMA");
+#ifdef USE_SIM_BUFFERING
+						}
+#endif
 			O_DESTROY(pdu);
 		    }
                 }
@@ -768,6 +835,134 @@
     return ret;
 }
 
+#ifdef USE_SIM_BUFFERING
+/******************************************************************************
+** at2_check_messages_in_memory
+** checks whether any messages are buffered in the SM or ME memory and extract
+** them.
+*/
+void at2_check_messages_in_memory(PrivAT2data* privdata)
+{
+	char cmd[20];
+
+	// get memory status
+	if (at2_memory_check(privdata) == -1) 
+	{
+		debug("bb.smsc.at2",0,"AT2[%s]: memory check error",octstr_get_cstr(privdata->device));
+		return;
+	}
+
+	if (privdata->input_mem_sms_used)
+	{ // that is - greater then 0, meaning there are some messages to fetch
+		// now - I used to just loop over the first input_mem_sms_used locations, but it doesn't
+		// hold, since under load, messages may be received while we're in the loop, and get stored
+		// in locations towards the end of the list, thus creating 'holes' in the memory.
+		// there are two ways we can fix this : (a) just read the last message location, delete it and return.
+		// it's not a complete solution since holes can still be created if messages are recieved between the
+		// memory check and the delete command, and anyway - it will slow us down and won't hold well under pressure
+		// (b) just scan the entire memory each call, bottom to top. this will be slow too, but it'll be reliable. 
+		//
+		// we can massivly improve performance by stopping after input_mem_sms_used messages have been read,
+		// but send_modem_command returns 0 for no message as well as for a message read, and the only other
+		// way to implement it is by doing memory_check after each read and stoping when input_mem_sms_used
+		// get to 0. this is slow (modem commands take time) so we improve speed only if there are less then
+		// 10 messages in memory.
+		// 
+		// I implemented the alternative - changed at2_wait_modem_command to return the number of messages it 
+		// collected
+		int i;
+		int message_count = 0; // cound number of messages collected
+
+		debug("bb.smsc.at2",0,"AT2[%s]: %d messages waiting in memory",octstr_get_cstr(privdata->device),privdata->input_mem_sms_used);
+		
+		for (i = 1; i <= privdata->input_mem_sms_capacity && 
+			message_count < privdata->input_mem_sms_used; ++i) 
+		{ // loop till end of memory or collected enouch messages
+			int old_message_count = message_count;
+			sprintf(cmd,"AT+CMGR=%d",i);
+			/* read one message from memory */
+			at2_write_line(privdata,cmd);
+			if (at2_wait_modem_command(privdata,0,0,&message_count) != 0)
+			{
+			   	debug("bb.smsc.at2",0,"AT2[%s]: failed to get message %d.",octstr_get_cstr(privdata->device),i);
+				continue; /* failed to read the message - skip to next message */
+			}
+
+			if (old_message_count == message_count) /* no need to delete if no message collected */
+				continue;
+
+			sprintf(cmd,"AT+CMGD=%d",i); /* delete the message we just read */
+			if (at2_send_modem_command(privdata,cmd,7,0) != 0)
+			{  /* 3 seconds is not enough with some modems if the message is large - so we'll give it 7 seconds */
+			   	debug("bb.smsc.at2",0,"AT2[%s]: failed to delete message %d.",octstr_get_cstr(privdata->device),i);
+				continue; /* failed to delete the message, we'll just ignore it for now - this is bad, since
+							if the message really didn't get deleted - we'll see it next time around. */
+			}
+		}
+	}
+	//at2_send_modem_command(privdata, ModemTypes[privdata->modemid].init1, 0, 0);
+}
+
+/*****************************************************************************
+ * Memory capacity and useage check
+ */
+int at2_memory_check(PrivAT2data *privdata) 
+{
+	long values[4]; // array to put response data in
+	int pos; // position of parser in data stream
+	int ret; // return code
+	Octstr* search_cpms = NULL;
+
+	if (at2_send_modem_command(privdata, "AT+CPMS?",0,0) != 0) //MEM_SELECT_COMMAND
+	{ // select memory type and get report
+		debug("bb.smsc.at2.memory_check",0,"failed to send mem select command to modem");
+		return -1;
+	}
+
+	search_cpms = octstr_create("+CPMS:");
+
+	if (pos = octstr_search(privdata->lines,search_cpms,0) != -1)
+	{ // got back a +CPMS response
+		int index = 0; // index in values array
+		pos += 6; // position of parser in the stream - start after header
+		
+		pos = octstr_search(privdata->lines,octstr_imm(","),pos) +1 ; // skip memory indication
+		while (index < 4 && 
+			pos < octstr_len(privdata->lines) && 
+			(pos = octstr_parse_long(&values[index],privdata->lines,pos,10)) != -1)
+		{ // find all the values
+			++pos; // skip number seperator;
+			++index; // increment array index
+			if (index == 2)
+				pos = octstr_search(privdata->lines,octstr_imm(","),pos)+1; // skip second memory indication
+		}
+		
+		if (index < 4)
+		{ // didn't get all memory data - I don't why, so I'll bail
+		   	debug("bb.smsc.at2",0,"AT2[%s]: couldn't parse all memory locations : %d:'%s'.",
+				octstr_get_cstr(privdata->device), index,octstr_get_cstr(privdata->lines)[pos]);
+			O_DESTROY(search_cpms);
+			return -1;
+		}
+
+		privdata->input_mem_sms_used = values[0];
+		privdata->input_mem_sms_capacity = values[1];
+		privdata->output_mem_sms_used = values[2];
+		privdata->output_mem_sms_capacity = values[3];
+
+		ret = 0; // everything cool
+		
+		//  clear the buffer
+		O_DESTROY(privdata->lines);
+	} else {
+	   	debug("bb.smsc.at2",0,"AT2[%s]: no correct header for CPMS response.",octstr_get_cstr(privdata->device));
+		ret = -1; // didn't get a +CPMS response - this is clearly an error
+	}
+
+	O_DESTROY(search_cpms);
+	return ret; 
+}
+#endif
 
 /******************************************************************************
 ** at2_set_speed
@@ -838,7 +1033,7 @@
     SMSCConn	*conn = arg;
     PrivAT2data	*privdata = conn->data;
 
-    int l;
+    int l,m;
    
     conn->status = SMSCCONN_CONNECTING;
     
@@ -869,7 +1064,15 @@
     	 if(l>0)
     	     at2_send_messages(privdata);
     	 else
-             at2_wait_modem_command(privdata,1,0);
+             at2_wait_modem_command(privdata,1,0,NULL);
+#ifdef USE_SIM_BUFFERING
+		 m++; 
+		 if (l == 0 && m > POLL_INTERVAL)
+		 {
+			 m = 0;
+			 at2_check_messages_in_memory(privdata);
+		 }
+#endif
    }
     at2_close_device(privdata);
     conn->status = SMSCCONN_DISCONNECTED;
@@ -1043,10 +1246,17 @@
 
     buffer = octstr_duplicate(line);
     /* find the beginning of a message from the modem*/ 
-    pos = octstr_search(buffer, octstr_imm("+CMT:"), 0);
-    if(pos == -1) 
+    if (pos = octstr_search(buffer, octstr_imm("+CMT:"), 0) != -1)
+		pos +=5;
+	else if (pos = octstr_search(buffer, octstr_imm("+CMGR:"), 0) != -1) {
+		pos += 6;
+		if ((pos = octstr_search(buffer,octstr_imm(","),pos)) != -1) // skip status field in +CMGR response
+			pos++;
+		else
+		goto nomsg;
+	}
+	else
 	goto nomsg;
-    pos += 5;
     pos = octstr_search(buffer, octstr_imm(","), pos);
     if(pos == -1)
 	goto nomsg;
@@ -1165,15 +1375,21 @@
         pos++;
         
         /* get the timestamp */
-        mtime.year   = octstr_get_char(pdu, pos) + 1900; pos++;
-        mtime.month  = octstr_get_char(pdu, pos); pos++;
-        mtime.day    = octstr_get_char(pdu, pos); pos++;
-        mtime.hour   = octstr_get_char(pdu, pos); pos++;
-        mtime.minute = octstr_get_char(pdu, pos); pos++;
-        mtime.second = octstr_get_char(pdu, pos); pos++;
+	mtime.year =  swap_nibbles(octstr_get_char(pdu, pos)); pos++;
+	mtime.year += (mtime.year < 70 ? 2000 : 1900);
+	mtime.month  = swap_nibbles(octstr_get_char(pdu, pos)) ; pos++;
+	mtime.day = swap_nibbles(octstr_get_char(pdu, pos)); pos++;
+	mtime.hour = swap_nibbles(octstr_get_char(pdu, pos)); pos++;
+	mtime.minute  = swap_nibbles(octstr_get_char(pdu, pos)); pos++;
+	mtime.second  = swap_nibbles(octstr_get_char(pdu, pos)); pos++;
+
         /* time zone: */
         /* XXX handle negative time zones */
-        mtime.hour  += octstr_get_char(pdu, pos); pos++;
+        /* time zone is not "swapped nibble", with the MSB as the sign (1 is negative). the problem is that
+        +1 means that we have to substract 1 from the hour to get GMT. also remember that the time zone is measured
+        in quarters of the hour and not in full hours.
+        FIXME: we need to get the module of the division by 4 to the minute field */
+	mtime.hour += ((octstr_get_char(pdu, pos) >> 7) ? 1 : -1) * (octstr_get_char(pdu, pos) & 127) / 4; pos++;
         stime = date_convert_universal(&mtime);
         
         /* get data length */
@@ -1310,8 +1526,11 @@
     
     do
     {
-    	if( ( msg = list_extract_first(privdata->outgoing_queue )))
+    	if ( msg = list_extract_first(privdata->outgoing_queue ))
 	    at2_send_one_message(privdata, msg);
+		 /* clear modem buffer, in case some leftovers from last send were left (you'd be surprised 
+			how often that happens, under load).*/
+		at2_wait_modem_command(privdata, 1, 0, NULL);
     } while (msg);
 }
 
@@ -1325,6 +1544,7 @@
     int ret = -1; 
     char sc[3];
     int retries = RETRY_SEND;
+	char tmp_id[500]; // message id for DLR
    
    /* the standard says you should be prepending the PDU with 00 to indicate to use the default SC.
       some older modems dont expect this so it can be disabled 
@@ -1354,15 +1587,60 @@
 	    at2_write(privdata,command);
 	    at2_write_ctrlz(privdata);
         /* wait 20 secs for modem command */
-	    ret = at2_wait_modem_command(privdata, 20, 0);
+	    ret = at2_wait_modem_command(privdata, 20, 0,NULL);
 	    debug("bb.at", 0, "send command status: %d", ret);
 	    if(ret != 0) /* OK only */
 	    	continue;
@@ -1826,4 +2104,11 @@
     return 0;
 }
 
-
+/* this silly thing here will just translate a "swapped nibble" pseodo Hex encoding
+   (from PDU) into something that people can actually understand.
+   implementation completly ripped off Dennis Malmstrom timestamp patches against 1.0.3.
+   thanks Dennis ! */
+int swap_nibbles(char byte)
+{
+	return ( ( byte & 15 ) * 10 ) + ( byte >> 4 );
+}