251882 - assertion in isctype.c when processing non us-ascii junk mail

Status: RESOLVED FIXED

(Thunderbird :: General, defect)

Description

[(not reading, please use [e-mail] instead)]

assertion when processing non us-ascii junk mail

here's my stack:

 	7ffe0304()	
 	user32.dll!77d43c53() 	
 	user32.dll!77d4b3f2() 	
 	user32.dll!77d4d9a0() 	
 	user32.dll!77d6ae8e() 	
 	ntdll.dll!77f59037() 	
 	user32.dll!77d45950() 	
 	user32.dll!77d4591f() 	
 	user32.dll!77d459c9() 	
 	uxtheme.dll!5ad744fa() 	
 	uxtheme.dll!5ad744bd() 	
 	uxtheme.dll!5ad7422f() 	
 	uxtheme.dll!5ad7450c() 	
 	user32.dll!77d43a50() 	
 	user32.dll!77d4a8fe() 	
 	user32.dll!77d43b4f() 	
 	user32.dll!77d43b33() 	
 	user32.dll!77d43b33() 	
 	user32.dll!77d444f5() 	
 	user32.dll!77d44500() 	
 	msvcr71d.dll!write_string(char * string=0x10268629, int len=1223328, _iobuf *
f=0x10214da4, int * pnumwritten=0x0012a800)  Line 1258 + 0x19	C
 	msvcr71d.dll!_output(_iobuf * stream=0x77d6b04a, const char *
format=0x0093028e, char * argptr=0x0012ac44)  Line 405 + 0x17	C
 	0012aaac()	
 	user32.dll!77d6b02e() 	
 	msvcr71d.dll!__crtMessageBoxA(const char * lpText=0x0012ac44, const char *
lpCaption=0x10268444, unsigned int uType=73746)  Line 119	C
 	msvcr71d.dll!CrtMessageWindow(int nRptType=2, const char * szFile=0x1026a314,
const char * szLine=0x0012bc88, const char * szModule=0x00000000, const char *
szUserMessage=0x0012bca8)  Line 617 + 0x16	C
 	msvcr71d.dll!_CrtDbgReport(int nRptType=2, const char * szFile=0x1026a314, int
nLine=68, const char * szModule=0x00000000, const char * szFormat=0x1026a320,
...)  Line 516 + 0x4c	C
 	msvcr71d.dll!_chvalidator_mt(threadlocaleinfostruct * p=0x003d4b90, int c=-47,
int mask=4)  Line 68 + 0x22	C
 	msvcr71d.dll!isdigit(int c=-47)  Line 101 + 0x2f	C
 	mail.dll!isDecimalNumber(const char * word=0x122b731d)  Line 276 + 0xb	C++
>	mail.dll!Tokenizer::addTokenForHeader(const char * aTokenPrefix=0x054419c0,
nsACString & aValue={...}, int aTokenizeValue=1)  Line 320 + 0x9	C++
 	mail.dll!Tokenizer::tokenizeHeaders(nsIUTF8StringEnumerator *
aHeaderNames=0x0544196c, nsIUTF8StringEnumerator * aHeaderValues=0x122b72ac) 
Line 396	C++
 	mail.dll!TokenStreamListener::ProcessHeaders(nsIUTF8StringEnumerator *
aHeaderNames=0x0544196c, nsIUTF8StringEnumerator * aHeaderValues=0x122b72ac, int
dontCollectAddress=0)  Line 638	C++
 	mail.dll!nsMimeHtmlDisplayEmitter::BroadcastHeaders(nsIMsgHeaderSink *
aHeaderSink=0x1232074c, int aHeaderMode=1, int aFromNewsgroup=0)  Line 216	C++
 	mail.dll!nsMimeHtmlDisplayEmitter::WriteHTMLHeaders()  Line 264 + 0x19	C++
 	mail.dll!nsMimeHtmlDisplayEmitter::EndHeader()  Line 361	C++
 	mail.dll!mimeEmitterEndHeader(MimeDisplayOptions * opt=0x1270ebc8)  Line
1926 + 0xc	C++
 	mail.dll!MimeMessage_write_headers_html(MimeObject * obj=0x12746680)  Line
804 + 0xc	C++
 	mail.dll!MimeMessage_close_headers(MimeObject * obj=0x12746680)  Line 436 +
0x9	C++
 	mail.dll!MimeMessage_parse_line(char * aLine=0x12315e48, int aLength=2,
MimeObject * obj=0x12746680)  Line 285 + 0x9	C++
 	mail.dll!convert_and_send_buffer(char * buf=0x12315e48, int length=2, int
convert_newlines_p=1, int (char *, unsigned int, void *)*
per_line_fn=0x023f2100, void * closure=0x12746680)  Line 168 + 0xf	C++
 	mail.dll!mime_LineBuffer(const char * net_buffer=0x04d7acc7, int
net_buffer_size=7957, char * * bufferP=0x127466a8, int *
buffer_sizeP=0x127466b0, unsigned int * buffer_fpP=0x127466b8, int
convert_newlines_p=1, int (char *, unsigned int, void *)*
per_line_fn=0x023f2100, void * closure=0x12746680)  Line 256 + 0x1d	C++
 	mail.dll!MimeObject_parse_buffer(const char * buffer=0x04d7a018, int
size=11204, MimeObject * obj=0x12746680)  Line 252 + 0x31	C++
 	mail.dll!mime_display_stream_write(_nsMIMESession * stream=0x1270ed20, const
char * buf=0x04d7a018, int size=11204)  Line 941 + 0x14	C++
 	mail.dll!nsStreamConverter::OnDataAvailable(nsIRequest * request=0x128a91c8,
nsISupports * ctxt=0x12493c44, nsIInputStream * aIStream=0x1235f968, unsigned
int sourceOffset=0, unsigned int aLength=11204)  Line 952 + 0x18	C++
 	necko.dll!nsStreamListenerTee::OnDataAvailable(nsIRequest *
request=0x128a91c8, nsISupports * context=0x12493c44, nsIInputStream *
input=0x1287ad84, unsigned int offset=0, unsigned int count=11204)  Line 97 +
0x33	C++
 	necko.dll!nsOnDataAvailableEvent0::HandleEvent()  Line 425 + 0x2f	C++
 	necko.dll!nsStreamListenerEvent0::HandlePLEvent(PLEvent * aEvent=0x05757e58) 
Line 113 + 0xc	C++
 	xpcom.dll!PL_HandleEvent(PLEvent * self=0x05757e58)  Line 673 + 0xa	C
 	xpcom.dll!PL_ProcessPendingEvents(PLEventQueue * self=0x00b48e28)  Line 608
+ 0x9	C
 	xpcom.dll!_md_TimerProc(HWND__ * hwnd=0x00b5028c, unsigned int uMsg=275,
unsigned int idEvent=0, unsigned long dwTime=1012166640)  Line 979 + 0x9	C
 	user32.dll!77d43a50() 	
 	user32.dll!77d442c5() 	
 	user32.dll!77d43e6f() 	
 	user32.dll!77d43fd4() 	
 	user32.dll!77d43ddf() 	
 	gkwidget.dll!nsAppShell::Run()  Line 135	C++
 	appshell.dll!nsAppShellService::Run()  Line 495	C++
 	thunderbird.exe!xre_main(int argc=1, char * * argv=0x003d8570, const
nsXREAppData * aAppData=0x0041e05c)  Line 1990 + 0x23	C++
 	thunderbird.exe!main(int argc=1, char * * argv=0x003d8570)  Line 58 + 0x12	C++
 	thunderbird.exe!mainCRTStartup()  Line 398 + 0x11	C
 	kernel32.dll!77e814c7()

Comment 1

[(not reading, please use [e-mail] instead)]

I think the problem is in Tokenizer::addTokenForHeader()


      while ((word = nsCRT::strtok(next, kBayesianFilterTokenDelimiters, &next))
!= NULL) 
      {
          if (word[0] == '\0') continue;
          if (isDecimalNumber(word)) continue;
          if (isASCII(word))
              add(PromiseFlatCString(nsDependentCString(aTokenPrefix) +
NS_LITERAL_CSTRING(":") + nsDependentCString(word)).get());
      }

in my case, word is: 

+	word	0x122b731d "управленuю"	char *

so when we call isDecimalNumber(), we end up calling isdigit() with -47

static PRBool isDecimalNumber(const char* word)
{
    const char* p = word;
    if (*p == '-') ++p;
    char c;
    while ((c = *p++)) {
        if (!isdigit(c))
            return PR_FALSE;
    }
    return PR_TRUE;
}

I think that the fix would be to what isASCII() does, which is first check for c
> 127

static PRBool isASCII(const char* word)
{
    const unsigned char* p = (const unsigned char*)word;
    unsigned char c;
    while ((c = *p++)) {
        if (c > 127)
            return PR_FALSE;
    }
    return PR_TRUE;
}

Comment 2

[(not reading, please use [e-mail] instead)]

hmm, checking for > 127 is probably wrong.

from http://www.mkssoftware.com/docs/man3/isdigit.3.asp

The behavior of the isdigit() function is affected by the current locale. To
modify the behavior, change the LC_CTYPE category in setlocale(), that is,
setlocale(LC_CTYPE, newlocale). In the C locale or in a locale where character
type information is not defined, characters are classified according to the
rules of the U.S. ASCII 7-bit coded character set.

see also http://blogs.msdn.com/oldnewthing/archive/2004/03/09/86555.aspx about
Arabic digits.

Comment 3

[(not reading, please use [e-mail] instead)]

Attachment: screen shot

Comment 4

[(not reading, please use [e-mail] instead)]

Attachment: spam that will assert (many, many times) when you classify it

Comment 5

[(not reading, please use [e-mail] instead)]

Attachment: patch

Comment 6

[David :Bienvenu]

I do see that occasionally, I think...patch seems OK, I guess. Cc'ing jshin.

Comment 7

[Jungshik Shin]

+        if ((((unsigned)(c+1)) > 256) || !isdigit(c))

You meant 128, didn't you? For now, that should more or less work. 


As for isdigit(), it's not generic enough even with the locale set properly (it
doesn't work 
with multibyte encodings). iswdigit() has to be used instead after converting to
wchar*. 
However, that's not portable because wchar_t is platform/compiler dependent.  

Comment 8

[David :Bienvenu]

Attachment: shutup assertion

we're just passing a char in, so make it unsigned.

Comment 9

[Chris Thomas (CTho) [formerly [e-mail] [e-mail]]]

*** Bug 325938 has been marked as a duplicate of this bug. ***

Comment 10

[Robin Grindley]

I'm seeing the exact same assertion, but I think mine is in Tokenizer::tokenize. Stack: 

 	msvcr80d.dll!1023c92d() 	
 	[Frames below may be incorrect and/or missing, no symbols loaded for msvcr80d.dll]	
 	msvcr80d.dll!1023cd81() 	
 	xpcom_core.dll!nsCRT::strtok(char * string=0x04829aa1, const char * delims=0x0152ee88, char * * newStr=0x00000000)  Line 172 + 0xa bytes	C++
>	thunderbird.exe!Tokenizer::tokenize(char * aText=0x044fd240)  Line 600 + 0x9 bytes	C++
 	thunderbird.exe!TokenStreamListener::OnDataAvailable(nsIRequest * aRequest=0x06939e88, nsISupports * aContext=0x06a42adc, nsIInputStream * aInputStream=0x04a6df18, unsigned int aOffset=0, unsigned int aCount=0)  Line 861	C++
 	thunderbird.exe!nsMimeBaseEmitter::Complete()  Line 948 + 0x31 bytes	C++
 	thunderbird.exe!nsStreamConverter::OnStopRequest(nsIRequest * request=0x06939e88, nsISupports * ctxt=0x06a42adc, unsigned int status=0)  Line 1034	C++
 	thunderbird.exe!nsStreamListenerTee::OnStopRequest(nsIRequest * request=0x06939e88, nsISupports * context=0x06a42adc, unsigned int status=0)  Line 66	C++
 	thunderbird.exe!nsOnStopRequestEvent0::HandleEvent()  Line 318 + 0x23 bytes	C++
 	thunderbird.exe!nsStreamListenerEvent0::HandlePLEvent(PLEvent * aEvent=0x06875f60)  Line 113 + 0xe bytes	C++
 	xpcom_core.dll!PL_HandleEvent(PLEvent * self=0x06875f60)  Line 688 + 0xc bytes	C
 	xpcom_core.dll!PL_ProcessPendingEvents(PLEventQueue * self=0x024ef8b8)  Line 623 + 0x9 bytes	C
 	xpcom_core.dll!_md_EventReceiverProc(HWND__ * hwnd=0x00470226, unsigned int uMsg=49559, unsigned int wParam=0, long lParam=38729912)  Line 1408 + 0x9 bytes	C
 	user32.dll!77d48734() 	
 	user32.dll!77d48816() 	
 	user32.dll!77d489cd() 	
 	user32.dll!77d49402() 	
 	user32.dll!77d48a10() 	
 	thunderbird.exe!nsAppShell::Run()  Line 135	C++
 	thunderbird.exe!nsAppStartup::Run()  Line 150 + 0x1c bytes	C++
 	thunderbird.exe!XRE_main(int argc=1, char * * argv=0x022b7a30, const nsXREAppData * aAppData=0x01439d34)  Line 2351 + 0x25 bytes	C++
 	thunderbird.exe!main(int argc=1, char * * argv=0x022b7a30)  Line 62 + 0x12 bytes	C++
 	thunderbird.exe!__tmainCRTStartup()  Line 586 + 0x19 bytes	C
 	thunderbird.exe!mainCRTStartup()  Line 403	C
 	kernel32.dll!7c816d4f() 	
 	kernel32.dll!7c8399f3() 	
 	thunderbird.exe!nsRuleNode::SetFont(nsPresContext * aPresContext=0x00530022, nsStyleContext * aContext=0x00730079, int aMinFontSize=6619252, int aUseDocumentFonts=2097261, int aIsGeneric=6619204, const nsRuleDataFont & aFontData={...}, const nsFont & aDefaultFont={...}, const nsStyleFont * aParentFont=0x00200074, nsStyleFont * aFont=0x006f0043, int & aInherited=1575412840)  Line 1847 + 0x10 bytes	C++
 	xpcom_core.dll!nsProxyObject::`scalar deleting destructor'()  + 0x20 bytes	C++
 	xpcom_core.dll!xptiManifest::Write(xptiInterfaceInfoManager * aMgr=0x00660064, xptiWorkingSet * aWorkingSet=0x002c0022)  Line 140 + 0x14 bytes	C++
 	xpcom_core.dll!xptiManifest::Write(xptiInterfaceInfoManager * aMgr=0x00660064, xptiWorkingSet * aWorkingSet=0x002c0022)  Line 140 + 0x14 bytes	C++
 	xpcom_core.dll!xptiManifest::Write(xptiInterfaceInfoManager * aMgr=0x002c0073, xptiWorkingSet * aWorkingSet=0x0061006c)  Line 140 + 0x14 bytes	C++
 	thunderbird.exe!nsTransferable::QueryInterface(const nsID & aIID={...}, void * * aInstancePtr=0x00000064)  Line 70 + 0x82 bytes	C++
 	8b0d7364()

Comment 11

[Robin Grindley]

Realized that the patch actually fixes all three possible occurrences of the assesrtion. Re-compiled with the patch and confirmed that the assertion is now gone. I'm new to debugging Mozilla, apologies for spam. 

Comment 12

[[e-mail]]

Comment on attachment 213113 [details] [diff] [review]
shutup assertion

I keep hitting this on branch :-(

Comment 13

[David :Bienvenu]

yes, that's fine for tb 2 - can you go ahead and land it, Neil? I don't have official bugzilla rights to + that, but I think my say-so is fine for this...

Comment 14

[Scott MacGregor]

Neil landed this patch on the branch.