401194 - crash in lg_FindObjects on win64

Status: RESOLVED FIXED

(NSS :: Libraries, defect, P2)

Description

[Julien Pierre]

There are three instances of cmsutil crashing in the QA with my port of NSS to Win64 (AMD64/x64). All three have a very similar stack, which is :

 	msvcr80.dll!memcpy(unsigned char * dst=, unsigned char * src=, unsigned long count=)  Line 284	Asm
>	nssdbm3.dll!lg_FindObjects(SDBStr * sdb=0x00000000025fe480, SDBFindStr * search=0x0000000002621270, unsigned long * phObject=0x0000000002648824, unsigned long ulMaxObjectCount=5, unsigned long * pulObjectCount=0x000000000023f5b0)  Line 923	C
 	softokn3.dll!sftkdb_FindObjects(SFTKDBHandleStr * handle=0x00000000025df0a0, SDBFindStr * find=0x0000000002621270, unsigned long * ids=0x0000000002648824, int arraySize=5, unsigned long * count=0x000000000023f5b0)  Line 765 + 0x29 bytes	C
 	softokn3.dll!sftk_searchDatabase(SFTKDBHandleStr * handle=0x00000000025df0a0, SFTKSearchResultsStr * search=0x0000000002621230, const CK_ATTRIBUTE * pTemplate=0x000000000023f708, long ulCount=2)  Line 3790 + 0x23 bytes	C
 	softokn3.dll!sftk_searchTokenList(SFTKSlotStr * slot=0x00000000025ec3c0, SFTKSearchResultsStr * search=0x0000000002621230, CK_ATTRIBUTE * pTemplate=0x000000000023f708, long ulCount=2, int * tokenOnly=0x000000000023f6a8, int isLoggedIn=0)  Line 3911 + 0x19 bytes	C
 	softokn3.dll!NSC_FindObjectsInit(unsigned long hSession=16777217, CK_ATTRIBUTE * pTemplate=0x000000000023f708, unsigned long ulCount=2)  Line 3962 + 0x31 bytes	C
 	nss3.dll!nssToken_TraverseCertificates(NSSTokenStr * token=0x0000000002633a20, nssSessionStr * sessionOpt=0x0000000002630a98, nssTokenSearchType searchType=nssTokenSearchType_TokenOnly, PRStatus (nssCryptokiInstanceStr *, void *)* callback=0x0000000000646600, void * arg=0x0000000002639930)  Line 1665 + 0x1d bytes	C
 	nss3.dll!NSSTrustDomain_TraverseCertificates(NSSTrustDomainStr * td=0x0000000002630920, PRStatus (NSSCertificateStr *, void *)* callback=0x00000000005f1f70, void * arg=0x000000000023f848)  Line 1077 + 0x28 bytes	C
 	nss3.dll!PK11_ListCerts(PK11CertListType type=PK11CertListUser, void * pwarg=0x00000000004241c0)  Line 2413	C
 	nss3.dll!pk11_keyIDHash_populate(void * wincx=0x00000000004241c0)  Line 1472 + 0xf bytes	C
 	libnspr4.dll!PR_CallOnceWithArg(PRCallOnceType * once=0x0000000000ba4a68, PRStatus (void *)* func=0x00000000005f0430, void * arg=0x00000000004241c0)  Line 844 + 0x9 bytes	C
 	nss3.dll!PK11_FindCertAndKeyByRecipientListNew(NSSCMSRecipientStr * * recipientlist=0x0000000002621210, void * wincx=0x00000000004241c0)  Line 1504 + 0x19 bytes	C
 	smime3.dll!NSS_CMSEnvelopedData_Decode_BeforeData(NSSCMSEnvelopedDataStr * envd=0x000000000262b1c8)  Line 345 + 0x20 bytes	C
 	smime3.dll!nss_cms_before_data(NSSCMSDecoderContextStr * p7dcx=0x000000000262f860)  Line 265 + 0xe bytes	C
 	smime3.dll!nss_cms_decoder_notify(void * arg=0x000000000262f860, int before=1, void * dest=0x000000000262b248, int depth=4)  Line 210 + 0xa bytes	C
 	nssutil3.dll!sec_asn1d_notify_before(sec_DecoderContext_struct * cx=0x0000000002637f90, void * dest=0x000000000262b248, int depth=4)  Line 452	C
 	nssutil3.dll!sec_asn1d_next_in_sequence(sec_asn1d_state_struct * state=0x0000000002638288)  Line 2018	C
 	nssutil3.dll!SEC_ASN1DecoderUpdate_Util(sec_DecoderContext_struct * cx=0x0000000002637f90, const char * buf=0x0000000002621aaf, unsigned long len=22)  Line 2673	C
 	smime3.dll!NSS_CMSDecoder_Update(NSSCMSDecoderContextStr * p7dcx=0x000000000262f860, const char * buf=0x0000000002621970, unsigned long len=341)  Line 670 + 0x17 bytes	C
 	cmsutil.exe!decode(_iobuf * out=0x0000000000000000, SECItemStr * input=0x000000000023fee8, const decodeOptionsStr * decodeOptions=0x000000000023fdb0)  Line 224 + 0x22 bytes	C
 	cmsutil.exe!main(int argc=12, char * * argv=0x00000000025e1550)  Line 1463 + 0x17 bytes	C
 	cmsutil.exe!__tmainCRTStartup()  Line 586 + 0x19 bytes	C

Comment 1

[Julien Pierre]

Attachment: Fix size for memcpy

On Windows 64-bit, sizeof(CK_OBJECT_HANDLE_PTR) is twice sizeof(CK_OBJECT_HANDLE) . The arrays, both source and targets, are actually of CK_OBJECT_HANDLE, not CK_OBJECT_HANDLE_PTR .

Comment 2

[Robert Relyea]

Comment on attachment 286253 [details] [diff] [review]
Fix size for memcpy

r+ yikes!

Comment 3

[Julien Pierre]

Bob,

Thanks for the review. I checked this in to the trunk.

Checking in lgfind.c;
/cvsroot/mozilla/security/nss/lib/softoken/legacydb/lgfind.c,v  <--  lgfind.c
new revision: 1.4; previous revision: 1.3
done