Bug 1758641 Comment 5 Edit History

Note: The actual edited comment in the bug view page will always show the original commenter’s name and original timestamp.

Original comment by

Yannis Juglaret [:yannis]

on 2023-01-11 08:54:21 PST

I have analyzed the most recurrent crashes together thanks to the DLL. The heap corruption is reported as a double free. I have identified that the last called functions in the stack belong to functions from the [JsonCpp](https://github.com/open-source-parsers/jsoncpp) library. I have also restored the Firefox part of the stack. The crash occurs in third-party code that gets called through `PR_GetAddrInfoByName`, they are probably hooking `getaddrinfo` from `ws2_32.dll` or something like this. The full call stack looks like this:

```
00 000000cd`343ce370 00007ffb`951cf673     ntdll!RtlReportFatalFailure+0x9
01 000000cd`343ce3c0 00007ffb`951d83f2     ntdll!RtlReportCriticalFailure+0x97
02 000000cd`343ce4b0 00007ffb`951d86da     ntdll!RtlpHeapHandleError+0x12
03 000000cd`343ce4e0 00007ffb`951de361     ntdll!RtlpHpHeapHandleError+0x7a
04 000000cd`343ce510 00007ffb`950f5bf0     ntdll!RtlpLogHeapFailure+0x45
05 000000cd`343ce540 00007ffb`950f47b1     ntdll!RtlpFreeHeapInternal+0x4e0
06 000000cd`343ce600 00007ffb`542850cc     ntdll!RtlFreeHeap+0x51
 wslbdhm64+0x350cc _free_base
 wslbdhm64+0x5eeb8 Json::Value::~Json::Value
 wslbdhm64+0x5ef84 Json::Value::"DestructiveMoveAssignment"
 wslbdhm64+0x657a6 DoStuff
 wslbdhm64+0x65e8b
 wslbdhm64+0x1de43
 wslbdhm64+0x1d9d8
 wslbdhm64+0x160db
 wslbdhm64+0x6829e
 wslbdhm64+0x64faf
 [ dynamic code ]
00 00000016`760dede0 00007ffb`49b4f32a     nss3!PR_GetAddrInfoByName+0xc3 [/builds/worker/checkouts/gecko/nsprpub/pr/src/misc/prnetdb.c @ 2171]
01 (Inline Function) --------`--------     xul!mozilla::net::_GetAddrInfo_Portable+0x5f [/builds/worker/checkouts/gecko/netwerk/dns/GetAddrInfo.cpp @ 241]
02 00000016`760dee60 00007ffb`4a2a284b     xul!mozilla::net::GetAddrInfo+0x15a [/builds/worker/checkouts/gecko/netwerk/dns/GetAddrInfo.cpp @ 374]
03 00000016`760df0a0 00007ffb`4a1e4471     xul!nsHostResolver::ThreadFunc+0x43b [/builds/worker/checkouts/gecko/netwerk/dns/nsHostResolver.cpp @ 1727]
04 (Inline Function) --------`--------     xul!mozilla::detail::RunnableMethodArguments<>::applyImpl+0x9 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1147]
05 (Inline Function) --------`--------     xul!mozilla::detail::RunnableMethodArguments<>::apply+0x9 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1153]
06 00000016`760df200 00007ffb`4a211f3e     xul!mozilla::detail::RunnableMethodImpl<RefPtr<nsObserverService>,void (nsObserverService::*)(),1,0>::Run+0x21 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1203]
07 00000016`760df230 00007ffb`4b1c1aa8     xul!nsThreadPool::Run+0x29e [/builds/worker/checkouts/gecko/xpcom/threads/nsThreadPool.cpp @ 311]
08 (Inline Function) --------`--------     xul!nsThread::ProcessNextEvent+0x17b0 [/builds/worker/checkouts/gecko/xpcom/threads/nsThread.cpp @ 1198]
09 00000016`760df3d0 00007ffb`4b1ad48f     xul!NS_ProcessNextEvent+0x1808 [/builds/worker/checkouts/gecko/xpcom/threads/nsThreadUtils.cpp @ 466]
0a 00000016`760df760 00007ffb`4a3aba5f     xul!mozilla::ipc::MessagePumpForNonMainThreads::Run+0xcf [/builds/worker/checkouts/gecko/ipc/glue/MessagePump.cpp @ 301]
0b (Inline Function) --------`--------     xul!MessageLoop::RunInternal+0x16 [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 381]
0c 00000016`760df800 00007ffb`499ef54e     xul!MessageLoop::RunHandler+0x2f [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 375]
0d 00000016`760df850 00007ffb`4a20d757     xul!MessageLoop::Run+0x4e [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 357]
0e 00000016`760df8b0 00007ffb`63d244bc     xul!nsThread::ThreadFunc+0xd7 [/builds/worker/checkouts/gecko/xpcom/threads/nsThread.cpp @ 385]
0f 00000016`760dfa60 00007ffb`63d39241     nss3!_PR_NativeRunThread+0x13c [/builds/worker/checkouts/gecko/nsprpub/pr/src/threads/combined/pruthr.c @ 421]
10 00000016`760dfae0 00007ffb`c2701bb2     nss3!pr_root+0x11 [/builds/worker/checkouts/gecko/nsprpub/pr/src/md/windows/w95thred.c @ 140]
11 00000016`760dfb10 00007ffb`c3657614     ucrtbase!thread_start<unsigned int (__cdecl*)(void *),1>+0x42
12 00000016`760dfb40 00007ffb`9782e8a8     kernel32!BaseThreadInitThunk+0x14
13 (Inline Function) --------`--------     mozglue!mozilla::interceptor::FuncHook<mozilla::interceptor::WindowsDllInterceptor<mozilla::interceptor::VMSharingPolicyShared>,void (*)(int, void *, void *)>::operator()+0x15 [/builds/worker/checkouts/gecko/toolkit/xre/dllservices/mozglue/nsWindowsDllInterceptor.h @ 150]
14 00000016`760dfb70 00007ffb`c4b426a1     mozglue!patched_BaseThreadInitThunk+0x28 [/builds/worker/checkouts/gecko/toolkit/xre/dllservices/mozglue/WindowsDllBlocklist.cpp @ 587]
15 00000016`760dfbe0 00000000`00000000     ntdll!RtlUserThreadStart+0x21
```

I believe the code is doing a move assignment to a global or static variable (that's what I called `DestructiveMoveAssignment` in the call stack), like the assignment in the `ThreadFunc` below:

```
#include "json/json.h"
#include <windows.h>

constexpr auto nThreads = 2;
constexpr auto nIterations = 0x1000;

Json::Value gValue;

DWORD ThreadFunc(void* aParam)
{
    for (size_t i = 0; i < 0x1000; ++i) {
        gValue[Json::String("")] = Json::Value("hello");
    }
    return 0;
}

int main() {
    auto bResult = HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0);
    if (!bResult) {
        return 1;
    }
    HANDLE threads[nThreads]{};
    for (auto& thread : threads) {
        DWORD threadId = 0;
        thread = CreateThread(nullptr, 0, ThreadFunc, nullptr, 0, &threadId);
        if (!thread) {
            return 1;
        }
    }
    WaitForMultipleObjects(nThreads, threads, TRUE, INFINITE);
    return 0;
}
```

Such a move assignment is racy, as the example program illustrates; the example program forces the race condition and crashes with a double free heap corruption:

```
00 00000038`b59ff3e0 00007fff`5989c213     ntdll!RtlReportFatalFailure+0x9
01 00000038`b59ff430 00007fff`598a52aa     ntdll!RtlReportCriticalFailure+0x97
02 00000038`b59ff520 00007fff`598a558a     ntdll!RtlpHeapHandleError+0x12
03 00000038`b59ff550 00007fff`598b1585     ntdll!RtlpHpHeapHandleError+0x7a
04 00000038`b59ff580 00007fff`597cc59c     ntdll!RtlpLogHeapFailure+0x45
05 00000038`b59ff5b0 00007fff`597cb1e1     ntdll!RtlpFreeHeapInternal+0x84c
06 00000038`b59ff670 00007fff`56c637eb     ntdll!RtlFreeHeap+0x51
07 00000038`b59ff6b0 00007ff7`49f34604     ucrtbase!_free_base+0x1b
08 00000038`b59ff6e0 00007ff7`49f354be     MyJson!Json::releasePrefixedStringValue+0x14 [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 182] 
09 00000038`b59ff710 00007ff7`49f350d3     MyJson!Json::Value::releasePayload+0x4e [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 1018] 
0a 00000038`b59ff760 00007ff7`49f315e1     MyJson!Json::Value::~Value+0x13 [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 443] 
0b 00000038`b59ff790 00007fff`58f726bd     MyJson!ThreadFunc+0xb1 [C:\repos\MyJson\MyJson\MyJson.cpp @ 13] 
0c 00000038`b59ff840 00007fff`597edfb8     KERNEL32!BaseThreadInitThunk+0x1d
0d 00000038`b59ff870 00000000`00000000     ntdll!RtlUserThreadStart+0x28
```

I will try to contact the vendor to confirm if they do have code that looks like this and if they could add a mutex around it.

Can anyone confirm that `PR_GetAddrInfoByName` can indeed get called by different threads?

Thanks!

(I haven't analyzed the other crashes for the moment.)

Revision 1 by

Yannis Juglaret [:yannis]

on 2023-01-11 08:55:21 PST

I have analyzed the most recurrent crashes together thanks to the DLL. The heap corruption is reported as a double free. I have identified that the last called functions in the stack belong to the statically-linked [JsonCpp](https://github.com/open-source-parsers/jsoncpp) library. I have also restored the Firefox part of the stack. The crash occurs in third-party code that gets called through `PR_GetAddrInfoByName`, they are probably hooking `getaddrinfo` from `ws2_32.dll` or something like this. The full call stack looks like this:

```
00 000000cd`343ce370 00007ffb`951cf673     ntdll!RtlReportFatalFailure+0x9
01 000000cd`343ce3c0 00007ffb`951d83f2     ntdll!RtlReportCriticalFailure+0x97
02 000000cd`343ce4b0 00007ffb`951d86da     ntdll!RtlpHeapHandleError+0x12
03 000000cd`343ce4e0 00007ffb`951de361     ntdll!RtlpHpHeapHandleError+0x7a
04 000000cd`343ce510 00007ffb`950f5bf0     ntdll!RtlpLogHeapFailure+0x45
05 000000cd`343ce540 00007ffb`950f47b1     ntdll!RtlpFreeHeapInternal+0x4e0
06 000000cd`343ce600 00007ffb`542850cc     ntdll!RtlFreeHeap+0x51
 wslbdhm64+0x350cc _free_base
 wslbdhm64+0x5eeb8 Json::Value::~Json::Value
 wslbdhm64+0x5ef84 Json::Value::"DestructiveMoveAssignment"
 wslbdhm64+0x657a6 DoStuff
 wslbdhm64+0x65e8b
 wslbdhm64+0x1de43
 wslbdhm64+0x1d9d8
 wslbdhm64+0x160db
 wslbdhm64+0x6829e
 wslbdhm64+0x64faf
 [ dynamic code ]
00 00000016`760dede0 00007ffb`49b4f32a     nss3!PR_GetAddrInfoByName+0xc3 [/builds/worker/checkouts/gecko/nsprpub/pr/src/misc/prnetdb.c @ 2171]
01 (Inline Function) --------`--------     xul!mozilla::net::_GetAddrInfo_Portable+0x5f [/builds/worker/checkouts/gecko/netwerk/dns/GetAddrInfo.cpp @ 241]
02 00000016`760dee60 00007ffb`4a2a284b     xul!mozilla::net::GetAddrInfo+0x15a [/builds/worker/checkouts/gecko/netwerk/dns/GetAddrInfo.cpp @ 374]
03 00000016`760df0a0 00007ffb`4a1e4471     xul!nsHostResolver::ThreadFunc+0x43b [/builds/worker/checkouts/gecko/netwerk/dns/nsHostResolver.cpp @ 1727]
04 (Inline Function) --------`--------     xul!mozilla::detail::RunnableMethodArguments<>::applyImpl+0x9 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1147]
05 (Inline Function) --------`--------     xul!mozilla::detail::RunnableMethodArguments<>::apply+0x9 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1153]
06 00000016`760df200 00007ffb`4a211f3e     xul!mozilla::detail::RunnableMethodImpl<RefPtr<nsObserverService>,void (nsObserverService::*)(),1,0>::Run+0x21 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1203]
07 00000016`760df230 00007ffb`4b1c1aa8     xul!nsThreadPool::Run+0x29e [/builds/worker/checkouts/gecko/xpcom/threads/nsThreadPool.cpp @ 311]
08 (Inline Function) --------`--------     xul!nsThread::ProcessNextEvent+0x17b0 [/builds/worker/checkouts/gecko/xpcom/threads/nsThread.cpp @ 1198]
09 00000016`760df3d0 00007ffb`4b1ad48f     xul!NS_ProcessNextEvent+0x1808 [/builds/worker/checkouts/gecko/xpcom/threads/nsThreadUtils.cpp @ 466]
0a 00000016`760df760 00007ffb`4a3aba5f     xul!mozilla::ipc::MessagePumpForNonMainThreads::Run+0xcf [/builds/worker/checkouts/gecko/ipc/glue/MessagePump.cpp @ 301]
0b (Inline Function) --------`--------     xul!MessageLoop::RunInternal+0x16 [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 381]
0c 00000016`760df800 00007ffb`499ef54e     xul!MessageLoop::RunHandler+0x2f [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 375]
0d 00000016`760df850 00007ffb`4a20d757     xul!MessageLoop::Run+0x4e [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 357]
0e 00000016`760df8b0 00007ffb`63d244bc     xul!nsThread::ThreadFunc+0xd7 [/builds/worker/checkouts/gecko/xpcom/threads/nsThread.cpp @ 385]
0f 00000016`760dfa60 00007ffb`63d39241     nss3!_PR_NativeRunThread+0x13c [/builds/worker/checkouts/gecko/nsprpub/pr/src/threads/combined/pruthr.c @ 421]
10 00000016`760dfae0 00007ffb`c2701bb2     nss3!pr_root+0x11 [/builds/worker/checkouts/gecko/nsprpub/pr/src/md/windows/w95thred.c @ 140]
11 00000016`760dfb10 00007ffb`c3657614     ucrtbase!thread_start<unsigned int (__cdecl*)(void *),1>+0x42
12 00000016`760dfb40 00007ffb`9782e8a8     kernel32!BaseThreadInitThunk+0x14
13 (Inline Function) --------`--------     mozglue!mozilla::interceptor::FuncHook<mozilla::interceptor::WindowsDllInterceptor<mozilla::interceptor::VMSharingPolicyShared>,void (*)(int, void *, void *)>::operator()+0x15 [/builds/worker/checkouts/gecko/toolkit/xre/dllservices/mozglue/nsWindowsDllInterceptor.h @ 150]
14 00000016`760dfb70 00007ffb`c4b426a1     mozglue!patched_BaseThreadInitThunk+0x28 [/builds/worker/checkouts/gecko/toolkit/xre/dllservices/mozglue/WindowsDllBlocklist.cpp @ 587]
15 00000016`760dfbe0 00000000`00000000     ntdll!RtlUserThreadStart+0x21
```

I believe the code is doing a move assignment to a global or static variable (that's what I called `DestructiveMoveAssignment` in the call stack), like the assignment in the `ThreadFunc` below:

```c++
#include "json/json.h"
#include <windows.h>

constexpr auto nThreads = 2;
constexpr auto nIterations = 0x1000;

Json::Value gValue;

DWORD ThreadFunc(void* aParam)
{
    for (size_t i = 0; i < 0x1000; ++i) {
        gValue[Json::String("")] = Json::Value("hello");
    }
    return 0;
}

int main() {
    auto bResult = HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0);
    if (!bResult) {
        return 1;
    }
    HANDLE threads[nThreads]{};
    for (auto& thread : threads) {
        DWORD threadId = 0;
        thread = CreateThread(nullptr, 0, ThreadFunc, nullptr, 0, &threadId);
        if (!thread) {
            return 1;
        }
    }
    WaitForMultipleObjects(nThreads, threads, TRUE, INFINITE);
    return 0;
}
```

Such a move assignment is racy, as the example program illustrates; the example program forces the race condition and crashes with a double free heap corruption:

```
00 00000038`b59ff3e0 00007fff`5989c213     ntdll!RtlReportFatalFailure+0x9
01 00000038`b59ff430 00007fff`598a52aa     ntdll!RtlReportCriticalFailure+0x97
02 00000038`b59ff520 00007fff`598a558a     ntdll!RtlpHeapHandleError+0x12
03 00000038`b59ff550 00007fff`598b1585     ntdll!RtlpHpHeapHandleError+0x7a
04 00000038`b59ff580 00007fff`597cc59c     ntdll!RtlpLogHeapFailure+0x45
05 00000038`b59ff5b0 00007fff`597cb1e1     ntdll!RtlpFreeHeapInternal+0x84c
06 00000038`b59ff670 00007fff`56c637eb     ntdll!RtlFreeHeap+0x51
07 00000038`b59ff6b0 00007ff7`49f34604     ucrtbase!_free_base+0x1b
08 00000038`b59ff6e0 00007ff7`49f354be     MyJson!Json::releasePrefixedStringValue+0x14 [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 182] 
09 00000038`b59ff710 00007ff7`49f350d3     MyJson!Json::Value::releasePayload+0x4e [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 1018] 
0a 00000038`b59ff760 00007ff7`49f315e1     MyJson!Json::Value::~Value+0x13 [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 443] 
0b 00000038`b59ff790 00007fff`58f726bd     MyJson!ThreadFunc+0xb1 [C:\repos\MyJson\MyJson\MyJson.cpp @ 13] 
0c 00000038`b59ff840 00007fff`597edfb8     KERNEL32!BaseThreadInitThunk+0x1d
0d 00000038`b59ff870 00000000`00000000     ntdll!RtlUserThreadStart+0x28
```

I will try to contact the vendor to confirm if they do have code that looks like this and if they could add a mutex around it.

Can anyone confirm that `PR_GetAddrInfoByName` can indeed get called by different threads?

Thanks!

(I haven't analyzed the other crashes for the moment.)

Revision 2 by

Yannis Juglaret [:yannis]

on 2023-01-11 08:56:45 PST

I have analyzed the most recurrent crashes together thanks to the DLL. The heap corruption is reported as a double free. I have identified that the last called functions in the stack belong to the statically-linked [JsonCpp](https://github.com/open-source-parsers/jsoncpp) library. I have also restored the Firefox part of the stack. The crash occurs in third-party code that gets called through `PR_GetAddrInfoByName`, they are probably hooking `getaddrinfo` from `ws2_32.dll` or something like this. The full call stack looks like this:

```
00 000000cd`343ce370 00007ffb`951cf673     ntdll!RtlReportFatalFailure+0x9
01 000000cd`343ce3c0 00007ffb`951d83f2     ntdll!RtlReportCriticalFailure+0x97
02 000000cd`343ce4b0 00007ffb`951d86da     ntdll!RtlpHeapHandleError+0x12
03 000000cd`343ce4e0 00007ffb`951de361     ntdll!RtlpHpHeapHandleError+0x7a
04 000000cd`343ce510 00007ffb`950f5bf0     ntdll!RtlpLogHeapFailure+0x45
05 000000cd`343ce540 00007ffb`950f47b1     ntdll!RtlpFreeHeapInternal+0x4e0
06 000000cd`343ce600 00007ffb`542850cc     ntdll!RtlFreeHeap+0x51
 wslbdhm64+0x350cc _free_base
 wslbdhm64+0x5eeb8 Json::Value::~Json::Value
 wslbdhm64+0x5ef84 Json::Value::"DestructiveMoveAssignment"
 wslbdhm64+0x657a6 DoStuff
 wslbdhm64+0x65e8b
 wslbdhm64+0x1de43
 wslbdhm64+0x1d9d8
 wslbdhm64+0x160db
 wslbdhm64+0x6829e
 wslbdhm64+0x64faf
 [ dynamic code ]
00 00000016`760dede0 00007ffb`49b4f32a     nss3!PR_GetAddrInfoByName+0xc3 [/builds/worker/checkouts/gecko/nsprpub/pr/src/misc/prnetdb.c @ 2171]
01 (Inline Function) --------`--------     xul!mozilla::net::_GetAddrInfo_Portable+0x5f [/builds/worker/checkouts/gecko/netwerk/dns/GetAddrInfo.cpp @ 241]
02 00000016`760dee60 00007ffb`4a2a284b     xul!mozilla::net::GetAddrInfo+0x15a [/builds/worker/checkouts/gecko/netwerk/dns/GetAddrInfo.cpp @ 374]
03 00000016`760df0a0 00007ffb`4a1e4471     xul!nsHostResolver::ThreadFunc+0x43b [/builds/worker/checkouts/gecko/netwerk/dns/nsHostResolver.cpp @ 1727]
04 (Inline Function) --------`--------     xul!mozilla::detail::RunnableMethodArguments<>::applyImpl+0x9 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1147]
05 (Inline Function) --------`--------     xul!mozilla::detail::RunnableMethodArguments<>::apply+0x9 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1153]
06 00000016`760df200 00007ffb`4a211f3e     xul!mozilla::detail::RunnableMethodImpl<RefPtr<nsObserverService>,void (nsObserverService::*)(),1,0>::Run+0x21 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1203]
07 00000016`760df230 00007ffb`4b1c1aa8     xul!nsThreadPool::Run+0x29e [/builds/worker/checkouts/gecko/xpcom/threads/nsThreadPool.cpp @ 311]
08 (Inline Function) --------`--------     xul!nsThread::ProcessNextEvent+0x17b0 [/builds/worker/checkouts/gecko/xpcom/threads/nsThread.cpp @ 1198]
09 00000016`760df3d0 00007ffb`4b1ad48f     xul!NS_ProcessNextEvent+0x1808 [/builds/worker/checkouts/gecko/xpcom/threads/nsThreadUtils.cpp @ 466]
0a 00000016`760df760 00007ffb`4a3aba5f     xul!mozilla::ipc::MessagePumpForNonMainThreads::Run+0xcf [/builds/worker/checkouts/gecko/ipc/glue/MessagePump.cpp @ 301]
0b (Inline Function) --------`--------     xul!MessageLoop::RunInternal+0x16 [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 381]
0c 00000016`760df800 00007ffb`499ef54e     xul!MessageLoop::RunHandler+0x2f [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 375]
0d 00000016`760df850 00007ffb`4a20d757     xul!MessageLoop::Run+0x4e [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 357]
0e 00000016`760df8b0 00007ffb`63d244bc     xul!nsThread::ThreadFunc+0xd7 [/builds/worker/checkouts/gecko/xpcom/threads/nsThread.cpp @ 385]
0f 00000016`760dfa60 00007ffb`63d39241     nss3!_PR_NativeRunThread+0x13c [/builds/worker/checkouts/gecko/nsprpub/pr/src/threads/combined/pruthr.c @ 421]
10 00000016`760dfae0 00007ffb`c2701bb2     nss3!pr_root+0x11 [/builds/worker/checkouts/gecko/nsprpub/pr/src/md/windows/w95thred.c @ 140]
11 00000016`760dfb10 00007ffb`c3657614     ucrtbase!thread_start<unsigned int (__cdecl*)(void *),1>+0x42
12 00000016`760dfb40 00007ffb`9782e8a8     kernel32!BaseThreadInitThunk+0x14
13 (Inline Function) --------`--------     mozglue!mozilla::interceptor::FuncHook<mozilla::interceptor::WindowsDllInterceptor<mozilla::interceptor::VMSharingPolicyShared>,void (*)(int, void *, void *)>::operator()+0x15 [/builds/worker/checkouts/gecko/toolkit/xre/dllservices/mozglue/nsWindowsDllInterceptor.h @ 150]
14 00000016`760dfb70 00007ffb`c4b426a1     mozglue!patched_BaseThreadInitThunk+0x28 [/builds/worker/checkouts/gecko/toolkit/xre/dllservices/mozglue/WindowsDllBlocklist.cpp @ 587]
15 00000016`760dfbe0 00000000`00000000     ntdll!RtlUserThreadStart+0x21
```

From analyzing the code in the DLL, I believe the code is doing a move assignment to a global or static variable (that's what I called `DestructiveMoveAssignment` in the call stack), like the assignment in the `ThreadFunc` below:

```c++
#include "json/json.h"
#include <windows.h>

constexpr auto nThreads = 2;
constexpr auto nIterations = 0x1000;

Json::Value gValue;

DWORD ThreadFunc(void* aParam)
{
    for (size_t i = 0; i < 0x1000; ++i) {
        gValue[Json::String("")] = Json::Value("hello");
    }
    return 0;
}

int main() {
    auto bResult = HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0);
    if (!bResult) {
        return 1;
    }
    HANDLE threads[nThreads]{};
    for (auto& thread : threads) {
        DWORD threadId = 0;
        thread = CreateThread(nullptr, 0, ThreadFunc, nullptr, 0, &threadId);
        if (!thread) {
            return 1;
        }
    }
    WaitForMultipleObjects(nThreads, threads, TRUE, INFINITE);
    return 0;
}
```

Such a move assignment is racy, as the example program illustrates; the example program forces the race condition and crashes with a double free heap corruption:

```
00 00000038`b59ff3e0 00007fff`5989c213     ntdll!RtlReportFatalFailure+0x9
01 00000038`b59ff430 00007fff`598a52aa     ntdll!RtlReportCriticalFailure+0x97
02 00000038`b59ff520 00007fff`598a558a     ntdll!RtlpHeapHandleError+0x12
03 00000038`b59ff550 00007fff`598b1585     ntdll!RtlpHpHeapHandleError+0x7a
04 00000038`b59ff580 00007fff`597cc59c     ntdll!RtlpLogHeapFailure+0x45
05 00000038`b59ff5b0 00007fff`597cb1e1     ntdll!RtlpFreeHeapInternal+0x84c
06 00000038`b59ff670 00007fff`56c637eb     ntdll!RtlFreeHeap+0x51
07 00000038`b59ff6b0 00007ff7`49f34604     ucrtbase!_free_base+0x1b
08 00000038`b59ff6e0 00007ff7`49f354be     MyJson!Json::releasePrefixedStringValue+0x14 [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 182] 
09 00000038`b59ff710 00007ff7`49f350d3     MyJson!Json::Value::releasePayload+0x4e [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 1018] 
0a 00000038`b59ff760 00007ff7`49f315e1     MyJson!Json::Value::~Value+0x13 [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 443] 
0b 00000038`b59ff790 00007fff`58f726bd     MyJson!ThreadFunc+0xb1 [C:\repos\MyJson\MyJson\MyJson.cpp @ 13] 
0c 00000038`b59ff840 00007fff`597edfb8     KERNEL32!BaseThreadInitThunk+0x1d
0d 00000038`b59ff870 00000000`00000000     ntdll!RtlUserThreadStart+0x28
```

I will try to contact the vendor to confirm if they do have code that looks like this and if they could add a mutex around it.

Can anyone confirm that `PR_GetAddrInfoByName` can indeed get called by different threads?

Thanks!

(I haven't analyzed the other crashes for the moment.)

Revision 3 by

Yannis Juglaret [:yannis]

on 2023-01-11 08:57:37 PST

I have analyzed the most recurrent crashes together thanks to the DLL. The heap corruption is reported as a double free. I have identified that the last called functions in the stack belong to the statically-linked [JsonCpp](https://github.com/open-source-parsers/jsoncpp) library. I have also restored the Firefox part of the stack. The crash occurs in third-party code that gets called through `PR_GetAddrInfoByName`, they are probably hooking `getaddrinfo` from `ws2_32.dll` or something like this. The full call stack looks like this:

```
00 000000cd`343ce370 00007ffb`951cf673     ntdll!RtlReportFatalFailure+0x9
01 000000cd`343ce3c0 00007ffb`951d83f2     ntdll!RtlReportCriticalFailure+0x97
02 000000cd`343ce4b0 00007ffb`951d86da     ntdll!RtlpHeapHandleError+0x12
03 000000cd`343ce4e0 00007ffb`951de361     ntdll!RtlpHpHeapHandleError+0x7a
04 000000cd`343ce510 00007ffb`950f5bf0     ntdll!RtlpLogHeapFailure+0x45
05 000000cd`343ce540 00007ffb`950f47b1     ntdll!RtlpFreeHeapInternal+0x4e0
06 000000cd`343ce600 00007ffb`542850cc     ntdll!RtlFreeHeap+0x51
 wslbdhm64+0x350cc _free_base
 wslbdhm64+0x5eeb8 Json::Value::~Json::Value
 wslbdhm64+0x5ef84 Json::Value::"DestructiveMoveAssignment"
 wslbdhm64+0x657a6 DoStuff
 wslbdhm64+0x65e8b
 wslbdhm64+0x1de43
 wslbdhm64+0x1d9d8
 wslbdhm64+0x160db
 wslbdhm64+0x6829e
 wslbdhm64+0x64faf
 [ dynamic code ]
00 00000016`760dede0 00007ffb`49b4f32a     nss3!PR_GetAddrInfoByName+0xc3 [/builds/worker/checkouts/gecko/nsprpub/pr/src/misc/prnetdb.c @ 2171]
01 (Inline Function) --------`--------     xul!mozilla::net::_GetAddrInfo_Portable+0x5f [/builds/worker/checkouts/gecko/netwerk/dns/GetAddrInfo.cpp @ 241]
02 00000016`760dee60 00007ffb`4a2a284b     xul!mozilla::net::GetAddrInfo+0x15a [/builds/worker/checkouts/gecko/netwerk/dns/GetAddrInfo.cpp @ 374]
03 00000016`760df0a0 00007ffb`4a1e4471     xul!nsHostResolver::ThreadFunc+0x43b [/builds/worker/checkouts/gecko/netwerk/dns/nsHostResolver.cpp @ 1727]
04 (Inline Function) --------`--------     xul!mozilla::detail::RunnableMethodArguments<>::applyImpl+0x9 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1147]
05 (Inline Function) --------`--------     xul!mozilla::detail::RunnableMethodArguments<>::apply+0x9 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1153]
06 00000016`760df200 00007ffb`4a211f3e     xul!mozilla::detail::RunnableMethodImpl<RefPtr<nsObserverService>,void (nsObserverService::*)(),1,0>::Run+0x21 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1203]
07 00000016`760df230 00007ffb`4b1c1aa8     xul!nsThreadPool::Run+0x29e [/builds/worker/checkouts/gecko/xpcom/threads/nsThreadPool.cpp @ 311]
08 (Inline Function) --------`--------     xul!nsThread::ProcessNextEvent+0x17b0 [/builds/worker/checkouts/gecko/xpcom/threads/nsThread.cpp @ 1198]
09 00000016`760df3d0 00007ffb`4b1ad48f     xul!NS_ProcessNextEvent+0x1808 [/builds/worker/checkouts/gecko/xpcom/threads/nsThreadUtils.cpp @ 466]
0a 00000016`760df760 00007ffb`4a3aba5f     xul!mozilla::ipc::MessagePumpForNonMainThreads::Run+0xcf [/builds/worker/checkouts/gecko/ipc/glue/MessagePump.cpp @ 301]
0b (Inline Function) --------`--------     xul!MessageLoop::RunInternal+0x16 [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 381]
0c 00000016`760df800 00007ffb`499ef54e     xul!MessageLoop::RunHandler+0x2f [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 375]
0d 00000016`760df850 00007ffb`4a20d757     xul!MessageLoop::Run+0x4e [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 357]
0e 00000016`760df8b0 00007ffb`63d244bc     xul!nsThread::ThreadFunc+0xd7 [/builds/worker/checkouts/gecko/xpcom/threads/nsThread.cpp @ 385]
0f 00000016`760dfa60 00007ffb`63d39241     nss3!_PR_NativeRunThread+0x13c [/builds/worker/checkouts/gecko/nsprpub/pr/src/threads/combined/pruthr.c @ 421]
10 00000016`760dfae0 00007ffb`c2701bb2     nss3!pr_root+0x11 [/builds/worker/checkouts/gecko/nsprpub/pr/src/md/windows/w95thred.c @ 140]
11 00000016`760dfb10 00007ffb`c3657614     ucrtbase!thread_start<unsigned int (__cdecl*)(void *),1>+0x42
12 00000016`760dfb40 00007ffb`9782e8a8     kernel32!BaseThreadInitThunk+0x14
13 (Inline Function) --------`--------     mozglue!mozilla::interceptor::FuncHook<mozilla::interceptor::WindowsDllInterceptor<mozilla::interceptor::VMSharingPolicyShared>,void (*)(int, void *, void *)>::operator()+0x15 [/builds/worker/checkouts/gecko/toolkit/xre/dllservices/mozglue/nsWindowsDllInterceptor.h @ 150]
14 00000016`760dfb70 00007ffb`c4b426a1     mozglue!patched_BaseThreadInitThunk+0x28 [/builds/worker/checkouts/gecko/toolkit/xre/dllservices/mozglue/WindowsDllBlocklist.cpp @ 587]
15 00000016`760dfbe0 00000000`00000000     ntdll!RtlUserThreadStart+0x21
```

From analyzing the code in the DLL, I believe the code is doing a move assignment to a global or static variable (that's what I called `DestructiveMoveAssignment` in the call stack), like the assignment in the `ThreadFunc` below:

```c++
#include "json/json.h"
#include <windows.h>

constexpr auto nThreads = 2;
constexpr auto nIterations = 0x1000;

Json::Value gValue;

DWORD ThreadFunc(void* aParam)
{
    for (size_t i = 0; i < nIterations; ++i) {
        gValue[Json::String("")] = Json::Value("hello");
    }
    return 0;
}

int main() {
    auto bResult = HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0);
    if (!bResult) {
        return 1;
    }
    HANDLE threads[nThreads]{};
    for (auto& thread : threads) {
        DWORD threadId = 0;
        thread = CreateThread(nullptr, 0, ThreadFunc, nullptr, 0, &threadId);
        if (!thread) {
            return 1;
        }
    }
    WaitForMultipleObjects(nThreads, threads, TRUE, INFINITE);
    return 0;
}
```

Such a move assignment is racy, as the example program illustrates; the example program forces the race condition and crashes with a double free heap corruption:

```
00 00000038`b59ff3e0 00007fff`5989c213     ntdll!RtlReportFatalFailure+0x9
01 00000038`b59ff430 00007fff`598a52aa     ntdll!RtlReportCriticalFailure+0x97
02 00000038`b59ff520 00007fff`598a558a     ntdll!RtlpHeapHandleError+0x12
03 00000038`b59ff550 00007fff`598b1585     ntdll!RtlpHpHeapHandleError+0x7a
04 00000038`b59ff580 00007fff`597cc59c     ntdll!RtlpLogHeapFailure+0x45
05 00000038`b59ff5b0 00007fff`597cb1e1     ntdll!RtlpFreeHeapInternal+0x84c
06 00000038`b59ff670 00007fff`56c637eb     ntdll!RtlFreeHeap+0x51
07 00000038`b59ff6b0 00007ff7`49f34604     ucrtbase!_free_base+0x1b
08 00000038`b59ff6e0 00007ff7`49f354be     MyJson!Json::releasePrefixedStringValue+0x14 [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 182] 
09 00000038`b59ff710 00007ff7`49f350d3     MyJson!Json::Value::releasePayload+0x4e [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 1018] 
0a 00000038`b59ff760 00007ff7`49f315e1     MyJson!Json::Value::~Value+0x13 [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 443] 
0b 00000038`b59ff790 00007fff`58f726bd     MyJson!ThreadFunc+0xb1 [C:\repos\MyJson\MyJson\MyJson.cpp @ 13] 
0c 00000038`b59ff840 00007fff`597edfb8     KERNEL32!BaseThreadInitThunk+0x1d
0d 00000038`b59ff870 00000000`00000000     ntdll!RtlUserThreadStart+0x28
```

I will try to contact the vendor to confirm if they do have code that looks like this and if they could add a mutex around it.

Can anyone confirm that `PR_GetAddrInfoByName` can indeed get called by different threads?

Thanks!

(I haven't analyzed the other crashes for the moment.)

Revision 4 by

Yannis Juglaret [:yannis]

on 2023-01-11 08:58:36 PST

I have analyzed the most recurrent crashes thanks to the DLL. The heap corruption is reported as a double free. I have identified that the last called functions in the stack belong to the statically-linked [JsonCpp](https://github.com/open-source-parsers/jsoncpp) library. I have also restored the Firefox part of the stack. The crash occurs in third-party code that gets called through `PR_GetAddrInfoByName`, they are probably hooking `getaddrinfo` from `ws2_32.dll` or something like this. The full call stack looks like this:

```
00 000000cd`343ce370 00007ffb`951cf673     ntdll!RtlReportFatalFailure+0x9
01 000000cd`343ce3c0 00007ffb`951d83f2     ntdll!RtlReportCriticalFailure+0x97
02 000000cd`343ce4b0 00007ffb`951d86da     ntdll!RtlpHeapHandleError+0x12
03 000000cd`343ce4e0 00007ffb`951de361     ntdll!RtlpHpHeapHandleError+0x7a
04 000000cd`343ce510 00007ffb`950f5bf0     ntdll!RtlpLogHeapFailure+0x45
05 000000cd`343ce540 00007ffb`950f47b1     ntdll!RtlpFreeHeapInternal+0x4e0
06 000000cd`343ce600 00007ffb`542850cc     ntdll!RtlFreeHeap+0x51
 wslbdhm64+0x350cc _free_base
 wslbdhm64+0x5eeb8 Json::Value::~Json::Value
 wslbdhm64+0x5ef84 Json::Value::"DestructiveMoveAssignment"
 wslbdhm64+0x657a6 DoStuff
 wslbdhm64+0x65e8b
 wslbdhm64+0x1de43
 wslbdhm64+0x1d9d8
 wslbdhm64+0x160db
 wslbdhm64+0x6829e
 wslbdhm64+0x64faf
 [ dynamic code ]
00 00000016`760dede0 00007ffb`49b4f32a     nss3!PR_GetAddrInfoByName+0xc3 [/builds/worker/checkouts/gecko/nsprpub/pr/src/misc/prnetdb.c @ 2171]
01 (Inline Function) --------`--------     xul!mozilla::net::_GetAddrInfo_Portable+0x5f [/builds/worker/checkouts/gecko/netwerk/dns/GetAddrInfo.cpp @ 241]
02 00000016`760dee60 00007ffb`4a2a284b     xul!mozilla::net::GetAddrInfo+0x15a [/builds/worker/checkouts/gecko/netwerk/dns/GetAddrInfo.cpp @ 374]
03 00000016`760df0a0 00007ffb`4a1e4471     xul!nsHostResolver::ThreadFunc+0x43b [/builds/worker/checkouts/gecko/netwerk/dns/nsHostResolver.cpp @ 1727]
04 (Inline Function) --------`--------     xul!mozilla::detail::RunnableMethodArguments<>::applyImpl+0x9 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1147]
05 (Inline Function) --------`--------     xul!mozilla::detail::RunnableMethodArguments<>::apply+0x9 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1153]
06 00000016`760df200 00007ffb`4a211f3e     xul!mozilla::detail::RunnableMethodImpl<RefPtr<nsObserverService>,void (nsObserverService::*)(),1,0>::Run+0x21 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1203]
07 00000016`760df230 00007ffb`4b1c1aa8     xul!nsThreadPool::Run+0x29e [/builds/worker/checkouts/gecko/xpcom/threads/nsThreadPool.cpp @ 311]
08 (Inline Function) --------`--------     xul!nsThread::ProcessNextEvent+0x17b0 [/builds/worker/checkouts/gecko/xpcom/threads/nsThread.cpp @ 1198]
09 00000016`760df3d0 00007ffb`4b1ad48f     xul!NS_ProcessNextEvent+0x1808 [/builds/worker/checkouts/gecko/xpcom/threads/nsThreadUtils.cpp @ 466]
0a 00000016`760df760 00007ffb`4a3aba5f     xul!mozilla::ipc::MessagePumpForNonMainThreads::Run+0xcf [/builds/worker/checkouts/gecko/ipc/glue/MessagePump.cpp @ 301]
0b (Inline Function) --------`--------     xul!MessageLoop::RunInternal+0x16 [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 381]
0c 00000016`760df800 00007ffb`499ef54e     xul!MessageLoop::RunHandler+0x2f [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 375]
0d 00000016`760df850 00007ffb`4a20d757     xul!MessageLoop::Run+0x4e [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 357]
0e 00000016`760df8b0 00007ffb`63d244bc     xul!nsThread::ThreadFunc+0xd7 [/builds/worker/checkouts/gecko/xpcom/threads/nsThread.cpp @ 385]
0f 00000016`760dfa60 00007ffb`63d39241     nss3!_PR_NativeRunThread+0x13c [/builds/worker/checkouts/gecko/nsprpub/pr/src/threads/combined/pruthr.c @ 421]
10 00000016`760dfae0 00007ffb`c2701bb2     nss3!pr_root+0x11 [/builds/worker/checkouts/gecko/nsprpub/pr/src/md/windows/w95thred.c @ 140]
11 00000016`760dfb10 00007ffb`c3657614     ucrtbase!thread_start<unsigned int (__cdecl*)(void *),1>+0x42
12 00000016`760dfb40 00007ffb`9782e8a8     kernel32!BaseThreadInitThunk+0x14
13 (Inline Function) --------`--------     mozglue!mozilla::interceptor::FuncHook<mozilla::interceptor::WindowsDllInterceptor<mozilla::interceptor::VMSharingPolicyShared>,void (*)(int, void *, void *)>::operator()+0x15 [/builds/worker/checkouts/gecko/toolkit/xre/dllservices/mozglue/nsWindowsDllInterceptor.h @ 150]
14 00000016`760dfb70 00007ffb`c4b426a1     mozglue!patched_BaseThreadInitThunk+0x28 [/builds/worker/checkouts/gecko/toolkit/xre/dllservices/mozglue/WindowsDllBlocklist.cpp @ 587]
15 00000016`760dfbe0 00000000`00000000     ntdll!RtlUserThreadStart+0x21
```

From analyzing the code in the DLL, I believe the code is doing a move assignment to a global or static variable (that's what I called `DestructiveMoveAssignment` in the call stack), like the assignment in the `ThreadFunc` below:

```c++
#include "json/json.h"
#include <windows.h>

constexpr auto nThreads = 2;
constexpr auto nIterations = 0x1000;

Json::Value gValue;

DWORD ThreadFunc(void* aParam)
{
    for (size_t i = 0; i < nIterations; ++i) {
        gValue[Json::String("")] = Json::Value("hello");
    }
    return 0;
}

int main() {
    auto bResult = HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0);
    if (!bResult) {
        return 1;
    }
    HANDLE threads[nThreads]{};
    for (auto& thread : threads) {
        DWORD threadId = 0;
        thread = CreateThread(nullptr, 0, ThreadFunc, nullptr, 0, &threadId);
        if (!thread) {
            return 1;
        }
    }
    WaitForMultipleObjects(nThreads, threads, TRUE, INFINITE);
    return 0;
}
```

Such a move assignment is racy, as the example program illustrates; the example program forces the race condition and crashes with a double free heap corruption:

```
00 00000038`b59ff3e0 00007fff`5989c213     ntdll!RtlReportFatalFailure+0x9
01 00000038`b59ff430 00007fff`598a52aa     ntdll!RtlReportCriticalFailure+0x97
02 00000038`b59ff520 00007fff`598a558a     ntdll!RtlpHeapHandleError+0x12
03 00000038`b59ff550 00007fff`598b1585     ntdll!RtlpHpHeapHandleError+0x7a
04 00000038`b59ff580 00007fff`597cc59c     ntdll!RtlpLogHeapFailure+0x45
05 00000038`b59ff5b0 00007fff`597cb1e1     ntdll!RtlpFreeHeapInternal+0x84c
06 00000038`b59ff670 00007fff`56c637eb     ntdll!RtlFreeHeap+0x51
07 00000038`b59ff6b0 00007ff7`49f34604     ucrtbase!_free_base+0x1b
08 00000038`b59ff6e0 00007ff7`49f354be     MyJson!Json::releasePrefixedStringValue+0x14 [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 182] 
09 00000038`b59ff710 00007ff7`49f350d3     MyJson!Json::Value::releasePayload+0x4e [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 1018] 
0a 00000038`b59ff760 00007ff7`49f315e1     MyJson!Json::Value::~Value+0x13 [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 443] 
0b 00000038`b59ff790 00007fff`58f726bd     MyJson!ThreadFunc+0xb1 [C:\repos\MyJson\MyJson\MyJson.cpp @ 13] 
0c 00000038`b59ff840 00007fff`597edfb8     KERNEL32!BaseThreadInitThunk+0x1d
0d 00000038`b59ff870 00000000`00000000     ntdll!RtlUserThreadStart+0x28
```

I will try to contact the vendor to confirm if they do have code that looks like this and if they could add a mutex around it.

Can anyone confirm that `PR_GetAddrInfoByName` can indeed get called by different threads?

Thanks!

(I haven't analyzed the other crashes for the moment.)

Revision 5 by

Yannis Juglaret [:yannis]

on 2023-01-11 08:59:01 PST

I have analyzed the most recurrent crash thanks to the DLL. The heap corruption is reported as a double free. I have identified that the last called functions in the stack belong to the statically-linked [JsonCpp](https://github.com/open-source-parsers/jsoncpp) library. I have also restored the Firefox part of the stack. The crash occurs in third-party code that gets called through `PR_GetAddrInfoByName`, they are probably hooking `getaddrinfo` from `ws2_32.dll` or something like this. The full call stack looks like this:

```
00 000000cd`343ce370 00007ffb`951cf673     ntdll!RtlReportFatalFailure+0x9
01 000000cd`343ce3c0 00007ffb`951d83f2     ntdll!RtlReportCriticalFailure+0x97
02 000000cd`343ce4b0 00007ffb`951d86da     ntdll!RtlpHeapHandleError+0x12
03 000000cd`343ce4e0 00007ffb`951de361     ntdll!RtlpHpHeapHandleError+0x7a
04 000000cd`343ce510 00007ffb`950f5bf0     ntdll!RtlpLogHeapFailure+0x45
05 000000cd`343ce540 00007ffb`950f47b1     ntdll!RtlpFreeHeapInternal+0x4e0
06 000000cd`343ce600 00007ffb`542850cc     ntdll!RtlFreeHeap+0x51
 wslbdhm64+0x350cc _free_base
 wslbdhm64+0x5eeb8 Json::Value::~Json::Value
 wslbdhm64+0x5ef84 Json::Value::"DestructiveMoveAssignment"
 wslbdhm64+0x657a6 DoStuff
 wslbdhm64+0x65e8b
 wslbdhm64+0x1de43
 wslbdhm64+0x1d9d8
 wslbdhm64+0x160db
 wslbdhm64+0x6829e
 wslbdhm64+0x64faf
 [ dynamic code ]
00 00000016`760dede0 00007ffb`49b4f32a     nss3!PR_GetAddrInfoByName+0xc3 [/builds/worker/checkouts/gecko/nsprpub/pr/src/misc/prnetdb.c @ 2171]
01 (Inline Function) --------`--------     xul!mozilla::net::_GetAddrInfo_Portable+0x5f [/builds/worker/checkouts/gecko/netwerk/dns/GetAddrInfo.cpp @ 241]
02 00000016`760dee60 00007ffb`4a2a284b     xul!mozilla::net::GetAddrInfo+0x15a [/builds/worker/checkouts/gecko/netwerk/dns/GetAddrInfo.cpp @ 374]
03 00000016`760df0a0 00007ffb`4a1e4471     xul!nsHostResolver::ThreadFunc+0x43b [/builds/worker/checkouts/gecko/netwerk/dns/nsHostResolver.cpp @ 1727]
04 (Inline Function) --------`--------     xul!mozilla::detail::RunnableMethodArguments<>::applyImpl+0x9 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1147]
05 (Inline Function) --------`--------     xul!mozilla::detail::RunnableMethodArguments<>::apply+0x9 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1153]
06 00000016`760df200 00007ffb`4a211f3e     xul!mozilla::detail::RunnableMethodImpl<RefPtr<nsObserverService>,void (nsObserverService::*)(),1,0>::Run+0x21 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1203]
07 00000016`760df230 00007ffb`4b1c1aa8     xul!nsThreadPool::Run+0x29e [/builds/worker/checkouts/gecko/xpcom/threads/nsThreadPool.cpp @ 311]
08 (Inline Function) --------`--------     xul!nsThread::ProcessNextEvent+0x17b0 [/builds/worker/checkouts/gecko/xpcom/threads/nsThread.cpp @ 1198]
09 00000016`760df3d0 00007ffb`4b1ad48f     xul!NS_ProcessNextEvent+0x1808 [/builds/worker/checkouts/gecko/xpcom/threads/nsThreadUtils.cpp @ 466]
0a 00000016`760df760 00007ffb`4a3aba5f     xul!mozilla::ipc::MessagePumpForNonMainThreads::Run+0xcf [/builds/worker/checkouts/gecko/ipc/glue/MessagePump.cpp @ 301]
0b (Inline Function) --------`--------     xul!MessageLoop::RunInternal+0x16 [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 381]
0c 00000016`760df800 00007ffb`499ef54e     xul!MessageLoop::RunHandler+0x2f [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 375]
0d 00000016`760df850 00007ffb`4a20d757     xul!MessageLoop::Run+0x4e [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 357]
0e 00000016`760df8b0 00007ffb`63d244bc     xul!nsThread::ThreadFunc+0xd7 [/builds/worker/checkouts/gecko/xpcom/threads/nsThread.cpp @ 385]
0f 00000016`760dfa60 00007ffb`63d39241     nss3!_PR_NativeRunThread+0x13c [/builds/worker/checkouts/gecko/nsprpub/pr/src/threads/combined/pruthr.c @ 421]
10 00000016`760dfae0 00007ffb`c2701bb2     nss3!pr_root+0x11 [/builds/worker/checkouts/gecko/nsprpub/pr/src/md/windows/w95thred.c @ 140]
11 00000016`760dfb10 00007ffb`c3657614     ucrtbase!thread_start<unsigned int (__cdecl*)(void *),1>+0x42
12 00000016`760dfb40 00007ffb`9782e8a8     kernel32!BaseThreadInitThunk+0x14
13 (Inline Function) --------`--------     mozglue!mozilla::interceptor::FuncHook<mozilla::interceptor::WindowsDllInterceptor<mozilla::interceptor::VMSharingPolicyShared>,void (*)(int, void *, void *)>::operator()+0x15 [/builds/worker/checkouts/gecko/toolkit/xre/dllservices/mozglue/nsWindowsDllInterceptor.h @ 150]
14 00000016`760dfb70 00007ffb`c4b426a1     mozglue!patched_BaseThreadInitThunk+0x28 [/builds/worker/checkouts/gecko/toolkit/xre/dllservices/mozglue/WindowsDllBlocklist.cpp @ 587]
15 00000016`760dfbe0 00000000`00000000     ntdll!RtlUserThreadStart+0x21
```

From analyzing the code in the DLL, I believe the code is doing a move assignment to a global or static variable (that's what I called `DestructiveMoveAssignment` in the call stack), like the assignment in the `ThreadFunc` below:

```c++
#include "json/json.h"
#include <windows.h>

constexpr auto nThreads = 2;
constexpr auto nIterations = 0x1000;

Json::Value gValue;

DWORD ThreadFunc(void* aParam)
{
    for (size_t i = 0; i < nIterations; ++i) {
        gValue[Json::String("")] = Json::Value("hello");
    }
    return 0;
}

int main() {
    auto bResult = HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0);
    if (!bResult) {
        return 1;
    }
    HANDLE threads[nThreads]{};
    for (auto& thread : threads) {
        DWORD threadId = 0;
        thread = CreateThread(nullptr, 0, ThreadFunc, nullptr, 0, &threadId);
        if (!thread) {
            return 1;
        }
    }
    WaitForMultipleObjects(nThreads, threads, TRUE, INFINITE);
    return 0;
}
```

Such a move assignment is racy, as the example program illustrates; the example program forces the race condition and crashes with a double free heap corruption:

```
00 00000038`b59ff3e0 00007fff`5989c213     ntdll!RtlReportFatalFailure+0x9
01 00000038`b59ff430 00007fff`598a52aa     ntdll!RtlReportCriticalFailure+0x97
02 00000038`b59ff520 00007fff`598a558a     ntdll!RtlpHeapHandleError+0x12
03 00000038`b59ff550 00007fff`598b1585     ntdll!RtlpHpHeapHandleError+0x7a
04 00000038`b59ff580 00007fff`597cc59c     ntdll!RtlpLogHeapFailure+0x45
05 00000038`b59ff5b0 00007fff`597cb1e1     ntdll!RtlpFreeHeapInternal+0x84c
06 00000038`b59ff670 00007fff`56c637eb     ntdll!RtlFreeHeap+0x51
07 00000038`b59ff6b0 00007ff7`49f34604     ucrtbase!_free_base+0x1b
08 00000038`b59ff6e0 00007ff7`49f354be     MyJson!Json::releasePrefixedStringValue+0x14 [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 182] 
09 00000038`b59ff710 00007ff7`49f350d3     MyJson!Json::Value::releasePayload+0x4e [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 1018] 
0a 00000038`b59ff760 00007ff7`49f315e1     MyJson!Json::Value::~Value+0x13 [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 443] 
0b 00000038`b59ff790 00007fff`58f726bd     MyJson!ThreadFunc+0xb1 [C:\repos\MyJson\MyJson\MyJson.cpp @ 13] 
0c 00000038`b59ff840 00007fff`597edfb8     KERNEL32!BaseThreadInitThunk+0x1d
0d 00000038`b59ff870 00000000`00000000     ntdll!RtlUserThreadStart+0x28
```

I will try to contact the vendor to confirm if they do have code that looks like this and if they could add a mutex around it.

Can anyone confirm that `PR_GetAddrInfoByName` can indeed get called by different threads?

Thanks!

(I haven't analyzed the other crashes for the moment.)

Revision 6 by

Yannis Juglaret [:yannis]

on 2023-01-11 09:26:14 PST

I have analyzed the most recurrent crash thanks to the DLL. The heap corruption is reported as a double free. I have identified that the last called functions in the stack belong to the statically-linked [JsonCpp](https://github.com/open-source-parsers/jsoncpp) library. I have also restored the Firefox part of the stack. The crash occurs in third-party code that gets called through `PR_GetAddrInfoByName`, they are probably hooking `getaddrinfo` from `ws2_32.dll` or something like this. The full call stack looks like this:

```
00 000000cd`343ce370 00007ffb`951cf673     ntdll!RtlReportFatalFailure+0x9
01 000000cd`343ce3c0 00007ffb`951d83f2     ntdll!RtlReportCriticalFailure+0x97
02 000000cd`343ce4b0 00007ffb`951d86da     ntdll!RtlpHeapHandleError+0x12
03 000000cd`343ce4e0 00007ffb`951de361     ntdll!RtlpHpHeapHandleError+0x7a
04 000000cd`343ce510 00007ffb`950f5bf0     ntdll!RtlpLogHeapFailure+0x45
05 000000cd`343ce540 00007ffb`950f47b1     ntdll!RtlpFreeHeapInternal+0x4e0
06 000000cd`343ce600 00007ffb`542850cc     ntdll!RtlFreeHeap+0x51
 wslbdhm64+0x350cc _free_base
 wslbdhm64+0x5eeb8 Json::Value::~Json::Value
 wslbdhm64+0x5ef84 Json::Value::"DestructiveMoveAssignment"
 wslbdhm64+0x657a6 DoStuff
 wslbdhm64+0x65e8b
 wslbdhm64+0x1de43
 wslbdhm64+0x1d9d8
 wslbdhm64+0x160db
 wslbdhm64+0x6829e
 wslbdhm64+0x64faf
 [ dynamic code ]
00 00000016`760dede0 00007ffb`49b4f32a     nss3!PR_GetAddrInfoByName+0xc3 [/builds/worker/checkouts/gecko/nsprpub/pr/src/misc/prnetdb.c @ 2171]
01 (Inline Function) --------`--------     xul!mozilla::net::_GetAddrInfo_Portable+0x5f [/builds/worker/checkouts/gecko/netwerk/dns/GetAddrInfo.cpp @ 241]
02 00000016`760dee60 00007ffb`4a2a284b     xul!mozilla::net::GetAddrInfo+0x15a [/builds/worker/checkouts/gecko/netwerk/dns/GetAddrInfo.cpp @ 374]
03 00000016`760df0a0 00007ffb`4a1e4471     xul!nsHostResolver::ThreadFunc+0x43b [/builds/worker/checkouts/gecko/netwerk/dns/nsHostResolver.cpp @ 1727]
04 (Inline Function) --------`--------     xul!mozilla::detail::RunnableMethodArguments<>::applyImpl+0x9 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1147]
05 (Inline Function) --------`--------     xul!mozilla::detail::RunnableMethodArguments<>::apply+0x9 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1153]
06 00000016`760df200 00007ffb`4a211f3e     xul!mozilla::detail::RunnableMethodImpl<RefPtr<nsObserverService>,void (nsObserverService::*)(),1,0>::Run+0x21 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1203]
07 00000016`760df230 00007ffb`4b1c1aa8     xul!nsThreadPool::Run+0x29e [/builds/worker/checkouts/gecko/xpcom/threads/nsThreadPool.cpp @ 311]
08 (Inline Function) --------`--------     xul!nsThread::ProcessNextEvent+0x17b0 [/builds/worker/checkouts/gecko/xpcom/threads/nsThread.cpp @ 1198]
09 00000016`760df3d0 00007ffb`4b1ad48f     xul!NS_ProcessNextEvent+0x1808 [/builds/worker/checkouts/gecko/xpcom/threads/nsThreadUtils.cpp @ 466]
0a 00000016`760df760 00007ffb`4a3aba5f     xul!mozilla::ipc::MessagePumpForNonMainThreads::Run+0xcf [/builds/worker/checkouts/gecko/ipc/glue/MessagePump.cpp @ 301]
0b (Inline Function) --------`--------     xul!MessageLoop::RunInternal+0x16 [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 381]
0c 00000016`760df800 00007ffb`499ef54e     xul!MessageLoop::RunHandler+0x2f [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 375]
0d 00000016`760df850 00007ffb`4a20d757     xul!MessageLoop::Run+0x4e [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 357]
0e 00000016`760df8b0 00007ffb`63d244bc     xul!nsThread::ThreadFunc+0xd7 [/builds/worker/checkouts/gecko/xpcom/threads/nsThread.cpp @ 385]
0f 00000016`760dfa60 00007ffb`63d39241     nss3!_PR_NativeRunThread+0x13c [/builds/worker/checkouts/gecko/nsprpub/pr/src/threads/combined/pruthr.c @ 421]
10 00000016`760dfae0 00007ffb`c2701bb2     nss3!pr_root+0x11 [/builds/worker/checkouts/gecko/nsprpub/pr/src/md/windows/w95thred.c @ 140]
11 00000016`760dfb10 00007ffb`c3657614     ucrtbase!thread_start<unsigned int (__cdecl*)(void *),1>+0x42
12 00000016`760dfb40 00007ffb`9782e8a8     kernel32!BaseThreadInitThunk+0x14
13 (Inline Function) --------`--------     mozglue!mozilla::interceptor::FuncHook<mozilla::interceptor::WindowsDllInterceptor<mozilla::interceptor::VMSharingPolicyShared>,void (*)(int, void *, void *)>::operator()+0x15 [/builds/worker/checkouts/gecko/toolkit/xre/dllservices/mozglue/nsWindowsDllInterceptor.h @ 150]
14 00000016`760dfb70 00007ffb`c4b426a1     mozglue!patched_BaseThreadInitThunk+0x28 [/builds/worker/checkouts/gecko/toolkit/xre/dllservices/mozglue/WindowsDllBlocklist.cpp @ 587]
15 00000016`760dfbe0 00000000`00000000     ntdll!RtlUserThreadStart+0x21
```

From analyzing the code in the DLL, I believe the code is doing a move assignment to a global or static variable (that's what I called `DestructiveMoveAssignment` in the call stack), like the assignment in the `ThreadFunc` below:

```c++
#include "json/json.h"
#include <windows.h>

constexpr auto nThreads = 2;
constexpr auto nIterations = 0x1000;

Json::Value gValue;

DWORD ThreadFunc(void* aParam)
{
    for (size_t i = 0; i < nIterations; ++i) {
        gValue[Json::String("")] = Json::Value("hello");
    }
    return 0;
}

int main() {
    auto bResult = HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0);
    if (!bResult) {
        return 1;
    }
    HANDLE threads[nThreads]{};
    for (auto& thread : threads) {
        DWORD threadId = 0;
        thread = CreateThread(nullptr, 0, ThreadFunc, nullptr, 0, &threadId);
        if (!thread) {
            return 1;
        }
    }
    WaitForMultipleObjects(nThreads, threads, TRUE, INFINITE);
    return 0;
}
```

Such a move assignment is racy, as the example program illustrates; the example program forces the race condition and crashes with a double free heap corruption:

```
00 00000038`b59ff3e0 00007fff`5989c213     ntdll!RtlReportFatalFailure+0x9
01 00000038`b59ff430 00007fff`598a52aa     ntdll!RtlReportCriticalFailure+0x97
02 00000038`b59ff520 00007fff`598a558a     ntdll!RtlpHeapHandleError+0x12
03 00000038`b59ff550 00007fff`598b1585     ntdll!RtlpHpHeapHandleError+0x7a
04 00000038`b59ff580 00007fff`597cc59c     ntdll!RtlpLogHeapFailure+0x45
05 00000038`b59ff5b0 00007fff`597cb1e1     ntdll!RtlpFreeHeapInternal+0x84c
06 00000038`b59ff670 00007fff`56c637eb     ntdll!RtlFreeHeap+0x51
07 00000038`b59ff6b0 00007ff7`49f34604     ucrtbase!_free_base+0x1b
08 00000038`b59ff6e0 00007ff7`49f354be     MyJson!Json::releasePrefixedStringValue+0x14 [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 182] 
09 00000038`b59ff710 00007ff7`49f350d3     MyJson!Json::Value::releasePayload+0x4e [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 1018] 
0a 00000038`b59ff760 00007ff7`49f315e1     MyJson!Json::Value::~Value+0x13 [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 443] 
0b 00000038`b59ff790 00007fff`58f726bd     MyJson!ThreadFunc+0xb1 [C:\repos\MyJson\MyJson\MyJson.cpp @ 13] 
0c 00000038`b59ff840 00007fff`597edfb8     KERNEL32!BaseThreadInitThunk+0x1d
0d 00000038`b59ff870 00000000`00000000     ntdll!RtlUserThreadStart+0x28
```

I will try to contact the vendor to confirm if they do have code that looks like this and if they could add a mutex around it.

Can anyone confirm that `PR_GetAddrInfoByName` can indeed potentially get called simultaneously by different threads?

Thanks!

(I haven't analyzed the other crashes for the moment.)

Revision 7 by

Yannis Juglaret [:yannis]

on 2023-01-11 09:28:59 PST

I have analyzed the most recurrent crash thanks to the DLL. The heap corruption is reported as a double free. I have identified that the last called functions in the stack belong to the statically-linked [JsonCpp](https://github.com/open-source-parsers/jsoncpp) library. I have also restored the Firefox part of the stack. The crash occurs in third-party code that gets called through `PR_GetAddrInfoByName`, they are probably hooking `getaddrinfo` from `ws2_32.dll` or something like this. The full call stack looks like this:

```
00 000000cd`343ce370 00007ffb`951cf673     ntdll!RtlReportFatalFailure+0x9
01 000000cd`343ce3c0 00007ffb`951d83f2     ntdll!RtlReportCriticalFailure+0x97
02 000000cd`343ce4b0 00007ffb`951d86da     ntdll!RtlpHeapHandleError+0x12
03 000000cd`343ce4e0 00007ffb`951de361     ntdll!RtlpHpHeapHandleError+0x7a
04 000000cd`343ce510 00007ffb`950f5bf0     ntdll!RtlpLogHeapFailure+0x45
05 000000cd`343ce540 00007ffb`950f47b1     ntdll!RtlpFreeHeapInternal+0x4e0
06 000000cd`343ce600 00007ffb`542850cc     ntdll!RtlFreeHeap+0x51
 wslbdhm64+0x350cc _free_base
 wslbdhm64+0x5eeb8 Json::Value::~Json::Value
 wslbdhm64+0x5ef84 Json::Value::"DestructiveMoveAssignment"
 wslbdhm64+0x657a6 DoStuff
 wslbdhm64+0x65e8b
 wslbdhm64+0x1de43
 wslbdhm64+0x1d9d8
 wslbdhm64+0x160db
 wslbdhm64+0x6829e
 wslbdhm64+0x64faf
 [ dynamic code ]
00 00000016`760dede0 00007ffb`49b4f32a     nss3!PR_GetAddrInfoByName+0xc3 [/builds/worker/checkouts/gecko/nsprpub/pr/src/misc/prnetdb.c @ 2171]
01 (Inline Function) --------`--------     xul!mozilla::net::_GetAddrInfo_Portable+0x5f [/builds/worker/checkouts/gecko/netwerk/dns/GetAddrInfo.cpp @ 241]
02 00000016`760dee60 00007ffb`4a2a284b     xul!mozilla::net::GetAddrInfo+0x15a [/builds/worker/checkouts/gecko/netwerk/dns/GetAddrInfo.cpp @ 374]
03 00000016`760df0a0 00007ffb`4a1e4471     xul!nsHostResolver::ThreadFunc+0x43b [/builds/worker/checkouts/gecko/netwerk/dns/nsHostResolver.cpp @ 1727]
04 (Inline Function) --------`--------     xul!mozilla::detail::RunnableMethodArguments<>::applyImpl+0x9 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1147]
05 (Inline Function) --------`--------     xul!mozilla::detail::RunnableMethodArguments<>::apply+0x9 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1153]
06 00000016`760df200 00007ffb`4a211f3e     xul!mozilla::detail::RunnableMethodImpl<RefPtr<nsObserverService>,void (nsObserverService::*)(),1,0>::Run+0x21 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1203]
07 00000016`760df230 00007ffb`4b1c1aa8     xul!nsThreadPool::Run+0x29e [/builds/worker/checkouts/gecko/xpcom/threads/nsThreadPool.cpp @ 311]
08 (Inline Function) --------`--------     xul!nsThread::ProcessNextEvent+0x17b0 [/builds/worker/checkouts/gecko/xpcom/threads/nsThread.cpp @ 1198]
09 00000016`760df3d0 00007ffb`4b1ad48f     xul!NS_ProcessNextEvent+0x1808 [/builds/worker/checkouts/gecko/xpcom/threads/nsThreadUtils.cpp @ 466]
0a 00000016`760df760 00007ffb`4a3aba5f     xul!mozilla::ipc::MessagePumpForNonMainThreads::Run+0xcf [/builds/worker/checkouts/gecko/ipc/glue/MessagePump.cpp @ 301]
0b (Inline Function) --------`--------     xul!MessageLoop::RunInternal+0x16 [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 381]
0c 00000016`760df800 00007ffb`499ef54e     xul!MessageLoop::RunHandler+0x2f [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 375]
0d 00000016`760df850 00007ffb`4a20d757     xul!MessageLoop::Run+0x4e [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 357]
0e 00000016`760df8b0 00007ffb`63d244bc     xul!nsThread::ThreadFunc+0xd7 [/builds/worker/checkouts/gecko/xpcom/threads/nsThread.cpp @ 385]
0f 00000016`760dfa60 00007ffb`63d39241     nss3!_PR_NativeRunThread+0x13c [/builds/worker/checkouts/gecko/nsprpub/pr/src/threads/combined/pruthr.c @ 421]
10 00000016`760dfae0 00007ffb`c2701bb2     nss3!pr_root+0x11 [/builds/worker/checkouts/gecko/nsprpub/pr/src/md/windows/w95thred.c @ 140]
11 00000016`760dfb10 00007ffb`c3657614     ucrtbase!thread_start<unsigned int (__cdecl*)(void *),1>+0x42
12 00000016`760dfb40 00007ffb`9782e8a8     kernel32!BaseThreadInitThunk+0x14
13 (Inline Function) --------`--------     mozglue!mozilla::interceptor::FuncHook<mozilla::interceptor::WindowsDllInterceptor<mozilla::interceptor::VMSharingPolicyShared>,void (*)(int, void *, void *)>::operator()+0x15 [/builds/worker/checkouts/gecko/toolkit/xre/dllservices/mozglue/nsWindowsDllInterceptor.h @ 150]
14 00000016`760dfb70 00007ffb`c4b426a1     mozglue!patched_BaseThreadInitThunk+0x28 [/builds/worker/checkouts/gecko/toolkit/xre/dllservices/mozglue/WindowsDllBlocklist.cpp @ 587]
15 00000016`760dfbe0 00000000`00000000     ntdll!RtlUserThreadStart+0x21
```

From analyzing the code in the DLL, I believe the code is doing a move assignment to a subvalue of a value stored as a global or static variable (that's what I called `DestructiveMoveAssignment` in the call stack), like the assignment in the `ThreadFunc` below:

```c++
#include "json/json.h"
#include <windows.h>

constexpr auto nThreads = 2;
constexpr auto nIterations = 0x1000;

Json::Value gValue;

DWORD ThreadFunc(void* aParam)
{
    for (size_t i = 0; i < nIterations; ++i) {
        gValue[Json::String("")] = Json::Value("hello");
    }
    return 0;
}

int main() {
    auto bResult = HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0);
    if (!bResult) {
        return 1;
    }
    HANDLE threads[nThreads]{};
    for (auto& thread : threads) {
        DWORD threadId = 0;
        thread = CreateThread(nullptr, 0, ThreadFunc, nullptr, 0, &threadId);
        if (!thread) {
            return 1;
        }
    }
    WaitForMultipleObjects(nThreads, threads, TRUE, INFINITE);
    return 0;
}
```

Such a move assignment is racy, as the example program illustrates; the example program forces the race condition and crashes with a double free heap corruption:

```
00 00000038`b59ff3e0 00007fff`5989c213     ntdll!RtlReportFatalFailure+0x9
01 00000038`b59ff430 00007fff`598a52aa     ntdll!RtlReportCriticalFailure+0x97
02 00000038`b59ff520 00007fff`598a558a     ntdll!RtlpHeapHandleError+0x12
03 00000038`b59ff550 00007fff`598b1585     ntdll!RtlpHpHeapHandleError+0x7a
04 00000038`b59ff580 00007fff`597cc59c     ntdll!RtlpLogHeapFailure+0x45
05 00000038`b59ff5b0 00007fff`597cb1e1     ntdll!RtlpFreeHeapInternal+0x84c
06 00000038`b59ff670 00007fff`56c637eb     ntdll!RtlFreeHeap+0x51
07 00000038`b59ff6b0 00007ff7`49f34604     ucrtbase!_free_base+0x1b
08 00000038`b59ff6e0 00007ff7`49f354be     MyJson!Json::releasePrefixedStringValue+0x14 [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 182] 
09 00000038`b59ff710 00007ff7`49f350d3     MyJson!Json::Value::releasePayload+0x4e [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 1018] 
0a 00000038`b59ff760 00007ff7`49f315e1     MyJson!Json::Value::~Value+0x13 [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 443] 
0b 00000038`b59ff790 00007fff`58f726bd     MyJson!ThreadFunc+0xb1 [C:\repos\MyJson\MyJson\MyJson.cpp @ 13] 
0c 00000038`b59ff840 00007fff`597edfb8     KERNEL32!BaseThreadInitThunk+0x1d
0d 00000038`b59ff870 00000000`00000000     ntdll!RtlUserThreadStart+0x28
```

I will try to contact the vendor to confirm if they do have code that looks like this and if they could add a mutex around it.

Can anyone confirm that `PR_GetAddrInfoByName` can indeed potentially get called simultaneously by different threads?

Thanks!

(I haven't analyzed the other crashes for the moment.)

Revision 8 by

Yannis Juglaret [:yannis]

on 2023-01-11 09:29:31 PST

I have analyzed the most recurrent crash thanks to the DLL. The heap corruption is reported as a double free. I have identified that the last called functions in the stack belong to the statically-linked [JsonCpp](https://github.com/open-source-parsers/jsoncpp) library. I have also restored the Firefox part of the stack. The crash occurs in third-party code that gets called through `PR_GetAddrInfoByName`, they are probably hooking `getaddrinfo` from `ws2_32.dll` or something like this. The full call stack looks like this:

```
00 000000cd`343ce370 00007ffb`951cf673     ntdll!RtlReportFatalFailure+0x9
01 000000cd`343ce3c0 00007ffb`951d83f2     ntdll!RtlReportCriticalFailure+0x97
02 000000cd`343ce4b0 00007ffb`951d86da     ntdll!RtlpHeapHandleError+0x12
03 000000cd`343ce4e0 00007ffb`951de361     ntdll!RtlpHpHeapHandleError+0x7a
04 000000cd`343ce510 00007ffb`950f5bf0     ntdll!RtlpLogHeapFailure+0x45
05 000000cd`343ce540 00007ffb`950f47b1     ntdll!RtlpFreeHeapInternal+0x4e0
06 000000cd`343ce600 00007ffb`542850cc     ntdll!RtlFreeHeap+0x51
 wslbdhm64+0x350cc _free_base
 wslbdhm64+0x5eeb8 Json::Value::~Json::Value
 wslbdhm64+0x5ef84 Json::Value::"DestructiveMoveAssignment"
 wslbdhm64+0x657a6 DoStuff
 wslbdhm64+0x65e8b
 wslbdhm64+0x1de43
 wslbdhm64+0x1d9d8
 wslbdhm64+0x160db
 wslbdhm64+0x6829e
 wslbdhm64+0x64faf
 [ dynamic code ]
00 00000016`760dede0 00007ffb`49b4f32a     nss3!PR_GetAddrInfoByName+0xc3 [/builds/worker/checkouts/gecko/nsprpub/pr/src/misc/prnetdb.c @ 2171]
01 (Inline Function) --------`--------     xul!mozilla::net::_GetAddrInfo_Portable+0x5f [/builds/worker/checkouts/gecko/netwerk/dns/GetAddrInfo.cpp @ 241]
02 00000016`760dee60 00007ffb`4a2a284b     xul!mozilla::net::GetAddrInfo+0x15a [/builds/worker/checkouts/gecko/netwerk/dns/GetAddrInfo.cpp @ 374]
03 00000016`760df0a0 00007ffb`4a1e4471     xul!nsHostResolver::ThreadFunc+0x43b [/builds/worker/checkouts/gecko/netwerk/dns/nsHostResolver.cpp @ 1727]
04 (Inline Function) --------`--------     xul!mozilla::detail::RunnableMethodArguments<>::applyImpl+0x9 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1147]
05 (Inline Function) --------`--------     xul!mozilla::detail::RunnableMethodArguments<>::apply+0x9 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1153]
06 00000016`760df200 00007ffb`4a211f3e     xul!mozilla::detail::RunnableMethodImpl<RefPtr<nsObserverService>,void (nsObserverService::*)(),1,0>::Run+0x21 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1203]
07 00000016`760df230 00007ffb`4b1c1aa8     xul!nsThreadPool::Run+0x29e [/builds/worker/checkouts/gecko/xpcom/threads/nsThreadPool.cpp @ 311]
08 (Inline Function) --------`--------     xul!nsThread::ProcessNextEvent+0x17b0 [/builds/worker/checkouts/gecko/xpcom/threads/nsThread.cpp @ 1198]
09 00000016`760df3d0 00007ffb`4b1ad48f     xul!NS_ProcessNextEvent+0x1808 [/builds/worker/checkouts/gecko/xpcom/threads/nsThreadUtils.cpp @ 466]
0a 00000016`760df760 00007ffb`4a3aba5f     xul!mozilla::ipc::MessagePumpForNonMainThreads::Run+0xcf [/builds/worker/checkouts/gecko/ipc/glue/MessagePump.cpp @ 301]
0b (Inline Function) --------`--------     xul!MessageLoop::RunInternal+0x16 [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 381]
0c 00000016`760df800 00007ffb`499ef54e     xul!MessageLoop::RunHandler+0x2f [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 375]
0d 00000016`760df850 00007ffb`4a20d757     xul!MessageLoop::Run+0x4e [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 357]
0e 00000016`760df8b0 00007ffb`63d244bc     xul!nsThread::ThreadFunc+0xd7 [/builds/worker/checkouts/gecko/xpcom/threads/nsThread.cpp @ 385]
0f 00000016`760dfa60 00007ffb`63d39241     nss3!_PR_NativeRunThread+0x13c [/builds/worker/checkouts/gecko/nsprpub/pr/src/threads/combined/pruthr.c @ 421]
10 00000016`760dfae0 00007ffb`c2701bb2     nss3!pr_root+0x11 [/builds/worker/checkouts/gecko/nsprpub/pr/src/md/windows/w95thred.c @ 140]
11 00000016`760dfb10 00007ffb`c3657614     ucrtbase!thread_start<unsigned int (__cdecl*)(void *),1>+0x42
12 00000016`760dfb40 00007ffb`9782e8a8     kernel32!BaseThreadInitThunk+0x14
13 (Inline Function) --------`--------     mozglue!mozilla::interceptor::FuncHook<mozilla::interceptor::WindowsDllInterceptor<mozilla::interceptor::VMSharingPolicyShared>,void (*)(int, void *, void *)>::operator()+0x15 [/builds/worker/checkouts/gecko/toolkit/xre/dllservices/mozglue/nsWindowsDllInterceptor.h @ 150]
14 00000016`760dfb70 00007ffb`c4b426a1     mozglue!patched_BaseThreadInitThunk+0x28 [/builds/worker/checkouts/gecko/toolkit/xre/dllservices/mozglue/WindowsDllBlocklist.cpp @ 587]
15 00000016`760dfbe0 00000000`00000000     ntdll!RtlUserThreadStart+0x21
```

From analyzing the code in the DLL, I believe the code is doing a move assignment to a subvalue of a `Json::Value` stored as a global or static variable (that's what I called `DestructiveMoveAssignment` in the call stack), like the assignment in the `ThreadFunc` below:

```c++
#include "json/json.h"
#include <windows.h>

constexpr auto nThreads = 2;
constexpr auto nIterations = 0x1000;

Json::Value gValue;

DWORD ThreadFunc(void* aParam)
{
    for (size_t i = 0; i < nIterations; ++i) {
        gValue[Json::String("")] = Json::Value("hello");
    }
    return 0;
}

int main() {
    auto bResult = HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0);
    if (!bResult) {
        return 1;
    }
    HANDLE threads[nThreads]{};
    for (auto& thread : threads) {
        DWORD threadId = 0;
        thread = CreateThread(nullptr, 0, ThreadFunc, nullptr, 0, &threadId);
        if (!thread) {
            return 1;
        }
    }
    WaitForMultipleObjects(nThreads, threads, TRUE, INFINITE);
    return 0;
}
```

Such a move assignment is racy, as the example program illustrates; the example program forces the race condition and crashes with a double free heap corruption:

```
00 00000038`b59ff3e0 00007fff`5989c213     ntdll!RtlReportFatalFailure+0x9
01 00000038`b59ff430 00007fff`598a52aa     ntdll!RtlReportCriticalFailure+0x97
02 00000038`b59ff520 00007fff`598a558a     ntdll!RtlpHeapHandleError+0x12
03 00000038`b59ff550 00007fff`598b1585     ntdll!RtlpHpHeapHandleError+0x7a
04 00000038`b59ff580 00007fff`597cc59c     ntdll!RtlpLogHeapFailure+0x45
05 00000038`b59ff5b0 00007fff`597cb1e1     ntdll!RtlpFreeHeapInternal+0x84c
06 00000038`b59ff670 00007fff`56c637eb     ntdll!RtlFreeHeap+0x51
07 00000038`b59ff6b0 00007ff7`49f34604     ucrtbase!_free_base+0x1b
08 00000038`b59ff6e0 00007ff7`49f354be     MyJson!Json::releasePrefixedStringValue+0x14 [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 182] 
09 00000038`b59ff710 00007ff7`49f350d3     MyJson!Json::Value::releasePayload+0x4e [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 1018] 
0a 00000038`b59ff760 00007ff7`49f315e1     MyJson!Json::Value::~Value+0x13 [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 443] 
0b 00000038`b59ff790 00007fff`58f726bd     MyJson!ThreadFunc+0xb1 [C:\repos\MyJson\MyJson\MyJson.cpp @ 13] 
0c 00000038`b59ff840 00007fff`597edfb8     KERNEL32!BaseThreadInitThunk+0x1d
0d 00000038`b59ff870 00000000`00000000     ntdll!RtlUserThreadStart+0x28
```

I will try to contact the vendor to confirm if they do have code that looks like this and if they could add a mutex around it.

Can anyone confirm that `PR_GetAddrInfoByName` can indeed potentially get called simultaneously by different threads?

Thanks!

(I haven't analyzed the other crashes for the moment.)

Revision 9 by

Yannis Juglaret [:yannis]

on 2023-01-11 22:37:00 PST

I have analyzed the most recurrent crash thanks to the DLL. The heap corruption is reported as a double free. I have identified that the last called functions in the stack belong to the statically-linked [JsonCpp](https://github.com/open-source-parsers/jsoncpp) library. I have also restored the Firefox part of the stack. The crash occurs in third-party code that gets called through `PR_GetAddrInfoByName`, they are probably hooking `getaddrinfo` from `ws2_32.dll` or something like this. The full call stack looks like this:

```
00 000000cd`343ce370 00007ffb`951cf673     ntdll!RtlReportFatalFailure+0x9
01 000000cd`343ce3c0 00007ffb`951d83f2     ntdll!RtlReportCriticalFailure+0x97
02 000000cd`343ce4b0 00007ffb`951d86da     ntdll!RtlpHeapHandleError+0x12
03 000000cd`343ce4e0 00007ffb`951de361     ntdll!RtlpHpHeapHandleError+0x7a
04 000000cd`343ce510 00007ffb`950f5bf0     ntdll!RtlpLogHeapFailure+0x45
05 000000cd`343ce540 00007ffb`950f47b1     ntdll!RtlpFreeHeapInternal+0x4e0
06 000000cd`343ce600 00007ffb`542850cc     ntdll!RtlFreeHeap+0x51
 wslbdhm64+0x350cc _free_base
 wslbdhm64+0x5eeb8 Json::Value::~Json::Value
 wslbdhm64+0x5ef84 Json::Value::"DestructiveMoveAssignment"
 wslbdhm64+0x657a6 DoStuff
 wslbdhm64+0x65e8b
 wslbdhm64+0x1de43
 wslbdhm64+0x1d9d8
 wslbdhm64+0x160db
 wslbdhm64+0x6829e
 wslbdhm64+0x64faf
 [ dynamic code ]
00 00000016`760dede0 00007ffb`49b4f32a     nss3!PR_GetAddrInfoByName+0xc3 [/builds/worker/checkouts/gecko/nsprpub/pr/src/misc/prnetdb.c @ 2171]
01 (Inline Function) --------`--------     xul!mozilla::net::_GetAddrInfo_Portable+0x5f [/builds/worker/checkouts/gecko/netwerk/dns/GetAddrInfo.cpp @ 241]
02 00000016`760dee60 00007ffb`4a2a284b     xul!mozilla::net::GetAddrInfo+0x15a [/builds/worker/checkouts/gecko/netwerk/dns/GetAddrInfo.cpp @ 374]
03 00000016`760df0a0 00007ffb`4a1e4471     xul!nsHostResolver::ThreadFunc+0x43b [/builds/worker/checkouts/gecko/netwerk/dns/nsHostResolver.cpp @ 1727]
04 (Inline Function) --------`--------     xul!mozilla::detail::RunnableMethodArguments<>::applyImpl+0x9 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1147]
05 (Inline Function) --------`--------     xul!mozilla::detail::RunnableMethodArguments<>::apply+0x9 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1153]
06 00000016`760df200 00007ffb`4a211f3e     xul!mozilla::detail::RunnableMethodImpl<RefPtr<nsObserverService>,void (nsObserverService::*)(),1,0>::Run+0x21 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1203]
07 00000016`760df230 00007ffb`4b1c1aa8     xul!nsThreadPool::Run+0x29e [/builds/worker/checkouts/gecko/xpcom/threads/nsThreadPool.cpp @ 311]
08 (Inline Function) --------`--------     xul!nsThread::ProcessNextEvent+0x17b0 [/builds/worker/checkouts/gecko/xpcom/threads/nsThread.cpp @ 1198]
09 00000016`760df3d0 00007ffb`4b1ad48f     xul!NS_ProcessNextEvent+0x1808 [/builds/worker/checkouts/gecko/xpcom/threads/nsThreadUtils.cpp @ 466]
0a 00000016`760df760 00007ffb`4a3aba5f     xul!mozilla::ipc::MessagePumpForNonMainThreads::Run+0xcf [/builds/worker/checkouts/gecko/ipc/glue/MessagePump.cpp @ 301]
0b (Inline Function) --------`--------     xul!MessageLoop::RunInternal+0x16 [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 381]
0c 00000016`760df800 00007ffb`499ef54e     xul!MessageLoop::RunHandler+0x2f [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 375]
0d 00000016`760df850 00007ffb`4a20d757     xul!MessageLoop::Run+0x4e [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 357]
0e 00000016`760df8b0 00007ffb`63d244bc     xul!nsThread::ThreadFunc+0xd7 [/builds/worker/checkouts/gecko/xpcom/threads/nsThread.cpp @ 385]
0f 00000016`760dfa60 00007ffb`63d39241     nss3!_PR_NativeRunThread+0x13c [/builds/worker/checkouts/gecko/nsprpub/pr/src/threads/combined/pruthr.c @ 421]
10 00000016`760dfae0 00007ffb`c2701bb2     nss3!pr_root+0x11 [/builds/worker/checkouts/gecko/nsprpub/pr/src/md/windows/w95thred.c @ 140]
11 00000016`760dfb10 00007ffb`c3657614     ucrtbase!thread_start<unsigned int (__cdecl*)(void *),1>+0x42
12 00000016`760dfb40 00007ffb`9782e8a8     kernel32!BaseThreadInitThunk+0x14
13 (Inline Function) --------`--------     mozglue!mozilla::interceptor::FuncHook<mozilla::interceptor::WindowsDllInterceptor<mozilla::interceptor::VMSharingPolicyShared>,void (*)(int, void *, void *)>::operator()+0x15 [/builds/worker/checkouts/gecko/toolkit/xre/dllservices/mozglue/nsWindowsDllInterceptor.h @ 150]
14 00000016`760dfb70 00007ffb`c4b426a1     mozglue!patched_BaseThreadInitThunk+0x28 [/builds/worker/checkouts/gecko/toolkit/xre/dllservices/mozglue/WindowsDllBlocklist.cpp @ 587]
15 00000016`760dfbe0 00000000`00000000     ntdll!RtlUserThreadStart+0x21
```

From analyzing the code in the DLL, I believe the code is doing a move assignment to a subvalue of a `Json::Value` stored as a global or static variable (that's what I called `DestructiveMoveAssignment` in the call stack), like the assignment in the `ThreadFunc` below:

```c++
#include "json/json.h"
#include <windows.h>

constexpr auto nThreads = 2;
constexpr auto nIterations = 0x1000;

Json::Value gValue;

DWORD ThreadFunc(void* aParam)
{
    for (size_t i = 0; i < nIterations; ++i) {
        gValue[Json::String("")] = Json::Value("hello");
    }
    return 0;
}

int main() {
    auto bResult = HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0);
    if (!bResult) {
        return 1;
    }
    HANDLE threads[nThreads]{};
    for (auto& thread : threads) {
        DWORD threadId = 0;
        thread = CreateThread(nullptr, 0, ThreadFunc, nullptr, 0, &threadId);
        if (!thread) {
            return 1;
        }
    }
    WaitForMultipleObjects(nThreads, threads, TRUE, INFINITE);
    return 0;
}
```

Such a move assignment is not thread-safe, as the example program illustrates; the example program forces the race condition and crashes with a double free heap corruption with a similar call stack:

```
00 00000038`b59ff3e0 00007fff`5989c213     ntdll!RtlReportFatalFailure+0x9
01 00000038`b59ff430 00007fff`598a52aa     ntdll!RtlReportCriticalFailure+0x97
02 00000038`b59ff520 00007fff`598a558a     ntdll!RtlpHeapHandleError+0x12
03 00000038`b59ff550 00007fff`598b1585     ntdll!RtlpHpHeapHandleError+0x7a
04 00000038`b59ff580 00007fff`597cc59c     ntdll!RtlpLogHeapFailure+0x45
05 00000038`b59ff5b0 00007fff`597cb1e1     ntdll!RtlpFreeHeapInternal+0x84c
06 00000038`b59ff670 00007fff`56c637eb     ntdll!RtlFreeHeap+0x51
07 00000038`b59ff6b0 00007ff7`49f34604     ucrtbase!_free_base+0x1b
08 00000038`b59ff6e0 00007ff7`49f354be     MyJson!Json::releasePrefixedStringValue+0x14 [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 182] 
09 00000038`b59ff710 00007ff7`49f350d3     MyJson!Json::Value::releasePayload+0x4e [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 1018] 
0a 00000038`b59ff760 00007ff7`49f315e1     MyJson!Json::Value::~Value+0x13 [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 443] 
0b 00000038`b59ff790 00007fff`58f726bd     MyJson!ThreadFunc+0xb1 [C:\repos\MyJson\MyJson\MyJson.cpp @ 13] 
0c 00000038`b59ff840 00007fff`597edfb8     KERNEL32!BaseThreadInitThunk+0x1d
0d 00000038`b59ff870 00000000`00000000     ntdll!RtlUserThreadStart+0x28
```

I will try to contact the vendor to confirm if they do have code that looks like this and if they could make their code thread-safe.

Can anyone confirm that `PR_GetAddrInfoByName` can indeed potentially get called simultaneously by different threads?

Thanks!

(I haven't analyzed the other crashes for the moment.)

Revision 10 by

Yannis Juglaret [:yannis]

on 2023-01-12 00:34:45 PST

I have analyzed the most recurrent crash thanks to the DLL. The heap corruption is reported as a double free. I have identified that the last called functions in the stack belong to the statically-linked [JsonCpp](https://github.com/open-source-parsers/jsoncpp) library. I have also restored the Firefox part of the stack. The crash occurs in third-party code that gets called through `PR_GetAddrInfoByName`, they are probably hooking `getaddrinfo` from `ws2_32.dll` or something like this. The full call stack looks like this:

```
00 000000cd`343ce370 00007ffb`951cf673     ntdll!RtlReportFatalFailure+0x9
01 000000cd`343ce3c0 00007ffb`951d83f2     ntdll!RtlReportCriticalFailure+0x97
02 000000cd`343ce4b0 00007ffb`951d86da     ntdll!RtlpHeapHandleError+0x12
03 000000cd`343ce4e0 00007ffb`951de361     ntdll!RtlpHpHeapHandleError+0x7a
04 000000cd`343ce510 00007ffb`950f5bf0     ntdll!RtlpLogHeapFailure+0x45
05 000000cd`343ce540 00007ffb`950f47b1     ntdll!RtlpFreeHeapInternal+0x4e0
06 000000cd`343ce600 00007ffb`542850cc     ntdll!RtlFreeHeap+0x51
 wslbdhm64+0x350cc _free_base
 wslbdhm64+0x5eeb8 Json::Value::~Json::Value
 wslbdhm64+0x5ef84 Json::Value::"DestructiveMoveAssignment"
 wslbdhm64+0x657a6 DoStuff
 wslbdhm64+0x65e8b
 wslbdhm64+0x1de43
 wslbdhm64+0x1d9d8
 wslbdhm64+0x160db
 wslbdhm64+0x6829e
 wslbdhm64+0x64faf
 [ dynamic code ]
00 00000016`760dede0 00007ffb`49b4f32a     nss3!PR_GetAddrInfoByName+0xc3 [/builds/worker/checkouts/gecko/nsprpub/pr/src/misc/prnetdb.c @ 2171]
01 (Inline Function) --------`--------     xul!mozilla::net::_GetAddrInfo_Portable+0x5f [/builds/worker/checkouts/gecko/netwerk/dns/GetAddrInfo.cpp @ 241]
02 00000016`760dee60 00007ffb`4a2a284b     xul!mozilla::net::GetAddrInfo+0x15a [/builds/worker/checkouts/gecko/netwerk/dns/GetAddrInfo.cpp @ 374]
03 00000016`760df0a0 00007ffb`4a1e4471     xul!nsHostResolver::ThreadFunc+0x43b [/builds/worker/checkouts/gecko/netwerk/dns/nsHostResolver.cpp @ 1727]
04 (Inline Function) --------`--------     xul!mozilla::detail::RunnableMethodArguments<>::applyImpl+0x9 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1147]
05 (Inline Function) --------`--------     xul!mozilla::detail::RunnableMethodArguments<>::apply+0x9 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1153]
06 00000016`760df200 00007ffb`4a211f3e     xul!mozilla::detail::RunnableMethodImpl<RefPtr<nsObserverService>,void (nsObserverService::*)(),1,0>::Run+0x21 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1203]
07 00000016`760df230 00007ffb`4b1c1aa8     xul!nsThreadPool::Run+0x29e [/builds/worker/checkouts/gecko/xpcom/threads/nsThreadPool.cpp @ 311]
08 (Inline Function) --------`--------     xul!nsThread::ProcessNextEvent+0x17b0 [/builds/worker/checkouts/gecko/xpcom/threads/nsThread.cpp @ 1198]
09 00000016`760df3d0 00007ffb`4b1ad48f     xul!NS_ProcessNextEvent+0x1808 [/builds/worker/checkouts/gecko/xpcom/threads/nsThreadUtils.cpp @ 466]
0a 00000016`760df760 00007ffb`4a3aba5f     xul!mozilla::ipc::MessagePumpForNonMainThreads::Run+0xcf [/builds/worker/checkouts/gecko/ipc/glue/MessagePump.cpp @ 301]
0b (Inline Function) --------`--------     xul!MessageLoop::RunInternal+0x16 [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 381]
0c 00000016`760df800 00007ffb`499ef54e     xul!MessageLoop::RunHandler+0x2f [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 375]
0d 00000016`760df850 00007ffb`4a20d757     xul!MessageLoop::Run+0x4e [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 357]
0e 00000016`760df8b0 00007ffb`63d244bc     xul!nsThread::ThreadFunc+0xd7 [/builds/worker/checkouts/gecko/xpcom/threads/nsThread.cpp @ 385]
0f 00000016`760dfa60 00007ffb`63d39241     nss3!_PR_NativeRunThread+0x13c [/builds/worker/checkouts/gecko/nsprpub/pr/src/threads/combined/pruthr.c @ 421]
10 00000016`760dfae0 00007ffb`c2701bb2     nss3!pr_root+0x11 [/builds/worker/checkouts/gecko/nsprpub/pr/src/md/windows/w95thred.c @ 140]
11 00000016`760dfb10 00007ffb`c3657614     ucrtbase!thread_start<unsigned int (__cdecl*)(void *),1>+0x42
12 00000016`760dfb40 00007ffb`9782e8a8     kernel32!BaseThreadInitThunk+0x14
13 (Inline Function) --------`--------     mozglue!mozilla::interceptor::FuncHook<mozilla::interceptor::WindowsDllInterceptor<mozilla::interceptor::VMSharingPolicyShared>,void (*)(int, void *, void *)>::operator()+0x15 [/builds/worker/checkouts/gecko/toolkit/xre/dllservices/mozglue/nsWindowsDllInterceptor.h @ 150]
14 00000016`760dfb70 00007ffb`c4b426a1     mozglue!patched_BaseThreadInitThunk+0x28 [/builds/worker/checkouts/gecko/toolkit/xre/dllservices/mozglue/WindowsDllBlocklist.cpp @ 587]
15 00000016`760dfbe0 00000000`00000000     ntdll!RtlUserThreadStart+0x21
```

From analyzing the code in the DLL, I believe the code is doing a move assignment to a subvalue of a `Json::Value` stored as a global or static variable (that's what I called `DestructiveMoveAssignment` in the call stack), like the assignment in the `ThreadFunc` below:

```c++
#include "json/json.h"
#include <windows.h>

constexpr auto nThreads = 2;
constexpr auto nIterations = 0x1000;

Json::Value gValue;

DWORD ThreadFunc(void* aParam)
{
    for (size_t i = 0; i < nIterations; ++i) {
        gValue[Json::String("")] = Json::Value("hello");
    }
    return 0;
}

int main() {
    auto bResult = HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0);
    if (!bResult) {
        return 1;
    }
    HANDLE threads[nThreads]{};
    for (auto& thread : threads) {
        DWORD threadId = 0;
        thread = CreateThread(nullptr, 0, ThreadFunc, nullptr, 0, &threadId);
        if (!thread) {
            return 1;
        }
    }
    WaitForMultipleObjects(nThreads, threads, TRUE, INFINITE);
    return 0;
}
```

Such a move assignment is not thread-safe, as the example program illustrates; the example program forces the race condition and crashes with a double free heap corruption with a similar call stack:

```
00 00000038`b59ff3e0 00007fff`5989c213     ntdll!RtlReportFatalFailure+0x9
01 00000038`b59ff430 00007fff`598a52aa     ntdll!RtlReportCriticalFailure+0x97
02 00000038`b59ff520 00007fff`598a558a     ntdll!RtlpHeapHandleError+0x12
03 00000038`b59ff550 00007fff`598b1585     ntdll!RtlpHpHeapHandleError+0x7a
04 00000038`b59ff580 00007fff`597cc59c     ntdll!RtlpLogHeapFailure+0x45
05 00000038`b59ff5b0 00007fff`597cb1e1     ntdll!RtlpFreeHeapInternal+0x84c
06 00000038`b59ff670 00007fff`56c637eb     ntdll!RtlFreeHeap+0x51
07 00000038`b59ff6b0 00007ff7`49f34604     ucrtbase!_free_base+0x1b
08 00000038`b59ff6e0 00007ff7`49f354be     MyJson!Json::releasePrefixedStringValue+0x14 [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 182] 
09 00000038`b59ff710 00007ff7`49f350d3     MyJson!Json::Value::releasePayload+0x4e [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 1018] 
0a 00000038`b59ff760 00007ff7`49f315e1     MyJson!Json::Value::~Value+0x13 [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 443] 
0b 00000038`b59ff790 00007fff`58f726bd     MyJson!ThreadFunc+0xb1 [C:\repos\MyJson\MyJson\MyJson.cpp @ 13] 
0c 00000038`b59ff840 00007fff`597edfb8     KERNEL32!BaseThreadInitThunk+0x1d
0d 00000038`b59ff870 00000000`00000000     ntdll!RtlUserThreadStart+0x28
```

I will try to contact the vendor to confirm if they do have code that looks like this and if they could make their code thread-safe.

Can anyone confirm that `PR_GetAddrInfoByName` can indeed potentially get called simultaneously by different threads? (Well, the call stack suggests that this is exactly what we are doing here -- using a specific thread each time we need to resolve a hostname.)

Thanks!

(I haven't analyzed the other crashes for the moment.)

Revision 11 by

Yannis Juglaret [:yannis]

on 2023-01-12 00:39:30 PST

I have analyzed the most recurrent crash thanks to the DLL. The heap corruption is reported as a double free. I have identified that the last called functions in the stack belong to the statically-linked [JsonCpp](https://github.com/open-source-parsers/jsoncpp) library. I have also restored the Firefox part of the stack. The crash occurs in third-party code that gets called through `PR_GetAddrInfoByName`, they are probably hooking `getaddrinfo` from `ws2_32.dll` or something like this. The full call stack looks like this:

```
00 000000cd`343ce370 00007ffb`951cf673     ntdll!RtlReportFatalFailure+0x9
01 000000cd`343ce3c0 00007ffb`951d83f2     ntdll!RtlReportCriticalFailure+0x97
02 000000cd`343ce4b0 00007ffb`951d86da     ntdll!RtlpHeapHandleError+0x12
03 000000cd`343ce4e0 00007ffb`951de361     ntdll!RtlpHpHeapHandleError+0x7a
04 000000cd`343ce510 00007ffb`950f5bf0     ntdll!RtlpLogHeapFailure+0x45
05 000000cd`343ce540 00007ffb`950f47b1     ntdll!RtlpFreeHeapInternal+0x4e0
06 000000cd`343ce600 00007ffb`542850cc     ntdll!RtlFreeHeap+0x51
 wslbdhm64+0x350cc _free_base
 wslbdhm64+0x5eeb8 Json::Value::~Json::Value
 wslbdhm64+0x5ef84 Json::Value::"DestructiveMoveAssignment"
 wslbdhm64+0x657a6 DoStuff
 wslbdhm64+0x65e8b
 wslbdhm64+0x1de43
 wslbdhm64+0x1d9d8
 wslbdhm64+0x160db
 wslbdhm64+0x6829e
 wslbdhm64+0x64faf
 [ dynamic code ]
00 00000016`760dede0 00007ffb`49b4f32a     nss3!PR_GetAddrInfoByName+0xc3 [/builds/worker/checkouts/gecko/nsprpub/pr/src/misc/prnetdb.c @ 2171]
01 (Inline Function) --------`--------     xul!mozilla::net::_GetAddrInfo_Portable+0x5f [/builds/worker/checkouts/gecko/netwerk/dns/GetAddrInfo.cpp @ 241]
02 00000016`760dee60 00007ffb`4a2a284b     xul!mozilla::net::GetAddrInfo+0x15a [/builds/worker/checkouts/gecko/netwerk/dns/GetAddrInfo.cpp @ 374]
03 00000016`760df0a0 00007ffb`4a1e4471     xul!nsHostResolver::ThreadFunc+0x43b [/builds/worker/checkouts/gecko/netwerk/dns/nsHostResolver.cpp @ 1727]
04 (Inline Function) --------`--------     xul!mozilla::detail::RunnableMethodArguments<>::applyImpl+0x9 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1147]
05 (Inline Function) --------`--------     xul!mozilla::detail::RunnableMethodArguments<>::apply+0x9 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1153]
06 00000016`760df200 00007ffb`4a211f3e     xul!mozilla::detail::RunnableMethodImpl<RefPtr<nsObserverService>,void (nsObserverService::*)(),1,0>::Run+0x21 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1203]
07 00000016`760df230 00007ffb`4b1c1aa8     xul!nsThreadPool::Run+0x29e [/builds/worker/checkouts/gecko/xpcom/threads/nsThreadPool.cpp @ 311]
08 (Inline Function) --------`--------     xul!nsThread::ProcessNextEvent+0x17b0 [/builds/worker/checkouts/gecko/xpcom/threads/nsThread.cpp @ 1198]
09 00000016`760df3d0 00007ffb`4b1ad48f     xul!NS_ProcessNextEvent+0x1808 [/builds/worker/checkouts/gecko/xpcom/threads/nsThreadUtils.cpp @ 466]
0a 00000016`760df760 00007ffb`4a3aba5f     xul!mozilla::ipc::MessagePumpForNonMainThreads::Run+0xcf [/builds/worker/checkouts/gecko/ipc/glue/MessagePump.cpp @ 301]
0b (Inline Function) --------`--------     xul!MessageLoop::RunInternal+0x16 [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 381]
0c 00000016`760df800 00007ffb`499ef54e     xul!MessageLoop::RunHandler+0x2f [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 375]
0d 00000016`760df850 00007ffb`4a20d757     xul!MessageLoop::Run+0x4e [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 357]
0e 00000016`760df8b0 00007ffb`63d244bc     xul!nsThread::ThreadFunc+0xd7 [/builds/worker/checkouts/gecko/xpcom/threads/nsThread.cpp @ 385]
0f 00000016`760dfa60 00007ffb`63d39241     nss3!_PR_NativeRunThread+0x13c [/builds/worker/checkouts/gecko/nsprpub/pr/src/threads/combined/pruthr.c @ 421]
10 00000016`760dfae0 00007ffb`c2701bb2     nss3!pr_root+0x11 [/builds/worker/checkouts/gecko/nsprpub/pr/src/md/windows/w95thred.c @ 140]
11 00000016`760dfb10 00007ffb`c3657614     ucrtbase!thread_start<unsigned int (__cdecl*)(void *),1>+0x42
12 00000016`760dfb40 00007ffb`9782e8a8     kernel32!BaseThreadInitThunk+0x14
13 (Inline Function) --------`--------     mozglue!mozilla::interceptor::FuncHook<mozilla::interceptor::WindowsDllInterceptor<mozilla::interceptor::VMSharingPolicyShared>,void (*)(int, void *, void *)>::operator()+0x15 [/builds/worker/checkouts/gecko/toolkit/xre/dllservices/mozglue/nsWindowsDllInterceptor.h @ 150]
14 00000016`760dfb70 00007ffb`c4b426a1     mozglue!patched_BaseThreadInitThunk+0x28 [/builds/worker/checkouts/gecko/toolkit/xre/dllservices/mozglue/WindowsDllBlocklist.cpp @ 587]
15 00000016`760dfbe0 00000000`00000000     ntdll!RtlUserThreadStart+0x21
```

From analyzing the code in the DLL, I believe the code is doing a move assignment to a subvalue of a `Json::Value` stored as a global or static variable (that's what I called `DestructiveMoveAssignment` in the call stack), like the assignment in the `ThreadFunc` below:

```c++
#include "json/json.h"
#include <windows.h>

constexpr auto nThreads = 2;
constexpr auto nIterations = 0x1000;

Json::Value gValue;

DWORD ThreadFunc(void* aParam)
{
    for (size_t i = 0; i < nIterations; ++i) {
        gValue[Json::String("")] = Json::Value("hello");
    }
    return 0;
}

int main() {
    auto bResult = HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0);
    if (!bResult) {
        return 1;
    }
    HANDLE threads[nThreads]{};
    for (auto& thread : threads) {
        DWORD threadId = 0;
        thread = CreateThread(nullptr, 0, ThreadFunc, nullptr, 0, &threadId);
        if (!thread) {
            return 1;
        }
    }
    WaitForMultipleObjects(nThreads, threads, TRUE, INFINITE);
    return 0;
}
```

Such a move assignment is not thread-safe, as the example program illustrates; the example program forces the race condition and crashes with a double free heap corruption with a similar call stack (the call to `releasePayload` is inlined but present in the original crash):

```
00 00000038`b59ff3e0 00007fff`5989c213     ntdll!RtlReportFatalFailure+0x9
01 00000038`b59ff430 00007fff`598a52aa     ntdll!RtlReportCriticalFailure+0x97
02 00000038`b59ff520 00007fff`598a558a     ntdll!RtlpHeapHandleError+0x12
03 00000038`b59ff550 00007fff`598b1585     ntdll!RtlpHpHeapHandleError+0x7a
04 00000038`b59ff580 00007fff`597cc59c     ntdll!RtlpLogHeapFailure+0x45
05 00000038`b59ff5b0 00007fff`597cb1e1     ntdll!RtlpFreeHeapInternal+0x84c
06 00000038`b59ff670 00007fff`56c637eb     ntdll!RtlFreeHeap+0x51
07 00000038`b59ff6b0 00007ff7`49f34604     ucrtbase!_free_base+0x1b
08 00000038`b59ff6e0 00007ff7`49f354be     MyJson!Json::releasePrefixedStringValue+0x14 [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 182] 
09 00000038`b59ff710 00007ff7`49f350d3     MyJson!Json::Value::releasePayload+0x4e [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 1018] 
0a 00000038`b59ff760 00007ff7`49f315e1     MyJson!Json::Value::~Value+0x13 [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 443] 
0b 00000038`b59ff790 00007fff`58f726bd     MyJson!ThreadFunc+0xb1 [C:\repos\MyJson\MyJson\MyJson.cpp @ 13] 
0c 00000038`b59ff840 00007fff`597edfb8     KERNEL32!BaseThreadInitThunk+0x1d
0d 00000038`b59ff870 00000000`00000000     ntdll!RtlUserThreadStart+0x28
```

I will try to contact the vendor to confirm if they do have code that looks like this and if they could make their code thread-safe.

Can anyone confirm that `PR_GetAddrInfoByName` can indeed potentially get called simultaneously by different threads? (Well, the call stack suggests that this is exactly what we are doing here -- using a specific thread each time we need to resolve a hostname.)

Thanks!

(I haven't analyzed the other crashes for the moment.)

Revision 12 by

Yannis Juglaret [:yannis]

on 2023-01-12 01:09:26 PST

I have analyzed the most recurrent crash thanks to the DLL. The heap corruption is reported as a double free. I have identified that the last called functions in the stack belong to the statically-linked [JsonCpp](https://github.com/open-source-parsers/jsoncpp) library. I have also restored the Firefox part of the stack. The crash occurs in third-party code that gets called through `PR_GetAddrInfoByName`, they are probably hooking `getaddrinfo` from `ws2_32.dll` or something like this. The full call stack looks like this:

```
00 000000cd`343ce370 00007ffb`951cf673     ntdll!RtlReportFatalFailure+0x9
01 000000cd`343ce3c0 00007ffb`951d83f2     ntdll!RtlReportCriticalFailure+0x97
02 000000cd`343ce4b0 00007ffb`951d86da     ntdll!RtlpHeapHandleError+0x12
03 000000cd`343ce4e0 00007ffb`951de361     ntdll!RtlpHpHeapHandleError+0x7a
04 000000cd`343ce510 00007ffb`950f5bf0     ntdll!RtlpLogHeapFailure+0x45
05 000000cd`343ce540 00007ffb`950f47b1     ntdll!RtlpFreeHeapInternal+0x4e0
06 000000cd`343ce600 00007ffb`542850cc     ntdll!RtlFreeHeap+0x51
 wslbdhm64+0x350cc _free_base
 wslbdhm64+0x5eeb8 Json::Value::~Json::Value
 wslbdhm64+0x5ef84 Json::Value::"DestructiveMoveAssignment"
 wslbdhm64+0x657a6 DoStuff
 wslbdhm64+0x65e8b
 wslbdhm64+0x1de43
 wslbdhm64+0x1d9d8
 wslbdhm64+0x160db
 wslbdhm64+0x6829e
 wslbdhm64+0x64faf
 [ dynamic code ]
00 00000016`760dede0 00007ffb`49b4f32a     nss3!PR_GetAddrInfoByName+0xc3 [/builds/worker/checkouts/gecko/nsprpub/pr/src/misc/prnetdb.c @ 2171]
01 (Inline Function) --------`--------     xul!mozilla::net::_GetAddrInfo_Portable+0x5f [/builds/worker/checkouts/gecko/netwerk/dns/GetAddrInfo.cpp @ 241]
02 00000016`760dee60 00007ffb`4a2a284b     xul!mozilla::net::GetAddrInfo+0x15a [/builds/worker/checkouts/gecko/netwerk/dns/GetAddrInfo.cpp @ 374]
03 00000016`760df0a0 00007ffb`4a1e4471     xul!nsHostResolver::ThreadFunc+0x43b [/builds/worker/checkouts/gecko/netwerk/dns/nsHostResolver.cpp @ 1727]
04 (Inline Function) --------`--------     xul!mozilla::detail::RunnableMethodArguments<>::applyImpl+0x9 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1147]
05 (Inline Function) --------`--------     xul!mozilla::detail::RunnableMethodArguments<>::apply+0x9 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1153]
06 00000016`760df200 00007ffb`4a211f3e     xul!mozilla::detail::RunnableMethodImpl<RefPtr<nsObserverService>,void (nsObserverService::*)(),1,0>::Run+0x21 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1203]
07 00000016`760df230 00007ffb`4b1c1aa8     xul!nsThreadPool::Run+0x29e [/builds/worker/checkouts/gecko/xpcom/threads/nsThreadPool.cpp @ 311]
08 (Inline Function) --------`--------     xul!nsThread::ProcessNextEvent+0x17b0 [/builds/worker/checkouts/gecko/xpcom/threads/nsThread.cpp @ 1198]
09 00000016`760df3d0 00007ffb`4b1ad48f     xul!NS_ProcessNextEvent+0x1808 [/builds/worker/checkouts/gecko/xpcom/threads/nsThreadUtils.cpp @ 466]
0a 00000016`760df760 00007ffb`4a3aba5f     xul!mozilla::ipc::MessagePumpForNonMainThreads::Run+0xcf [/builds/worker/checkouts/gecko/ipc/glue/MessagePump.cpp @ 301]
0b (Inline Function) --------`--------     xul!MessageLoop::RunInternal+0x16 [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 381]
0c 00000016`760df800 00007ffb`499ef54e     xul!MessageLoop::RunHandler+0x2f [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 375]
0d 00000016`760df850 00007ffb`4a20d757     xul!MessageLoop::Run+0x4e [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 357]
0e 00000016`760df8b0 00007ffb`63d244bc     xul!nsThread::ThreadFunc+0xd7 [/builds/worker/checkouts/gecko/xpcom/threads/nsThread.cpp @ 385]
0f 00000016`760dfa60 00007ffb`63d39241     nss3!_PR_NativeRunThread+0x13c [/builds/worker/checkouts/gecko/nsprpub/pr/src/threads/combined/pruthr.c @ 421]
10 00000016`760dfae0 00007ffb`c2701bb2     nss3!pr_root+0x11 [/builds/worker/checkouts/gecko/nsprpub/pr/src/md/windows/w95thred.c @ 140]
11 00000016`760dfb10 00007ffb`c3657614     ucrtbase!thread_start<unsigned int (__cdecl*)(void *),1>+0x42
12 00000016`760dfb40 00007ffb`9782e8a8     kernel32!BaseThreadInitThunk+0x14
13 (Inline Function) --------`--------     mozglue!mozilla::interceptor::FuncHook<mozilla::interceptor::WindowsDllInterceptor<mozilla::interceptor::VMSharingPolicyShared>,void (*)(int, void *, void *)>::operator()+0x15 [/builds/worker/checkouts/gecko/toolkit/xre/dllservices/mozglue/nsWindowsDllInterceptor.h @ 150]
14 00000016`760dfb70 00007ffb`c4b426a1     mozglue!patched_BaseThreadInitThunk+0x28 [/builds/worker/checkouts/gecko/toolkit/xre/dllservices/mozglue/WindowsDllBlocklist.cpp @ 587]
15 00000016`760dfbe0 00000000`00000000     ntdll!RtlUserThreadStart+0x21
```

From analyzing the code in the DLL, I believe the code is doing a move assignment to a subvalue of a `Json::Value` stored as a global or static variable (that's what I called `DestructiveMoveAssignment` in the call stack), like the assignment in the `ThreadFunc` below:

```c++
#include "json/json.h"
#include <windows.h>

constexpr auto nThreads = 2;
constexpr auto nIterations = 0x1000;

Json::Value gValue;

DWORD ThreadFunc(void* aParam)
{
    for (size_t i = 0; i < nIterations; ++i) {
        gValue[Json::String("")] = Json::Value("hello");
    }
    return 0;
}

int main() {
    auto bResult = HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0);
    if (!bResult) {
        return 1;
    }
    HANDLE threads[nThreads]{};
    for (auto& thread : threads) {
        DWORD threadId = 0;
        thread = CreateThread(nullptr, 0, ThreadFunc, nullptr, 0, &threadId);
        if (!thread) {
            return 1;
        }
    }
    WaitForMultipleObjects(nThreads, threads, TRUE, INFINITE);
    return 0;
}
```

Such a move assignment is not thread-safe, as the example program illustrates; the example program forces the race condition and crashes with a double free heap corruption with a similar call stack (the call to `releasePayload` and `releasePrefixedStringValue` are inlined but present in the original crash):

```
00 00000038`b59ff3e0 00007fff`5989c213     ntdll!RtlReportFatalFailure+0x9
01 00000038`b59ff430 00007fff`598a52aa     ntdll!RtlReportCriticalFailure+0x97
02 00000038`b59ff520 00007fff`598a558a     ntdll!RtlpHeapHandleError+0x12
03 00000038`b59ff550 00007fff`598b1585     ntdll!RtlpHpHeapHandleError+0x7a
04 00000038`b59ff580 00007fff`597cc59c     ntdll!RtlpLogHeapFailure+0x45
05 00000038`b59ff5b0 00007fff`597cb1e1     ntdll!RtlpFreeHeapInternal+0x84c
06 00000038`b59ff670 00007fff`56c637eb     ntdll!RtlFreeHeap+0x51
07 00000038`b59ff6b0 00007ff7`49f34604     ucrtbase!_free_base+0x1b
08 00000038`b59ff6e0 00007ff7`49f354be     MyJson!Json::releasePrefixedStringValue+0x14 [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 182] 
09 00000038`b59ff710 00007ff7`49f350d3     MyJson!Json::Value::releasePayload+0x4e [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 1018] 
0a 00000038`b59ff760 00007ff7`49f315e1     MyJson!Json::Value::~Value+0x13 [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 443] 
0b 00000038`b59ff790 00007fff`58f726bd     MyJson!ThreadFunc+0xb1 [C:\repos\MyJson\MyJson\MyJson.cpp @ 13] 
0c 00000038`b59ff840 00007fff`597edfb8     KERNEL32!BaseThreadInitThunk+0x1d
0d 00000038`b59ff870 00000000`00000000     ntdll!RtlUserThreadStart+0x28
```

I will try to contact the vendor to confirm if they do have code that looks like this and if they could make their code thread-safe.

Can anyone confirm that `PR_GetAddrInfoByName` can indeed potentially get called simultaneously by different threads? (Well, the call stack suggests that this is exactly what we are doing here -- using a specific thread each time we need to resolve a hostname.)

Thanks!

(I haven't analyzed the other crashes for the moment.)

Revision 13 by

Yannis Juglaret [:yannis]

on 2023-01-12 01:31:30 PST

I have analyzed the most recurrent crash thanks to the DLL. The heap corruption is reported as a double free. I have identified that the last called functions in the stack belong to the statically-linked [JsonCpp](https://github.com/open-source-parsers/jsoncpp) library. I have also restored the Firefox part of the stack. The crash occurs in third-party code that gets called through `PR_GetAddrInfoByName`, they are probably hooking `getaddrinfo` from `ws2_32.dll` or something like this. The full call stack looks like this:

```
00 000000cd`343ce370 00007ffb`951cf673     ntdll!RtlReportFatalFailure+0x9
01 000000cd`343ce3c0 00007ffb`951d83f2     ntdll!RtlReportCriticalFailure+0x97
02 000000cd`343ce4b0 00007ffb`951d86da     ntdll!RtlpHeapHandleError+0x12
03 000000cd`343ce4e0 00007ffb`951de361     ntdll!RtlpHpHeapHandleError+0x7a
04 000000cd`343ce510 00007ffb`950f5bf0     ntdll!RtlpLogHeapFailure+0x45
05 000000cd`343ce540 00007ffb`950f47b1     ntdll!RtlpFreeHeapInternal+0x4e0
06 000000cd`343ce600 00007ffb`542850cc     ntdll!RtlFreeHeap+0x51
 wslbdhm64+0x350cc _free_base
 wslbdhm64+0x5eeb8 Json::Value::~Value
 wslbdhm64+0x5ef84 Json::Value::"DestructiveMoveAssignment"
 wslbdhm64+0x657a6 DoStuff
 wslbdhm64+0x65e8b
 wslbdhm64+0x1de43
 wslbdhm64+0x1d9d8
 wslbdhm64+0x160db
 wslbdhm64+0x6829e
 wslbdhm64+0x64faf
 [ dynamic code ]
00 00000016`760dede0 00007ffb`49b4f32a     nss3!PR_GetAddrInfoByName+0xc3 [/builds/worker/checkouts/gecko/nsprpub/pr/src/misc/prnetdb.c @ 2171]
01 (Inline Function) --------`--------     xul!mozilla::net::_GetAddrInfo_Portable+0x5f [/builds/worker/checkouts/gecko/netwerk/dns/GetAddrInfo.cpp @ 241]
02 00000016`760dee60 00007ffb`4a2a284b     xul!mozilla::net::GetAddrInfo+0x15a [/builds/worker/checkouts/gecko/netwerk/dns/GetAddrInfo.cpp @ 374]
03 00000016`760df0a0 00007ffb`4a1e4471     xul!nsHostResolver::ThreadFunc+0x43b [/builds/worker/checkouts/gecko/netwerk/dns/nsHostResolver.cpp @ 1727]
04 (Inline Function) --------`--------     xul!mozilla::detail::RunnableMethodArguments<>::applyImpl+0x9 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1147]
05 (Inline Function) --------`--------     xul!mozilla::detail::RunnableMethodArguments<>::apply+0x9 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1153]
06 00000016`760df200 00007ffb`4a211f3e     xul!mozilla::detail::RunnableMethodImpl<RefPtr<nsObserverService>,void (nsObserverService::*)(),1,0>::Run+0x21 [/builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h @ 1203]
07 00000016`760df230 00007ffb`4b1c1aa8     xul!nsThreadPool::Run+0x29e [/builds/worker/checkouts/gecko/xpcom/threads/nsThreadPool.cpp @ 311]
08 (Inline Function) --------`--------     xul!nsThread::ProcessNextEvent+0x17b0 [/builds/worker/checkouts/gecko/xpcom/threads/nsThread.cpp @ 1198]
09 00000016`760df3d0 00007ffb`4b1ad48f     xul!NS_ProcessNextEvent+0x1808 [/builds/worker/checkouts/gecko/xpcom/threads/nsThreadUtils.cpp @ 466]
0a 00000016`760df760 00007ffb`4a3aba5f     xul!mozilla::ipc::MessagePumpForNonMainThreads::Run+0xcf [/builds/worker/checkouts/gecko/ipc/glue/MessagePump.cpp @ 301]
0b (Inline Function) --------`--------     xul!MessageLoop::RunInternal+0x16 [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 381]
0c 00000016`760df800 00007ffb`499ef54e     xul!MessageLoop::RunHandler+0x2f [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 375]
0d 00000016`760df850 00007ffb`4a20d757     xul!MessageLoop::Run+0x4e [/builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc @ 357]
0e 00000016`760df8b0 00007ffb`63d244bc     xul!nsThread::ThreadFunc+0xd7 [/builds/worker/checkouts/gecko/xpcom/threads/nsThread.cpp @ 385]
0f 00000016`760dfa60 00007ffb`63d39241     nss3!_PR_NativeRunThread+0x13c [/builds/worker/checkouts/gecko/nsprpub/pr/src/threads/combined/pruthr.c @ 421]
10 00000016`760dfae0 00007ffb`c2701bb2     nss3!pr_root+0x11 [/builds/worker/checkouts/gecko/nsprpub/pr/src/md/windows/w95thred.c @ 140]
11 00000016`760dfb10 00007ffb`c3657614     ucrtbase!thread_start<unsigned int (__cdecl*)(void *),1>+0x42
12 00000016`760dfb40 00007ffb`9782e8a8     kernel32!BaseThreadInitThunk+0x14
13 (Inline Function) --------`--------     mozglue!mozilla::interceptor::FuncHook<mozilla::interceptor::WindowsDllInterceptor<mozilla::interceptor::VMSharingPolicyShared>,void (*)(int, void *, void *)>::operator()+0x15 [/builds/worker/checkouts/gecko/toolkit/xre/dllservices/mozglue/nsWindowsDllInterceptor.h @ 150]
14 00000016`760dfb70 00007ffb`c4b426a1     mozglue!patched_BaseThreadInitThunk+0x28 [/builds/worker/checkouts/gecko/toolkit/xre/dllservices/mozglue/WindowsDllBlocklist.cpp @ 587]
15 00000016`760dfbe0 00000000`00000000     ntdll!RtlUserThreadStart+0x21
```

From analyzing the code in the DLL, I believe the code is doing a move assignment to a subvalue of a `Json::Value` stored as a global or static variable (that's what I called `DestructiveMoveAssignment` in the call stack), like the assignment in the `ThreadFunc` below:

```c++
#include "json/json.h"
#include <windows.h>

constexpr auto nThreads = 2;
constexpr auto nIterations = 0x1000;

Json::Value gValue;

DWORD ThreadFunc(void* aParam)
{
    for (size_t i = 0; i < nIterations; ++i) {
        gValue[Json::String("")] = Json::Value("hello");
    }
    return 0;
}

int main() {
    auto bResult = HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0);
    if (!bResult) {
        return 1;
    }
    HANDLE threads[nThreads]{};
    for (auto& thread : threads) {
        DWORD threadId = 0;
        thread = CreateThread(nullptr, 0, ThreadFunc, nullptr, 0, &threadId);
        if (!thread) {
            return 1;
        }
    }
    WaitForMultipleObjects(nThreads, threads, TRUE, INFINITE);
    return 0;
}
```

Such a move assignment is not thread-safe, as the example program illustrates; the example program forces the race condition and crashes with a double free heap corruption with a similar call stack (the call to `releasePayload` and `releasePrefixedStringValue` are inlined but present in the original crash):

```
00 00000038`b59ff3e0 00007fff`5989c213     ntdll!RtlReportFatalFailure+0x9
01 00000038`b59ff430 00007fff`598a52aa     ntdll!RtlReportCriticalFailure+0x97
02 00000038`b59ff520 00007fff`598a558a     ntdll!RtlpHeapHandleError+0x12
03 00000038`b59ff550 00007fff`598b1585     ntdll!RtlpHpHeapHandleError+0x7a
04 00000038`b59ff580 00007fff`597cc59c     ntdll!RtlpLogHeapFailure+0x45
05 00000038`b59ff5b0 00007fff`597cb1e1     ntdll!RtlpFreeHeapInternal+0x84c
06 00000038`b59ff670 00007fff`56c637eb     ntdll!RtlFreeHeap+0x51
07 00000038`b59ff6b0 00007ff7`49f34604     ucrtbase!_free_base+0x1b
08 00000038`b59ff6e0 00007ff7`49f354be     MyJson!Json::releasePrefixedStringValue+0x14 [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 182] 
09 00000038`b59ff710 00007ff7`49f350d3     MyJson!Json::Value::releasePayload+0x4e [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 1018] 
0a 00000038`b59ff760 00007ff7`49f315e1     MyJson!Json::Value::~Value+0x13 [C:\repos\MyJson\packages\JsonCpp-source.1.0.2\build\native\src\json\json_value.cpp @ 443] 
0b 00000038`b59ff790 00007fff`58f726bd     MyJson!ThreadFunc+0xb1 [C:\repos\MyJson\MyJson\MyJson.cpp @ 13] 
0c 00000038`b59ff840 00007fff`597edfb8     KERNEL32!BaseThreadInitThunk+0x1d
0d 00000038`b59ff870 00000000`00000000     ntdll!RtlUserThreadStart+0x28
```

I will try to contact the vendor to confirm if they do have code that looks like this and if they could make their code thread-safe.

Can anyone confirm that `PR_GetAddrInfoByName` can indeed potentially get called simultaneously by different threads? (Well, the call stack suggests that this is exactly what we are doing here -- using a specific thread each time we need to resolve a hostname.)

Thanks!

(I haven't analyzed the other crashes for the moment.)

Back to Bug 1758641 Comment 5