1620671 - -nan is outside the range of representable values of type 'int' in src/layout/generic/nsFloatManager.cpp:2813

Status: RESOLVED DUPLICATE of bug 1758824

(Core :: Layout: Floats, defect, P3)

Description

[Tyson Smith [:tsmith]]

Attachment: testcase.html

This was found with m-c 20200305-c991b81a7833

To enable this check add the following to your mozconfig:

ac_add_options --enable-undefined-sanitizer="float-cast-overflow"

src/layout/generic/nsFloatManager.cpp:2813:10: runtime error: -nan is outside the range of representable values of type 'int'
    #0 0x7fd86a6d9a73 in nsFloatManager::ShapeInfo::XInterceptAtY(int, int, int) src/layout/generic/nsFloatManager.cpp:2813:10
    #1 0x7fd86a6da048 in nsFloatManager::ShapeInfo::ComputeEllipseLineInterceptDiff(int, int, int, int, int, int, int, int) src/layout/generic/nsFloatManager.cpp:2792:9
    #2 0x7fd86a6d9bea in nsFloatManager::EllipseShapeInfo::LineEdge(int, int, bool) const src/layout/generic/nsFloatManager.cpp:935:24
    #3 0x7fd86a6d65d2 in nsFloatManager::FloatInfo::LineRight(nsFloatManager::ShapeType, int, int) const src/layout/generic/nsFloatManager.cpp:2371:44
    #4 0x7fd86a6d59c8 in nsFloatManager::GetFlowArea(mozilla::WritingMode, int, int, nsFloatManager::BandInfoType, nsFloatManager::ShapeType, mozilla::LogicalRect, nsFloatManager::SavedState*, nsSize const&) const src/layout/generic/nsFloatManager.cpp:208:16
    #5 0x7fd86a5f6c47 in mozilla::BlockReflowInput::GetFloatAvailableSpaceWithState(int, nsFloatManager::ShapeType, nsFloatManager::SavedState*) const src/layout/generic/BlockReflowInput.cpp:298:43
    #6 0x7fd86a62005a in mozilla::BlockReflowInput::GetFloatAvailableSpace(int) const src/layout/generic/BlockReflowInput.h:122:12
    #7 0x7fd86a5f80ac in mozilla::BlockReflowInput::AddFloat(nsLineLayout*, nsIFrame*, int) src/layout/generic/BlockReflowInput.cpp:572:36
    #8 0x7fd86a8416db in nsLineLayout::ReflowFrame(nsIFrame*, nsReflowStatus&, mozilla::ReflowOutput*, bool&) src/layout/generic/nsLineLayout.cpp:922:25
    #9 0x7fd86a65f49d in nsBlockFrame::ReflowInlineFrame(mozilla::BlockReflowInput&, nsLineLayout&, nsLineList_iterator, nsIFrame*, LineReflowStatus*) src/layout/generic/nsBlockFrame.cpp:4477:15
    #10 0x7fd86a65e591 in nsBlockFrame::DoReflowInlineFrames(mozilla::BlockReflowInput&, nsLineLayout&, nsLineList_iterator, nsFlowAreaRect&, int&, nsFloatManager::SavedState*, bool*, LineReflowStatus*, bool) src/layout/generic/nsBlockFrame.cpp:4279:5
    #11 0x7fd86a65961a in nsBlockFrame::ReflowInlineFrames(mozilla::BlockReflowInput&, nsLineList_iterator, bool*) src/layout/generic/nsBlockFrame.cpp:4164:9
    #12 0x7fd86a654666 in nsBlockFrame::ReflowLine(mozilla::BlockReflowInput&, nsLineList_iterator, bool*) src/layout/generic/nsBlockFrame.cpp:3145:5
    #13 0x7fd86a64cbe5 in nsBlockFrame::ReflowDirtyLines(mozilla::BlockReflowInput&) src/layout/generic/nsBlockFrame.cpp:2685:7
    #14 0x7fd86a648686 in nsBlockFrame::Reflow(nsPresContext*, mozilla::ReflowOutput&, mozilla::ReflowInput const&, nsReflowStatus&) src/layout/generic/nsBlockFrame.cpp:1398:3
    #15 0x7fd86a65cba9 in nsBlockReflowContext::ReflowBlock(mozilla::LogicalRect const&, bool, nsCollapsingMargin&, int, bool, nsLineBox*, mozilla::ReflowInput&, nsReflowStatus&, mozilla::BlockReflowInput&) src/layout/generic/nsBlockReflowContext.cpp:293:11
    #16 0x7fd86a6570f5 in nsBlockFrame::ReflowBlockFrame(mozilla::BlockReflowInput&, nsLineList_iterator, bool*) src/layout/generic/nsBlockFrame.cpp:3795:11
    #17 0x7fd86a654620 in nsBlockFrame::ReflowLine(mozilla::BlockReflowInput&, nsLineList_iterator, bool*) src/layout/generic/nsBlockFrame.cpp:3142:5
    #18 0x7fd86a64cbe5 in nsBlockFrame::ReflowDirtyLines(mozilla::BlockReflowInput&) src/layout/generic/nsBlockFrame.cpp:2685:7
    #19 0x7fd86a648686 in nsBlockFrame::Reflow(nsPresContext*, mozilla::ReflowOutput&, mozilla::ReflowInput const&, nsReflowStatus&) src/layout/generic/nsBlockFrame.cpp:1398:3
    #20 0x7fd86a67da1a in nsContainerFrame::ReflowChild(nsIFrame*, nsPresContext*, mozilla::ReflowOutput&, mozilla::ReflowInput const&, mozilla::WritingMode const&, mozilla::LogicalPoint const&, nsSize const&, nsIFrame::ReflowChildFlags, nsReflowStatus&, nsOverflowContinuationTracker*) src/layout/generic/nsContainerFrame.cpp:906:14
    #21 0x7fd86a67cbf5 in nsCanvasFrame::Reflow(nsPresContext*, mozilla::ReflowOutput&, mozilla::ReflowInput const&, nsReflowStatus&) src/layout/generic/nsCanvasFrame.cpp:750:5
    #22 0x7fd86a67da1a in nsContainerFrame::ReflowChild(nsIFrame*, nsPresContext*, mozilla::ReflowOutput&, mozilla::ReflowInput const&, mozilla::WritingMode const&, mozilla::LogicalPoint const&, nsSize const&, nsIFrame::ReflowChildFlags, nsReflowStatus&, nsOverflowContinuationTracker*) src/layout/generic/nsContainerFrame.cpp:906:14
    #23 0x7fd86a742515 in nsHTMLScrollFrame::ReflowScrolledFrame(mozilla::ScrollReflowInput*, bool, bool, mozilla::ReflowOutput*) src/layout/generic/nsGfxScrollFrame.cpp:653:3
    #24 0x7fd86a743c5c in nsHTMLScrollFrame::ReflowContents(mozilla::ScrollReflowInput*, mozilla::ReflowOutput const&) src/layout/generic/nsGfxScrollFrame.cpp:767:3
    #25 0x7fd86a74723d in nsHTMLScrollFrame::Reflow(nsPresContext*, mozilla::ReflowOutput&, mozilla::ReflowInput const&, nsReflowStatus&) src/layout/generic/nsGfxScrollFrame.cpp:1154:3   
    #26 0x7fd86a63da66 in nsContainerFrame::ReflowChild(nsIFrame*, nsPresContext*, mozilla::ReflowOutput&, mozilla::ReflowInput const&, int, int, nsIFrame::ReflowChildFlags, nsReflowStatus&, nsOverflowContinuationTracker*) src/layout/generic/nsContainerFrame.cpp:946:14
    #27 0x7fd86a63d38f in mozilla::ViewportFrame::Reflow(nsPresContext*, mozilla::ReflowOutput&, mozilla::ReflowInput const&, nsReflowStatus&) src/layout/generic/ViewportFrame.cpp:299:7
    #28 0x7fd86a4624d1 in mozilla::PresShell::DoReflow(nsIFrame*, bool, mozilla::OverflowChangedTracker*) src/layout/base/PresShell.cpp:9251:11
    #29 0x7fd86a470c50 in mozilla::PresShell::ProcessReflowCommands(bool) src/layout/base/PresShell.cpp:9424:24
    #30 0x7fd86a46ffb5 in mozilla::PresShell::DoFlushPendingNotifications(mozilla::ChangesToFlush) src/layout/base/PresShell.cpp:4121:11
    #31 0x7fd86a410532 in nsRefreshDriver::Tick(mozilla::layers::BaseTransactionId<mozilla::VsyncIdType>, mozilla::TimeStamp) src/layout/base/nsRefreshDriver.cpp:2059:20
    #32 0x7fd86a42026e in mozilla::RefreshDriverTimer::TickRefreshDrivers(mozilla::layers::BaseTransactionId<mozilla::VsyncIdType>, mozilla::TimeStamp, nsTArray<RefPtr<nsRefreshDriver> >&) src/layout/base/nsRefreshDriver.cpp:351:7
    #33 0x7fd86a41ffd1 in mozilla::RefreshDriverTimer::Tick(mozilla::layers::BaseTransactionId<mozilla::VsyncIdType>, mozilla::TimeStamp) src/layout/base/nsRefreshDriver.cpp:368:5
    #34 0x7fd86a41e94f in mozilla::VsyncRefreshDriverTimer::RefreshDriverVsyncObserver::TickRefreshDriver(mozilla::layers::BaseTransactionId<mozilla::VsyncIdType>, mozilla::TimeStamp) src/layout/base/nsRefreshDriver.cpp:743:16
    #35 0x7fd86a41dcce in mozilla::VsyncRefreshDriverTimer::RefreshDriverVsyncObserver::NotifyVsync(mozilla::VsyncEvent const&) src/layout/base/nsRefreshDriver.cpp:642:9
    #36 0x7fd86aaf8966 in mozilla::layout::VsyncChild::RecvNotify(mozilla::VsyncEvent const&) src/layout/ipc/VsyncChild.cpp:55:16
    #37 0x7fd8639307f6 in mozilla::layout::PVsyncChild::OnMessageReceived(IPC::Message const&) src/objdir-ff-ubsan/ipc/ipdl/PVsyncChild.cpp:187:54
    #38 0x7fd863309390 in mozilla::ipc::PBackgroundChild::OnMessageReceived(IPC::Message const&) src/objdir-ff-ubsan/ipc/ipdl/PBackgroundChild.cpp:5970:32
    #39 0x7fd862acd46b in mozilla::ipc::MessageChannel::DispatchAsyncMessage(mozilla::ipc::ActorLifecycleProxy*, IPC::Message const&) src/ipc/glue/MessageChannel.cpp:2187:25
    #40 0x7fd862ac8cb6 in mozilla::ipc::MessageChannel::DispatchMessage(IPC::Message&&) src/ipc/glue/MessageChannel.cpp:2111:9
    #41 0x7fd862aca873 in mozilla::ipc::MessageChannel::RunMessage(mozilla::ipc::MessageChannel::MessageTask&) src/ipc/glue/MessageChannel.cpp:1959:3
    #42 0x7fd862acb7a8 in mozilla::ipc::MessageChannel::MessageTask::Run() src/ipc/glue/MessageChannel.cpp:1990:13
    #43 0x7fd861692d63 in nsThread::ProcessNextEvent(bool, bool*) src/xpcom/threads/nsThread.cpp:1220:14
    #44 0x7fd861699ab6 in NS_ProcessNextEvent(nsIThread*, bool) src/xpcom/threads/nsThreadUtils.cpp:481:10
    #45 0x7fd862ada5ee in mozilla::ipc::MessagePump::Run(base::MessagePump::Delegate*) src/ipc/glue/MessagePump.cpp:87:21
    #46 0x7fd862914444 in MessageLoop::Run() src/ipc/chromium/src/base/message_loop.cc:290:3
    #47 0x7fd86a0045fa in nsBaseAppShell::Run() src/widget/nsBaseAppShell.cpp:137:27
    #48 0x7fd86e0f0431 in XRE_RunAppShell() src/toolkit/xre/nsEmbedFunctions.cpp:926:20
    #49 0x7fd862adbbe1 in mozilla::ipc::MessagePumpForChildProcess::Run(base::MessagePump::Delegate*) src/ipc/glue/MessagePump.cpp:237:9
    #50 0x7fd862914444 in MessageLoop::Run() src/ipc/chromium/src/base/message_loop.cc:290:3
    #51 0x7fd86e0efa56 in XRE_InitChildProcess(int, char**, XREChildData const*) src/toolkit/xre/nsEmbedFunctions.cpp:761:34
    #52 0x56383f7631bd in content_process_main(mozilla::Bootstrap*, int, char**) src/browser/app/../../ipc/contentproc/plugin-container.cpp:56:28
    #53 0x56383f7634e6 in main src/browser/app/nsBrowserApp.cpp:303:18

Comment 1

[Tyson Smith [:tsmith]]

A Pernosco session is available here: https://pernos.co/debug/xIYtXJLmrHyT01_xIOPpvA/index.html

Comment 2

[Tyson Smith [:tsmith]]

This issue is currently triggered while fuzzing with the 'float-cast-overflow' UBSan check enabled. This issue will need to be addressed before the check can be enabled by default.

If it requires too much effort to fix immediately please ni? me and let me know. If necessary it will be added to a suppression list. Thank you :)

Comment 3

[Ting-Yu Lin [:TYLin] (PST, UTC-8)]

I believe this bug is the same as bug 1758824. Although I cannot reproduce the runtime error using the testcase in comment 0, I still add the test as a wpt test in my patch in bug 1758824.