Open Bug 1468125 Opened 2 years ago Updated 2 years ago

UBSan: signed integer overflow in [@ mozilla::FrameLayerBuilder::PaintItems]

Categories

(Core :: Web Painting, defect, P3)

defect

Tracking

()

Tracking Status
firefox62 --- affected

People

(Reporter: tsmith, Unassigned)

References

(Blocks 1 open bug)

Details

(Keywords: csectype-undefined, testcase)

Attachments

(1 file)

Attached file testcase.html
Found with commit 422090:874dedd55599

gfx/src/nsCoord.h:373:23: runtime error: signed integer overflow: 126024528 * 60 cannot be represented in type 'int'
    #0 0x7fd0d72113bf in NSIntPixelsToAppUnits(int, int) gfx/src/nsCoord.h:373:23
    #1 0x7fd0d95eb46f in nsRect ToAppUnits<mozilla::gfx::UnknownUnits>(mozilla::gfx::IntRectTyped<mozilla::gfx::UnknownUnits> const&, int) objdir-ff-ubsan/dist/include/nsRect.h:490:17
    #2 0x7fd0d9921640 in mozilla::FrameLayerBuilder::PaintItems(nsTArray<mozilla::AssignedDisplayItem>&, mozilla::gfx::IntRectTyped<mozilla::gfx::UnknownUnits> const&, gfxContext*, nsDisplayListBuilder*, nsPresContext*, mozilla::gfx::IntPointTyped<mozilla::gfx::UnknownUnits> const&, float, float) layout/painting/FrameLayerBuilder.cpp:6334:22
    #3 0x7fd0d992224f in mozilla::FrameLayerBuilder::DrawPaintedLayer(mozilla::layers::PaintedLayer*, gfxContext*, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, mozilla::layers::DrawRegionClip, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, void*) layout/painting/FrameLayerBuilder.cpp:6595:19
    #4 0x7fd0d7469870 in mozilla::layers::BasicPaintedLayer::PaintThebes(gfxContext*, mozilla::layers::Layer*, void (*)(mozilla::layers::PaintedLayer*, gfxContext*, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, mozilla::layers::DrawRegionClip, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, void*), void*) gfx/layers/basic/BasicPaintedLayer.cpp:94:9
    #5 0x7fd0d7467d03 in mozilla::layers::BasicLayerManager::PaintLayer(gfxContext*, mozilla::layers::Layer*, void (*)(mozilla::layers::PaintedLayer*, gfxContext*, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, mozilla::layers::DrawRegionClip, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, void*), void*) gfx/layers/basic/BasicLayerManager.cpp
    #6 0x7fd0d7468497 in mozilla::layers::BasicLayerManager::PaintSelfOrChildren(mozilla::layers::PaintLayerContext&, gfxContext*) gfx/layers/basic/BasicLayerManager.cpp:729:7
    #7 0x7fd0d7467d03 in mozilla::layers::BasicLayerManager::PaintLayer(gfxContext*, mozilla::layers::Layer*, void (*)(mozilla::layers::PaintedLayer*, gfxContext*, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, mozilla::layers::DrawRegionClip, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, void*), void*) gfx/layers/basic/BasicLayerManager.cpp
    #8 0x7fd0d7468497 in mozilla::layers::BasicLayerManager::PaintSelfOrChildren(mozilla::layers::PaintLayerContext&, gfxContext*) gfx/layers/basic/BasicLayerManager.cpp:729:7
    #9 0x7fd0d7467d03 in mozilla::layers::BasicLayerManager::PaintLayer(gfxContext*, mozilla::layers::Layer*, void (*)(mozilla::layers::PaintedLayer*, gfxContext*, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, mozilla::layers::DrawRegionClip, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, void*), void*) gfx/layers/basic/BasicLayerManager.cpp
    #10 0x7fd0d7468497 in mozilla::layers::BasicLayerManager::PaintSelfOrChildren(mozilla::layers::PaintLayerContext&, gfxContext*) gfx/layers/basic/BasicLayerManager.cpp:729:7
    #11 0x7fd0d7467d03 in mozilla::layers::BasicLayerManager::PaintLayer(gfxContext*, mozilla::layers::Layer*, void (*)(mozilla::layers::PaintedLayer*, gfxContext*, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, mozilla::layers::DrawRegionClip, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, void*), void*) gfx/layers/basic/BasicLayerManager.cpp
    #12 0x7fd0d7468497 in mozilla::layers::BasicLayerManager::PaintSelfOrChildren(mozilla::layers::PaintLayerContext&, gfxContext*) gfx/layers/basic/BasicLayerManager.cpp:729:7
    #13 0x7fd0d7467d03 in mozilla::layers::BasicLayerManager::PaintLayer(gfxContext*, mozilla::layers::Layer*, void (*)(mozilla::layers::PaintedLayer*, gfxContext*, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, mozilla::layers::DrawRegionClip, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, void*), void*) gfx/layers/basic/BasicLayerManager.cpp
    #14 0x7fd0d7468497 in mozilla::layers::BasicLayerManager::PaintSelfOrChildren(mozilla::layers::PaintLayerContext&, gfxContext*) gfx/layers/basic/BasicLayerManager.cpp:729:7
    #15 0x7fd0d7467d03 in mozilla::layers::BasicLayerManager::PaintLayer(gfxContext*, mozilla::layers::Layer*, void (*)(mozilla::layers::PaintedLayer*, gfxContext*, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, mozilla::layers::DrawRegionClip, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, void*), void*) gfx/layers/basic/BasicLayerManager.cpp
    #16 0x7fd0d7468497 in mozilla::layers::BasicLayerManager::PaintSelfOrChildren(mozilla::layers::PaintLayerContext&, gfxContext*) gfx/layers/basic/BasicLayerManager.cpp:729:7
    #17 0x7fd0d7467d03 in mozilla::layers::BasicLayerManager::PaintLayer(gfxContext*, mozilla::layers::Layer*, void (*)(mozilla::layers::PaintedLayer*, gfxContext*, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, mozilla::layers::DrawRegionClip, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, void*), void*) gfx/layers/basic/BasicLayerManager.cpp
    #18 0x7fd0d7468497 in mozilla::layers::BasicLayerManager::PaintSelfOrChildren(mozilla::layers::PaintLayerContext&, gfxContext*) gfx/layers/basic/BasicLayerManager.cpp:729:7
    #19 0x7fd0d7467d03 in mozilla::layers::BasicLayerManager::PaintLayer(gfxContext*, mozilla::layers::Layer*, void (*)(mozilla::layers::PaintedLayer*, gfxContext*, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, mozilla::layers::DrawRegionClip, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, void*), void*) gfx/layers/basic/BasicLayerManager.cpp
    #20 0x7fd0d74670ea in mozilla::layers::BasicLayerManager::EndTransactionInternal(void (*)(mozilla::layers::PaintedLayer*, gfxContext*, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, mozilla::layers::DrawRegionClip, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, void*), void*, mozilla::layers::LayerManager::EndTransactionFlags) gfx/layers/basic/BasicLayerManager.cpp:616:5
    #21 0x7fd0d9921cc1 in mozilla::PaintInactiveLayer(nsDisplayListBuilder*, mozilla::layers::LayerManager*, nsDisplayItem*, gfxContext*, gfxContext*) layout/painting/FrameLayerBuilder.cpp:4055:12
    #22 0x7fd0d992191f in mozilla::FrameLayerBuilder::PaintItems(nsTArray<mozilla::AssignedDisplayItem>&, mozilla::gfx::IntRectTyped<mozilla::gfx::UnknownUnits> const&, gfxContext*, nsDisplayListBuilder*, nsPresContext*, mozilla::gfx::IntPointTyped<mozilla::gfx::UnknownUnits> const&, float, float) layout/painting/FrameLayerBuilder.cpp:6424:7
    #23 0x7fd0d992224f in mozilla::FrameLayerBuilder::DrawPaintedLayer(mozilla::layers::PaintedLayer*, gfxContext*, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, mozilla::layers::DrawRegionClip, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, void*) layout/painting/FrameLayerBuilder.cpp:6595:19
    #24 0x7fd0d747e9cd in mozilla::layers::ClientPaintedLayer::PaintThebes(nsTArray<mozilla::layers::ReadbackProcessor::Update>*) gfx/layers/client/ClientPaintedLayer.cpp:158:5
    #25 0x7fd0d747f59f in mozilla::layers::ClientPaintedLayer::RenderLayerWithReadback(mozilla::layers::ReadbackProcessor*) gfx/layers/client/ClientPaintedLayer.cpp:314:3
    #26 0x7fd0d7491b06 in mozilla::layers::ClientContainerLayer::RenderLayer() gfx/layers/client/ClientContainerLayer.h:58:29
    #27 0x7fd0d747c866 in mozilla::layers::ClientLayerManager::EndTransactionInternal(void (*)(mozilla::layers::PaintedLayer*, gfxContext*, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, mozilla::layers::DrawRegionClip, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, void*), void*, mozilla::layers::LayerManager::EndTransactionFlags) gfx/layers/client/ClientLayerManager.cpp:372:13
    #28 0x7fd0d747cc40 in mozilla::layers::ClientLayerManager::EndTransaction(void (*)(mozilla::layers::PaintedLayer*, gfxContext*, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, mozilla::layers::DrawRegionClip, mozilla::gfx::IntRegionTyped<mozilla::gfx::UnknownUnits> const&, void*), void*, mozilla::layers::LayerManager::EndTransactionFlags) gfx/layers/client/ClientLayerManager.cpp:430:3
    #29 0x7fd0d99444d3 in nsDisplayList::PaintRoot(nsDisplayListBuilder*, gfxContext*, unsigned int) layout/painting/nsDisplayList.cpp:2798:19
    #30 0x7fd0d9633fbf in nsLayoutUtils::PaintFrame(gfxContext*, nsIFrame*, nsRegion const&, unsigned int, nsDisplayListBuilderMode, nsLayoutUtils::PaintFrameFlags) layout/base/nsLayoutUtils.cpp:3843:12
    #31 0x7fd0d95d48c6 in mozilla::PresShell::Paint(nsView*, nsRegion const&, unsigned int) layout/base/PresShell.cpp:6314:5
    #32 0x7fd0d9357106 in nsViewManager::ProcessPendingUpdatesPaint(nsIWidget*) view/nsViewManager.cpp:480:19
    #33 0x7fd0d9356d5a in nsViewManager::ProcessPendingUpdatesForView(nsView*, bool) view/nsViewManager.cpp:412:33
    #34 0x7fd0d9357c6a in nsViewManager::ProcessPendingUpdates() view/nsViewManager.cpp:1102:5
    #35 0x7fd0d95a4e0e in nsRefreshDriver::Tick(long, mozilla::TimeStamp) layout/base/nsRefreshDriver.cpp:2039:11
    #36 0x7fd0d95aabe1 in mozilla::RefreshDriverTimer::TickRefreshDrivers(long, mozilla::TimeStamp, nsTArray<RefPtr<nsRefreshDriver> >&) layout/base/nsRefreshDriver.cpp:301:7
    #37 0x7fd0d95aaac8 in mozilla::RefreshDriverTimer::Tick(long, mozilla::TimeStamp) layout/base/nsRefreshDriver.cpp:320:5
    #38 0x7fd0d95ace4b in mozilla::VsyncRefreshDriverTimer::RunRefreshDrivers(mozilla::TimeStamp) layout/base/nsRefreshDriver.cpp:760:5
    #39 0x7fd0d95ac4ad in mozilla::VsyncRefreshDriverTimer::RefreshDriverVsyncObserver::TickRefreshDriver(mozilla::TimeStamp) layout/base/nsRefreshDriver.cpp:673:35
    #40 0x7fd0d95ac125 in mozilla::VsyncRefreshDriverTimer::RefreshDriverVsyncObserver::NotifyVsync(mozilla::TimeStamp) layout/base/nsRefreshDriver.cpp:574:9
    #41 0x7fd0d98d5aee in mozilla::layout::VsyncChild::RecvNotify(mozilla::TimeStamp const&) layout/ipc/VsyncChild.cpp:68:16
    #42 0x7fd0d6bf1e70 in mozilla::layout::PVsyncChild::OnMessageReceived(IPC::Message const&) objdir-ff-ubsan/ipc/ipdl/PVsyncChild.cpp:167:20
    #43 0x7fd0d685d84c in mozilla::ipc::MessageChannel::DispatchAsyncMessage(IPC::Message const&) ipc/glue/MessageChannel.cpp:2134:25
    #44 0x7fd0d685c85c in mozilla::ipc::MessageChannel::DispatchMessage(IPC::Message&&) ipc/glue/MessageChannel.cpp:2064:17
    #45 0x7fd0d685d2b0 in mozilla::ipc::MessageChannel::MessageTask::Run() ipc/glue/MessageChannel.cpp:1943:15
    #46 0x7fd0d605c472 in nsThread::ProcessNextEvent(bool, bool*) xpcom/threads/nsThread.cpp:1088:14
    #47 0x7fd0d607af5e in NS_ProcessNextEvent(nsIThread*, bool) xpcom/threads/nsThreadUtils.cpp:519:10
    #48 0x7fd0d686037f in mozilla::ipc::MessagePump::Run(base::MessagePump::Delegate*) ipc/glue/MessagePump.cpp:97:21
    #49 0x7fd0d67c9a89 in MessageLoop::Run() ipc/chromium/src/base/message_loop.cc:298:3
    #50 0x7fd0d9397fff in nsBaseAppShell::Run() widget/nsBaseAppShell.cpp:157:27
    #51 0x7fd0db11a993 in XRE_RunAppShell() toolkit/xre/nsEmbedFunctions.cpp:896:22
    #52 0x7fd0d6860a98 in mozilla::ipc::MessagePumpForChildProcess::Run(base::MessagePump::Delegate*) ipc/glue/MessagePump.cpp:269:9
    #53 0x7fd0d67c9a89 in MessageLoop::Run() ipc/chromium/src/base/message_loop.cc:298:3
    #54 0x7fd0db11a552 in XRE_InitChildProcess(int, char**, XREChildData const*) toolkit/xre/nsEmbedFunctions.cpp:722:34
    #55 0x430a7a in content_process_main(mozilla::Bootstrap*, int, char**) browser/app/../../ipc/contentproc/plugin-container.cpp:50:30
    #56 0x430b40 in main browser/app/nsBrowserApp.cpp:287:18
    #57 0x7fd0f8c8d1c0 in __libc_start_main /build/glibc-itYbWN/glibc-2.26/csu/../csu/libc-start.c:308
    #58 0x4092e9 in _start (firefox+0x4092e9)
Flags: in-testsuite?
This has a lot of nested frames, eaching scaling by 7, until the innermost one gets a size that overflows a 32bit signed int.

Would be nice to check for this, but unlikely to cause any serious issues (other than just not rendering).
Priority: -- → P3
You need to log in before you can comment on or make changes to this bug.