Found while fuzzing m-c 20220609-d6fefbd370cd (--enable-address-sanitizer --enable-fuzzing)

This was found by enabling the float-cast-overflow check in UBSan. This type of issue can create inconsistencies across platforms, architectures and optimization levels.

To enable this check add the following to your mozconfig:

ac_add_options --enable-undefined-sanitizer="float-cast-overflow"

To reproduce with the attached test case use the following commands:

$ pip install grizzly-framework
$ python -m grizzly.replay <ubsan-build>/firefox ./testcase.html

src/layout/painting/nsCSSRenderingBorders.cpp:2117:20: runtime error: -4 is outside the range of representable values of type 'unsigned long'
    #0 0x7f8f75c61a82 in nsCSSBorderRenderer::DrawDottedSideSlow(mozilla::Side) src/layout/painting/nsCSSRenderingBorders.cpp:2117:20
    #1 0x7f8f75c5fd7b in nsCSSBorderRenderer::DrawDashedOrDottedSide(mozilla::Side) src/layout/painting/nsCSSRenderingBorders.cpp:1770:5
    #2 0x7f8f75c36639 in nsCSSBorderRenderer::DrawBorders() src/layout/painting/nsCSSRenderingBorders.cpp:3321:9
    #3 0x7f8f75c2c980 in nsCSSRendering::PaintBorderWithStyleBorder(nsPresContext*, gfxContext&, nsIFrame*, nsRect const&, nsRect const&, nsStyleBorder const&, mozilla::ComputedStyle*, mozilla::PaintBorderFlags, mozilla::Sides) src/layout/painting/nsCSSRendering.cpp:888:6
    #4 0x7f8f75c2bf26 in nsCSSRendering::PaintBorder(nsPresContext*, gfxContext&, nsIFrame*, nsRect const&, nsRect const&, mozilla::ComputedStyle*, mozilla::PaintBorderFlags, mozilla::Sides) src/layout/painting/nsCSSRendering.cpp:647:10
    #5 0x7f8f75c9d744 in mozilla::nsDisplayBorder::Paint(mozilla::nsDisplayListBuilder*, gfxContext*) src/layout/painting/nsDisplayList.cpp:4277:26
    #6 0x7f8f75c2b697 in mozilla::nsDisplayList::Paint(mozilla::nsDisplayListBuilder*, gfxContext*, int) src/layout/painting/nsDisplayList.cpp:2198:11
    #7 0x7f8f75cb6f0b in mozilla::nsDisplayTransform::Paint(mozilla::nsDisplayListBuilder*, gfxContext*, mozilla::Maybe<mozilla::gfx::PolygonTyped<mozilla::gfx::UnknownUnits> > const&) src/layout/painting/nsDisplayList.cpp:6861:20
    #8 0x7f8f75cb6666 in mozilla::nsDisplayTransform::Paint(mozilla::nsDisplayListBuilder*, gfxContext*) src/layout/painting/nsDisplayList.cpp:6828:3
    #9 0x7f8f75c2b697 in mozilla::nsDisplayList::Paint(mozilla::nsDisplayListBuilder*, gfxContext*, int) src/layout/painting/nsDisplayList.cpp:2198:11
    #10 0x7f8f75cb6f0b in mozilla::nsDisplayTransform::Paint(mozilla::nsDisplayListBuilder*, gfxContext*, mozilla::Maybe<mozilla::gfx::PolygonTyped<mozilla::gfx::UnknownUnits> > const&) src/layout/painting/nsDisplayList.cpp:6861:20
    #11 0x7f8f75cb6666 in mozilla::nsDisplayTransform::Paint(mozilla::nsDisplayListBuilder*, gfxContext*) src/layout/painting/nsDisplayList.cpp:6828:3
    #12 0x7f8f75c2b697 in mozilla::nsDisplayList::Paint(mozilla::nsDisplayListBuilder*, gfxContext*, int) src/layout/painting/nsDisplayList.cpp:2198:11
    #13 0x7f8f75cb6f0b in mozilla::nsDisplayTransform::Paint(mozilla::nsDisplayListBuilder*, gfxContext*, mozilla::Maybe<mozilla::gfx::PolygonTyped<mozilla::gfx::UnknownUnits> > const&) src/layout/painting/nsDisplayList.cpp:6861:20
    #14 0x7f8f75cb6666 in mozilla::nsDisplayTransform::Paint(mozilla::nsDisplayListBuilder*, gfxContext*) src/layout/painting/nsDisplayList.cpp:6828:3
    #15 0x7f8f75c2b697 in mozilla::nsDisplayList::Paint(mozilla::nsDisplayListBuilder*, gfxContext*, int) src/layout/painting/nsDisplayList.cpp:2198:11
    #16 0x7f8f75c87f8c in mozilla::nsDisplayList::PaintRoot(mozilla::nsDisplayListBuilder*, gfxContext*, unsigned int, mozilla::Maybe<double>) src/layout/painting/nsDisplayList.cpp:2263:5
    #17 0x7f8f755aa599 in nsLayoutUtils::PaintFrame(gfxContext*, nsIFrame*, nsRegion const&, unsigned int, mozilla::nsDisplayListBuilderMode, nsLayoutUtils::PaintFrameFlags) src/layout/base/nsLayoutUtils.cpp:3456:9
    #18 0x7f8f758e2bb8 in nsPageSequenceFrame::PrintNextSheet() src/layout/generic/nsPageSequenceFrame.cpp:693:3
    #19 0x7f8f75d32973 in nsPrintJob::PrintSheet(nsPrintObject*, bool&) src/layout/printing/nsPrintJob.cpp:1824:31
    #20 0x7f8f75d32421 in nsPagePrintTimer::Run() src/layout/printing/nsPagePrintTimer.cpp:92:43
    #21 0x7f8f6c96d27c in mozilla::SchedulerGroup::Runnable::Run() src/xpcom/threads/SchedulerGroup.cpp:140:20
    #22 0x7f8f6c9c328a in mozilla::RunnableTask::Run() src/xpcom/threads/TaskController.cpp:475:16
    #23 0x7f8f6c9850e1 in mozilla::TaskController::DoExecuteNextTaskOnlyMainThreadInternal(mozilla::detail::BaseAutoLock<mozilla::Mutex&> const&) src/xpcom/threads/TaskController.cpp:788:26
    #24 0x7f8f6c98295e in mozilla::TaskController::ExecuteNextTaskOnlyMainThreadInternal(mozilla::detail::BaseAutoLock<mozilla::Mutex&> const&) src/xpcom/threads/TaskController.cpp:620:15
    #25 0x7f8f6c9830cb in mozilla::TaskController::ProcessPendingMTTask(bool) src/xpcom/threads/TaskController.cpp:398:36
    #26 0x7f8f6c9b4dd4 in mozilla::TaskController::InitializeInternal()::$_1::operator()() const src/xpcom/threads/TaskController.cpp:127:37
    #27 0x7f8f6c9b4dd4 in mozilla::detail::RunnableFunction<mozilla::TaskController::InitializeInternal()::$_1>::Run() src/objdir-ff-ubsan/dist/include/nsThreadUtils.h:531:5
    #28 0x7f8f6c9a1263 in nsThread::ProcessNextEvent(bool, bool*) src/xpcom/threads/nsThread.cpp:1180:16
    #29 0x7f8f6c9a95a4 in NS_ProcessNextEvent(nsIThread*, bool) src/xpcom/threads/nsThreadUtils.cpp:465:10
    #30 0x7f8f6f900917 in bool mozilla::SpinEventLoopUntil<(mozilla::ProcessFailureBehavior)1, nsGlobalWindowOuter::Print(nsIPrintSettings*, mozilla::layout::RemotePrintJobChild*, nsIWebProgressListener*, nsIDocShell*, nsGlobalWindowOuter::IsPreview, nsGlobalWindowOuter::IsForWindowDotPrint, std::function<void (mozilla::dom::PrintPreviewResultInfo const&)>&&, mozilla::ErrorResult&)::$_3>(nsTSubstring<char> const&, nsGlobalWindowOuter::Print(nsIPrintSettings*, mozilla::layout::RemotePrintJobChild*, nsIWebProgressListener*, nsIDocShell*, nsGlobalWindowOuter::IsPreview, nsGlobalWindowOuter::IsForWindowDotPrint, std::function<void (mozilla::dom::PrintPreviewResultInfo const&)>&&, mozilla::ErrorResult&)::$_3&&, nsIThread*) src/objdir-ff-ubsan/dist/include/mozilla/SpinEventLoopUntil.h:176:25
    #31 0x7f8f6f8fbf6d in nsGlobalWindowOuter::Print(nsIPrintSettings*, mozilla::layout::RemotePrintJobChild*, nsIWebProgressListener*, nsIDocShell*, nsGlobalWindowOuter::IsPreview, nsGlobalWindowOuter::IsForWindowDotPrint, std::function<void (mozilla::dom::PrintPreviewResultInfo const&)>&&, mozilla::ErrorResult&) src/dom/base/nsGlobalWindowOuter.cpp:5347:5
    #32 0x7f8f6f8fa24c in nsGlobalWindowOuter::PrintOuter(mozilla::ErrorResult&) src/dom/base/nsGlobalWindowOuter.cpp:5146:3
    #33 0x7f8f6f891b3f in nsGlobalWindowInner::Print(mozilla::ErrorResult&) src/dom/base/nsGlobalWindowInner.cpp:3840:3
    #34 0x7f8f71063763 in mozilla::dom::Window_Binding::print(JSContext*, JS::Handle<JSObject*>, void*, JSJitMethodCallArgs const&) src/objdir-ff-ubsan/dom/bindings/WindowBinding.cpp:3101:24
    #35 0x7f8f7192ffd6 in bool mozilla::dom::binding_detail::GenericMethod<mozilla::dom::binding_detail::MaybeCrossOriginObjectThisPolicy, mozilla::dom::binding_detail::ThrowExceptions>(JSContext*, unsigned int, JS::Value*) src/dom/bindings/BindingUtils.cpp:3272:13
    #36 0x7f8f7a1ddec0 in CallJSNative(JSContext*, bool (*)(JSContext*, unsigned int, JS::Value*), js::CallReason, JS::CallArgs const&) src/js/src/vm/Interpreter.cpp:420:13
    #37 0x7f8f7a1ddec0 in js::InternalCallOrConstruct(JSContext*, JS::CallArgs const&, js::MaybeConstruct, js::CallReason) src/js/src/vm/Interpreter.cpp:507:12
    #38 0x7f8f7a1cbc55 in InternalCall(JSContext*, js::AnyInvokeArgs const&, js::CallReason) src/js/src/vm/Interpreter.cpp:574:10
    #39 0x7f8f7a1cbc55 in js::CallFromStack(JSContext*, JS::CallArgs const&) src/js/src/vm/Interpreter.cpp:578:10
    #40 0x7f8f7a1cbc55 in Interpret(JSContext*, js::RunState&) src/js/src/vm/Interpreter.cpp:3324:16
    #41 0x7f8f7a1b08c3 in js::RunScript(JSContext*, js::RunState&) src/js/src/vm/Interpreter.cpp:389:13
    #42 0x7f8f7a1ddfbd in js::InternalCallOrConstruct(JSContext*, JS::CallArgs const&, js::MaybeConstruct, js::CallReason) src/js/src/vm/Interpreter.cpp:539:13
    #43 0x7f8f7a1df545 in InternalCall(JSContext*, js::AnyInvokeArgs const&, js::CallReason) src/js/src/vm/Interpreter.cpp:574:10
    #44 0x7f8f7a1df545 in js::Call(JSContext*, JS::Handle<JS::Value>, JS::Handle<JS::Value>, js::AnyInvokeArgs const&, JS::MutableHandle<JS::Value>, js::CallReason) src/js/src/vm/Interpreter.cpp:605:8
    #45 0x7f8f7a2dde94 in JS::Call(JSContext*, JS::Handle<JS::Value>, JS::Handle<JS::Value>, JS::HandleValueArray const&, JS::MutableHandle<JS::Value>) src/js/src/vm/CallAndConstruct.cpp:117:10
    #46 0x7f8f70ebed76 in mozilla::dom::IdleRequestCallback::Call(mozilla::dom::BindingCallContext&, JS::Handle<JS::Value>, mozilla::dom::IdleDeadline&, mozilla::ErrorResult&) src/objdir-ff-ubsan/dom/bindings/WindowBinding.cpp:836:8
    #47 0x7f8f6fa0bf80 in mozilla::dom::IdleRequestCallback::Call(mozilla::dom::IdleDeadline&, mozilla::ErrorResult&, char const*, mozilla::dom::CallbackObject::ExceptionHandling, JS::Realm*) src/objdir-ff-ubsan/dist/include/mozilla/dom/WindowBinding.h:691:12
    #48 0x7f8f6fc391f0 in mozilla::dom::IdleRequestCallback::Call(mozilla::dom::IdleDeadline&, char const*) src/objdir-ff-ubsan/dist/include/mozilla/dom/WindowBinding.h:704:12
    #49 0x7f8f6fc391f0 in mozilla::dom::IdleRequest::IdleRun(nsPIDOMWindowInner*, double, bool) src/dom/base/IdleRequest.cpp:61:13
    #50 0x7f8f6f8699bf in nsGlobalWindowInner::RunIdleRequest(mozilla::dom::IdleRequest*, double, bool) src/dom/base/nsGlobalWindowInner.cpp:721:12
    #51 0x7f8f6f8683fc in nsGlobalWindowInner::ExecuteIdleRequest(mozilla::TimeStamp) src/dom/base/nsGlobalWindowInner.cpp:749:3
    #52 0x7f8f6f868114 in IdleRequestExecutor::Run() src/dom/base/nsGlobalWindowInner.cpp:590:13
    #53 0x7f8f6c9c328a in mozilla::RunnableTask::Run() src/xpcom/threads/TaskController.cpp:475:16
    #54 0x7f8f6c9850e1 in mozilla::TaskController::DoExecuteNextTaskOnlyMainThreadInternal(mozilla::detail::BaseAutoLock<mozilla::Mutex&> const&) src/xpcom/threads/TaskController.cpp:788:26
    #55 0x7f8f6c982cb4 in mozilla::TaskController::ExecuteNextTaskOnlyMainThreadInternal(mozilla::detail::BaseAutoLock<mozilla::Mutex&> const&) src/xpcom/threads/TaskController.cpp:662:15
    #56 0x7f8f6c9830cb in mozilla::TaskController::ProcessPendingMTTask(bool) src/xpcom/threads/TaskController.cpp:398:36
    #57 0x7f8f6c9b4da1 in mozilla::TaskController::InitializeInternal()::$_0::operator()() const src/xpcom/threads/TaskController.cpp:124:37
    #58 0x7f8f6c9b4da1 in mozilla::detail::RunnableFunction<mozilla::TaskController::InitializeInternal()::$_0>::Run() src/objdir-ff-ubsan/dist/include/nsThreadUtils.h:531:5
    #59 0x7f8f6c9a1263 in nsThread::ProcessNextEvent(bool, bool*) src/xpcom/threads/nsThread.cpp:1180:16
    #60 0x7f8f6c9a95a4 in NS_ProcessNextEvent(nsIThread*, bool) src/xpcom/threads/nsThreadUtils.cpp:465:10
    #61 0x7f8f6e0dcdf2 in mozilla::ipc::MessagePump::Run(base::MessagePump::Delegate*) src/ipc/glue/MessagePump.cpp:85:21
    #62 0x7f8f6e0de3d2 in mozilla::ipc::MessagePumpForChildProcess::Run(base::MessagePump::Delegate*) src/ipc/glue/MessagePump.cpp:268:30
    #63 0x7f8f6df51411 in MessageLoop::RunInternal() src/ipc/chromium/src/base/message_loop.cc:380:10
    #64 0x7f8f6df51411 in MessageLoop::RunHandler() src/ipc/chromium/src/base/message_loop.cc:373:3
    #65 0x7f8f6df51411 in MessageLoop::Run() src/ipc/chromium/src/base/message_loop.cc:355:3
    #66 0x7f8f74ed2208 in nsBaseAppShell::Run() src/widget/nsBaseAppShell.cpp:137:27
    #67 0x7f8f79e094c7 in XRE_RunAppShell() src/toolkit/xre/nsEmbedFunctions.cpp:875:20
    #68 0x7f8f6e0de3b1 in mozilla::ipc::MessagePumpForChildProcess::Run(base::MessagePump::Delegate*) src/ipc/glue/MessagePump.cpp:235:9
    #69 0x7f8f6df51411 in MessageLoop::RunInternal() src/ipc/chromium/src/base/message_loop.cc:380:10
    #70 0x7f8f6df51411 in MessageLoop::RunHandler() src/ipc/chromium/src/base/message_loop.cc:373:3
    #71 0x7f8f6df51411 in MessageLoop::Run() src/ipc/chromium/src/base/message_loop.cc:355:3
    #72 0x7f8f79e08598 in XRE_InitChildProcess(int, char**, XREChildData const*) src/toolkit/xre/nsEmbedFunctions.cpp:734:34
    #73 0x7f8f79e1d5c0 in mozilla::BootstrapImpl::XRE_InitChildProcess(int, char**, XREChildData const*) src/toolkit/xre/Bootstrap.cpp:67:12
    #74 0x55a332afdcc5 in content_process_main(mozilla::Bootstrap*, int, char**) src/browser/app/../../ipc/contentproc/plugin-container.cpp:57:28
    #75 0x55a332afe0d5 in main src/browser/app/nsBrowserApp.cpp:338:18
    #76 0x7f8f972e5c86 in __libc_start_main /build/glibc-uZu3wS/glibc-2.27/csu/../csu/libc-start.c:310
    #77 0x55a332a3e0a8 in _start (src/objdir-ff-ubsan/dist/bin/firefox+0xf80a8) (BuildId: 50d3c19e0caef8fd433f807e7ae2abab68767b4e)

This issue is currently triggered while fuzzing with the 'float-cast-overflow' UBSan check enabled. This issue will need to be addressed before the check can be enabled by default.

If it requires too much effort to fix immediately please ni? me and let me know. If necessary it will be added to a suppression list. Thank you :)