1692394 - Include NSException stack in the app notes

Status: RESOLVED FIXED

(Toolkit :: Crash Reporting, task)

Description

[Markus Stange [:mstange]]

We have a number of places in the tree that catch Objective C exceptions and log them to the app notes. However, we currently only log exception name and "reason", but not the stack at which the exception was thrown.

We should add the exception stack to the app notes.

This is a different stack than the crash stack; after an exception occurs, we usually just move on. We may later crash in a totally different place, or not at all.

NSException has a callStackReturnAddresses property which contains a callstack from the point where the exception was thrown; it seems that NSException eagerly performs a framepointer stackwalk when it is created. Also, getting the callStackReturnAddresses is fast because it does not perform symbolication.
Since it's using framepointer stackwalk, this means we can get the ~full stack in Nightly builds, and partial stacks in Release builds that will at least include the Objective C part of the stack.

This stack will let us see where the exception is triggered and should make it much easier to stop them from firing, or to catch them in a more tightly-scoped fashion.

This bug is step 2 of bug 1692375.

Comment 1

[Markus Stange [:mstange]]

Attachment: Bug 1692394 - Add the stack at which an NSException was thrown to the app notes, as unsymbolicated hex addresses. r=gsvelto!,haik

This will let us divine where the exception fired. At the moment, we only see
the stack at which a later crash happened, which could happen much later or even
never. If a crash does happen, it'll usually be in a completely different stack.

We will need to symbolicate these addresses manually, on a per crash report basis.

Depends on D104960

Comment 2

[Pulsebot]

Pushed by [e-mail]:
https://hg.mozilla.org/integration/autoland/rev/e76de8fe86a2
Add the stack at which an NSException was thrown to the app notes, as unsymbolicated hex addresses. r=gsvelto

Comment 3

[Cristian Brindusan [:cbrindusan]]

https://hg.mozilla.org/mozilla-central/rev/e76de8fe86a2

Comment 4

[Markus Stange [:mstange]]

We have our first crash report with this data!
https://crash-stats.mozilla.org/report/index/ecc5657a-9edf-49b2-b486-a8faa0210219
(I filed bug 1693872 about it.)

Now I'll try to write a script that does the symbolication.

Comment 5

[Markus Stange [:mstange]]

Here's a user script that you can add to Violentmonkey to have the symbolication done automatically:

// ==UserScript==
// @name         Symbolicate Objective C exception stacks in crash reports
// @namespace    https://crash-stats.mozilla.org/
// @version      0.1
// @description  When navigating to a crash report on crash-stats.mozilla.org, this script automatically looks for Objective C exception stacks in the app notes and replaces them with symbols from the Mozilla symbol server.
// @author       Markus Stange <[e-mail]>
// @match        https://crash-stats.mozilla.org/report/index/*
// @connect      symbolication.services.mozilla.com
// @grant        GM.xmlHttpRequest
// ==/UserScript==

(async function() {
  'use strict';

  function moduleContainsAddress(module, address) {
    const baseAddress = parseInt(module.base_addr, 16);
    const endAddress = parseInt(module.end_addr, 16);
    return baseAddress <= address && address < endAddress;
  }

  async function symbolicateAddresses(addresses, modules) {
    const map = new Map();
    const addressToStackPayloadIndexMap = new Map();
    const stackPayload = [];
    for (const address of addresses) {
      const moduleIndex = modules.findIndex(module => moduleContainsAddress(module, address));
      if (moduleIndex === -1) {
        map.set(address, [`0x${address.toString(16)} (unknown module)`]);
        continue;
      }

      const module = modules[moduleIndex];
      const baseAddress = parseInt(module.base_addr, 16);
      const relativeAddress = address - baseAddress;
      const stackPayloadIndex = stackPayload.length;
      stackPayload.push([moduleIndex, relativeAddress]);
      addressToStackPayloadIndexMap.set(address, stackPayloadIndex);
    }

    const body = {
      "jobs": [
        {
          "memoryMap": modules.map(m => ([m.debug_file, m.debug_id])),
          "stacks": [stackPayload]
        }
      ]
    };

    const response = await GM.xmlHttpRequest({
      url: 'https://symbolication.services.mozilla.com/symbolicate/v5',
      method: 'POST',
      headers: {'User-Agent': 'Bug1692394TampermonkeyScript/1.0'},
      mode: 'cors',
      data: JSON.stringify(body),
    });
    // console.log(response.responseText);
    const result = JSON.parse(response.responseText);

    const addressResults = result.results[0].stacks[0];
    for (const [address, stackPayloadIndex] of addressToStackPayloadIndexMap) {
      const addressResult = addressResults[stackPayloadIndex];
      const frames = [];
      if (addressResult.inlines) {
        for (const inline of addressResult.inlines) {
          let frame = inline.function;
          if (inline.file) {
            frame += ` ${inline.file}:${inline.line ?? "?"}`;
          }
          frames.push(frame);
        }
      }
      const symbolName = addressResult.function ?? addressResult.module_offset;
      const moduleName = addressResult.module;
      let frame = `${symbolName} (in ${moduleName})`;
      if (addressResult.file) {
        frame += ` ${addressResult.file}:${addressResult.line ?? "?"}`;
      } else {
        const functionOffset = addressResult.function_offset;
        frame += ` + ${functionOffset}`;
      }
      frames.push(frame);
      map.set(address, frames);
    }

    return map;
  }

  async function symbolicateAppNotes(appNotes, modules) {
    const re = /(?<=Obj-C Exception data.*Thrown at stack\:\n)((0x[0-9a-f]+\n)+)/gs;
    const stackMatches = appNotes.matchAll(re);
    const addresses = new Set();
    for (const stackMatch of stackMatches) {
      const stack = stackMatch[1];
      const stackAddresses = stack.trimEnd().split("\n").map((s) => parseInt(s, 16));
      const pc = stackAddresses[0];
      addresses.add(pc);
      for (let i = 1; i < stackAddresses.length; i++) {
        const returnAddress = stackAddresses[i];
        addresses.add(returnAddress - 1);
      }
    }
    const symbolMap = await symbolicateAddresses(addresses, modules);
    return appNotes.replaceAll(re, stack => {
      const stackAddresses = stack.trimEnd().split("\n").map((s) => parseInt(s, 16));
      const stackSymbols = [];
      const pc = stackAddresses[0];
      stackSymbols[0] = (symbolMap.get(pc) ?? [`<unknown> for ${pc}`]).join("\n");
      for (let i = 1; i < stackAddresses.length; i++) {
        const returnAddress = stackAddresses[i];
        stackSymbols[i] = (symbolMap.get(returnAddress - 1) ?? [`<unknown> for ${returnAddress}`]).join("\n");
      }
      return stackSymbols.join("\n") + "\n";
    });
  }

  const modules = JSON.parse(document.querySelector("#minidump-stackwalk-json").dataset.minidumpstackwalk).modules;
  const appNotesElem = document.querySelector('tr[title~="app_notes"] > td > pre');
  appNotesElem.textContent = await symbolicateAppNotes(appNotesElem.textContent, modules);
})();

It takes about 30 seconds when loading the crash report, and it doesn't show any indication while it's happening, it just replaces the app notes with the symbolicated stack once it's finished.