From Userspace(''Bug'') into iOS Kernelcache & IOKit

Prologue

之前发在内网分享下(字数超了)

原链接在这里 https://github.com/jmpews/NoteZ/issues/43

这里会分析下如何从 iOS 的 Userspace 分析到 Kernelspace 的 IOkit 的大致思路.

0. 问题简述

在使用 glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, texWidth, texHeight, 0, GL_RGBA, GL_UNSIGNED_BYTE, textureData); 创建 Texture 时, 会导致 task_info 获取到统计信息中 internal 增加, 但是 phys_footprint 并不增加.

1. PreTools

以下工具在分析的过程中起到了非常重要的作用(包括但不限于).

0. IDA
# 辅助分析 iOS KernelCache
1. https://github.com/bazad/ida_kernelcache
# 辅助分析 iOS KernelCache
2. Joker http://newosxbook.com/tools/joker.html
# IOKit Driver Class Dumper
3. iometa https://github.com/Siguza/iometa
# IOKit Runtime Dump
4. IOKitUser-1445.71.1/iodisplayregistry.c
# KernelCache 解压
5. lzssdec http://nah6.com/~itsme/cvs-xdadevtools/iphone/tools/lzssdec.cpp
# Runtime DBI(插桩) 分析
6. HookZz http://github.com/jmpews/HookZz

2. Refer Source

以下源码在分析的过程中起到了非常重要的作用(包括但不限于).

# AppleOpenSource
xnu-4570.71.2
IOKitUser-1445.71.1
IOKitTools-108
IOHIDFamily-1035.70.7
IOGraphics-519.20
# GithubSource
# macOS Display Driver with VMware
vmsvga2
# macOS Simple Display Driver
https://github.com/tSoniq/displayx

3. Analysis Process

在分析的过程, 每一个阶段, 都存在问题的确定/转化.

0. 确定 iOS Userspace 下导致 `internal` 增加, 但是 `phys_footprint` 没有增加的函数.
1. 逆向分析对应的 iOS Kernel `AGXAcceleratorG10P_B0` 内核驱动
2. 根据 `AGXAcceleratorG10P_B0` 驱动中使用 `IOBufferMemoryDescriptor` IOKit 方法. 转化为 macOS 对应的调用
3. 编写 macOS 测试驱动, 复现问题. 
4. 对 XNU 进行内核调试, 确定最终发生位置.

前置知识

本文缺少 ARM64 架构下的异常/中断机制的详细分析, 将在后续 <iOS Kernel Debugger> 详细补充.

xnu 系统调用机制
Mach 消息机制, 以及 MIG
IOKit 机制和内存布局
iOS Kernelcache 解密分析

1. xnu 系统调用机制

本质根据 IDT 做 handler 分发, 不关心的可以跳过此部分.

1.1. Legacy 传统的 int 0x80 (only for 32-bit)

在 xnu-4570.71.2/osfmk/x86_64/idt_table.h 可以看到.

TRAP(0x00,idt64_zero_div)
TRAP_IST1(0x01,idt64_debug)
TRAP_IST2(0x02,idt64_nmi)
USER_TRAP(0x03,idt64_int3)
USER_TRAP(0x04,idt64_into)
USER_TRAP(0x05,idt64_bounds)
TRAP(0x06,idt64_invop)
TRAP(0x07,idt64_nofpu)
TRAP_IST1(0x08,idt64_double_fault)
TRAP(0x09,idt64_fpu_over)
TRAP_ERR(0x0a,idt64_inv_tss)
TRAP_IST1(0x0b,idt64_segnp)

对于系统调用, Mach 调用.

USER_TRAP_SPC(0x80,idt64_unix_scall)
USER_TRAP_SPC(0x81,idt64_mach_scall)
USER_TRAP_SPC(0x82,idt64_mdep_scall)

对应的分发处理函数.

/*
 * Legacy interrupt gate System call handlers.
 * These are entered via a syscall interrupt. The system call number in %rax
 * is saved to the error code slot in the stack frame. We then branch to the
 * common state saving code.
 */

#ifndef UNIX_INT
#error NO UNIX INT!!!
#endif
Entry(idt64_unix_scall)
    pushq   %rax            /* save system call number */
    pushq   $(HNDL_UNIX_SCALL)
    pushq   $(UNIX_INT)
    jmp L_dispatch
    
Entry(idt64_mach_scall)
    pushq   %rax            /* save system call number */
    pushq   $(HNDL_MACH_SCALL)
    pushq   $(MACH_INT)
    jmp L_dispatch
    
Entry(idt64_mdep_scall)
    pushq   %rax            /* save system call number */
    pushq   $(HNDL_MDEP_SCALL)
    pushq   $(MACHDEP_INT)
    jmp L_dispatch

此时的 x86_64_intr_stack_frame 状态为 trapfn == %rax. (Cpu在处理中断会存在部分寄存器自动压栈, 具体参考 6.4.1 Call and Return Operation for Interrupt or Exception Handling Procedures)

struct x86_64_intr_stack_frame {
    uint16_t    trapno;
    uint16_t    cpu;
    uint32_t    _pad;
    uint64_t    trapfn;
    uint64_t    err;
    uint64_t    rip;
    uint64_t    cs;
    uint64_t    rflags;
    uint64_t    rsp;
    uint64_t    ss;
};
typedef struct x86_64_intr_stack_frame x86_64_intr_stack_frame_t;

int 0x80 后的处理流程

L_dispatch
-> *idt64_hndl_table0
-> ks_dispatch
-> ks_dispatch_user
-> L_dispatch_U32
-> L_common_dispatch
-> *(idt64_hndl_table1 + 8*(trapfn = HNDL_UNIX_SCALL))
-> hndl_unix_scall

1.2. syscall 快速系统调用 (only for x86_64)

# xnu-4570.71.2/osfmk/i386/mp_desc.c

/*
 * Set MSRs for sysenter/sysexit and syscall/sysret for 64-bit.
 */
void
cpu_syscall_init(cpu_data_t *cdp)
{
    ...
    wrmsr64(MSR_IA32_LSTAR, DBLMAP((uintptr_t) hi64_syscall));
    ...
}

Entry(hi64_syscall)
Entry(idt64_syscall)
    swapgs
     /* Use RAX as a temporary by shifting its contents into R11[32:63]
      * The systemcall number is defined to be a 32-bit quantity, as is
      * RFLAGS.
      */
    shlq    $32, %rax
    or  %rax, %r11
.globl EXT(dblsyscall_patch_point)
EXT(dblsyscall_patch_point):
//  movabsq $0x12345678ABCDEFFFULL, %rax
     /* Generate offset to the double-mapped per-CPU data shadow
      * into RAX
      */
    leaq    EXT(idt64_hndl_table0)(%rip), %rax
    mov 16(%rax), %rax
    mov     %rsp, %gs:CPU_UBER_TMP(%rax)  /* save user stack */
    mov     %gs:CPU_ESTACK(%rax), %rsp  /* switch stack to per-cpu estack */
    sub $(ISF64_SIZE), %rsp

    /*
     * Synthesize an ISF frame on the exception stack
     */
    movl    $(USER_DS), ISF64_SS(%rsp)
    mov %rcx, ISF64_RIP(%rsp)       /* rip */

    mov %gs:CPU_UBER_TMP(%rax), %rcx
    mov %rcx, ISF64_RSP(%rsp)       /* user stack --changed */

    mov %r11, %rax
    shrq    $32, %rax       /* Restore RAX */
    mov %r11d, %r11d        /* Clear r11[32:63] */

    mov %r11, ISF64_RFLAGS(%rsp)    /* rflags */
    movl    $(SYSCALL_CS), ISF64_CS(%rsp)   /* cs - a pseudo-segment */
    mov %rax, ISF64_ERR(%rsp)       /* err/rax - syscall code */
    movq    $(HNDL_SYSCALL), ISF64_TRAPFN(%rsp)
    movq    $(T_SYSCALL), ISF64_TRAPNO(%rsp)    /* trapno */
    swapgs
    jmp L_dispatch          /* this can only be 64-bit */

syscall 之后的处理流程

L_dispatch
-> *idt64_hndl_table0
-> ks_dispatch
-> ks_dispatch_user
-> L_dispatch_64bit
-> L_common_dispatch
-> *(idt64_hndl_table1 + 8*(trapfn = HNDL_SYSCALL))
-> hndl_unix_scall

在 hndl_syscall 中根据 rax 判断 syscall 类型, 例如正常系统调用会在 hndl_unix_scall64 处进行跳转执行.

Entry(hndl_syscall)
    TIME_TRAP_UENTRY

    movq    %gs:CPU_ACTIVE_THREAD,%rcx  /* get current thread     */
    movl    $-1, TH_IOTIER_OVERRIDE(%rcx)   /* Reset IO tier override to -1 before handling syscall */
    movq    TH_TASK(%rcx),%rbx      /* point to current task  */

    /* Check for active vtimers in the current task */
    TASK_VTIMER_CHECK(%rbx,%rcx)

    /*
     * We can be here either for a mach, unix machdep or diag syscall,
     * as indicated by the syscall class:
     */
    movl    R64_RAX(%r15), %eax     /* syscall number/class */
    movl    %eax, %edx
    andl    $(SYSCALL_CLASS_MASK), %edx /* syscall class */
    cmpl    $(SYSCALL_CLASS_MACH<<SYSCALL_CLASS_SHIFT), %edx
    je  EXT(hndl_mach_scall64)
    cmpl    $(SYSCALL_CLASS_UNIX<<SYSCALL_CLASS_SHIFT), %edx
    je  EXT(hndl_unix_scall64)
    cmpl    $(SYSCALL_CLASS_MDEP<<SYSCALL_CLASS_SHIFT), %edx
    je  EXT(hndl_mdep_scall64)
    cmpl    $(SYSCALL_CLASS_DIAG<<SYSCALL_CLASS_SHIFT), %edx
    je  EXT(hndl_diag_scall64)

    /* Syscall class unknown */
    sti
    CCALL3(i386_exception, $(EXC_SYSCALL), %rax, $1)
    /* no return */


Entry(hndl_unix_scall64)
    incl    TH_SYSCALLS_UNIX(%rcx)      /* increment call count   */
    sti

    CCALL1(unix_syscall64, %r15)
    /*
     * always returns through thread_exception_return
     */

1.3. userspace 空间的 vm_fault

* thread #96, name = '0xffffff8026811650', queue = 'cpu-1', stop reason = breakpoint 3.1
  * frame #0: 0xffffff801dd6d5ee kernel`pmap_enter_options(pmap=0xffffff80269de250, vaddr=<unavailable>, pn=<unavailable>, prot=3, fault_type=<unavailable>, flags=<unavailable>, wired=<unavailable>, options=<unavailable>, arg=<unavailable>) at pmap_x86_common.c:986 [opt]
    frame #1: 0xffffff801dcec990 kernel`vm_fault_enter(m=0xffffff80215f0750, pmap=0xffffff80269de250, vaddr=4353159168, prot=3, caller_prot=<unavailable>, wired=0, change_wiring=<unavailable>, wire_tag=<unavailable>, no_cache=<unavailable>, cs_bypass=<unavailable>, user_tag=-2139079024, pmap_options=<unavailable>, need_retry=<unavailable>, type_of_fault=<unavailable>) at vm_fault.c:3279 [opt]
    frame #2: 0xffffff801dcee19b kernel`vm_fault_internal(map=0xffffff802918b700, vaddr=4353159168, caller_prot=3, change_wiring=<unavailable>, wire_tag=0, interruptible=<unavailable>, caller_pmap=<unavailable>, caller_pmap_addr=<unavailable>, physpage_p=<unavailable>) at vm_fault.c:0 [opt]
    frame #3: 0xffffff801dd891a2 kernel`user_trap [inlined] vm_fault(map=<unavailable>, fault_type=<unavailable>, change_wiring=0, wire_tag=0, interruptible=2, caller_pmap_addr=0) at vm_fault.c:3416 [opt]
    frame #4: 0xffffff801dd8917f kernel`user_trap(saved_state=0xffffff8025ad2460) at trap.c:1118 [opt]
    frame #5: 0xffffff801dc1d0a5 kernel`hndl_alltraps + 229

2. Mach 消息机制

所有的 mach 消息, 都要经过 mach_msg.

可以通过 MIG(Mach Interface Generator) 对 .defs 生成 mach 接口. 例如:

xcrun -sdk iphoneos mig \
  -arch arm64 \
  -DIOKIT \
  -I/xxx/macOS_10.13.6/xnu-4570.71.2/osfmk \
  iokitmig.defs

这里以 IOConnectCallMethod 举例.

Usermode
---
IOConnectCallMethod => io_connect_method => mach_msg() =>

Kernelmode
---
_Xio_connect_method => is_io_connect_method

3. IOKit 机制和内存布局

xnu 里的驱动是基于 libkern 的 c++ IO Kit 框架开发, 但只是一个 c++ 子集.

所以可以通过获取驱动类的继承关系和虚表关系帮助分析 iOS Kernelcache 中的各种无符号的驱动.

例如 AGXAcceleratorG10P_B0, 这也是下文要分析的驱动:

vtab=0xfffffff006fe2b68 size=0x00001998 meta=0xfffffff007866850 parent=0xfffffff0078666b0 AGXAcceleratorG10P_B0 (com.apple.AGXG10P)
       0x0 func=0xfffffff006d6e264 overrides=0xfffffff006d67308 pac=0x0000 AGXAcceleratorG10P_B0::fn_0x0()
       0x8 func=0xfffffff006d6e268 overrides=0xfffffff006d6730c pac=0x0000 AGXAcceleratorG10P_B0::~AGXAcceleratorG10P_B0()
      0x38 func=0xfffffff006d6e280 overrides=0xfffffff006d67324 pac=0x0000 AGXAcceleratorG10P_B0::getMetaClass() const
     0x2a8 func=0xfffffff006d6e28c overrides=0xfffffff006d67330 pac=0x0000 AGXAcceleratorG10P_B0::start()
     0x2b0 func=0xfffffff006d6e350 overrides=0xfffffff006d67494 pac=0x0000 AGXAcceleratorG10P_B0::stop()
     0x370 func=0xfffffff006d6e388 overrides=0xfffffff006d674cc pac=0x0000 AGXAcceleratorG10P_B0::getWorkLoop() const
     0x548 func=0xfffffff006d6e390 overrides=0xfffffff006d675d8 pac=0x0000 AGXAcceleratorG10P_B0::fn_0x548()
     0x710 func=0xfffffff006d6e398 overrides=0xfffffff006d6779c pac=0x0000 AGXAcceleratorG10P_B0::fn_0x710()
    ...
vtab=0xfffffff006fe0068 size=0x00001998 meta=0xfffffff0078666b0 parent=0xfffffff007866540 AGXAcceleratorG10 (com.apple.AGXG10P)
       0x0 func=0xfffffff006d67308 overrides=0xfffffff006d3648c pac=0x0000 AGXAcceleratorG10::fn_0x0()
       0x8 func=0xfffffff006d6730c overrides=0xfffffff006d36490 pac=0x0000 AGXAcceleratorG10::~AGXAcceleratorG10()
      0x38 func=0xfffffff006d67324 overrides=0xfffffff006d36494 pac=0x0000 AGXAcceleratorG10::getMetaClass() const
     0x140 func=0xfffffff006d454c0 overrides=0xfffffff007564e08 pac=0x0000 AGXAcceleratorG10::copyProperty() const
     0x158 func=0xfffffff006d4644c overrides=0xfffffff007564f9c pac=0x0000 AGXAcceleratorG10::setProperties()
     0x270 func=0xfffffff006d48198 overrides=0xfffffff00758b464 pac=0x0000 AGXAcceleratorG10::systemWillShutdown()
     0x280 func=0xfffffff006d481d4 overrides=0xfffffff00756a538 pac=0x0000 AGXAcceleratorG10::configureReport()
     0x288 func=0xfffffff006d481f0 overrides=0xfffffff00756a76c pac=0x0000 AGXAcceleratorG10::updateReport()
     0x2a0 func=0xfffffff006d57ba4 overrides=0xfffffff00756ac30 pac=0x0000 AGXAcceleratorG10::probe()
     0x2a8 func=0xfffffff006d67330 overrides=0xfffffff006d366ec pac=0x0000 AGXAcceleratorG10::start()
     0x2b0 func=0xfffffff006d67494 overrides=0xfffffff006d37a0c pac=0x0000 AGXAcceleratorG10::stop()
     0x370 func=0xfffffff006d674cc overrides=0xfffffff006d3adc4 pac=0x0000 AGXAcceleratorG10::getWorkLoop() const
     0x3a0 func=0xfffffff006d674d4 overrides=0xfffffff00756c31c pac=0x0000 AGXAcceleratorG10::callPlatformFunction()
     0x4d0 func=0xfffffff006d4ada0 overrides=0xfffffff00758b2e4 pac=0x0000 AGXAcceleratorG10::setPowerState()
     0x548 func=0xfffffff006d675d8 overrides=0xfffffff006d3adcc pac=0x0000 AGXAcceleratorG10::fn_0x548()
    ...
vtab=0x0000000000000000 size=0x00001990 meta=0xfffffff007866540 parent=0xfffffff007866358 AGXFamilyAccelerator (com.apple.AGXG10P)
vtab=0x0000000000000000 size=0x00001990 meta=0xfffffff007866358 parent=0xfffffff0078658a8 AGXAccelerator (com.apple.AGXG10P)
vtab=0xfffffff006fda6a8 size=0x00000c78 meta=0xfffffff0078658a8 parent=0xfffffff007865880 IOGraphicsAccelerator2 (com.apple.iokit.IOAcceleratorFamily2)
       0x0 func=0xfffffff006d3648c overrides=0xfffffff006d363b0 pac=0x0000 IOGraphicsAccelerator2::fn_0x0()
       0x8 func=0xfffffff006d36490 overrides=0xfffffff006d363b4 pac=0x0000 IOGraphicsAccelerator2::~IOGraphicsAccelerator2()
      0x38 func=0xfffffff006d36494 overrides=0xfffffff006d363cc pac=0x0000 IOGraphicsAccelerator2::getMetaClass() const
      0x68 func=0xfffffff006d37f74 overrides=0xfffffff007569c4c pac=0x0000 IOGraphicsAccelerator2::free()
     0x2a8 func=0xfffffff006d366ec overrides=0xfffffff00756ac34 pac=0x0000 IOGraphicsAccelerator2::start()
     0x2b0 func=0xfffffff006d37a0c overrides=0xfffffff00756ac3c pac=0x0000 IOGraphicsAccelerator2::stop()
     0x370 func=0xfffffff006d3adc4 overrides=0xfffffff00756c120 pac=0x0000 IOGraphicsAccelerator2::getWorkLoop() const
     0x418 func=0xfffffff006d3a444 overrides=0xfffffff00756c998 pac=0x0000 IOGraphicsAccelerator2::message()
     0x460 func=0xfffffff006d38560 overrides=0xfffffff00756cf88 pac=0x0000 IOGraphicsAccelerator2::newUserClient()
     0x538 func=0xfffffff006d3a550 overrides=0x0000000000000000 pac=0x0000 IOGraphicsAccelerator2::fn_0x538()
    ...

3.1. Connnect(Open) Driver(Service)

将在分析 AGXAcceleratorG10P_B0 时用到.

Kernelspace
---
_Xio_service_open_extended => is_io_service_open_extended =>
---
provider_service->newUserClient() => userClient_instantiate_class->initWithTask()

Userspace
---
io_service_t _io_get_service(const char *name) {
  static io_service_t service = MACH_PORT_NULL;
  if (service == MACH_PORT_NULL) {
    DLOG("Getting IO service handle...");
    service = _IOServiceGetMatchingService(get_io_master_port(), _IOServiceMatching(name));
    if (!MACH_PORT_VALID(service)) {
      FATAL("Failed to get IO service handle (port = 0x%08x)", service);
    }
  }
  return service;
}

io_service_t platformExpertDevice = _io_get_service("IOPlatformExpertDevice");
io_connect_t connect;
kr = _IOServiceOpen(platformExpertDevice, mach_task_self(), 0, &connect);
---
-> io_service_open_extended => mach_msg

3.2. Invoke driver methods

将在分析 AGXAcceleratorG10P_B0 时用到.

Usermode
---
IOConnectCallMethod => io_connect_method => mach_msg() =>

Kernelmode
---
_Xio_connect_method => is_io_connect_method =>
---
client->externalMethod => dispatch_by_selector

(lldb) bt
* thread #6, name = '0xffffff8013341520', queue = 'cpu-1', stop reason = breakpoint 1.1
  * frame #0: 0xffffff800bf3b8d6 kernel`mach_make_memory_entry_64(target_map=<unavailable>, size=0xffffff886e9d3790, offset=0, permission=409603, object_handle=0xffffff8017546c28, parent_handle=<unavailable>) at vm_user.c:2339 [opt]
    frame #1: 0xffffff800c4b52a9 kernel`IOGeneralMemoryDescriptor::memoryReferenceCreate(this=<unavailable>, options=0, reference=0xffffff8017546c28) at IOMemoryDescriptor.cpp:611 [opt]
    frame #2: 0xffffff800c4b3651 kernel`IOGeneralMemoryDescriptor::initWithOptions(this=<unavailable>, buffers=0x0000000000000001, count=1, offset=310765616, task=<unavailable>, options=<unavailable>, mapper=0x0000000000000000) at IOMemoryDescriptor.cpp:1709 [opt]
    frame #3: 0xffffff800c4a9d58 kernel`IOBufferMemoryDescriptor::initWithPhysicalMask(this=0xffffff8017546c00, inTask=<unavailable>, options=<unavailable>, capacity=2048000, alignment=<unavailable>, physicalMask=<unavailable>) at IOBufferMemoryDescriptor.cpp:289 [opt]
    frame #4: 0xffffff800c4aa9b7 kernel`IOBufferMemoryDescriptor::inTaskWithOptions(inTask=0x0000000000000000, options=65635, capacity=0x00000000001f4000, alignment=0x0000000000001000) at IOBufferMemoryDescriptor.cpp:339 [opt]
    frame #5: 0xffffff7f8e3344ac IOKitDemoDriver`SharedMemoryAlloc(buffer=0xffffff886e9d3a28, options=65635, size=2048000) at IOKitDemoDriver.cpp:87
    frame #6: 0xffffff7f8e334595 IOKitDemoDriver`IOKitDemoDriver::userClientMap(this=0xffffff8012b77360, owningTask=0xffffff8014b4dc28, map=0xffffff8012f93610, mapSize=0xffffff886e9d3a80) at IOKitDemoDriver.cpp:112
    frame #7: 0xffffff7f8e334aac IOKitDemoDriver`IOKitDemoDriverUserClient::userClientMap(target=0xffffff801745c000, reference=0x0000000000000000, args=0xffffff886e9d3b80) at IOKitDemoDriverUserClient.cc:53
    frame #8: 0xffffff800c4cb428 kernel`IOUserClient::externalMethod(this=<unavailable>, selector=<unavailable>, args=0xffffff886e9d3b80, dispatch=0xffffff7f8e336438, target=0xffffff801745c000, reference=0x0000000000000000) at IOUserClient.cpp:5289 [opt]
    frame #9: 0xffffff7f8e334b66 IOKitDemoDriver`IOKitDemoDriverUserClient::externalMethod(this=0xffffff801745c000, selector=1, arguments=0xffffff886e9d3b80, dispatch=0xffffff7f8e336438, target=0xffffff801745c000, reference=0x0000000000000000) at IOKitDemoDriverUserClient.cc:74
    frame #10: 0xffffff800c4d4237 kernel`::is_io_connect_method(connection=0xffffff801745c000, selector=1, scalar_input=<unavailable>, scalar_inputCnt=<unavailable>, inband_input=<unavailable>, inband_inputCnt=0, ool_input=<unavailable>, ool_input_size=<unavailable>, inband_output=<unavailable>, inband_outputCnt=<unavailable>, scalar_output=<unavailable>, scalar_outputCnt=<unavailable>, ool_output=<unavailable>, ool_output_size=<unavailable>) at IOUserClient.cpp:3943 [opt]
    frame #11: 0xffffff800bf44514 kernel`_Xio_connect_method(InHeadP=<unavailable>, OutHeadP=0xffffff8012f935e0) at device_server.c:8376 [opt]
    frame #12: 0xffffff800be70d9e kernel`ipc_kobject_server(request=0xffffff8017555380, option=<unavailable>) at ipc_kobject.c:351 [opt]
    frame #13: 0xffffff800be4dc2d kernel`ipc_kmsg_send(kmsg=0xffffff8017555380, option=3, send_timeout=0) at ipc_kmsg.c:1861 [opt]
    frame #14: 0xffffff800be60d1b kernel`mach_msg_overwrite_trap(args=<unavailable>) at mach_msg.c:570 [opt]
    frame #15: 0xffffff800bf7388d kernel`mach_call_munger64(state=0xffffff80144e1640) at bsd_i386.c:573 [opt]
    frame #16: 0xffffff800be1d996 kernel`hndl_mach_scall64 + 22

4. iOS(10+) Kernelcache 解密分析

这里以 iPhone8 - 11.4.1 为例子.

4.1. 解密 iOS Kernelcache

1. download `iPhone_4.7_P3_11.0_11.4.1_15G77_Restore.ipsw` from `https://ipsw.me/`
2. wget http://nah6.com/~itsme/cvs-xdadevtools/iphone/tools/lzssdec.cpp
3. clang++ lzssdec.cpp -o lzssdec
4. check the offset which lzssdec needed
[0] % hexdump -C -n 600 /Users/jmpews/Downloads/kernelcache.release.iphone10
00000000  30 83 fd 63 59 16 04 49  4d 34 50 16 04 6b 72 6e  |0..cY..IM4P..krn|
00000010  6c 16 1c 4b 65 72 6e 65  6c 43 61 63 68 65 42 75  |l..KernelCacheBu|
00000020  69 6c 64 65 72 2d 31 33  34 38 2e 37 30 2e 31 04  |ilder-1348.70.1.|
00000030  83 fd 63 2a 63 6f 6d 70  6c 7a 73 73 28 31 d7 4d  |..c*complzss(1.M|
00000040  02 02 40 00 00 fd 61 aa  00 00 00 01 00 00 00 00  |..@...a.........|
00000050  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  |................|
*
000001b0  00 00 00 00 ff cf fa ed  fe 0c 00 00 01 d5 00 f6  |................|
000001c0  f0 02 f6 f0 15 f6 f0 70  0f 5a f3 f1 20 f6 f1 00  |.......p.Z.. ...|
000001d0  19 f6 f0 38 f5 f0 3f 5f  5f 54 45 58 54 09 02 1c  |...8..?__TEXT...|
5. ./lzssdec -o 0x1b4 < kernelcache.release.iphone10 > kernelcache.release.iphone10.decrypt

4.2. 获取 iOS dyld_shared_cache

cp /Volumes/xxx/System/Library/Caches/com.apple.dyld/dyld_shared_cache_arm64 /xxx/xxx

4.3 分析 iOS Pirvate Framework

可以通过 Users/jmpews/Library/Developer/Xcode/iOS DeviceSupport/11.4.1 (15G77)/Symbols/System/Library/Extensions/AGXMetalA11.bundle/ 获取到 AGXMetalA11 由于 shared cache 但会存在大量未知地址引用, 所以如需更进一步的具体分析请使用 IDA 分析 dyld_shared_cache_arm64, 尽量避免完全分析, 只要分析对应 Framework 以及其依赖即可.

5. XNU Kernel 调试

这里的 XNU Kernel 调试并没有使用 Apple 提供的 KDP, 而是基于 VMware + LLDB 的调试, 可以参考我的上篇文章 From UEFI Reversing to Xnu Kernel Debug, 这里贴一下 lldb command script.

#Help lldb figure out we're debugging x86_64
settings set plugin.process.gdb-remote.target-definition-file ~/.lldb/x86_64_target_definition.py

#Use a reasonable disassembly syntax
settings set target.x86-disassembly-flavor intel

#Tell load any lldb scripts and macros hidden inside .dSYM files
settings set target.load-script-from-symbol-file true

#Tell lldb where the source directory really is
settings set target.source-map  /BuildRoot/Library/Caches/com.apple.xbs/Sources/xnu/xnu-4570.20.62 /Users/jmpews/Downloads/xnu-4570.20.62

#This should get loaded automatically when we set the target executable
#command script import "/Library/Developer/KDKs/KDK_10.13.1_17B48.kdk/System/Library/Kernels/kernel.debug.dSYM/Contents/Resources/Python/lldbmacros/xnu.py"

#This does not appear to get loaded automatically, so we load it here.
#command script import "/Library/Developer/KDKs/KDK_10.13.1_17B48.kdk/System/Library/Kernels/kernel.debug.dSYM/Contents/Resources/Python/lldbmacros/memory.py"

# Load the kernel binary we're going to be debugging.
target create /Library/Developer/KDKs/KDK_10.13.1_17B48.kdk/System/Library/Kernels/kernel.debug

6. HookZz Runtime DBI 分析

HookZz 是本人编写的 hook framework, 在分析过程中, 为了确定了 Userspace 下问题问题的发生点, 使用了 HookZz 大量进行了插桩分析.

ZzWrap API 允许对任何函数添加 pre_call 和 post_call, 可以非常方便确定某一个函数是否是导致问题发生的点.

ZzDynamicBinaryInstrument API 允许对任意指令地址增加 dbi_call, 可以非常方便确定某一个函数内的哪一个区间导致问题发生的点.

分析思路

1.1 `vm_fault` 导致 `internal+compressed` 增加

首先判断问题的发生位置:

1). Userspace 下的内存分配 malloc / vm_allocate / mmap(这里其实并不用考虑, 因为 userspace 下在进行分配的时其 vm_object 一定统计在自身的 task->map)
2). IOKit 驱动内部内存分配

对于 1) 这里直接 Inlinehook malloc / vm_allocate / mmap, 并没有发现异常点. 接着怀疑是调用了驱动方法, 直接断点 mach_msg. 可以发现确实调用了 mach 系统调用.

(lldb) bt
* thread #1, queue = 'com.apple.main-thread', stop reason = breakpoint 9.1
  * frame #0: 0x0000000184b9fc18 libsystem_kernel.dylib`mach_msg
    frame #1: 0x0000000185407bb8 IOKit`io_connect_method + 416
    frame #2: 0x00000001853a4038 IOKit`IOConnectCallMethod + 232
    frame #3: 0x0000000186f399c0 IOAccelerator`IOAccelResourceCreate + 148
    frame #4: 0x0000000186f75228 Metal`-[MTLIOAccelResource initWithDevice:options:args:argsSize:] + 264
    frame #5: 0x0000000186f6ad48 Metal`-[MTLIOAccelTexture initWithDevice:descriptor:sysMemSize:sysMemRowBytes:vidMemSize:vidMemRowBytes:args:argsSize:] + 176
    frame #6: 0x00000001ad604360 AGXMetalA11`___lldb_unnamed_symbol1181$$AGXMetalA11 + 580
    frame #7: 0x00000001a56ef41c AppleMetalGLRenderer`GLDTextureRec::allocMetalTexture() + 176
    frame #8: 0x00000001a56ef58c AppleMetalGLRenderer`GLDTextureRec::loadPrivateTexture(unsigned int, unsigned short*) + 56
    frame #9: 0x00000001a56ed8e8 AppleMetalGLRenderer`GLDTextureRec::update(unsigned int, unsigned short*) + 168
    frame #10: 0x00000001a56ed7f4 AppleMetalGLRenderer`GLDTextureRec::load() + 96
    frame #11: 0x00000001a56fe1f4 AppleMetalGLRenderer`gldModifyTexSubImage + 88
    frame #12: 0x00000001a6467328 GLEngine`glTexImage2D_Exec + 1752
    frame #13: 0x0000000102941934 libglInterpose.dylib`tex_image2D(__GLIContextRec*, unsigned int, int, unsigned int, int, int, int, unsigned int, unsigned int, void const*) + 388
    frame #14: 0x000000018814c738 OpenGLES`glTexImage2D + 108

这里需要判断是否确实这个 mach 调用, 导致的 internal 增加, 但 footprint 不增加, 尝试使用 HookZz 的 ZzWrap 对 IOAccelResourceCreate 进行 PreCall 和 PostCall 的 trace.

void common_pre_call(RegisterContext *reg_ctx, const HookEntryInfo *info)
{
  NSByteCountFormatter *formatter            = [[NSByteCountFormatter alloc] init];
  formatter.countStyle = NSByteCountFormatterCountStyleMemory;
  MemoryInfo *mem_info = memoryFootprint();
  NSLog(@"PRE==:footprint: %@, internal+compressed: %@, virtual:%@",
        [formatter stringFromByteCount:mem_info.footprint],
        [formatter stringFromByteCount:mem_info.internal + mem_info.compressed],
        [formatter stringFromByteCount:mem_info.virtual_size]);
  printf("virtual:%d\n", mem_info.virtual_size);
}

void common_post_call(RegisterContext *reg_ctx, const HookEntryInfo *info)
{
  NSByteCountFormatter *formatter            = [[NSByteCountFormatter alloc] init];
  formatter.countStyle = NSByteCountFormatterCountStyleMemory;
  MemoryInfo *mem_info = memoryFootprint();
  NSLog(@"POST==:footprint: %@, internal+compressed: %@, virtual:%@",
        [formatter stringFromByteCount:mem_info.footprint],
        [formatter stringFromByteCount:mem_info.internal + mem_info.compressed],
        [formatter stringFromByteCount:mem_info.virtual_size]);
  
  printf("virtual:%d\n", mem_info.virtual_size);
}

ZzWrap((void *)0x1a46b619c, common_pre_call, common_post_call);

但是发现 internal+compressed 没有任何增加, 那是不是确实跟这个 mach 系统调用无关呢? 接下来继续通过 Dynamic Binary Instrumentation (DBI) 来确定到底是哪个函数导致了 internal+compressed 的增加.

如下是 -[MTLIOAccelResource initWithDevice:options:args:argsSize:] 的实现, 请注意图中的标注点.

最终在 dyld_shared_cache_arm64 的 AGXMetalA11 如下函数确定导致 internal+compressed 的增加, 但是这段函数没有任何调用, 并且推测是一段内存拷贝的代码. 所以这里推测是 vm_fault 导致.

__int64 __fastcall sub_1A93455C8(__int64 result, __int64 a2, int a3, int a4, int a5, int a6, __int64 a7, unsigned int a8, unsigned int a9, unsigned int a10, unsigned int a11)
{
    ...
      do
      {
        v39 = 0;
        v40 = v38 + 4LL * a8;
        v41 = v26;
        do
        {
          v42 = result + 4 * (v41 + v33);
          v43 = *(v40 + 16);
          v45 = *(v40 + a7);
          v46 = *(v40 + a7 + 16);
          v44 = (v40 + a7 + a7);
          v48 = *v44;
          v49 = v44[1];
          v47 = (v44 + a7);
          v50 = *v47;
          v51 = v47[1];
          v52 = vzip2q_s32(*v40, v45);
          *v42 = vzip1q_s32(*v40, v45);
          *(v42 + 16) = vzip1q_s32(v48, v50);
          *(v42 + 32) = v52;
          *(v42 + 48) = vzip2q_s32(v48, v50);
          v42 += 128LL;
          *v42 = vzip1q_s32(v43, v46);
          *(v42 + 16) = vzip1q_s32(v49, v51);
          *(v42 + 32) = vzip2q_s32(v43, v46);
          *(v42 + 48) = vzip2q_s32(v49, v51);
          v41 = (v41 - v12) & v12;
          v40 += 32LL;
          v39 += 8;
        }
        while ( v39 < a10 );
        v33 = (v33 - v19) & v19;
        v38 += 4 * a7;
        v37 += 4;
      }
      while ( v37 < a11 );
    ...
}

1.2. 内核驱动定位

这里先确定要分析的驱动.

kern_return_t
(*orig_IOServiceOpen)(
                   io_service_t    service,
                   task_port_t  owningTask,
                   uint32_t  type,
                   io_connect_t  *  connect );

kern_return_t
fake_IOServiceOpen(
              io_service_t    service,
              task_port_t  owningTask,
              uint32_t  type,
              io_connect_t  *  connect ) {
  
  kern_return_t kr;
  kr = orig_IOServiceOpen(service, owningTask, type, connect);
  io_name_t name;
  io_string_t path;
  
  kr = IORegistryEntryGetName(service, name);
  kr = IORegistryEntryGetPath(service, "IOService", path );
  if (kIOReturnSuccess != kr) {
    FATAL("open %s (0x%x)\n", mach_error_string(kr), kr);
  }
  CFMutableDictionaryRef properties = NULL;
  kr = _IORegistryEntryCreateCFProperties(service, &properties, kCFAllocatorDefault, kNilOptions);
  NSLog(@"%@", properties);
  printf(">>> service: %d, connect: %d, type: %d, name %s, path: %s\n", service, *connect, type, name, path);
  return kr;
}

// IOServiceOpen
void *IOServiceOpen_addr = dlsym(dlopen(0, RTLD_NOW), "IOServiceOpen");
ZzReplace((void *)IOServiceOpen_addr, (void *)fake_IOServiceOpen, (void **)&orig_IOServiceOpen);

通过 Hook 获取到期间使用到的驱动 AGXAcceleratorG10P_B0.

2018-12-24 10:52:04.671007+0800 ios_memory_usage_demo[4289:2499148] {
    AGXParameterBufferMaxSize = 201326592;
    AGXParameterBufferMaxSizeEverMemless = 134217728;
    AGXParameterBufferMaxSizeNeverMemless = 67108864;
    IOClass = "AGXAcceleratorG10P_B0";
    IOGLESBundleName = AppleMetalGLRenderer;
    InternalStatistics =     {
        "Allocated PB Size" = 1048576;
        agpTextureCreationCount = 0;
        agprefTextureCreationCount = 0;
        bufferSwapCount = 0;
        clientGLWaitTime = 0;
        dataBufferCount = 0;
        dataBytesPerSample = 0;
        freeDataBufferWaitTime = 0;
        gartCacheBytes = 33554432;
        gartFreeBytes = 1361772544;
        gartSizeBytes = 1393639424;
        hardwareSubmitWaitTime = 0;
        inUseSysMemoryBytes = 0;
        iosurfaceTextureCreationCount = 0;
        oolTextureCreationCount = 0;
        orphanedNonReusableSysMemoryBytes = 0;
        orphanedNonReusableSysMemoryCount = 0;
        orphanedReusableSysMemoryBytes = 0;
        orphanedReusableSysMemoryCount = 0;
        orphanedReusableSysMemoryHitRate = 0;
        stdTextureCreationCount = 0;
    };
    InternalStatisticsAccm =     {
        "Allocated PB Size" = 1048576;
        agpTextureCreationCount = 0;
        agprefTextureCreationCount = 0;
        bufferSwapCount = 0;
        clientGLWaitTime = 0;
        dataBufferCount = 0;
        dataBytesPerSample = 0;
        freeDataBufferWaitTime = 0;
        gartCacheBytes = 33554432;
        gartFreeBytes = 1361772544;
        gartSizeBytes = 1393639424;
        hardwareSubmitWaitTime = 0;
        inUseSysMemoryBytes = 0;
        iosurfaceTextureCreationCount = 0;
        oolTextureCreationCount = 0;
        orphanedNonReusableSysMemoryBytes = 0;
        orphanedNonReusableSysMemoryCount = 0;
        orphanedReusableSysMemoryBytes = 0;
        orphanedReusableSysMemoryCount = 0;
        orphanedReusableSysMemoryHitRate = 0;
        stdTextureCreationCount = 0;
    };
    MetalPluginClassName = AGXA11Device;
    MetalPluginName = AGXMetalA11;
    MetalStatisticsName = AGXMetalStatisticsExternalA11;
    PerformanceStatistics =     {
        CommandBufferRenderCount = 0;
        "Device Utilization %" = 0;
        "Renderer Utilization %" = 0;
        SplitSceneCount = 0;
        TiledSceneBytes = 0;
        "Tiler Utilization %" = 0;
        agpTextureCreationBytes = 0;
        agprefTextureCreationBytes = 0;
        contextGLCount = 0;
        finishGLWaitTime = 0;
        freeToAllocGPUAddressWaitTime = 0;
        gartMapInBytesPerSample = 0;
        gartMapOutBytesPerSample = 0;
        gartUsedBytes = 31866880;
        hardwareWaitTime = 0;
        iosurfaceTextureCreationBytes = 0;
        oolTextureCreationBytes = 0;
        recoveryCount = 0;
        stdTextureCreationBytes = 0;
        textureCount = 873;
    };
    PerformanceStatisticsAccum =     {
        CommandBufferRenderCount = 0;
        "Device Utilization %" = 0;
        "Renderer Utilization %" = 0;
        SplitSceneCount = 0;
        TiledSceneBytes = 0;
        "Tiler Utilization %" = 0;
        agpTextureCreationBytes = 0;
        agprefTextureCreationBytes = 0;
        contextGLCount = 0;
        finishGLWaitTime = 0;
        freeToAllocGPUAddressWaitTime = 0;
        gartMapInBytesPerSample = 0;
        gartMapOutBytesPerSample = 0;
        gartUsedBytes = 31866880;
        hardwareWaitTime = 0;
        iosurfaceTextureCreationBytes = 0;
        oolTextureCreationBytes = 0;
        recoveryCount = 0;
        stdTextureCreationBytes = 0;
        textureCount = 873;
    };
}
>>> service: 9219, connect: 39939, type: 2, name AGXAcceleratorG10P_B0, path: IOService:/AppleARMPE/arm-io@8F00000/AppleT8015IO/sgx@4000000/AGXAcceleratorG10P_B0

通过 hook IOConnectCallMethod 发现, 调用的是 selector == 0 的驱动方法

connnection: 8719, selector: 0, input: 0x0, inputCnt: 0,inputStruct: 0x16ba34cf0, inputStructCnt: 96, outputStruct: 0x16ba34b30, outputStructCnt: 72

接下来需要定位 selector = 0 对应的内核驱动方法实现. 通过使用 iometa -n -bmosv ./kernelcache.release.iphone10.decrypt > IOKitDump.mosv dump 到 IOKit 所有类的关系和 vtable 信息.

通过分析 AGXAcceleratorG10P_B0 继承于 IOGraphicsAccelerator2, 并使用 0xfffffff006d38560 IOGraphicsAccelerator2::newUserClient() 在 Connnect Servevice 时创建 UserClient.

vtab=0xfffffff006fda6a8 size=0x00000c78 meta=0xfffffff0078658a8 parent=0xfffffff007865880 IOGraphicsAccelerator2 (com.apple.iokit.IOAcceleratorFamily2)
     0x460 func=0xfffffff006d38560 overrides=0xfffffff00756cf88 pac=0x0000 IOGraphicsAccelerator2::newUserClient()

signed __int64 __fastcall sub_FFFFFFF006D38560(__int64 a1, __int64 owningTask, __int64 security_id, __int64 type, IOUserClient **handler)
{
... ignore

  switch ( (_DWORD)type_1 )
  {
    case 1:
      v14 = (*(__int64 (__fastcall **)(__int64))(*(_QWORD *)v8 + 0x690LL))(v8);
      v15 = v14;
      if ( v14 )
      {
        if ( sub_FFFFFFF006D23004(v14, 0LL, owningTask_1) & 1 )
          goto LABEL_17;
        goto LABEL_22;
      }
      break;
    case 2:
      v16 = (*(__int64 (__fastcall **)(__int64))(*(_QWORD *)v8 + 0x688LL))(v8);
      v15 = v16;
      if ( v16 )
      {
        if ( sub_FFFFFFF006D30AE4(v16, 0LL, owningTask_1) & 1 )
          goto LABEL_17;
        goto LABEL_22;
      }
      break;
    case 3:

... ignore
}

由于之前 IOServiceOpen 是以 type = 2 打开的服务, 根据 IDA 里的伪代码, 调用 vtable 里 offset == 0x688 的虚方法, 这里

vtab=0xfffffff006fe0068 size=0x00001998 meta=0xfffffff0078666b0 parent=0xfffffff007866540 AGXAcceleratorG10 (com.apple.AGXG10P)
     0x688 func=0xfffffff006d4bf74 overrides=0xfffffff006d38994 pac=0x0000 AGXAcceleratorG10::fn_0x688()

_QWORD *sub_FFFFFFF006D4BF74()
{
  _QWORD *v0; // x19

  v0 = (_QWORD *)OSObject::operator new(336LL);
  sub_FFFFFFF006D7ABD4();
  *v0 = &off_FFFFFFF006FE41F8;
  OSMetaClass::instanceConstructed(&AGXSharedUserClient::gMetaClass);
  return v0;
}

直接找 AGXSharedUserClient 的 externalMethod 看看怎么实现的 dispatch.

vtab=0xfffffff006fe41f8 size=0x00000150 meta=0xfffffff007866a98 parent=0xfffffff0078657b8 AGXSharedUserClient (com.apple.AGXG10P)
     0x538 func=0xfffffff006d768f8 overrides=0xfffffff006d32098 pac=0x0000 AGXSharedUserClient::externalMethod()

可以看到之后分发给了父类 IOAccelSharedUserClient2, 接着跟踪, 可以看到 IOAccelSharedUserClient2 的 externalMethod 则是直接转交给 IOUserClient 的 externalMethod 进行处理.

vtab=0xfffffff006fd9228 size=0x00000150 meta=0xfffffff0078657b8 parent=0xfffffff007646dc8 IOAccelSharedUserClient2 (com.apple.iokit.IOAcceleratorFamily2)
     0x538 func=0xfffffff006d32098 overrides=0xfffffff0075c033c pac=0x0000 IOAccelSharedUserClient2::externalMethod()

这里分两种情况分析 1. w1 == #0 2. w1 != #0, 这里 w1 == selector

1). w1 == #0

经过 CSEL X3, X8, X3, EQ, 会将参数 dipatch 设置为 off_FFFFFFF006FD9A38, 经过 IOUserClient::externalMethod 此时 dipatch == off_FFFFFFF006FD9A38, target == AGXSharedUserClient.

2). w1 != #0

那继续分析 getTargetAndMethodForIndex, 因为在 IOUserClient::externalMethod 会调用到 getTargetAndMethodForIndex 进行 dispatch.

分析 AGXSharedUserClient::getTargetAndMethodForIndex, w2 也就是 selector = #unknown, 直接转发给父类 IOAccelSharedUserClient2::getTargetAndMethodForIndex

vtab=0xfffffff006fe41f8 size=0x00000150 meta=0xfffffff007866a98 parent=0xfffffff0078657b8 AGXSharedUserClient (com.apple.AGXG10P)
     0x5a8 func=0xfffffff006d7691c overrides=0xfffffff006d320bc pac=0x0000 AGXSharedUserClient::getTargetAndMethodForIndex()

通过分析 IOAccelSharedUserClient2::getTargetAndMethodForIndex 发现通过取得 [x0,#0xF0] 中的 IOExternalMethod ** 基址, 之后通过 UMADDL X0, W2, W9, X8 取得对应 selector = #unknown 偏移的 IOExternalMethod *. 究竟 [X0,#0xF0] 里的地址到底指向哪里, 这里可以找找相关的方法实现.

vtab=0xfffffff006fd9228 size=0x00000150 meta=0xfffffff0078657b8 parent=0xfffffff007646dc8 IOAccelSharedUserClient2 (com.apple.iokit.IOAcceleratorFamily2)
     0x5a8 func=0xfffffff006d320bc overrides=0xfffffff0075c1824 pac=0x0000 IOAccelSharedUserClient2::getTargetAndMethodForIndex()

通过对 AGXSharedUserClient 和它父类 IOAccelSharedUserClient2 部分方法进行查看, 最后发现在 0xfffffff006d30e28 初始化了 [X0,#0xF0] 为 qword_FFFFFFF006FD9888.

vtab=0xfffffff006fd9228 size=0x00000150 meta=0xfffffff0078657b8 parent=0xfffffff007646dc8 IOAccelSharedUserClient2 (com.apple.iokit.IOAcceleratorFamily2)
     0x5c8 func=0xfffffff006d30e28 overrides=0x0000000000000000 pac=0x0000 IOAccelSharedUserClient2::fn_0x5c8()

这里因为 selector == 0 所以这里直接分析 sub_FFFFFFF006D31FB4 即可

但是由于 iOS Kernel 里很多驱动的数据结构是未知的, 分析难度很大.

1.3. 内核驱动函数分析.

这里将部分符号化的驱动处理函数. 由于缺乏 runtime 的分析, 通过上面的静态分析, 这里也只能分析出使用到 AGXResource 等未知内存分布的内核类. 以及会在内核层使用 IOBufferMemoryDescriptor::createMappingInTask 创建内存 mapping, 并将 VirtualAddress 作为 OutputStruct 的第二个成员返回给 Userspace, 这个和我们在 userspace 看到的行为是一样的.

1.4. IOMemoryDescriptor 调用分析

通过查找一些 IOKit 资料/驱动, 在 IOKit 中会大量使用 IOBufferMemoryDescriptor 进行内存页分配,

所以这里对 IOBufferMemoryDescriptor 特别关注下, 这里看下 IOBufferMemoryDescriptor::inTaskWithOptions 的 options

options = input_1->x1.x_1_1_2 | 0x10063; 其中 0x10063 == kIOMemoryKernelUserShared | kIODirectionInOut | kIOMemoryPageable | kIOMemoryPurgeable, 这里通过对比一些 IOKit 驱动, 例如 VMsvga2 等, 发现这个参数仅在 Texture 绘制时出现, 那么是否和这个参数有关, 需要写一个 DemoDriver 测试下, 这里讲 iOS Kernel IOKit 的问题转化为 macOS Kernel IOKit 的问题.

1.5. 利用 DemoDriver 复现问题

// ===== FrameBufffer Test Case =====

IOReturn SharedMemoryAlloc(IOBufferMemoryDescriptor **buffer, IOOptionBits options, unsigned size) {
  IOKIT_AUTO_LOG_FLAG();

  *buffer = IOBufferMemoryDescriptor::inTaskWithOptions(0, options, size, PAGE_SIZE);
  if (!*buffer) {
    IOLog("failed to allocate buffer memory");
    return kIOReturnNoMemory;
  } else {
    return kIOReturnSuccess;
  }
}

IOReturn IOKitDemoDriver::userClientMap(task_t owningTask, DisplayXMap *map, uint32_t *mapSize) {
  IOKIT_AUTO_LOG_FLAG();
  IOReturn kr = kIOReturnSuccess;

  if (!map || !mapSize || *mapSize != sizeof *map)
    kr = kIOReturnBadArgument;

  FrameBuffferEmu *frameBuffer     = new FrameBuffferEmu;
  IOBufferMemoryDescriptor *buffer = frameBuffer->buffer;
  IOOptionBits options;
#if 0
  options = kIOMemoryKernelUserShared | kIODirectionInOut | kIOMemoryPageable | kIOMemoryPurgeable;
#elif 1
  options = kIOMemoryKernelUserShared | kIODirectionInOut | kIOMemoryPageable;
#endif

  kr = SharedMemoryAlloc(&buffer, options, PAGE_SIZE * 500);

#if 0
#elif 1
  options = kIOMapAnywhere;
#endif

  IOMemoryMap *iomap = buffer->createMappingInTask(owningTask, 0, options, 0, 0);

  map->address = iomap->getVirtualAddress();
  return kr;
}

加载刚才写的 IOKitDemoDriver.kext

sudo rm -rf IOKitDemoDriver.kext
sudo chown -R root:wheel IOKitDemoDriver.kext
sudo kextload IOKitDemoDriver.kext
kernel-exploitdeMac:Desktop kernel_exploit$ sudo chown -R root:wheel IOKitDemoDriver.kext
kernel-exploitdeMac:Desktop kernel_exploit$ sudo kextload IOKitDemoDriver.kext

这是增加了这个 flag 之后的调用结果, 很明显, 我们浮现了这个问题.


kernel-exploitdeMac:Desktop kernel_exploit$ ./IOKitDemoClient
[*] IOKitDemoDriver Open Success, Version: 1.1, Task: 0xffffff80123997e8
2019-02-15 00:03:31.787 IOKitDemoClient:footprint: 712 KB, internal: 712 KB, purges: 328, purgeable_count:5196
[*] userClientMap Address: 0x111984000
2019-02-15 00:03:31.789 IOKitDemoClient:footprint: 3 MB, internal: 4.9 MB, purges: 328, purgeable_count:5196
2019-02-15 00:03:31.789 IOKitDemoClient:footprint: 3 MB, internal: 4.9 MB, purges: 328, purgeable_count:5196
[*] userClientMap Address: 0x111b78000
2019-02-15 00:03:31.791 IOKitDemoClient:footprint: 3 MB, internal: 6.9 MB, purges: 328, purgeable_count:5196
2019-02-15 00:03:31.791 IOKitDemoClient:footprint: 3 MB, internal: 6.9 MB, purges: 328, purgeable_count:5196
[*] userClientMap Address: 0x111d6c000
2019-02-15 00:03:31.793 IOKitDemoClient:footprint: 3 MB, internal: 8.8 MB, purges: 328, purgeable_count:5196
2019-02-15 00:03:31.793 IOKitDemoClient:footprint: 3 MB, internal: 8.8 MB, purges: 328, purgeable_count:5196
[*] userClientMap Address: 0x111f60000
2019-02-15 00:03:31.794 IOKitDemoClient:footprint: 3 MB, internal: 10.8 MB, purges: 328, purgeable_count:5196
2019-02-15 00:03:31.795 IOKitDemoClient:footprint: 3 MB, internal: 10.8 MB, purges: 328, purgeable_count:5196
[*] userClientMap Address: 0x112154000
2019-02-15 00:03:31.798 IOKitDemoClient:footprint: 3 MB, internal: 12.7 MB, purges: 328, purgeable_count:5196
2019-02-15 00:03:31.798 IOKitDemoClient:footprint: 3 MB, internal: 12.7 MB, purges: 328, purgeable_count:5196
[*] userClientMap Address: 0x112348000
2019-02-15 00:03:31.800 IOKitDemoClient:footprint: 3 MB, internal: 14.7 MB, purges: 328, purgeable_count:5196
2019-02-15 00:03:31.800 IOKitDemoClient:footprint: 3 MB, internal: 14.7 MB, purges: 328, purgeable_count:5196
[*] userClientMap Address: 0x11253c000
2019-02-15 00:03:31.802 IOKitDemoClient:footprint: 3 MB, internal: 16.6 MB, purges: 328, purgeable_count:5196
2019-02-15 00:03:31.802 IOKitDemoClient:footprint: 3 MB, internal: 16.6 MB, purges: 328, purgeable_count:5196
[*] userClientMap Address: 0x112730000
2019-02-15 00:03:31.804 IOKitDemoClient:footprint: 3 MB, internal: 18.6 MB, purges: 328, purgeable_count:5196
2019-02-15 00:03:31.804 IOKitDemoClient:footprint: 3 MB, internal: 18.6 MB, purges: 328, purgeable_count:5196
[*] userClientMap Address: 0x112924000
2019-02-15 00:03:31.806 IOKitDemoClient:footprint: 3 MB, internal: 20.6 MB, purges: 328, purgeable_count:5196
2019-02-15 00:03:31.806 IOKitDemoClient:footprint: 3 MB, internal: 20.6 MB, purges: 328, purgeable_count:5196
[*] userClientMap Address: 0x112b18000
2019-02-15 00:03:31.808 IOKitDemoClient:footprint: 3 MB, internal: 22.5 MB, purges: 328, purgeable_count:5196

以下是移除该 flag 后的调用结果, phys_footprint 与 internal 相同.

kernel-exploitdeMac:Desktop kernel_exploit$ ./IOKitDemoClient
[*] IOKitDemoDriver Open Success, Version: 1.1, Task: 0xffffff801122c7e8
2019-02-14 23:57:29.417 IOKitDemoClient:footprint: 712 KB, internal: 712 KB, purges: 328, purgeable_count:5180
[*] userClientMap Address: 0x110584000
2019-02-14 23:57:29.422 IOKitDemoClient:footprint: 4.9 MB, internal: 4.9 MB, purges: 328, purgeable_count:5180
2019-02-14 23:57:29.422 IOKitDemoClient:footprint: 4.9 MB, internal: 4.9 MB, purges: 328, purgeable_count:5180
[*] userClientMap Address: 0x110778000
2019-02-14 23:57:29.424 IOKitDemoClient:footprint: 6.9 MB, internal: 6.9 MB, purges: 328, purgeable_count:5180
2019-02-14 23:57:29.425 IOKitDemoClient:footprint: 6.9 MB, internal: 6.9 MB, purges: 328, purgeable_count:5180
[*] userClientMap Address: 0x11096c000
2019-02-14 23:57:29.427 IOKitDemoClient:footprint: 8.8 MB, internal: 8.8 MB, purges: 328, purgeable_count:5180
2019-02-14 23:57:29.427 IOKitDemoClient:footprint: 8.8 MB, internal: 8.8 MB, purges: 328, purgeable_count:5180
[*] userClientMap Address: 0x110b60000
2019-02-14 23:57:29.428 IOKitDemoClient:footprint: 10.8 MB, internal: 10.8 MB, purges: 328, purgeable_count:5180
2019-02-14 23:57:29.428 IOKitDemoClient:footprint: 10.8 MB, internal: 10.8 MB, purges: 328, purgeable_count:5180
[*] userClientMap Address: 0x110d54000
2019-02-14 23:57:29.430 IOKitDemoClient:footprint: 12.8 MB, internal: 12.8 MB, purges: 328, purgeable_count:5180
2019-02-14 23:57:29.430 IOKitDemoClient:footprint: 12.8 MB, internal: 12.8 MB, purges: 328, purgeable_count:5180
[*] userClientMap Address: 0x110f48000
2019-02-14 23:57:29.432 IOKitDemoClient:footprint: 14.7 MB, internal: 14.7 MB, purges: 328, purgeable_count:5180
2019-02-14 23:57:29.433 IOKitDemoClient:footprint: 14.7 MB, internal: 14.7 MB, purges: 328, purgeable_count:5180
[*] userClientMap Address: 0x11113c000
2019-02-14 23:57:29.434 IOKitDemoClient:footprint: 16.7 MB, internal: 16.7 MB, purges: 328, purgeable_count:5180
2019-02-14 23:57:29.434 IOKitDemoClient:footprint: 16.7 MB, internal: 16.7 MB, purges: 328, purgeable_count:5180
[*] userClientMap Address: 0x111330000
2019-02-14 23:57:29.436 IOKitDemoClient:footprint: 18.6 MB, internal: 18.6 MB, purges: 328, purgeable_count:5180
2019-02-14 23:57:29.436 IOKitDemoClient:footprint: 18.6 MB, internal: 18.6 MB, purges: 328, purgeable_count:5180
[*] userClientMap Address: 0x111524000
2019-02-14 23:57:29.439 IOKitDemoClient:footprint: 20.6 MB, internal: 20.6 MB, purges: 328, purgeable_count:5180
2019-02-14 23:57:29.439 IOKitDemoClient:footprint: 20.6 MB, internal: 20.6 MB, purges: 328, purgeable_count:5180
[*] userClientMap Address: 0x111718000
2019-02-14 23:57:29.441 IOKitDemoClient:footprint: 22.5 MB, internal: 22.5 MB, purges: 328, purgeable_count:5180

1.6. kIOMemoryPurgeable 作用

对着 xnu-4570.20.62 一顿操作, 发现当存在 kIOMemoryPurgeable 时, 会设置 object->vo_purgeable_owner = owner 为 kernel_task, 但是这是否就是因为这个愿意导致 phys_footprint 不增加的原因呢, 这里在手动调试下正常情况下 phys_footprint 增加的原因.

1.7. 内核硬断调试

如果没有 kIOMemoryPurgeable, 进程 task 的 phys_footprint 为何增加呢?

这里通过获取进程 task_t task, 对应的保存 phys_footprint 的 le_credit, 并对设置硬件写断点. (注意: lldb 内核调试的时, 硬断会失败, 具体原因由于时间原因尚未深究, 这里可以换成 GDB 或者 IDA remote GDB Debugger 两者是一样的)

p/x &(&(((task_t)0xffffff800fab09b0)->ledger->l_entries[task_ledgers.phys_footprint]))->le_credit

获取到保存 phys_footprint 的 le_credit 后, 在 IDA 进行硬断.

调试器挂载

触发硬断后, 这里直接根据堆栈和调用规约, 根据图中的标注找到 caller. (不可以单步走)

__text:FFFFFF8012606C73 mov     rdi, [r12+168h]                 ; ledger
__text:FFFFFF8012606C7B mov     esi, cs:task_ledgers.iokit_mapped ; entry
__text:FFFFFF8012606C81 mov     rdx, rbx                        ; amount
__text:FFFFFF8012606C84 call    ledger_credit
__text:FFFFFF8012606C89 mov     rdi, [r12+168h]                 ; ledger
__text:FFFFFF8012606C91 mov     esi, cs:task_ledgers.phys_footprint ; entry
__text:FFFFFF8012606C97 mov     rdx, rbx                        ; amount
__text:FFFFFF8012606C9A call    ledger_credit
__text:FFFFFF8012606C9F mov     r9, [rbp+hint_offset]
__text:FFFFFF8012606CA6 mov     r10, qword ptr [rbp+var_68]
__text:FFFFFF8012606CAA mov     r12, [rbp+object]
__text:FFFFFF8012606CB1 test    r15d, r15d
__text:FFFFFF8012606CB4 jz      loc_FFFFFF8012606D8F

通过 lldb 看下对应的源码位置.

(lldb) b 0xFFFFFF8012606C9F
Breakpoint 2: where = kernel.development`vm_map_enter + 5407 [inlined] vm_map_iokit_mapped_region + 48 at vm_map.c:2737, address = 0xffffff8012606c9f

在 xnu 源码里发现实现位置.

1.8. kIOMemoryPurgeable 的对比

最终通过这里的不同点, 确定了问题的发生位置.

1.9. 总结

当调用 glTexImage2D 函数时, iOS 的 AGXAcceleratorG10P_B0 驱动, 会调用 IOBufferMemoryDescriptor::inTaskWithOptions 其中 options 为 0x10063 == kIOMemoryKernelUserShared | kIODirectionInOut | kIOMemoryPageable | kIOMemoryPurgeable, 由于 options 中包含 kIOMemoryPurgeable flag, 导致 object->vo_purgeable_owner 被设置为 kernel_task, 因而当发生 #PF 即 vm_fault 时, 进入 vm_map_enter 函数进行统计的时, 没有将该内存统计给对应进程的 task_t task, 而是统计给 kernel_task.

这里有一篇文章讲解了为什么 Texture 下要使用 purgeable memory. https://www.khronos.org/registry/OpenGL/extensions/APPLE/APPLE_object_purgeable.txt

Rerfer

https://www.khronos.org/registry/OpenGL/extensions/APPLE/APPLE_object_purgeable.txt

Posts: 1

Participants: 1

Read full topic