121151 – XFS filesystem crashes due to NULL pointer dereference

Bug 121151 - XFS filesystem crashes due to NULL pointer dereference

Summary: XFS filesystem crashes due to NULL pointer dereference

Status:	RESOLVED WILL_NOT_FIX

Alias:	None

Product:	File System
Classification:	Unclassified
Component:	XFS (show other bugs)
Hardware:	x86-64 Linux

Importance:	P1 high
Assignee:	XFS Guru

URL:
Keywords:

Depends on:
Blocks:

Reported:	2016-06-29 09:32 UTC by Vladimir
Modified:	2016-06-30 11:06 UTC (History)
CC List:	1 user (show)

See Also:
Kernel Version:	3.10
Tree:	Mainline
Regression:	No

Attachments
Add an attachment (proposed patch, testcase, etc.)

Description Vladimir 2016-06-29 09:32:52 UTC

I'm getting a kernel oops indicating a null pointer dereference in xfs_trans_mod_dquot:

crash> bt
PID: 484420  TASK: ffff88003914e580  CPU: 0   COMMAND: "webalizer"
 #0 [ffff8801657ab758] machine_kexec at ffffffff8105249b
 #1 [ffff8801657ab7b8] crash_kexec at ffffffff811034f2
 #2 [ffff8801657ab888] oops_end at ffffffff8163d9e8
 #3 [ffff8801657ab8b0] no_context at ffffffff8162e64b
 #4 [ffff8801657ab900] __bad_area_nosemaphore at ffffffff8162e6e1
 #5 [ffff8801657ab950] bad_area at ffffffff8162ea24
 #6 [ffff8801657ab978] __do_page_fault at ffffffff8164091c
 #7 [ffff8801657ab9d8] do_page_fault at ffffffff81640993
 #8 [ffff8801657aba00] page_fault at ffffffff8163cb88
    [exception RIP: xfs_trans_mod_dquot+56]
    RIP: ffffffffa0305768  RSP: ffff8801657abab0  RFLAGS: 00010246
    RAX: 0000000000000000  RBX: ffff88019f3842b8  RCX: 000000000000002a
    RDX: 0000000000010000  RSI: ffff8800c235df58  RDI: ffff88019f3842f8
    RBP: ffff8801657abad8   R8: ffff8800c235e088   R9: 0000000000000000
    R10: 000000000000002a  R11: ffff880413c93800  R12: 0000000000010000
    R13: ffff8800c235df58  R14: 000000000000002a  R15: ffff88019f3842f8
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018
 #9 [ffff8801657abae0] xfs_trans_dqresv at ffffffffa0305c47 [xfs]
#10 [ffff8801657abb50] xfs_trans_reserve_quota_bydquots at ffffffffa03062ee [xfs]
#11 [ffff8801657abb90] xfs_create at ffffffffa02e76f2 [xfs]
#12 [ffff8801657abc50] xfs_vn_mknod at ffffffffa02e3e99 [xfs]
#13 [ffff8801657abcb8] xfs_vn_create at ffffffffa02e4043 [xfs]
#14 [ffff8801657abcc8] vfs_create at ffffffff81207b7c
#15 [ffff8801657abd00] do_last at ffffffff812096ed
#16 [ffff8801657abda8] path_openat at ffffffff8120ab12
#17 [ffff8801657abe48] do_filp_open at ffffffff8120d82b
#18 [ffff8801657abf18] do_sys_open at ffffffff811fa3a3
#19 [ffff8801657abf70] sys_open at ffffffff811fa4be
#20 [ffff8801657abf80] system_call_fastpath at ffffffff81645189
    RIP: 00007f1067c3c850  RSP: 00007ffd6cb59bd8  RFLAGS: 00000246
    RAX: 0000000000000002  RBX: ffffffff81645189  RCX: ffffffffffffffff
    RDX: 00000000000001b6  RSI: 0000000000000241  RDI: 00007ffd6cb58ad0
    RBP: 00007ffd6cb58a60   R8: 000000000041cf15   R9: 0000000000000240
    R10: 0000000000000024  R11: 0000000000000246  R12: ffffffff811fa4be
    R13: ffff8801657abf78  R14: 0000000000000023  R15: 0000000000000001
    ORIG_RAX: 0000000000000002  CS: 0033  SS: 002b

I'm using OpenVZ kernel rh7-3.10.0-327.10.1.vz7.12.14 located at https://github.com/OpenVZ/vzkernel/releases/tag/rh7-3.10.0-327.10.1.vz7.12.14

Here is some info from coredump (hope it will be useful):

crash> bt -f
..
 #8 [ffff8801657aba00] page_fault at ffffffff8163cb88
    [exception RIP: xfs_trans_mod_dquot+56]
    RIP: ffffffffa0305768  RSP: ffff8801657abab0  RFLAGS: 00010246
    RAX: 0000000000000000  RBX: ffff88019f3842b8  RCX: 000000000000002a
    RDX: 0000000000010000  RSI: ffff8800c235df58  RDI: ffff88019f3842f8
    RBP: ffff8801657abad8   R8: ffff8800c235e088   R9: 0000000000000000
    R10: 000000000000002a  R11: ffff880413c93800  R12: 0000000000010000
    R13: ffff8800c235df58  R14: 000000000000002a  R15: ffff88019f3842f8
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018
    ffff8801657aba08: ffff88019f3842f8 000000000000002a 
    ffff8801657aba18: ffff8800c235df58 0000000000010000 
    ffff8801657aba28: ffff8801657abad8 ffff88019f3842b8 
    ffff8801657aba38: ffff880413c93800 000000000000002a 
    ffff8801657aba48: 0000000000000000 ffff8800c235e088 
    ffff8801657aba58: 0000000000000000 000000000000002a 
    ffff8801657aba68: 0000000000010000 ffff8800c235df58 
    ffff8801657aba78: ffff88019f3842f8 ffffffffffffffff 
    ffff8801657aba88: ffffffffa0305768 0000000000000010 
    ffff8801657aba98: 0000000000010246 ffff8801657abab0 
    ffff8801657abaa8: 0000000000000018 ffff8800c235df58 
    ffff8801657abab8: 0000000000000001 0000000000010000 
    ffff8801657abac8: ffff8800c235e0c8 ffff88019f3842b8 
    ffff8801657abad8: ffff8801657abb48 ffffffffa0305c47 
 #9 [ffff8801657abae0] xfs_trans_dqresv at ffffffffa0305c47 [xfs]
    ffff8801657abae8: ffff880411cf6100 0000000000000000 
    ffff8801657abaf8: 0000000000000000 ffff8801657abb30 
    ffff8801657abb08: 000000000000002a ffff880413c93800 
    ffff8801657abb18: ffff880415decc00 ffff880413c93800 
    ffff8801657abb28: ffff88019f3842b8 ffff88041546c000 
    ffff8801657abb38: 000000000000002a 0000000000000000 
    ffff8801657abb48: ffff8801657abb88 ffffffffa03062ee 
#10 [ffff8801657abb50] xfs_trans_reserve_quota_bydquots at ffffffffa03062ee [xfs]
    ffff8801657abb58: ffff8800c235df58 ffff8801a642dd80 
    ffff8801657abb68: ffff880413c93800 ffff8801657abc70 
    ffff8801657abb78: 000000000000000a 0000000000000000 
    ffff8801657abb88: ffff8801657abc48 ffffffffa02e76f2 
#11 [ffff8801657abb90] xfs_create at ffffffffa02e76f2 [xfs]
    ffff8801657abb98: 0000000000000001 ffff880100010000 
    ffff8801657abba8: ffff8801657abbd8 ffff8801657abc68 
    ffff8801657abbb8: ffff880413c93b48 0000000000000000 
    ffff8801657abbc8: 0000002a000081a4 ffffffff81213ec2 
    ffff8801657abbd8: 0000000000000000 ffff88019f3842b8 
    ffff8801657abbe8: ffffffffffffffff ffff88041546c000 
    ffff8801657abbf8: 0000000000000000 ffff8800c235df58 
    ffff8801657abc08: 0000000000000000 0000000000000000 
    ffff8801657abc18: 0000000051c3d868 ffff8801610e1680 
    ffff8801657abc28: 0000000000000000 ffff8801a642df38 
    ffff8801657abc38: 00000000000081a4 00000000000081b6 
    ffff8801657abc48: ffff8801657abcb0 ffffffffa02e3e99 
#12 [ffff8801657abc50] xfs_vn_mknod at ffffffffa02e3e99 [xfs]
    ffff8801657abc58: ffffffff81206900 ffff880100000000 
    ffff8801657abc68: 0000000000000000 ffff8801610e16b8 
    ffff8801657abc78: 0000000100000015 0000000051c3d868 
    ffff8801657abc88: 0000000000000000 ffff8801a642df38 
    ffff8801657abc98: ffff8801610e1680 00000000000081b6 
    ffff8801657abca8: 0000000000000000 ffff8801657abcc0 
    ffff8801657abcb8: ffffffffa02e4043 
#13 [ffff8801657abcb8] xfs_vn_create at ffffffffa02e4043 [xfs]
    ffff8801657abcc0: ffff8801657abcf8 ffffffff81207b7c 
#14 [ffff8801657abcc8] vfs_create at ffffffff81207b7c
    ffff8801657abcd0: ffff8801657abf28 0000000000008241 
    ffff8801657abce0: ffff8801657abe50 ffff8801013255c0 
    ffff8801657abcf0: ffff8801610e1680 ffff8801657abda0 
    ffff8801657abd00: ffffffff812096ed 
#15 [ffff8801657abd00] do_last at ffffffff812096ed
    ffff8801657abd08: ffffea000d264240 0000000000000000 
    ffff8801657abd18: ffff8801657abd68 ffff8801657abd90 
    ffff8801657abd28: ffff8801a642df38 ffff8803f5e18000 
    ffff8801657abd38: ffff88001f9cd000 ffff8801657abde4 
    ffff8801657abd48: ffff8801013255c0 0100000100000022 
    ffff8801657abd58: ffff8801657abdf0 00ff88001f9cd000 
    ffff8801657abd68: ffff8801d7d1a000 0000000051c3d868 
    ffff8801657abd78: ffff8801657abe50 ffff88001f9cd000 
    ffff8801657abd88: ffff8803f5e18000 ffff8801657abf28 
    ffff8801657abd98: ffff88003914e580 ffff8801657abe40 
    ffff8801657abda8: ffffffff8120ab12 
#16 [ffff8801657abda8] path_openat at ffffffff8120ab12
    ffff8801657abdb0: ffff8801657abe80 ffffffff8120d732 
    ffff8801657abdc0: ffff880413bcb720 ffff8801610e1680 
    ffff8801657abdd0: 0000001553594ce5 00000041f5e18020 
    ffff8801657abde0: 0000000100000000 0000000000000000 
    ffff8801657abdf0: ffff8801a642df38 0000000200000000 
    ffff8801657abe00: 0000000000000000 00007f1068e7c000 
    ffff8801657abe10: 0000000051c3d868 00000000ffffff9c 
    ffff8801657abe20: ffff8803f5e18000 ffff8801657abf28 
    ffff8801657abe30: 0000000000000001 0000000000000023 
    ffff8801657abe40: ffff8801657abf10 ffffffff8120d82b 
#17 [ffff8801657abe48] do_filp_open at ffffffff8120d82b
    ffff8801657abe50: ffff880413bcb720 ffff8801013255c0 
    ffff8801657abe60: 0000001553594ce5 ffff8803f5e18020 
    ffff8801657abe70: 0000000000000000 00007ffd6cb58ad0 
    ffff8801657abe80: ffff8801a642df38 0000000200000301 
    ffff8801657abe90: 0000000000000000 0000000000000001 
    ffff8801657abea0: 00007ffd6cb58ad0 0000000000000000 
    ffff8801657abeb0: 0000000000000000 ffff8801657abf00 
    ffff8801657abec0: ffffffff8121a867 ffff880413ca91c0 
    ffff8801657abed0: 0000ffff00008241 0000000000000001 
    ffff8801657abee0: 0000000000008241 0000000051c3d868 
    ffff8801657abef0: 0000000000000001 0000000000000005 
    ffff8801657abf00: 00000000ffffff9c ffff8803f5e18000 
    ffff8801657abf10: ffff8801657abf68 ffffffff811fa3a3 
#18 [ffff8801657abf18] do_sys_open at ffffffff811fa3a3
    ffff8801657abf20: ffff880286c5b648 ffff81b600008241 
    ffff8801657abf30: 0000030000000022 0000000051c3d868 
    ffff8801657abf40: 000000000041cf0f 0000000001aa8300 
    ffff8801657abf50: 0000000000000004 0000000000000001 
    ffff8801657abf60: 0000000000000023 ffff8801657abf78 
    ffff8801657abf70: ffffffff811fa4be 
#19 [ffff8801657abf70] sys_open at ffffffff811fa4be
    ffff8801657abf78: 00007ffd6cb58a60 ffffffff81645189 
#20 [ffff8801657abf80] system_call_fastpath at ffffffff81645189
    RIP: 00007f1067c3c850  RSP: 00007ffd6cb59bd8  RFLAGS: 00000246
    RAX: 0000000000000002  RBX: ffffffff81645189  RCX: ffffffffffffffff
    RDX: 00000000000001b6  RSI: 0000000000000241  RDI: 00007ffd6cb58ad0
    RBP: 00007ffd6cb58a60   R8: 000000000041cf15   R9: 0000000000000240
    R10: 0000000000000024  R11: 0000000000000246  R12: ffffffff811fa4be
    R13: ffff8801657abf78  R14: 0000000000000023  R15: 0000000000000001
    ORIG_RAX: 0000000000000002  CS: 0033  SS: 002b

crash> bt -F
...
 #8 [ffff8801657aba00] page_fault at ffffffff8163cb88
    [exception RIP: xfs_trans_mod_dquot+56]
    RIP: ffffffffa0305768  RSP: ffff8801657abab0  RFLAGS: 00010246
    RAX: 0000000000000000  RBX: ffff88019f3842b8  RCX: 000000000000002a
    RDX: 0000000000010000  RSI: ffff8800c235df58  RDI: ffff88019f3842f8
    RBP: ffff8801657abad8   R8: ffff8800c235e088   R9: 0000000000000000
    R10: 000000000000002a  R11: ffff880413c93800  R12: 0000000000010000
    R13: ffff8800c235df58  R14: 000000000000002a  R15: ffff88019f3842f8
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018
    ffff8801657aba08: [cfq_queue]      000000000000002a 
    ffff8801657aba18: ffff8800c235df58 0000000000010000 
    ffff8801657aba28: ffff8801657abad8 [cfq_queue]      
    ffff8801657aba38: ffff880413c93800 000000000000002a 
    ffff8801657aba48: 0000000000000000 ffff8800c235e088 
    ffff8801657aba58: 0000000000000000 000000000000002a 
    ffff8801657aba68: 0000000000010000 ffff8800c235df58 
    ffff8801657aba78: [cfq_queue]      ffffffffffffffff 
    ffff8801657aba88: xfs_trans_mod_dquot+56 0000000000000010 
    ffff8801657aba98: 0000000000010246 ffff8801657abab0 
    ffff8801657abaa8: 0000000000000018 ffff8800c235df58 
    ffff8801657abab8: 0000000000000001 0000000000010000 
    ffff8801657abac8: ffff8800c235e0c8 [cfq_queue]      
    ffff8801657abad8: ffff8801657abb48 xfs_trans_dqresv+647 
 #9 [ffff8801657abae0] xfs_trans_dqresv at ffffffffa0305c47 [xfs]
    ffff8801657abae8: [kmem_cache]     0000000000000000 
    ffff8801657abaf8: 0000000000000000 ffff8801657abb30 
    ffff8801657abb08: 000000000000002a ffff880413c93800 
    ffff8801657abb18: [kmalloc-512]    ffff880413c93800 
    ffff8801657abb28: [cfq_queue]      [xfs_dquot]      
    ffff8801657abb38: 000000000000002a 0000000000000000 
    ffff8801657abb48: ffff8801657abb88 xfs_trans_reserve_quota_bydquots+286 
#10 [ffff8801657abb50] xfs_trans_reserve_quota_bydquots at ffffffffa03062ee [xfs]
    ffff8801657abb58: ffff8800c235df58 ffff8801a642dd80 
    ffff8801657abb68: ffff880413c93800 ffff8801657abc70 
    ffff8801657abb78: 000000000000000a 0000000000000000 
    ffff8801657abb88: ffff8801657abc48 xfs_create+546   
#11 [ffff8801657abb90] xfs_create at ffffffffa02e76f2 [xfs]
    ffff8801657abb98: 0000000000000001 ffff880100010000 
    ffff8801657abba8: ffff8801657abbd8 ffff8801657abc68 
    ffff8801657abbb8: ffff880413c93b48 0000000000000000 
    ffff8801657abbc8: 0000002a000081a4 __d_instantiate+146 
    ffff8801657abbd8: 0000000000000000 [cfq_queue]      
    ffff8801657abbe8: ffffffffffffffff [xfs_dquot]      
    ffff8801657abbf8: 0000000000000000 ffff8800c235df58 
    ffff8801657abc08: 0000000000000000 0000000000000000 
    ffff8801657abc18: 0000000051c3d868 ffff8801610e1680 
    ffff8801657abc28: 0000000000000000 ffff8801a642df38 
    ffff8801657abc38: 00000000000081a4 00000000000081b6 
    ffff8801657abc48: ffff8801657abcb0 xfs_vn_mknod+185 
#12 [ffff8801657abc50] xfs_vn_mknod at ffffffffa02e3e99 [xfs]
    ffff8801657abc58: generic_permission+272 ffff880100000000 
    ffff8801657abc68: 0000000000000000 ffff8801610e16b8 
    ffff8801657abc78: 0000000100000015 0000000051c3d868 
    ffff8801657abc88: 0000000000000000 ffff8801a642df38 
    ffff8801657abc98: ffff8801610e1680 00000000000081b6 
    ffff8801657abca8: 0000000000000000 ffff8801657abcc0 
    ffff8801657abcb8: xfs_vn_create+19 
#13 [ffff8801657abcb8] xfs_vn_create at ffffffffa02e4043 [xfs]
    ffff8801657abcc0: ffff8801657abcf8 vfs_create+140   
#14 [ffff8801657abcc8] vfs_create at ffffffff81207b7c
    ffff8801657abcd0: ffff8801657abf28 0000000000008241 
    ffff8801657abce0: ffff8801657abe50 ffff8801013255c0 
    ffff8801657abcf0: ffff8801610e1680 ffff8801657abda0 
    ffff8801657abd00: do_last+3085     
#15 [ffff8801657abd00] do_last at ffffffff812096ed
    ffff8801657abd08: ffffea000d264240 0000000000000000 
    ffff8801657abd18: ffff8801657abd68 ffff8801657abd90 
    ffff8801657abd28: ffff8801a642df38 [kmalloc-4096]   
    ffff8801657abd38: ffff88001f9cd000 ffff8801657abde4 
    ffff8801657abd48: ffff8801013255c0 0100000100000022 
    ffff8801657abd58: ffff8801657abdf0 00ff88001f9cd000 
    ffff8801657abd68: [kmalloc-192]    0000000051c3d868 
    ffff8801657abd78: ffff8801657abe50 ffff88001f9cd000 
    ffff8801657abd88: [kmalloc-4096]   ffff8801657abf28 
    ffff8801657abd98: ffff88003914e580 ffff8801657abe40 
    ffff8801657abda8: path_openat+194  
#16 [ffff8801657abda8] path_openat at ffffffff8120ab12
    ffff8801657abdb0: ffff8801657abe80 user_path_at_empty+114 
    ffff8801657abdc0: ffff880413bcb720 ffff8801610e1680 
    ffff8801657abdd0: 0000001553594ce5 00000041f5e18020 
    ffff8801657abde0: 0000000100000000 0000000000000000 
    ffff8801657abdf0: ffff8801a642df38 0000000200000000 
    ffff8801657abe00: 0000000000000000 00007f1068e7c000 
    ffff8801657abe10: 0000000051c3d868 00000000ffffff9c 
    ffff8801657abe20: [kmalloc-4096]   ffff8801657abf28 
    ffff8801657abe30: 0000000000000001 0000000000000023 
    ffff8801657abe40: ffff8801657abf10 do_filp_open+75  
#17 [ffff8801657abe48] do_filp_open at ffffffff8120d82b
    ffff8801657abe50: ffff880413bcb720 ffff8801013255c0 
    ffff8801657abe60: 0000001553594ce5 [kmalloc-4096]   
    ffff8801657abe70: 0000000000000000 00007ffd6cb58ad0 
    ffff8801657abe80: ffff8801a642df38 0000000200000301 
    ffff8801657abe90: 0000000000000000 0000000000000001 
    ffff8801657abea0: 00007ffd6cb58ad0 0000000000000000 
    ffff8801657abeb0: 0000000000000000 ffff8801657abf00 
    ffff8801657abec0: __alloc_fd+167   ffff880413ca91c0 
    ffff8801657abed0: 0000ffff00008241 0000000000000001 
    ffff8801657abee0: 0000000000008241 0000000051c3d868 
    ffff8801657abef0: 0000000000000001 0000000000000005 
    ffff8801657abf00: 00000000ffffff9c [kmalloc-4096]   
    ffff8801657abf10: ffff8801657abf68 do_sys_open+243  
#18 [ffff8801657abf18] do_sys_open at ffffffff811fa3a3
    ffff8801657abf20: ffff880286c5b648 ffff81b600008241 
    ffff8801657abf30: 0000030000000022 0000000051c3d868 
    ffff8801657abf40: 000000000041cf0f 0000000001aa8300 
    ffff8801657abf50: 0000000000000004 0000000000000001 
    ffff8801657abf60: 0000000000000023 ffff8801657abf78 
    ffff8801657abf70: sys_open+30      
#19 [ffff8801657abf70] sys_open at ffffffff811fa4be
    ffff8801657abf78: 00007ffd6cb58a60 system_call_fastpath+22 
#20 [ffff8801657abf80] system_call_fastpath at ffffffff81645189
    RIP: 00007f1067c3c850  RSP: 00007ffd6cb59bd8  RFLAGS: 00000246
    RAX: 0000000000000002  RBX: ffffffff81645189  RCX: ffffffffffffffff
    RDX: 00000000000001b6  RSI: 0000000000000241  RDI: 00007ffd6cb58ad0
    RBP: 00007ffd6cb58a60   R8: 000000000041cf15   R9: 0000000000000240
    R10: 0000000000000024  R11: 0000000000000246  R12: ffffffff811fa4be
    R13: ffff8801657abf78  R14: 0000000000000023  R15: 0000000000000001
    ORIG_RAX: 0000000000000002  CS: 0033  SS: 002b
crash>

I was trying to analyse the coredump and found a strange flag value at pdquot:

crash> struct xfs_mount -x ffff880413c93800
struct xfs_mount {
  m_super = 0xffff880413c93000, 
  m_tid = 0x0, 
  m_ail = 0xffff880416fc7d80, 
  m_sb = {
    sb_magicnum = 0x58465342, 
    sb_blocksize = 0x1000, 
    sb_dblocks = 0xda28800, 
    sb_rblocks = 0x0, 
    sb_rextents = 0x0, 
    sb_uuid = {
..

crash> struct xfs_mount.m_qflags -x ffff880413c93800
  m_qflags = 0x560f

crash> struct xfs_inode -x ffff8801a642dd80
struct xfs_inode {
  i_mount = 0xffff880413c93800, 
  i_udquot = 0xffff88035e460000, 
  i_gdquot = 0x0, 
  i_pdquot = 0xffff8800c235df58, 
  i_ino = 0x833d6095, 
  i_imap = {
    im_blkno = 0x3828d040, 
    im_len = 0x10, 
    im_boffset = 0x1500
  }, 
...


crash> struct xfs_dquot -x 0xffff88035e460000
struct xfs_dquot {
  dq_flags = 0x1, 
  q_lru = {
    next = 0xffff88035e460008, 
    prev = 0xffff88035e460008
  }, 
  q_mount = 0xffff880413c93800,
...

crash> struct xfs_dquot -x 0xffff8800c235df58
struct xfs_dquot {
  dq_flags = 0xc235e308,  <<--------------------------- ???????
  q_lru = {
    next = 0xffff8800c235df60, 
    prev = 0xffff8800c235df60
  }, 
  q_mount = 0xffff880413c93800,
...

The value dq_flags=0xc235e308 looks like a part of address.

Would you please help to find the root cause of the issue?

Thank you.

Comment 1 Dave Chinner 2016-06-29 22:07:11 UTC

openvz kernels are not supported by upstream developers. Please report teh problem to the openvz maintainers and close this bug.

-Dave.

Note You need to log in before you can comment on or make changes to this bug.