Hi all,

To the ext3 maintainer, I got a new crash on this box today. This time it
seems to be due to the /var/idp file system being full... I noticed the
partition was not full yesterday when I  encountered the previous crash. See
details below.

Any help appreciated,
Regards,
__
Eric

# df -h
Filesystem            Size  Used Avail Use% Mounted on
/dev/sda1             2.1G  329M  1.7G  16% /
/dev/sda2             2.1G  345M  1.6G  17% /tmp
/dev/sda5             2.1G   64M  1.9G   4% /var
/dev/sda6              23G   22G   83M 100% /var/idp 
# ksymoops -m /boot/System.map-2.4.22-9P3idpSMP /tmp/oops.txt
ksymoops 2.4.9 on i686 2.4.22-9P3idpSMP.  Options used
     -V (default)
     -k /proc/ksyms (default)
     -l /proc/modules (default)
     -o /lib/modules/2.4.22-9P3idpSMP/ (default)
     -m /boot/System.map-2.4.22-9P3idpSMP (specified)

kernel BUG at transaction.c:1416!] [ave: 0]
invalid operand: 0000
CPU:    0Packets    Flows      Sessions   Peak       Peak Time
EIP:    0010:[<4017659d>]    Tainted: PF  2          04/28/2004 11:52:59
Using defaults from ksymoops -t elf32-i386 -a i386
EFLAGS: 00010202    4          2          2          04/27/2004 10:20:54
eax: 00000063   ebx: 4029c066   ecx: 47ad2000   edx: 7f6aff7c04 11:42:39
Warning (Oops_set_regs): garbage '11:42:39' at end of register line ignored
esi: 717636c0   edi: 41cb07c0   ebp: 47ad3bbc   esp: 47ad3b9404 11:52:58
Warning (Oops_set_regs): garbage '11:52:58' at end of register line ignored
ds: 0018   es: 0018   ss: 0018
Process dLogPurger (pid: 2871, stackpage=47ad3000)
Stack: 4029de80 4029c066 4029bea6 00000588 4029ff00 00000002 41c8f000
00000000
       41cb07c0 7fdd0ca0 47ad3bdc 4016f391 41cb07c0 7fdd0ca0 00000000
00000001
       7fdd0ca0 41c87000 47ad3bf4 40157a3a 7fdd0ca0 00000001 47ad2000
40174b60
Call Trace:    [<4016f391>] [<40157a3a>] [<40174b60>] [<40158cbb>]
[<4014ca2f>]
  [<4014ce3a>] [<40149be1>] [<40149cfa>] [<40160400>] [<4023ca4c>]
[<4023ce55>]
  [<4023d105>] [<4010b553>] [<402852c1>] [<4015f810>] [<4014a7c3>]
[<4014a9a3>]
  [<40107d2f>] [<4010961f>]
Code: 0f 0b 88 05 a6 be 29 40 0f b6 57 18 f6 c2 04 75 0d 8b 07 8b


>>EIP; 4017659d <journal_stop+5d/200>   <=====

>>ebx; 4029c066 <large_digits.1+ad06/2f900>
>>ecx; 47ad2000 <_end+76d30ec/4051a0ec>
>>esi; 717636c0 <_end+313647ac/4051a0ec>
>>edi; 41cb07c0 <_end+18b18ac/4051a0ec>
>>ebp; 47ad3bbc <_end+76d4ca8/4051a0ec>

Trace; 4016f391 <ext3_dirty_inode+a1/140>
Trace; 40157a3a <__mark_inode_dirty+ba/c0>
Trace; 40174b60 <ext3_follow_link+0/20>
Trace; 40158cbb <update_atime+6b/70>
Trace; 4014ca2f <link_path_walk+4ef/700>
Trace; 4014ce3a <path_lookup+3a/40>
Trace; 40149be1 <open_exec+21/e0>
Trace; 40149cfa <kernel_read+5a/70>
Trace; 40160400 <load_elf_binary+bf0/ca0>
Trace; 4023ca4c <netif_rx+8c/1b0>
Trace; 4023ce55 <netif_receive_skb+d5/1a0>
Trace; 4023d105 <net_rx_action+b5/170>
Trace; 4010b553 <do_IRQ+e3/f0>
Trace; 402852c1 <strnlen_user+31/4c>
Trace; 4015f810 <load_elf_binary+0/ca0>
Trace; 4014a7c3 <search_binary_handler+133/1d0>
Trace; 4014a9a3 <do_execve+143/1d0>
Trace; 40107d2f <sys_execve+3f/70>
Trace; 4010961f <system_call+33/38>

Code;  4017659d <journal_stop+5d/200>
00000000 <_EIP>:
Code;  4017659d <journal_stop+5d/200>   <=====
   0:   0f 0b                     ud2a      <=====
Code;  4017659f <journal_stop+5f/200>
   2:   88 05 a6 be 29 40         mov    %al,0x4029bea6
Code;  401765a5 <journal_stop+65/200>
   8:   0f b6 57 18               movzbl 0x18(%edi),%edx
Code;  401765a9 <journal_stop+69/200>
   c:   f6 c2 04                  test   $0x4,%dl
Code;  401765ac <journal_stop+6c/200>
   f:   75 0d                     jne    1e <_EIP+0x1e> 401765bb
<journal_stop+7b/200>
Code;  401765ae <journal_stop+6e/200>
  11:   8b 07                     mov    (%edi),%eax
Code;  401765b0 <journal_stop+70/200>
  13:   8b 00                     mov    (%eax),%eax

dump: Dump: Rebooting in 5 seconds ...NMI Watchdog detected LOCKUP on CPU0,
eip 40284f86, registers:
CPU:    0
EIP:    0010:[<40284f86>]    Tainted: PF
EFLAGS: 00000012
eax: 0000e952   ebx: 000f1874   ecx: af4bf9df   edx: 00006151
esi: 40292a7f   edi: 4010a0f0   ebp: 47ad3a60   esp: 47ad3a5c
ds: 0018   es: 0018   ss: 0018
Process dLogPurger (pid: 2871, stackpage=47ad3000)
Stack: 00000017 47ad3a74 40228fad 000f1874 00000005 00000001 47ad3a90
40229a42
       40366a88 00000002 4038c100 00000000 47ad3b60 47ad3aac 40109f8b
40292a7f
       47ad3b60 00000000 47ad2000 00000000 47ad3b50 4010a14d 40292a7f
47ad3b60
Call Trace:    [<40228fad>] [<40229a42>] [<40109f8b>] [<4010a14d>]
[<4017659d>]
  [<401b17f1>] [<40109710>] [<4017659d>] [<4016f391>] [<40157a3a>]
[<40174b60>]
  [<40158cbb>] [<4014ca2f>] [<4014ce3a>] [<40149be1>] [<40149cfa>]
[<40160400>]
  [<4023ca4c>] [<4023ce55>] [<4023d105>] [<4010b553>] [<402852c1>]
[<4015f810>]
  [<4014a7c3>] [<4014a9a3>] [<40107d2f>] [<4010961f>]
Code: 39 d8 72 f6 5b 5d c3 8d 76 00 55 89 e5 8b 45 08 eb 08 90 8d


>>EIP; 40284f86 <__rdtsc_delay+16/20>   <=====

>>esi; 40292a7f <large_digits.1+171f/2f900>
>>edi; 4010a0f0 <do_invalid_op+0/70>
>>ebp; 47ad3a60 <_end+76d4b4c/4051a0ec>
>>esp; 47ad3a5c <_end+76d4b48/4051a0ec>

Trace; 40228fad <dump_resume_system+cd/e0>
Trace; 40229a42 <dump_execute+82/1f0>
Trace; 40109f8b <die+9b/a0>
Trace; 4010a14d <do_invalid_op+5d/70>
Trace; 4017659d <journal_stop+5d/200>
Trace; 401b17f1 <serial_console_write+121/220>
Trace; 40109710 <error_code+34/3c>
Trace; 4017659d <journal_stop+5d/200>
Trace; 4016f391 <ext3_dirty_inode+a1/140>
Trace; 40157a3a <__mark_inode_dirty+ba/c0>
Trace; 40174b60 <ext3_follow_link+0/20>
Trace; 40158cbb <update_atime+6b/70>
Trace; 4014ca2f <link_path_walk+4ef/700>
Trace; 4014ce3a <path_lookup+3a/40>
Trace; 40149be1 <open_exec+21/e0>
Trace; 40149cfa <kernel_read+5a/70>
Trace; 40160400 <load_elf_binary+bf0/ca0>
Trace; 4023ca4c <netif_rx+8c/1b0>
Trace; 4023ce55 <netif_receive_skb+d5/1a0>
Trace; 4023d105 <net_rx_action+b5/170>
Trace; 4010b553 <do_IRQ+e3/f0>
Trace; 402852c1 <strnlen_user+31/4c>
Trace; 4015f810 <load_elf_binary+0/ca0>
Trace; 4014a7c3 <search_binary_handler+133/1d0>
Trace; 4014a9a3 <do_execve+143/1d0>
Trace; 40107d2f <sys_execve+3f/70>
Trace; 4010961f <system_call+33/38>

Code;  40284f86 <__rdtsc_delay+16/20>
00000000 <_EIP>:
Code;  40284f86 <__rdtsc_delay+16/20>   <=====
   0:   39 d8                     cmp    %ebx,%eax   <=====
Code;  40284f88 <__rdtsc_delay+18/20>
   2:   72 f6                     jb     fffffffa <_EIP+0xfffffffa> 40284f80
<__rdtsc_delay+10/20>
Code;  40284f8a <__rdtsc_delay+1a/20>
   4:   5b                        pop    %ebx
Code;  40284f8b <__rdtsc_delay+1b/20>
   5:   5d                        pop    %ebp
Code;  40284f8c <__rdtsc_delay+1c/20>
   6:   c3                        ret
Code;  40284f8d <__rdtsc_delay+1d/20>
   7:   8d 76 00                  lea    0x0(%esi),%esi
Code;  40284f90 <__loop_delay+0/30>
   a:   55                        push   %ebp
Code;  40284f91 <__loop_delay+1/30>
   b:   89 e5                     mov    %esp,%ebp
Code;  40284f93 <__loop_delay+3/30>
   d:   8b 45 08                  mov    0x8(%ebp),%eax
Code;  40284f96 <__loop_delay+6/30>
  10:   eb 08                     jmp    1a <_EIP+0x1a> 40284fa0
<__loop_delay+10/30>
Code;  40284f98 <__loop_delay+8/30>
  12:   90                        nop
Code;  40284f99 <__loop_delay+9/30>
  13:   8d 00                     lea    (%eax),%eax


2 warnings issued.  Results may not be reliable.


-----Original Message-----
From: Eric Moret 
Sent: Thursday, April 29, 2004 6:10 PM
To: linux-kernel@vger.kernel.org
Subject: Ext3 kernel crash on 2.4.22

Hi,

Please CC me as I only read lkml occasionally.

I have a crash similar to the one described in a previous post at
http://testing.lkml.org/slashdot.php?mid=326628 unfortunately I have not
seen a resolution on this issue... 

Hardware detail:
Dell PowerEdge 1550 PIII 1GHz, single CPU running SMP kernel 1Gb RAM Adaptec
aic7899 Ultra160 SCSI adapter 18Gb Hard drive SEAGATE ST318395LC

__
Eric


---- OOPS MESSAGE decoded with ksymoops

ksymoops 2.4.9 on i686 2.4.22-9P3idpSMP.  Options used
     -V (default)
     -k /proc/ksyms (default)
     -l /proc/modules (default)
     -o /lib/modules/2.4.22-9P3idpSMP/ (default)
     -m /boot/System.map-2.4.22-9P3idpSMP (specified)

kernel BUG at transaction.c:1416!] [ave: 0] invalid operand: 0000
CPU:    0Packets    Flows      Sessions   Peak       Peak Time
EIP:    0010:[<4017659d>]    Tainted: PF  2          04/28/2004 11:52:59
Using defaults from ksymoops -t elf32-i386 -a i386
EFLAGS: 00010202    4          2          2          04/27/2004 10:20:54
eax: 00000063   ebx: 4029c066   ecx: 47ad2000   edx: 7f6aff7c04 11:42:39
Warning (Oops_set_regs): garbage '11:42:39' at end of register line ignored
esi: 717636c0   edi: 41cb07c0   ebp: 47ad3bbc   esp: 47ad3b9404 11:52:58
Warning (Oops_set_regs): garbage '11:52:58' at end of register line ignored
ds: 0018   es: 0018   ss: 0018
Process dLogPurger (pid: 2871, stackpage=47ad3000)
Stack: 4029de80 4029c066 4029bea6 00000588 4029ff00 00000002 41c8f000
00000000
       41cb07c0 7fdd0ca0 47ad3bdc 4016f391 41cb07c0 7fdd0ca0 00000000
00000001
       7fdd0ca0 41c87000 47ad3bf4 40157a3a 7fdd0ca0 00000001 47ad2000
40174b60
Call Trace:    [<4016f391>] [<40157a3a>] [<40174b60>] [<40158cbb>]
[<4014ca2f>]
  [<4014ce3a>] [<40149be1>] [<40149cfa>] [<40160400>] [<4023ca4c>]
[<4023ce55>]
  [<4023d105>] [<4010b553>] [<402852c1>] [<4015f810>] [<4014a7c3>]
[<4014a9a3>]
  [<40107d2f>] [<4010961f>]
Code: 0f 0b 88 05 a6 be 29 40 0f b6 57 18 f6 c2 04 75 0d 8b 07 8b


>>EIP; 4017659d <journal_stop+5d/200>   <=====

>>ebx; 4029c066 <large_digits.1+ad06/2f900> ecx; 47ad2000 
>><_end+76d30ec/4050e0ec> esi; 717636c0 <_end+313647ac/4050e0ec> edi; 
>>41cb07c0 <_end+18b18ac/4050e0ec> ebp; 47ad3bbc <_end+76d4ca8/4050e0ec>

Trace; 4016f391 <ext3_dirty_inode+a1/140> Trace; 40157a3a
<__mark_inode_dirty+ba/c0> Trace; 40174b60 <ext3_follow_link+0/20> Trace;
40158cbb <update_atime+6b/70> Trace; 4014ca2f <link_path_walk+4ef/700>
Trace; 4014ce3a <path_lookup+3a/40> Trace; 40149be1 <open_exec+21/e0> Trace;
40149cfa <kernel_read+5a/70> Trace; 40160400 <load_elf_binary+bf0/ca0>
Trace; 4023ca4c <netif_rx+8c/1b0> Trace; 4023ce55 <netif_receive_skb+d5/1a0>
Trace; 4023d105 <net_rx_action+b5/170> Trace; 4010b553 <do_IRQ+e3/f0> Trace;
402852c1 <strnlen_user+31/4c> Trace; 4015f810 <load_elf_binary+0/ca0> Trace;
4014a7c3 <search_binary_handler+133/1d0> Trace; 4014a9a3 <do_execve+143/1d0>
Trace; 40107d2f <sys_execve+3f/70> Trace; 4010961f <system_call+33/38>

Code;  4017659d <journal_stop+5d/200>
00000000 <_EIP>:
Code;  4017659d <journal_stop+5d/200>   <=====
   0:   0f 0b                     ud2a      <=====
Code;  4017659f <journal_stop+5f/200>
   2:   88 05 a6 be 29 40         mov    %al,0x4029bea6
Code;  401765a5 <journal_stop+65/200>
   8:   0f b6 57 18               movzbl 0x18(%edi),%edx
Code;  401765a9 <journal_stop+69/200>
   c:   f6 c2 04                  test   $0x4,%dl
Code;  401765ac <journal_stop+6c/200>
   f:   75 0d                     jne    1e <_EIP+0x1e> 401765bb
<journal_stop+7b/200>
Code;  401765ae <journal_stop+6e/200>
  11:   8b 07                     mov    (%edi),%eax
Code;  401765b0 <journal_stop+70/200>
  13:   8b 00                     mov    (%eax),%eax

dump: Dump: Rebooting in 5 seconds ...NMI Watchdog detected LOCKUP on CPU0,
eip 40284f86, registers:
CPU:    0
EIP:    0010:[<40284f86>]    Tainted: PF
EFLAGS: 00000012
eax: 0000e952   ebx: 000f1874   ecx: af4bf9df   edx: 00006151
esi: 40292a7f   edi: 4010a0f0   ebp: 47ad3a60   esp: 47ad3a5c
ds: 0018   es: 0018   ss: 0018
Process dLogPurger (pid: 2871, stackpage=47ad3000)
Stack: 00000017 47ad3a74 40228fad 000f1874 00000005 00000001 47ad3a90
40229a42
       40366a88 00000002 4038c100 00000000 47ad3b60 47ad3aac 40109f8b
40292a7f
       47ad3b60 00000000 47ad2000 00000000 47ad3b50 4010a14d 40292a7f
47ad3b60
Call Trace:    [<40228fad>] [<40229a42>] [<40109f8b>] [<4010a14d>]
[<4017659d>]
  [<401b17f1>] [<40109710>] [<4017659d>] [<4016f391>] [<40157a3a>]
[<40174b60>]
  [<40158cbb>] [<4014ca2f>] [<4014ce3a>] [<40149be1>] [<40149cfa>]
[<40160400>]
  [<4023ca4c>] [<4023ce55>] [<4023d105>] [<4010b553>] [<402852c1>]
[<4015f810>]
  [<4014a7c3>] [<4014a9a3>] [<40107d2f>] [<4010961f>]
Code: 39 d8 72 f6 5b 5d c3 8d 76 00 55 89 e5 8b 45 08 eb 08 90 8d


>>EIP; 40284f86 <__rdtsc_delay+16/20>   <=====

>>esi; 40292a7f <large_digits.1+171f/2f900> edi; 4010a0f0 
>><do_invalid_op+0/70> ebp; 47ad3a60 <_end+76d4b4c/4050e0ec> esp; 
>>47ad3a5c <_end+76d4b48/4050e0ec>

Trace; 40228fad <dump_resume_system+cd/e0> Trace; 40229a42
<dump_execute+82/1f0> Trace; 40109f8b <die+9b/a0> Trace; 4010a14d
<do_invalid_op+5d/70> Trace; 4017659d <journal_stop+5d/200> Trace; 401b17f1
<serial_console_write+121/220> Trace; 40109710 <error_code+34/3c> Trace;
4017659d <journal_stop+5d/200> Trace; 4016f391 <ext3_dirty_inode+a1/140>
Trace; 40157a3a <__mark_inode_dirty+ba/c0> Trace; 40174b60
<ext3_follow_link+0/20> Trace; 40158cbb <update_atime+6b/70> Trace; 4014ca2f
<link_path_walk+4ef/700> Trace; 4014ce3a <path_lookup+3a/40> Trace; 40149be1
<open_exec+21/e0> Trace; 40149cfa <kernel_read+5a/70> Trace; 40160400
<load_elf_binary+bf0/ca0> Trace; 4023ca4c <netif_rx+8c/1b0> Trace; 4023ce55
<netif_receive_skb+d5/1a0> Trace; 4023d105 <net_rx_action+b5/170> Trace;
4010b553 <do_IRQ+e3/f0> Trace; 402852c1 <strnlen_user+31/4c> Trace; 4015f810
<load_elf_binary+0/ca0> Trace; 4014a7c3 <search_binary_handler+133/1d0>
Trace; 4014a9a3 <do_execve+143/1d0> Trace; 40107d2f <sys_execve+3f/70>
Trace; 4010961f <system_call+33/38>

Code;  40284f86 <__rdtsc_delay+16/20>
00000000 <_EIP>:
Code;  40284f86 <__rdtsc_delay+16/20>   <=====
   0:   39 d8                     cmp    %ebx,%eax   <=====
Code;  40284f88 <__rdtsc_delay+18/20>
   2:   72 f6                     jb     fffffffa <_EIP+0xfffffffa> 40284f80
<__rdtsc_delay+10/20>
Code;  40284f8a <__rdtsc_delay+1a/20>
   4:   5b                        pop    %ebx
Code;  40284f8b <__rdtsc_delay+1b/20>
   5:   5d                        pop    %ebp
Code;  40284f8c <__rdtsc_delay+1c/20>
   6:   c3                        ret
Code;  40284f8d <__rdtsc_delay+1d/20>
   7:   8d 76 00                  lea    0x0(%esi),%esi
Code;  40284f90 <__loop_delay+0/30>
   a:   55                        push   %ebp
Code;  40284f91 <__loop_delay+1/30>
   b:   89 e5                     mov    %esp,%ebp
Code;  40284f93 <__loop_delay+3/30>
   d:   8b 45 08                  mov    0x8(%ebp),%eax
Code;  40284f96 <__loop_delay+6/30>
  10:   eb 08                     jmp    1a <_EIP+0x1a> 40284fa0
<__loop_delay+10/30>
Code;  40284f98 <__loop_delay+8/30>
  12:   90                        nop
Code;  40284f99 <__loop_delay+9/30>
  13:   8d 00                     lea    (%eax),%eax


2 warnings issued.  Results may not be reliable.