I disassembled modes 0/1 of the syscall to be sure (... I was a bit bored admittedly):
Code: cg_3.60 @ 0x80000000> d %2b2
# r4: &area (struct { uint32_t x1 y1 x2 y2; uint8_t mode; })
# for mode: 0=fill white, 1=fill color, 2=shade color, 4=invert
# r5: target (#1: VRAM, #2: DD, can be combined)
# r6: color
# stack> (y2) (?:u32) (?:u32) pr r14 r13 r12 r11 r10 r9 r8
<%2b2 Bdisp_AreaClr>
# Save registers, r8=x2, r9=target, r12=x1
80056d7c: 2f86 mov.l r8, @-r15
80056d7e: 2f96 mov.l r9, @-r15
80056d80: 2fa6 mov.l r10, @-r15
80056d82: 2fb6 mov.l r11, @-r15
80056d84: 2fc6 mov.l r12, @-r15
80056d86: 2fd6 mov.l r13, @-r15
80056d88: 2fe6 mov.l r14, @-r15
80056d8a: 4f22 sts.l pr, @-r15
80056d8c: 7ff4 add #-12, r15
80056d8e: 6c42 mov.l @r4, r12
80056d90: 6953 mov r5, r9
80056d92: 5242 mov.l @(8,r4), r2
# Swap r8/r12 (x1/x2) to that r8<r12
80056d94: 3c27 cmp/gt r2, r12
80056d96: 8d02 bt.s <0x80056d9e>
80056d98: 6823 mov r2, r8
80056d9a: 68c3 mov r12, r8
80056d9c: 6c23 mov r2, r12
# Swap y1/y2 so that y1<y2, then r11=y1 and r1/@r15=y2
80056d9e: 5141 mov.l @(4,r4), r1
80056da0: 5243 mov.l @(12,r4), r2
80056da2: 3127 cmp/gt r2, r1
80056da4: 8b02 bf <0x80056dac>
80056da6: 6b23 mov r2, r11
80056da8: a003 bra <0x80056db2>
80056daa: 2f12 mov.l r1, @r15
80056dac: 6b13 mov r1, r11
80056dae: 6123 mov r2, r1
80056db0: 2f22 mov.l r2, @r15
# If x1 < 0, return (no clipping is performed)
80056db2: 4811 cmp/pz r8
80056db4: 8901 bt <0x80056dba>
80056db6: a08a bra <0x80056ece>
80056db8: 0009 nop
# If x2 >= 384, also give up
80056dba: e260 mov #96, r2
80056dbc: 4208 shll2 r2
80056dbe: 3c23 cmp/ge r2, r12
80056dc0: 8b01 bf <0x80056dc6>
80056dc2: a084 bra <0x80056ece>
80056dc4: 0009 nop
# If y1 < 0, also give up
80056dc6: 4b11 cmp/pz r11
80056dc8: 8901 bt <0x80056dce>
80056dca: a080 bra <0x80056ece>
80056dcc: 0009 nop
# If y2 >= 216, also give up
80056dce: e2d8 mov #-40, r2
80056dd0: 622c extu.b r2, r2
80056dd2: 3123 cmp/ge r2, r1
80056dd4: 897b bt <0x80056ece>
# If mode=1, set r10=color, else r10=0xffff (white)
80056dd6: e010 mov #16, r0
80056dd8: 024c mov.b @(r0,r4), r2
80056dda: 622c extu.b r2, r2
80056ddc: 6023 mov r2, r0
80056dde: 8801 cmp/eq #1, r0
80056de0: 8b01 bf <0x80056de6>
80056de2: a002 bra <0x80056dea>
80056de4: 6a63 mov r6, r10
80056de6: eaff mov #-1, r10
80056de8: 6aad extu.w r10, r10
# Switch on mode
80056dea: 6023 mov r2, r0
80056dec: 8804 cmp/eq #4, r0
80056dee: 8903 bt <0x80056df8>
80056df0: 8802 cmp/eq #2, r0
80056df2: 892e bt <0x80056e52>
80056df4: a04f bra <0x80056e96>
80056df6: 0009 nop
#---
# mode=4
#---
80056df8: 679c extu.b r9, r7
80056dfa: 1f72 mov.l r7, @(8,r15)
80056dfc: 61f2 mov.l @r15, r1
80056dfe: 3b17 cmp/gt r1, r11
80056e00: 8965 bt <0x80056ece>
80056e02: a022 bra <0x80056e4a>
80056e04: 6d83 mov r8, r13
80056e06: 6093 mov r9, r0
80056e08: c902 and #2, r0
80056e0a: 6e03 mov r0, r14
80056e0c: 6093 mov r9, r0
80056e0e: c801 tst #1, r0
80056e10: 890a bt <0x80056e28>
80056e12: d14f mov.l %266 Bdisp_GetPoint_VRAM_WB, r1
80056e14: 65b3 mov r11, r5
80056e16: 410b jsr @r1
80056e18: 64d3 mov r13, r4
80056e1a: 660d extu.w r0, r6
80056e1c: d74d mov.l %263 Bdisp_SetPoint_VRAM, r7
80056e1e: 6a67 not r6, r10
80056e20: 65b3 mov r11, r5
80056e22: 64d3 mov r13, r4
80056e24: 470b jsr @r7
80056e26: 66a3 mov r10, r6
80056e28: 2ee8 tst r14, r14
80056e2a: 890d bt <0x80056e48>
80056e2c: 50f2 mov.l @(8,r15), r0
80056e2e: 8803 cmp/eq #3, r0
80056e30: 8905 bt <0x80056e3e>
80056e32: da49 mov.l %26e Bdisp_GetPoint_DD_WB, r10
80056e34: 65b3 mov r11, r5
80056e36: 4a0b jsr @r10
80056e38: 64d3 mov r13, r4
80056e3a: 6e0d extu.w r0, r14
80056e3c: 6ae7 not r14, r10
80056e3e: d247 mov.l %26b Bdisp_SetPoint_DD, r2
80056e40: 66a3 mov r10, r6
80056e42: 65b3 mov r11, r5
80056e44: 420b jsr @r2
80056e46: 64d3 mov r13, r4
80056e48: 7d01 add #1, r13
80056e4a: 3dc7 cmp/gt r12, r13
80056e4c: 8bdb bf <0x80056e06>
80056e4e: afd5 bra <0x80056dfc>
80056e50: 7b01 add #1, r11
#---
# mode=2
#---
80056e52: 666d extu.w r6, r6
80056e54: 1f61 mov.l r6, @(4,r15)
80056e56: 61f2 mov.l @r15, r1
80056e58: 3b17 cmp/gt r1, r11
80056e5a: 8938 bt <0x80056ece>
80056e5c: a017 bra <0x80056e8e>
80056e5e: 6d83 mov r8, r13
80056e60: 60d3 mov r13, r0
80056e62: 30bc add r11, r0
80056e64: c801 tst #1, r0
80056e66: 8b11 bf <0x80056e8c>
80056e68: 6093 mov r9, r0
80056e6a: c902 and #2, r0
80056e6c: 6e03 mov r0, r14
80056e6e: 6093 mov r9, r0
80056e70: c801 tst #1, r0
80056e72: 8904 bt <0x80056e7e>
80056e74: d237 mov.l %263 Bdisp_SetPoint_VRAM, r2
80056e76: 65b3 mov r11, r5
80056e78: 56f1 mov.l @(4,r15), r6
80056e7a: 420b jsr @r2
80056e7c: 64d3 mov r13, r4
80056e7e: 2ee8 tst r14, r14
80056e80: 8904 bt <0x80056e8c>
80056e82: d236 mov.l %26b Bdisp_SetPoint_DD, r2
80056e84: 65b3 mov r11, r5
80056e86: 56f1 mov.l @(4,r15), r6
80056e88: 420b jsr @r2
80056e8a: 64d3 mov r13, r4
80056e8c: 7d01 add #1, r13
80056e8e: 3dc7 cmp/gt r12, r13
80056e90: 8be6 bf <0x80056e60>
80056e92: afe0 bra <0x80056e56>
80056e94: 7b01 add #1, r11
#---
# mode=0 and mode=1
#---
# Outer loop: r11 initialized to y1, increases until y2
# If r11>y2, return
80056e96: 61f2 mov.l @r15, r1
80056e98: 3b17 cmp/gt r1, r11
80056e9a: 8918 bt <0x80056ece>
80056e9c: a013 bra <0x80056ec6>
80056e9e: 6d83 mov r8, r13
# Inner loop: r13 initialized to x1, increases until x2
# Save (target&2) in r14
80056ea0: 6093 mov r9, r0
80056ea2: c902 and #2, r0
80056ea4: 6e03 mov r0, r14
# If (target&1), set the current point in VRAM with the selected color
80056ea6: 6093 mov r9, r0
80056ea8: c801 tst #1, r0
80056eaa: 8904 bt <0x80056eb6>
80056eac: d229 mov.l %263 Bdisp_SetPoint_VRAM, r2
80056eae: 66a3 mov r10, r6
80056eb0: 65b3 mov r11, r5
80056eb2: 420b jsr @r2
80056eb4: 64d3 mov r13, r4
# If (target&2), set the same point on the DD
80056eb6: 2ee8 tst r14, r14
80056eb8: 8904 bt <0x80056ec4>
80056eba: d228 mov.l %26b Bdisp_SetPoint_DD, r2
80056ebc: 66a3 mov r10, r6
80056ebe: 65b3 mov r11, r5
80056ec0: 420b jsr @r2
80056ec2: 64d3 mov r13, r4
# End of inner loop, x1++
80056ec4: 7d01 add #1, r13
# While guard of inner loop, when r13 > x2 go back to outer loop with y1++
80056ec6: 3dc7 cmp/gt r12, r13
80056ec8: 8bea bf <0x80056ea0>
80056eca: afe4 bra <0x80056e96>
80056ecc: 7b01 add #1, r11
# Epilogue
80056ece: 7f0c add #12, r15
80056ed0: 4f26 lds.l @r15+, pr
80056ed2: 6ef6 mov.l @r15+, r14
80056ed4: 6df6 mov.l @r15+, r13
80056ed6: 6cf6 mov.l @r15+, r12
80056ed8: 6bf6 mov.l @r15+, r11
80056eda: 6af6 mov.l @r15+, r10
80056edc: 69f6 mov.l @r15+, r9
80056ede: 000b rts
80056ee0: 68f6 mov.l @r15+, r8
The color argument must be 16-bit since it's passed directly to Bdisp_SetPoint_VRAM(), so COLOR_LIGHTGRAY should work; apart from that the color is the same for both VRAM and DD.
Does it give you a white rectangle for every color? Did you make sure you're not overwriting the white afterwards?