dynamobi/dynamobi — Issue #830: [FRG-40] Java calculator has inconsistent behavior on decimal overflow
Author: dynamobi-build
Created: 2011-12-30 03:51:28+00:00
State: open
Labels: FRG

[reporter="schoi", created="Tue, 14 Feb 2006 23:01:19 -0500 (GMT-05:00)"]
Given a table and a column in that table of type decimal (P,1). Trying to insert value (P+1):

Fennel calculator will consistently output an error message.

Java calculator will sometimes output an error message, and sometimes overflow.

Repro:

----------

create schema s;

set schema 's';

-- using Fennel calculator

-----------------------------------------

alter system set "calcVirtualMachine"='CALCVM_FENNEL';

create table t1(d decimal(10,1) primary key);

insert into t1 values(123456789);

insert into t1 values(1234567890);

-- Error: could not calculate results for the following row:

-- [ 0 ]

-- Messages:

-- [0]:PC=5 Code=22003 (state=,code=0)

-- using Java calculator

-------------------------------------

alter system set "calcVirtualMachine"='CALCVM_JAVA';

create table t2(d decimal(9,1) primary key);

insert into t2 values(12345678);

insert into t2 values(123456789);

-- Error: Overflow during calculation or cast. (state=,code=0)

select * from t2;

--

-- +-------------+

-- | D |

-- +-------------+

-- | 12345678.0 |

-- +-------------+

--

create table t3(d decimal(10,1) primary key);

insert into t3 values(123456789);

insert into t3 values(1234567890);

select * from t3;

--

-- +--------------+

-- | D |

-- +--------------+

-- | -53922298.8 |

-- | 123456789.0 |

-- +--------------+

--