RefPack

**RefPack** is an LZ77/LZSS compression format made by Frank Barchard of EA Canada for the Gimex library used by many older games by EA. In The Sims Online, RefPack bitstreams are held in Stream files. In Need For Speed II SE, the first game to use RefPack, RefPack data is found in Gimex (GMX) files, and in Command & Conquer 3, this data is found in BIG files. Other games may keep RefPack-compressed data in standalone files with no container.

KUDr is credited with reverse-engineering RefPack for Command & Conquer 3.

The values for RefPack are occasionally tweaked on a per-game basis (as observed in SimCity 4). The Sims Online uses the normal values defined in EA's default implementation of RefPack: it is these values that are described in this article.

## Functions

In The Sims Online New & Improved Trial version, the RefPack decompression code is duplicated verbatim across many of the game dlls. The copy used for decompressing tuning.dat and censor.dat is located at TSOServiceClientD\_base+0x724fd.

Following is a verbatim translation of TSOServiceClientD\_base+0x724fd from x86 assembly into C.

```
/\*\*
 \* @brief Decompress a RefPack bitstream
 \* @param indata - (optional) Pointer to the input RefPack bitstream; may be
 \* NULL
 \* @param bytes\_read\_out - (optional) Pointer to a size\_t which will be filled
 \* with the total number of bytes read from the RefPack bitstream; may be
 \* NULL
 \* @param outdata - Pointer to the output buffer which will be filled with the
 \* decompressed data; outdata may be NULL only if indata is also NULL
 \* @return The value of the "decompressed size" field in the RefPack bitstream,
 \* or 0 if indata is NULL
 \*
 \* This function is a verbatim translation from x86 assembly into C (with
 \* new names and comments supplied) of the RefPack decompression function
 \* located at TSOServiceClientD\_base+0x724fd in The Sims Online New & Improved
 \* Trial.
 \*
 \* This function \*\*\*does not\*\*\* perform any bounds-checking on reading or
 \* writing. It is inappropriate to use this function on untrusted data obtained
 \* from the internet (even though that is exactly what The Sims Online does...).
 \* Here are the potential problems:
 \* - This function will read past the end of indata if the last command in
 \* indata tells it to.
 \* - This function will write past the end of outdata if indata tells it to.
 \* - This function will read before the beginning of outdata if indata tells
 \* it to.
 \*/
size\_t refpack\_decompress\_unsafe(const uint8\_t \*indata, size\_t \*bytes\_read\_out,
 uint8\_t \*outdata)
{
 const uint8\_t \*in\_ptr;
 uint8\_t \*out\_ptr;
 uint16\_t signature;
 uint32\_t decompressed\_size = 0;
 uint8\_t byte\_0, byte\_1, byte\_2, byte\_3;
 uint32\_t proc\_len, ref\_len;
 uint8\_t \*ref\_ptr;
 uint32\_t i;

 in\_ptr = indata, out\_ptr = outdata;
 if (!in\_ptr)
 goto done;

 signature = ((in\_ptr[0] << 8) | in\_ptr[1]), in\_ptr += 2;
 if (signature & 0x0100)
 in\_ptr += 3; /\* skip over the compressed size field \*/

 decompressed\_size = ((in\_ptr[0] << 16) | (in\_ptr[1] << 8) | in\_ptr[2]);
 in\_ptr += 3;

 while (1) {
 byte\_0 = \*in\_ptr++;
 if (!(byte\_0 & 0x80)) {
 /\* 2-byte command: 0DDRRRPP DDDDDDDD \*/
 byte\_1 = \*in\_ptr++;

 proc\_len = byte\_0 & 0x03;
 for (i = 0; i < proc\_len; i++)
 \*out\_ptr++ = \*in\_ptr++;

 ref\_ptr = out\_ptr - ((byte\_0 & 0x60) << 3) - byte\_1 - 1;
 ref\_len = ((byte\_0 >> 2) & 0x07) + 3;
 for (i = 0; i < ref\_len; i++)
 \*out\_ptr++ = \*ref\_ptr++;
 } else if(!(byte\_0 & 0x40)) {
 /\* 3-byte command: 10RRRRRR PPDDDDDD DDDDDDDD \*/
 byte\_1 = \*in\_ptr++;
 byte\_2 = \*in\_ptr++;

 proc\_len = byte\_1 >> 6;
 for (i = 0; i < proc\_len; i++)
 \*out\_ptr++ = \*in\_ptr++;

 ref\_ptr = out\_ptr - ((byte\_1 & 0x3f) << 8) - byte\_2 - 1;
 ref\_len = (byte\_0 & 0x3f) + 4;
 for (i = 0; i < ref\_len; i++)
 \*out\_ptr++ = \*ref\_ptr++;
 } else if(!(byte\_0 & 0x20)) {
 /\* 4-byte command: 110DRRPP DDDDDDDD DDDDDDDD RRRRRRRR\*/
 byte\_1 = \*in\_ptr++;
 byte\_2 = \*in\_ptr++;
 byte\_3 = \*in\_ptr++;

 proc\_len = byte\_0 & 0x03;
 for (i = 0; i < proc\_len; i++)
 \*out\_ptr++ = \*in\_ptr++;

 ref\_ptr = out\_ptr - ((byte\_0 & 0x10) << 12)
 - (byte\_1 << 8) - byte\_2 - 1;
 ref\_len = ((byte\_0 & 0x0c) << 6) + byte\_3 + 5;
 for (i = 0; i < ref\_len; i++)
 \*out\_ptr++ = \*ref\_ptr++;
 } else {
 /\* 1-byte command: 111PPPPP \*/
 proc\_len = (byte\_0 & 0x1f) \* 4 + 4;
 if (proc\_len <= 0x70) {
 /\* no stop flag \*/
 for (i = 0; i < proc\_len; i++)
 \*out\_ptr++ = \*in\_ptr++;
 } else {
 /\* stop flag \*/
 proc\_len = byte\_0 & 0x3;
 for (i = 0; i < proc\_len; i++)
 \*out\_ptr++ = \*in\_ptr++;

 break;
 }
 }
 }

done:
 if (bytes\_read\_out)
 \*bytes\_read\_out = in\_ptr - indata;
 return decompressed\_size;
}

```

Here is a safe reimplementation of the function:

```
struct scanner {
 uint8\_t \*start;
 uint8\_t \*ptr;
 uint8\_t \*end;
 uint8\_t overflow;
};

static \_\_inline size\_t position(const struct scanner \*ctx)
{
 return (size\_t)(ctx->ptr - ctx->start);
}

static \_\_inline size\_t remaining(const struct scanner \*ctx)
{
 return (size\_t)(ctx->end - ctx->ptr);
}

static \_\_inline uint8\_t overflowed(const struct scanner \*ctx)
{
 return ctx->overflow;
}

static \_\_inline void scanner\_init(struct scanner \*ctx, uint8\_t \*start,
 size\_t size)
{
 ctx->start = ctx->ptr = start;
 ctx->end = start + size;
}

static uint8\_t read\_u8(struct scanner \*ctx)
{
 if (ctx->end == ctx->ptr) {
 ctx->overflow = 1;
 return 0;
 }
 return \*(ctx->ptr++);
}

static uint16\_t read\_u16(struct scanner \*ctx)
{
 uint16\_t x;
 if (remaining(ctx) < 2) {
 ctx->ptr = ctx->end, ctx->overflow = 1;
 return 0;
 }
 x = (ctx->ptr[0] << 8) | ctx->ptr[1];
 ctx->ptr += 2;
 return x;
}

static uint32\_t read\_u24(struct scanner \*ctx)
{
 uint32\_t x;
 if (remaining(ctx) < 3) {
 ctx->ptr = ctx->end, ctx->overflow = 1;
 return 0;
 }
 x = (ctx->ptr[0] << 16) | (ctx->ptr[1] << 8) | ctx->ptr[2];
 ctx->ptr += 3;
 return x;
}

static void append(struct scanner \*out, struct scanner \*in, size\_t length)
{
 if (!length)
 return;
 else if (length > 4)
 length = 4;

 if (remaining(in) < length || remaining(out) < length) {
 if (remaining(in) < length)
 in->ptr = in->end, in->overflow = 1;
 if (remaining(out) < length)
 out->ptr = out->end, out->overflow = 1;
 return;
 }

 memcpy(out->ptr, in->ptr, length);
 out->ptr += length, in->ptr += length;
}

static void self\_copy(struct scanner \*ctx, size\_t distance, size\_t length)
{
 size\_t i;
 uint8\_t \*in\_ptr, \*out\_ptr;

 if (position(ctx) < distance || remaining(ctx) < length) {
 ctx->ptr = ctx->end, ctx->overflow = 1;
 return;
 }

 in\_ptr = ctx->ptr - distance, out\_ptr = ctx->ptr;
 /\* neither memcpy nor memmove implement an LZ77 overlapping copy; a
 \*\* one-byte-at-a-time copy is the most efficient thing which does what
 \*\* we want \*/
 for (i = 0; i < length; i++)
 \*out\_ptr++ = \*in\_ptr++;
 ctx->ptr += length;
}

/\*\*
 \* @brief Decompress a RefPack bitstream
 \* @param indata - Pointer to the input RefPack bitstream
 \* @param insize - Size of RefPack bitstream
 \* @param bytes\_read\_out - (optional) Pointer to a size\_t which will be filled
 \* with the total number of bytes read from the RefPack bitstream; may be
 \* NULL
 \* @param outdata - Pointer to the output buffer which will be filled with the
 \* decompressed data
 \* @param outsize - Size of output buffer
 \* @param bytes\_written\_out - (optional) Pointer to a size\_t which will be
 \* filled with the total number of bytes written to the output buffer; may
 \* be NULL
 \* @param compressed\_size\_out - (optional) Pointer to a uint32\_t which will be
 \* set to the "compressed size" field in the RefPack bitstream
 \* @param decompressed\_size\_out - (optional) Pointer to a uint32\_t which will be
 \* set to the "decompressed size" field in the RefPack bitstream
 \* @return 0 on success; -1 on read overflow of the input buffer; and -2 on
 \* read/write overflow of the output buffer. As an input overflow can
 \* \*cause\* an output overflow, an input overflow is considered more
 \* significant by this function and takes precedence in the return value;
 \* thus, if both an input overflow and an output overflow occur, this
 \* function returns -1.
 \*
 \* This is a safe reimplementation of refpack\_decompress\_unsafe.
 \*/
int refpack\_decompress\_safe(const uint8\_t \*indata, size\_t insize,
 size\_t \*bytes\_read\_out, uint8\_t \*outdata, size\_t outsize,
 size\_t \*bytes\_written\_out, uint32\_t \*compressed\_size\_out,
 uint32\_t \*decompressed\_size\_out)
{
 struct scanner in, out;
 uint16\_t signature;
 uint32\_t compressed\_size, decompressed\_size;
 uint8\_t byte\_0, byte\_1, byte\_2, byte\_3;
 uint32\_t proc\_len, ref\_dis, ref\_len;

 scanner\_init(&in, (uint8\_t\*)indata, insize);
 scanner\_init(&out, outdata, outsize);

 signature = read\_u16(&in);
 compressed\_size = (signature & 0x0100) ? read\_u24(&in) : 0;
 if (compressed\_size\_out)
 \*compressed\_size\_out = compressed\_size;

 decompressed\_size = read\_u24(&in);
 if (decompressed\_size\_out)
 \*decompressed\_size\_out = decompressed\_size;

 while (!overflowed(&in) && !overflowed(&out)) {
 byte\_0 = read\_u8(&in);
 if (!(byte\_0 & 0x80)) {
 /\* 2-byte command: 0DDRRRPP DDDDDDDD \*/
 byte\_1 = read\_u8(&in);

 proc\_len = byte\_0 & 0x03;
 append(&out, &in, proc\_len);

 ref\_dis = ((byte\_0 & 0x60) << 3) + byte\_1 + 1;
 ref\_len = ((byte\_0 >> 2) & 0x07) + 3;
 self\_copy(&out, ref\_dis, ref\_len);
 } else if(!(byte\_0 & 0x40)) {
 /\* 3-byte command: 10RRRRRR PPDDDDDD DDDDDDDD \*/
 byte\_1 = read\_u8(&in);
 byte\_2 = read\_u8(&in);

 proc\_len = byte\_1 >> 6;
 append(&out, &in, proc\_len);

 ref\_dis = ((byte\_1 & 0x3f) << 8) + byte\_2 + 1;
 ref\_len = (byte\_0 & 0x3f) + 4;
 self\_copy(&out, ref\_dis, ref\_len);
 } else if(!(byte\_0 & 0x20)) {
 /\* 4-byte command: 110DRRPP DDDDDDDD DDDDDDDD RRRRRRRR\*/
 byte\_1 = read\_u8(&in);
 byte\_2 = read\_u8(&in);
 byte\_3 = read\_u8(&in);

 proc\_len = byte\_0 & 0x03;
 append(&out, &in, proc\_len);

 ref\_dis = ((byte\_0 & 0x10) << 12)
 + (byte\_1 << 8) + byte\_2 + 1;
 ref\_len = ((byte\_0 & 0x0c) << 6) + byte\_3 + 5;
 self\_copy(&out, ref\_dis, ref\_len);
 } else {
 /\* 1-byte command \*/
 proc\_len = (byte\_0 & 0x1f) \* 4 + 4;
 if (proc\_len <= 0x70) {
 /\* no stop flag \*/
 append(&out, &in, proc\_len);
 } else {
 /\* stop flag \*/
 proc\_len = byte\_0 & 0x3;
 append(&out, &in, proc\_len);

 break;
 }
 }
 }

 if (bytes\_read\_out)
 \*bytes\_read\_out = position(&in);
 if (bytes\_written\_out)
 \*bytes\_written\_out = position(&out);

 if (overflowed(&in))
 return -1;
 else if (overflowed(&out))
 return -2;
 else
 return 0;
}

```

## Bitstream specification

In this article, bytes are rendered in big-endian bit order, that is, from the most significant bit of the first byte to the least significant bit of the first byte, followed by the most significant bit of the second byte, and so on.

### Header

The RefPack header begins with a 1-byte Flags field, followed by a 1-byte magic number equal to 0xFB:

```
LU01000C

```
```
11111011

```

The large-files flag ("L") specifies that the decompressed field and (if applicable) the compressed size field are 4-byte fields; if this flag is unset, both of these fields are 3-byte fields. The unknown flag ("U") is currently unknown. The compressed-size-present flag ("C") specifies that the compressed size field is present.

Note that The Sims Online only recognizes the compressed-size-present flag; the large-files flag and the unknown flag were not introduced until SimCity 4 and Command & Conquer 3.

Afterwards, if the compressed-size-present flag is both set and recognized by the decoder, the compressed size field follows in big-endian format. It is unknown if this field starts counting from the beginning of the RefPack bitstream or from after this field, because no file in the game specifies this field, and the game appears to always ignore this field.

Afterwards, the decompressed size field follows in big-endian format.

### Commands

Following the header is a series of commands, each containing a 1- to 4-byte opcode followed by its proceeding data.

Each opcode specifies three parameters:

* **Proceeding data length** - The length of the data that follows the opcode and which is to be appended to the output buffer; this data is called the proceeding data.
* **Referenced data length** - The length of the data which is to be copied one byte at a time from earlier in the output buffer and appended to the output buffer directly after the proceeding data; this data is called the referenced data.
* **Referenced data distance** - The distance to the beginning of the referenced data from the current position in the output buffer, after the proceeding data.

Like any other LZ77 algorithm, the referenced data's source pointer may overrun into the initial value of the destination pointer. This happens when *Referenced data length* is longer than *Referenced data distance*. This is considered legal: in this case, the decoder shall copy one byte at a time, or, equivalently, the decoder shall copy and paste the first *Referenced data length* bytes of the referenced data repeatedly until the correct length is met.

The *Referenced data distance* field of any command must not be greater than the total number of bytes in the output buffer at the time of the reference copy. (If this rule is violated, the decoder in The Sims Online will break.)

Additionally, the stop command terminates the decoder after the command's proceeding data has been copied. The stop command must appear exactly once after all ordinary (non-stop) commands. (If the stop command does not appear, the decoder in The Sims Online will break.)

SimCity 4 uses a modified definition for the 4-byte opcode; it appears that this is the only game to use a modified version of RefPack.

 `D` | Referenced Data Distance bit

#### 2-byte command

```
0DDRRRPP

```
```
DDDDDDDD

```

 Referenced Data Length
 Referenced Data Distance

#### 3-byte command

```
10RRRRRR

```
```
PPDDDDDD

```
```
DDDDDDDD

```

 Referenced Data Length
 Referenced Data Distance

#### 4-byte command

```
110DRRPP

```
```
DDDDDDDD

```
```
DDDDDDDD

```
```
RRRRRRRR

```

 Referenced Data Length
 Referenced Data Distance

#### 1-byte command

```
111PPPPP

```

The 1-byte command follows one of two definitions: the ordinary 1-byte command and the stop command.

If the numerical value of the opcode byte is less than 0xFC, then the command is the ordinary 1-byte command and follows this format:

 Referenced Data Length
 Referenced Data Distance

Otherwise, if the numerical value of the opcode is greater than or equal to 0xFC, then the command is the stop command and follows this format:

 Referenced Data Length
 Referenced Data Distance

## Naming notes

The original RefPack.cpp written by Frank Barchard was released by WCNews: http://download.wcnews.com/files/documents/sourcecode/shadowforce/transfer/asommers/mfcapp\_src/engine/compress/RefPack.cpp

In The Sims Online, this file was split into refencode.cpp and refdecode.cpp, according to TSOGimexD.dll, although these files have not been leaked.