Edge Zlib
Edge Zlib is the compression algorithm used for every file compressed in their Home archives.
This section shows the only zlib-level change Sony made; the primary characteristic of Edge is the segmented/chunked storage described below rather than any major change to the compression algorithm.
Edits
- return deflateInit2_(strm, level, Z_DEFLATED, MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, version, stream_size);
+ return deflateInit2_(strm, level, Z_DEFLATED, -15, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, version, stream_size);- return inflateInit2_(strm, DEF_WBITS, version, stream_size);
+ return inflateInit2_(strm, -15, version, stream_size);In short, the diffs simply switch inflate/deflate to raw (no-header) deflate by using -15 for the window bits. Likely to reduce strain on the already small computing power of the PS3/PSP hardware.
They have also set the maximum CHUNK size to 0x4000 bytes.
Usage
The Edge prefix denotes Sony’s segmented variant: compressed data is stored as a sequence of independent chunks (up to 0xFFFF bytes each). This design lets tools decompress only a portion of a file (for example, the start or end) without processing the entire stream.
Each chunk is stored with a small header of two big-endian u16s: the uncompressed (source) size and the compressed size.
// Read header as BE bytes
let src_size: u16 = u16::from_be_bytes([data[i], data[i + 1]]);
let comp_size: u16 = u16::from_be_bytes([data[i + 2], data[i + 3]]);Sometimes, those two match. This means the data hasn’t been compressed and you can just extract the chunk directly as-is, then continue with the rest.
C Implementation of edge.c
Below is an implementation in C of two functions usable for deflating and inflating data.
#include <stdio.h>
#include <string.h>
#include <assert.h>
#include <stdlib.h>
#include "zlib.h"
#define CHUNK 0x4000
int def(const unsigned char *source, size_t source_size, unsigned char **dest, size_t *dest_size, int level) {
int ret, flush;
unsigned have;
z_stream strm;
unsigned char in[CHUNK];
unsigned char out[CHUNK];
/* allocate deflate state */
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
ret = deflateInit(&strm, level);
if (ret != Z_OK)
return ret;
size_t dest_offset = 0;
/* compress until end of input */
do {
// Copy data from source buffer to input buffer
size_t bytes_to_copy = (source_size < CHUNK) ? source_size : CHUNK;
memcpy(in, source, bytes_to_copy);
source += bytes_to_copy;
source_size -= bytes_to_copy;
strm.avail_in = bytes_to_copy;
flush = (source_size == 0) ? Z_FINISH : Z_NO_FLUSH;
strm.next_in = in;
/* run deflate() on input until output buffer not full, finish
compression if all of source has been processed */
do {
strm.avail_out = CHUNK;
strm.next_out = out;
ret = deflate(&strm, flush); /* no bad return value */
assert(ret != Z_STREAM_ERROR); /* state not clobbered */
have = CHUNK - strm.avail_out;
// Copy compressed data to the output buffer
memcpy(*dest + dest_offset, out, have);
dest_offset += have;
} while (strm.avail_out == 0);
assert(strm.avail_in == 0);
} while (flush != Z_FINISH);
// Set the actual size of the compressed data in dest_size
if (dest_size) {
*dest_size = dest_offset;
}
assert(ret == Z_STREAM_END);
/* clean up and return */
(void)deflateEnd(&strm);
return Z_OK;
}
/* Decompress from file source to file dest until stream ends or EOF.
inf() returns Z_OK on success, Z_MEM_ERROR if memory could not be
allocated for processing, Z_DATA_ERROR if the deflate data is
invalid or incomplete, Z_VERSION_ERROR if the version of zlib.h and
the version of the library linked dgo not match, or Z_ERRNO if there
is an error reading or writing the files. */
int inf(unsigned char *source, size_t source_size, unsigned char **dest, size_t dest_size)
{
int ret;
unsigned have;
z_stream strm;
unsigned char in[CHUNK];
unsigned char out[CHUNK];
// Allocate memory for the destination buffer
*dest = (unsigned char*)malloc(dest_size); // Allocate enough space for the decompressed data
dest_size = 0; // Initialize the output size
/* allocate inflate state */
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
strm.avail_in = 0;
strm.next_in = Z_NULL;
ret = inflateInit(&strm);
if (ret != Z_OK)
return ret;
/* decompress until deflate stream ends or end of file */
strm.avail_in = source_size;
strm.next_in = source;
do {
/* run inflate() on input until output buffer not full */
do {
strm.avail_out = CHUNK;
strm.next_out = out;
ret = inflate(&strm, Z_NO_FLUSH);
assert(ret != Z_STREAM_ERROR); /* state not clobbered */
switch (ret) {
case Z_NEED_DICT:
ret = Z_DATA_ERROR; /* and fall through */
case Z_DATA_ERROR:
case Z_MEM_ERROR:
(void)inflateEnd(&strm);
return ret;
}
have = CHUNK - strm.avail_out;
// Copy decompressed data to the output buffer
memcpy(*dest + dest_size, out, have);
dest_size += have;
} while (strm.avail_out == 0);
/* done when inflate() says it's done */
} while (ret != Z_STREAM_END);
/* clean up and return */
(void)inflateEnd(&strm);
return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR;
}
/* report a zlib or i/o error */
void zerr(int ret)
{
fputs("zpipe: ", stderr);
switch (ret) {
case Z_ERRNO:
if (ferror(stdin))
fputs("error reading stdin\n", stderr);
if (ferror(stdout))
fputs("error writing stdout\n", stderr);
break;
case Z_STREAM_ERROR:
fputs("invalid compression level\n", stderr);
break;
case Z_DATA_ERROR:
fputs("invalid or incomplete deflate data\n", stderr);
break;
case Z_MEM_ERROR:
fputs("out of memory\n", stderr);
break;
case Z_VERSION_ERROR:
fputs("zlib version mismatch!\n", stderr);
}
}
// take input buffer and input buffer size
#if __cplusplus
extern "C" {
#endif
ZEXTERN int ZEXPORT inflate_noheader(unsigned char *source, size_t source_size, unsigned char **dest, size_t dest_size)
{
return inf(source, source_size, dest, dest_size);
}
ZEXTERN int ZEXPORT deflate_noheader(const unsigned char *source, size_t source_size, unsigned char **dest, size_t *dest_size)
{
return def(source, source_size, dest, dest_size, 9);
}
#if __cplusplus
}
#endif