use old fuzziqer prs implementation to fix some .qst file issues

using the current libsylverant prs implementation to re-compress the
.bin file after setting the download flag in it MAY result in a .qst
file which when loaded by the gamecube client, will cause it to lock-up
with a black screen

simply switching to this older fuzziqer prs implementation fixed this
issue
This commit is contained in:
Gered 2021-03-21 19:29:17 -04:00
parent 588a7b45f1
commit 2e6d446b1c
4 changed files with 476 additions and 8 deletions

View file

@ -20,5 +20,5 @@ target_compile_definitions(gen_qst_header PRIVATE ICONV_CONST=${ICONV_CONST})
target_include_directories(gen_qst_header PRIVATE ${ICONV_INCLUDE_DIR})
# bindat_to_gcdl
add_executable(bindat_to_gcdl bindat_to_gcdl.c quests.c utils.c)
add_executable(bindat_to_gcdl bindat_to_gcdl.c quests.c fuzziqer_prs.c utils.c)
target_link_libraries(bindat_to_gcdl ${SYLVERANT_LIBRARY})

View file

@ -6,11 +6,11 @@
#include <malloc.h>
#include <sylverant/encryption.h>
#include <sylverant/prs.h>
//#include <sylverant/prs.h>
#include "fuzziqer_prs.h"
#include "quests.h"
#include "utils.h"
#include "retvals.h"
typedef struct __attribute__((packed)) {
uint32_t decompressed_size;
@ -64,14 +64,14 @@ int main(int argc, char *argv[]) {
uint8_t *decompressed_bin;
uint32_t decompressed_bin_size, decompressed_dat_size;
result = prs_decompress_buf(compressed_bin, &decompressed_bin, compressed_bin_size);
result = fuzziqer_prs_decompress_buf(compressed_bin, &decompressed_bin, compressed_bin_size);
if (result < 0) {
printf("prs_decompress_buf() error %d with bin file data: %s\n", result, bin_filename);
return 1;
}
decompressed_bin_size = (uint32_t)result;
result = prs_decompress_size(compressed_dat, compressed_dat_size);
result = fuzziqer_prs_decompress_size(compressed_dat, compressed_dat_size);
if (result < 0) {
printf("prs_decompress_size() error %d with dat file data: %s\n", result, dat_filename);
return 1;
@ -82,8 +82,6 @@ int main(int argc, char *argv[]) {
/** parse quest .bin header from decompressed .bin file data. also set the "download" flag in the .bin header **/
QUEST_BIN_HEADER *bin_header = (QUEST_BIN_HEADER*)decompressed_bin;
bin_header->download = 1;
printf("Quest: id=%d, download=%d, language=0x%02x, name=%s\n", bin_header->quest_number, bin_header->download, bin_header->language, bin_header->name);
// TODO: validations might need tweaking ...
if (bin_header->object_code_offset != 468) {
@ -103,11 +101,16 @@ int main(int argc, char *argv[]) {
return 1;
}
bin_header->download = 1; // gamecube pso client will not find quests on a memory card if this is not set!
printf("Quest: id=%d, download=%d, language=0x%02x, name=%s\n", bin_header->quest_number, bin_header->download, bin_header->language, bin_header->name);
/** re-compress bin data, so it includes our modified header "download" flag **/
uint8_t *recompressed_bin;
result = prs_compress(decompressed_bin, &recompressed_bin, decompressed_bin_size);
result = fuzziqer_prs_compress(decompressed_bin, &recompressed_bin, decompressed_bin_size);
if (result < 0) {
printf("prs_compress() error %d with modified bin file data: %s\n", result, bin_filename);
return 1;
@ -177,6 +180,11 @@ int main(int argc, char *argv[]) {
uint8_t bin_counter = 0, dat_counter = 0;
QST_DATA_CHUNK chunk;
// note: .qst files actually do NOT need to be interleaved like this to work with the gamecube pso client. the
// khyller server did not do this. it is possible that some .qst file tools (qedit?) expect it though? so, meh,
// we'll just do it here because it's easy enough. also worth mentioning that khyller also put the .dat file data
// first. so the order seems unimportant too ... ?
while (!bin_done || !dat_done) {
if (!bin_done) {
uint32_t size = (final_bin_size - bin_pos >= 1024) ? 1024 : (final_bin_size - bin_pos);
@ -205,6 +213,9 @@ int main(int argc, char *argv[]) {
fclose(fp);
free(decompressed_bin);
free(final_bin);
free(final_dat);
free(compressed_bin);
free(compressed_dat);

447
fuzziqer_prs.c Normal file
View file

@ -0,0 +1,447 @@
/*
* This implementation of PRS compression/decompression comes from here:
* https://github.com/Sylverant/libsylverant/blob/67074b719e5b52e6cf55898578e40c2dbccc0839/src/utils/prs.c
*
* The reason it is being brought back out into use instead of using
* libsylverant's current PRS compression/decompression implementation (which is
* by far the cleanest one out there in my opinion) is due to apparent
* incompatibilities when used to generate Gamecube download quest .qst files.
*
* When using libsylverant's implementation, some generated .qst files work fine.
* But some others were resulting in black-screen crashes when loaded from the
* memory card. As soon as I switched to this older PRS implementation to
* generate the same .qst file (using the same source .bin/.dat files in both
* cases, obviously) the black-screen crashes disappeared.
*
* The externally accessible functions, prefixed "fuzziqer_" so as not to
* conflict with libsylverant (which I am still using for encryption), are just
* libsylverant-API-compatible wrappers over the original "prs_" functions found
* in this source file. This will make it easier to switch away from this older
* implementation in the future if a fix is found for these apparent
* incompatibilities.
*
* March 2021, Gered King. Original header comment from 2011 follows.
*/
/*
The PRS compressor/decompressor that this file implements to was originally
written by Fuzziqer Software. The file was distributed with the message that
it could be used in anything/for any purpose as long as credit was given. I
have incorporated it into libsylverant for use with the Sylverant PSO server
and related utilities.
Other than minor changes (making it compile cleanly as C) this file has been
left relatively intact from its original distribution, which was obtained on
June 21st, 2009 from http://www.fuzziqersoftware.com/files/prsutil.zip
Modified June 30, 2011 by Lawrence Sebald:
Make the code work properly when compiled for a 64-bit target.
*/
#include <stdint.h>
#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <malloc.h>
#include "fuzziqer_prs.h"
////////////////////////////////////////////////////////////////////////////////
typedef struct {
uint8_t bitpos;
uint8_t *controlbyteptr;
uint8_t *srcptr_orig;
uint8_t *dstptr_orig;
uint8_t *srcptr;
uint8_t *dstptr;
} PRS_COMPRESSOR;
static void prs_put_control_bit(PRS_COMPRESSOR *pc, uint8_t bit) {
*pc->controlbyteptr = *pc->controlbyteptr >> 1;
*pc->controlbyteptr |= ((!!bit) << 7);
pc->bitpos++;
if (pc->bitpos >= 8) {
pc->bitpos = 0;
pc->controlbyteptr = pc->dstptr;
pc->dstptr++;
}
}
static void prs_put_control_bit_nosave(PRS_COMPRESSOR *pc, uint8_t bit) {
*pc->controlbyteptr = *pc->controlbyteptr >> 1;
*pc->controlbyteptr |= ((!!bit) << 7);
pc->bitpos++;
}
static void prs_put_control_save(PRS_COMPRESSOR *pc) {
if (pc->bitpos >= 8) {
pc->bitpos = 0;
pc->controlbyteptr = pc->dstptr;
pc->dstptr++;
}
}
static void prs_put_static_data(PRS_COMPRESSOR *pc, uint8_t data) {
*pc->dstptr = data;
pc->dstptr++;
}
static uint8_t prs_get_static_data(PRS_COMPRESSOR *pc) {
uint8_t data = *pc->srcptr;
pc->srcptr++;
return data;
}
////////////////////////////////////////////////////////////////////////////////
static void prs_init(PRS_COMPRESSOR *pc, const void *src, void *dst) {
pc->bitpos = 0;
pc->srcptr = (uint8_t *) src;
pc->srcptr_orig = (uint8_t *) src;
pc->dstptr = (uint8_t *) dst;
pc->dstptr_orig = (uint8_t *) dst;
pc->controlbyteptr = pc->dstptr;
pc->dstptr++;
}
static void prs_finish(PRS_COMPRESSOR *pc) {
prs_put_control_bit(pc, 0);
prs_put_control_bit(pc, 1);
if (pc->bitpos != 0) {
*pc->controlbyteptr = ((*pc->controlbyteptr << pc->bitpos) >> 8);
}
prs_put_static_data(pc, 0);
prs_put_static_data(pc, 0);
}
static void prs_rawbyte(PRS_COMPRESSOR *pc) {
prs_put_control_bit_nosave(pc, 1);
prs_put_static_data(pc, prs_get_static_data(pc));
prs_put_control_save(pc);
}
static void prs_shortcopy(PRS_COMPRESSOR *pc, int offset, uint8_t size) {
size -= 2;
prs_put_control_bit(pc, 0);
prs_put_control_bit(pc, 0);
prs_put_control_bit(pc, (size >> 1) & 1);
prs_put_control_bit_nosave(pc, size & 1);
prs_put_static_data(pc, offset & 0xFF);
prs_put_control_save(pc);
}
static void prs_longcopy(PRS_COMPRESSOR *pc, int offset, uint8_t size) {
uint8_t byte1, byte2;
if (size <= 9) {
prs_put_control_bit(pc, 0);
prs_put_control_bit_nosave(pc, 1);
prs_put_static_data(pc, ((offset << 3) & 0xF8) | ((size - 2) & 0x07));
prs_put_static_data(pc, (offset >> 5) & 0xFF);
prs_put_control_save(pc);
} else {
prs_put_control_bit(pc, 0);
prs_put_control_bit_nosave(pc, 1);
prs_put_static_data(pc, (offset << 3) & 0xF8);
prs_put_static_data(pc, (offset >> 5) & 0xFF);
prs_put_static_data(pc, size - 1);
prs_put_control_save(pc);
}
}
static void prs_copy(PRS_COMPRESSOR *pc, int offset, uint8_t size) {
if ((offset > -0x100) && (size <= 5)) {
prs_shortcopy(pc, offset, size);
} else {
prs_longcopy(pc, offset, size);
}
pc->srcptr += size;
}
////////////////////////////////////////////////////////////////////////////////
static uint32_t prs_compress(const void *source, void *dest, uint32_t size) {
PRS_COMPRESSOR pc;
int x, y, z;
uint32_t xsize;
int lsoffset, lssize;
uint8_t *src = (uint8_t *) source, *dst = (uint8_t *) dest;
prs_init(&pc, source, dest);
for (x = 0; x < size; x++) {
lsoffset = lssize = xsize = 0;
for (y = x - 3; (y > 0) && (y > (x - 0x1FF0)) && (xsize < 255); y--) {
xsize = 3;
if (!memcmp(src + y, src + x, xsize)) {
do xsize++;
while (!memcmp(src + y, src + x, xsize) &&
(xsize < 256) &&
((y + xsize) < x) &&
((x + xsize) <= size)
);
xsize--;
if (xsize > lssize) {
lsoffset = -(x - y);
lssize = xsize;
}
}
}
if (lssize == 0) {
prs_rawbyte(&pc);
} else {
prs_copy(&pc, lsoffset, lssize);
x += (lssize - 1);
}
}
prs_finish(&pc);
return pc.dstptr - pc.dstptr_orig;
}
////////////////////////////////////////////////////////////////////////////////
static uint32_t prs_decompress(const void *source, void *dest) // 800F7CB0 through 800F7DE4 in mem
{
uint32_t r0, r3, r6, r9; // 6 unnamed registers
uint32_t bitpos = 9; // 4 named registers
uint8_t *sourceptr = (uint8_t *) source;
uint8_t *sourceptr_orig = (uint8_t *) source;
uint8_t *destptr = (uint8_t *) dest;
uint8_t *destptr_orig = (uint8_t *) dest;
uint8_t *ptr_reg;
uint8_t currentbyte;
int flag;
int32_t offset;
uint32_t x, t; // 2 placed variables
currentbyte = sourceptr[0];
sourceptr++;
for (;;) {
bitpos--;
if (bitpos == 0) {
currentbyte = sourceptr[0];
bitpos = 8;
sourceptr++;
}
flag = currentbyte & 1;
currentbyte = currentbyte >> 1;
if (flag) {
destptr[0] = sourceptr[0];
sourceptr++;
destptr++;
continue;
}
bitpos--;
if (bitpos == 0) {
currentbyte = sourceptr[0];
bitpos = 8;
sourceptr++;
}
flag = currentbyte & 1;
currentbyte = currentbyte >> 1;
if (flag) {
r3 = sourceptr[0] & 0xFF;
offset = ((sourceptr[1] & 0xFF) << 8) | r3;
sourceptr += 2;
if (offset == 0) return (uint32_t) (destptr - destptr_orig);
r3 = r3 & 0x00000007;
//r5 = (offset >> 3) | 0xFFFFE000;
if (r3 == 0) {
flag = 0;
r3 = sourceptr[0] & 0xFF;
sourceptr++;
r3++;
} else r3 += 2;
//r5 += (uint32_t)destptr;
ptr_reg = destptr + ((int32_t) ((offset >> 3) | 0xFFFFE000));
} else {
r3 = 0;
for (x = 0; x < 2; x++) {
bitpos--;
if (bitpos == 0) {
currentbyte = sourceptr[0];
bitpos = 8;
sourceptr++;
}
flag = currentbyte & 1;
currentbyte = currentbyte >> 1;
offset = r3 << 1;
r3 = offset | flag;
}
offset = sourceptr[0] | 0xFFFFFF00;
r3 += 2;
sourceptr++;
//r5 = offset + (uint32_t)destptr;
ptr_reg = destptr + offset;
}
if (r3 == 0) continue;
t = r3;
for (x = 0; x < t; x++) {
//destptr[0] = *(uint8_t*)r5;
//r5++;
*destptr++ = *ptr_reg++;
r3++;
//destptr++;
}
}
}
static uint32_t prs_decompress_size(const void *source) {
uint32_t r0, r3, r6, r9; // 6 unnamed registers
uint32_t bitpos = 9; // 4 named registers
uint8_t *sourceptr = (uint8_t *) source;
uint8_t *destptr = NULL;
uint8_t *destptr_orig = NULL;
uint8_t *ptr_reg;
uint8_t currentbyte, lastbyte;
int flag;
int32_t offset;
uint32_t x, t; // 2 placed variables
currentbyte = sourceptr[0];
sourceptr++;
for (;;) {
bitpos--;
if (bitpos == 0) {
lastbyte = currentbyte = sourceptr[0];
bitpos = 8;
sourceptr++;
}
flag = currentbyte & 1;
currentbyte = currentbyte >> 1;
if (flag) {
sourceptr++;
destptr++;
continue;
}
bitpos--;
if (bitpos == 0) {
lastbyte = currentbyte = sourceptr[0];
bitpos = 8;
sourceptr++;
}
flag = currentbyte & 1;
currentbyte = currentbyte >> 1;
if (flag) {
r3 = sourceptr[0];
offset = (sourceptr[1] << 8) | r3;
sourceptr += 2;
if (offset == 0) return (uint32_t) (destptr - destptr_orig);
r3 = r3 & 0x00000007;
//r5 = (offset >> 3) | 0xFFFFE000;
if (r3 == 0) {
r3 = sourceptr[0];
sourceptr++;
r3++;
} else r3 += 2;
//r5 += (uint32_t)destptr;
ptr_reg = destptr + ((int32_t) ((offset >> 3) | 0xFFFFE000));
} else {
r3 = 0;
for (x = 0; x < 2; x++) {
bitpos--;
if (bitpos == 0) {
lastbyte = currentbyte = sourceptr[0];
bitpos = 8;
sourceptr++;
}
flag = currentbyte & 1;
currentbyte = currentbyte >> 1;
offset = r3 << 1;
r3 = offset | flag;
}
offset = sourceptr[0] | 0xFFFFFF00;
r3 += 2;
sourceptr++;
//r5 = offset + (uint32_t)destptr;
ptr_reg = destptr + offset;
}
if (r3 == 0) continue;
t = r3;
for (x = 0; x < t; x++) {
//r5++;
ptr_reg++;
r3++;
destptr++;
}
}
}
////////////////////////////////////////////////////////////////////////////////
// borrowed from libsylverant: https://github.com/Sylverant/libsylverant/blob/master/src/utils/prs-comp.c
static size_t prs_max_compressed_size(size_t len) {
len += 2;
return len + (len >> 3) + ((len & 0x07) ? 1 : 0);
}
/*
* The below functions are included as wrappers for the above "prs_" functions in order to provide
* API compatibility with libsylverant's PRS functions, with the goal being to make it easier to
* switch back to that implementation of PRS compression/decompression in the future. Do note that
* the error handling is NOT as robust as libsylverant's PRS functions!
*/
int fuzziqer_prs_compress(const uint8_t *src, uint8_t **dst, size_t src_len) {
if (!src || !dst)
return -EFAULT;
if (!src_len)
return -EINVAL;
if (src_len < 3)
return -EBADMSG;
/* Allocate probably more than enough space for the compressed output. */
uint8_t *temp_dst;
size_t max_compressed_size = prs_max_compressed_size(src_len);
if (!(temp_dst = (uint8_t *)malloc(max_compressed_size)))
return -errno;
/* TODO: this version of prs_compress doesn't really do much in the way of error checking ... */
uint32_t size = prs_compress(src, temp_dst, src_len);
/* Resize the output (if realloc fails to resize it, then just use the
unshortened buffer). */
if(!(*dst = realloc(temp_dst, size)))
*dst = temp_dst;
return size;
}
int fuzziqer_prs_decompress_buf(const uint8_t *src, uint8_t **dst, size_t src_len) {
if (!src || !dst)
return -EFAULT;
if (!src_len)
return -EINVAL;
/* The minimum length of a PRS compressed file (if you were to "compress" a
zero-byte file) is 3 bytes. If we don't have that, then bail out now. */
if (src_len < 3)
return -EBADMSG;
uint32_t dst_len = prs_decompress_size(src);
if (!(*dst = malloc(dst_len)))
return -errno;
/* TODO: this version of prs_decompress doesn't really do much in the way of error checking ... */
uint32_t size = prs_decompress(src, *dst);
return size;
}
int fuzziqer_prs_decompress_size(const uint8_t *src, size_t src_len) {
if (!src)
return -EFAULT;
if (!src_len)
return -EINVAL;
/* The minimum length of a PRS compressed file (if you were to "compress" a
zero-byte file) is 3 bytes. If we don't have that, then bail out now. */
if(src_len < 3)
return -EBADMSG;
return prs_decompress_size(src);
}

10
fuzziqer_prs.h Normal file
View file

@ -0,0 +1,10 @@
#ifndef PRS_H_INCLUDED
#define PRS_H_INCLUDED
#include <stdint.h>
int fuzziqer_prs_compress(const uint8_t *src, uint8_t **dst, size_t src_len);
int fuzziqer_prs_decompress_buf(const uint8_t *src, uint8_t **dst, size_t src_len);
int fuzziqer_prs_decompress_size(const uint8_t *src, size_t src_len);
#endif