From d26d804cc271879670c63b58f39aea0d464c6b53 Mon Sep 17 00:00:00 2001 From: topjohnwu Date: Thu, 21 Nov 2019 06:08:02 -0500 Subject: [PATCH] Migrate to generic stream implementation --- native/jni/magiskboot/bootimg.cpp | 19 +- native/jni/magiskboot/compress.cpp | 1036 ++++++++++++++++------------ native/jni/magiskboot/compress.h | 168 +---- native/jni/magiskboot/ramdisk.cpp | 32 +- native/jni/utils/cpio.cpp | 16 +- native/jni/utils/include/cpio.h | 6 +- native/jni/utils/include/stream.h | 122 +--- native/jni/utils/stream.cpp | 19 + 8 files changed, 648 insertions(+), 770 deletions(-) diff --git a/native/jni/magiskboot/bootimg.cpp b/native/jni/magiskboot/bootimg.cpp index d31f7d807..1e8d3bffe 100644 --- a/native/jni/magiskboot/bootimg.cpp +++ b/native/jni/magiskboot/bootimg.cpp @@ -21,19 +21,18 @@ using namespace std; uint32_t dyn_img_hdr::j32 = 0; uint64_t dyn_img_hdr::j64 = 0; -static int64_t one_step(unique_ptr &&ptr, int fd, const void *in, size_t size) { - ptr->setOut(make_unique(fd)); - if (!ptr->write(in, size)) - return -1; - return ptr->finalize(); -} - -static int64_t decompress(format_t type, int fd, const void *in, size_t size) { - return one_step(unique_ptr(get_decoder(type)), fd, in, size); +static void decompress(format_t type, int fd, const void *in, size_t size) { + unique_ptr ptr(get_decoder(type, open_stream(fd))); + ptr->write(in, size); } static int64_t compress(format_t type, int fd, const void *in, size_t size) { - return one_step(unique_ptr(get_encoder(type)), fd, in, size); + auto prev = lseek(fd, 0, SEEK_CUR); + unique_ptr ptr(get_encoder(type, open_stream(fd))); + ptr->write(in, size); + ptr->close(); + auto now = lseek(fd, 0, SEEK_CUR); + return now - prev; } static void dump(void *buf, size_t size, const char *filename) { diff --git a/native/jni/magiskboot/compress.cpp b/native/jni/magiskboot/compress.cpp index 4f37ac591..0694766ba 100644 --- a/native/jni/magiskboot/compress.cpp +++ b/native/jni/magiskboot/compress.cpp @@ -6,6 +6,13 @@ #include #include +#include +#include +#include +#include +#include +#include + #include #include @@ -14,32 +21,565 @@ using namespace std; -static bool read_file(FILE *fp, const function &fn) { - char buf[4096]; - size_t len; - while ((len = fread(buf, 1, sizeof(buf), fp))) - fn(buf, len); - return true; +#define bwrite filter_stream::write +#define bclose filter_stream::close + +constexpr size_t CHUNK = 0x40000; +constexpr size_t LZ4_UNCOMPRESSED = 0x800000; +constexpr size_t LZ4_COMPRESSED = LZ4_COMPRESSBOUND(LZ4_UNCOMPRESSED); + +class cpr_stream : public filter_stream { +public: + explicit cpr_stream(FILE *fp) : filter_stream(fp) {} + + int read(void *buf, size_t len) final { + return stream::read(buf, len); + } + + int close() final { + finish(); + return bclose(); + } + +protected: + // If finish is overridden, destroy should be called in the destructor + virtual void finish() {} + void destroy() { if (fp) finish(); } +}; + +class gz_strm : public cpr_stream { +public: + ~gz_strm() override { destroy(); } + + int write(const void *buf, size_t len) override { + return len ? write(buf, len, Z_NO_FLUSH) : 0; + } + +protected: + enum mode_t { + DECODE, + ENCODE + } mode; + + gz_strm(mode_t mode, FILE *fp) : cpr_stream(fp), mode(mode) { + switch(mode) { + case DECODE: + inflateInit2(&strm, 15 | 16); + break; + case ENCODE: + deflateInit2(&strm, 9, Z_DEFLATED, 15 | 16, 8, Z_DEFAULT_STRATEGY); + break; + } + } + + void finish() override { + write(nullptr, 0, Z_FINISH); + switch(mode) { + case DECODE: + inflateEnd(&strm); + break; + case ENCODE: + deflateEnd(&strm); + break; + } + } + +private: + z_stream strm; + uint8_t outbuf[CHUNK]; + + int write(const void *buf, size_t len, int flush) { + strm.next_in = (Bytef *) buf; + strm.avail_in = len; + do { + int code; + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + switch(mode) { + case DECODE: + code = inflate(&strm, flush); + break; + case ENCODE: + code = deflate(&strm, flush); + break; + } + if (code == Z_STREAM_ERROR) { + LOGW("gzip %s failed (%d)\n", mode ? "encode" : "decode", code); + return -1; + } + bwrite(outbuf, sizeof(outbuf) - strm.avail_out); + } while (strm.avail_out == 0); + return len; + } +}; + +class gz_decoder : public gz_strm { +public: + explicit gz_decoder(FILE *fp) : gz_strm(DECODE, fp) {}; +}; + +class gz_encoder : public gz_strm { +public: + explicit gz_encoder(FILE *fp) : gz_strm(ENCODE, fp) {}; +}; + +class bz_strm : public cpr_stream { +public: + ~bz_strm() override { destroy(); } + + int write(const void *buf, size_t len) override { + return len ? write(buf, len, BZ_RUN) : 0; + } + +protected: + enum mode_t { + DECODE, + ENCODE + } mode; + + bz_strm(mode_t mode, FILE *fp) : cpr_stream(fp), mode(mode) { + switch(mode) { + case DECODE: + BZ2_bzDecompressInit(&strm, 0, 0); + break; + case ENCODE: + BZ2_bzCompressInit(&strm, 9, 0, 0); + break; + } + } + + void finish() override { + switch(mode) { + case DECODE: + BZ2_bzDecompressEnd(&strm); + break; + case ENCODE: + write(nullptr, 0, BZ_FINISH); + BZ2_bzCompressEnd(&strm); + break; + } + } + +private: + bz_stream strm; + char outbuf[CHUNK]; + + int write(const void *buf, size_t len, int flush) { + strm.next_in = (char *) buf; + strm.avail_in = len; + do { + int code; + strm.avail_out = sizeof(outbuf); + strm.next_out = outbuf; + switch(mode) { + case DECODE: + code = BZ2_bzDecompress(&strm); + break; + case ENCODE: + code = BZ2_bzCompress(&strm, flush); + break; + } + if (code < 0) { + LOGW("bzip2 %s failed (%d)\n", mode ? "encode" : "decode", code); + return -1; + } + bwrite(outbuf, sizeof(outbuf) - strm.avail_out); + } while (strm.avail_out == 0); + return len; + } +}; + +class bz_decoder : public bz_strm { +public: + explicit bz_decoder(FILE *fp) : bz_strm(DECODE, fp) {}; +}; + +class bz_encoder : public bz_strm { +public: + explicit bz_encoder(FILE *fp) : bz_strm(ENCODE, fp) {}; +}; + +class lzma_strm : public cpr_stream { +public: + ~lzma_strm() override { destroy(); } + + int write(const void *buf, size_t len) override { + return len ? write(buf, len, LZMA_RUN) : 0; + } + +protected: + enum mode_t { + DECODE, + ENCODE_XZ, + ENCODE_LZMA + } mode; + + lzma_strm(mode_t mode, FILE *fp) : cpr_stream(fp), mode(mode), strm(LZMA_STREAM_INIT) { + lzma_options_lzma opt; + + // Initialize preset + lzma_lzma_preset(&opt, 9); + lzma_filter filters[] = { + { .id = LZMA_FILTER_LZMA2, .options = &opt }, + { .id = LZMA_VLI_UNKNOWN, .options = nullptr }, + }; + + lzma_ret ret; + switch(mode) { + case DECODE: + ret = lzma_auto_decoder(&strm, UINT64_MAX, 0); + break; + case ENCODE_XZ: + ret = lzma_stream_encoder(&strm, filters, LZMA_CHECK_CRC32); + break; + case ENCODE_LZMA: + ret = lzma_alone_encoder(&strm, &opt); + break; + } + } + + void finish() override { + write(nullptr, 0, LZMA_FINISH); + lzma_end(&strm); + } + +private: + lzma_stream strm; + uint8_t outbuf[CHUNK]; + + int write(const void *buf, size_t len, lzma_action flush) { + strm.next_in = (uint8_t *) buf; + strm.avail_in = len; + do { + strm.avail_out = sizeof(outbuf); + strm.next_out = outbuf; + int code = lzma_code(&strm, flush); + if (code != LZMA_OK && code != LZMA_STREAM_END) { + LOGW("LZMA %s failed (%d)\n", mode ? "encode" : "decode", code); + return -1; + } + bwrite(outbuf, sizeof(outbuf) - strm.avail_out); + } while (strm.avail_out == 0); + return len; + } +}; + +class lzma_decoder : public lzma_strm { +public: + lzma_decoder(FILE *fp) : lzma_strm(DECODE, fp) {} +}; + +class xz_encoder : public lzma_strm { +public: + xz_encoder(FILE *fp) : lzma_strm(ENCODE_XZ, fp) {} +}; + +class lzma_encoder : public lzma_strm { +public: + lzma_encoder(FILE *fp) : lzma_strm(ENCODE_LZMA, fp) {} +}; + +class LZ4F_decoder : public cpr_stream { +public: + explicit LZ4F_decoder(FILE *fp) : cpr_stream(fp), outbuf(nullptr) { + LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION); + } + + ~LZ4F_decoder() override { + LZ4F_freeDecompressionContext(ctx); + delete[] outbuf; + } + + int write(const void *buf, size_t len) override { + auto ret = len; + auto inbuf = reinterpret_cast(buf); + if (!outbuf) + read_header(inbuf, len); + size_t read, write; + LZ4F_errorCode_t code; + do { + read = len; + write = outCapacity; + code = LZ4F_decompress(ctx, outbuf, &write, inbuf, &read, nullptr); + if (LZ4F_isError(code)) { + LOGW("LZ4F decode error: %s\n", LZ4F_getErrorName(code)); + return -1; + } + len -= read; + inbuf += read; + bwrite(outbuf, write); + } while (len != 0 || write != 0); + return ret; + } + +private: + LZ4F_decompressionContext_t ctx; + uint8_t *outbuf; + size_t outCapacity; + + void read_header(const uint8_t *&in, size_t &size) { + size_t read = size; + LZ4F_frameInfo_t info; + LZ4F_getFrameInfo(ctx, &info, in, &read); + switch (info.blockSizeID) { + case LZ4F_default: + case LZ4F_max64KB: outCapacity = 1 << 16; break; + case LZ4F_max256KB: outCapacity = 1 << 18; break; + case LZ4F_max1MB: outCapacity = 1 << 20; break; + case LZ4F_max4MB: outCapacity = 1 << 22; break; + } + outbuf = new uint8_t[outCapacity]; + in += read; + size -= read; + } +}; + +class LZ4F_encoder : public cpr_stream { +public: + explicit LZ4F_encoder(FILE *fp) : cpr_stream(fp), outbuf(nullptr), outCapacity(0) { + LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); + } + + ~LZ4F_encoder() override { + destroy(); + LZ4F_freeCompressionContext(ctx); + delete[] outbuf; + } + + int write(const void *buf, size_t len) override { + auto ret = len; + if (!outbuf) + write_header(); + if (len == 0) + return 0; + auto inbuf = reinterpret_cast(buf); + size_t read, write; + do { + read = len > BLOCK_SZ ? BLOCK_SZ : len; + write = LZ4F_compressUpdate(ctx, outbuf, outCapacity, inbuf, read, nullptr); + if (LZ4F_isError(write)) { + LOGW("LZ4F encode error: %s\n", LZ4F_getErrorName(write)); + return -1; + } + len -= read; + inbuf += read; + bwrite(outbuf, write); + } while (len != 0); + return ret; + } + +protected: + void finish() override { + size_t len = LZ4F_compressEnd(ctx, outbuf, outCapacity, nullptr); + bwrite(outbuf, len); + } + +private: + LZ4F_compressionContext_t ctx; + uint8_t *outbuf; + size_t outCapacity; + + static constexpr size_t BLOCK_SZ = 1 << 22; + + void write_header() { + LZ4F_preferences_t prefs { + .autoFlush = 1, + .compressionLevel = 9, + .frameInfo = { + .blockMode = LZ4F_blockIndependent, + .blockSizeID = LZ4F_max4MB, + .blockChecksumFlag = LZ4F_noBlockChecksum, + .contentChecksumFlag = LZ4F_contentChecksumEnabled + } + }; + outCapacity = LZ4F_compressBound(BLOCK_SZ, &prefs); + outbuf = new uint8_t[outCapacity]; + size_t write = LZ4F_compressBegin(ctx, outbuf, outCapacity, &prefs); + bwrite(outbuf, write); + } +}; + +class LZ4_decoder : public cpr_stream { +public: + explicit LZ4_decoder(FILE *fp) + : cpr_stream(fp), out_buf(new char[LZ4_UNCOMPRESSED]), buffer(new char[LZ4_COMPRESSED]), + init(false), block_sz(0), buf_off(0) {} + + ~LZ4_decoder() override { + delete[] out_buf; + delete[] buffer; + } + + int write(const void *in, size_t size) override { + auto ret = size; + auto inbuf = static_cast(in); + if (!init) { + // Skip magic + inbuf += 4; + size -= 4; + init = true; + } + int write; + size_t consumed; + do { + if (block_sz == 0) { + block_sz = *((unsigned *) inbuf); + inbuf += sizeof(unsigned); + size -= sizeof(unsigned); + } else if (buf_off + size >= block_sz) { + consumed = block_sz - buf_off; + memcpy(buffer + buf_off, inbuf, consumed); + inbuf += consumed; + size -= consumed; + + write = LZ4_decompress_safe(buffer, out_buf, block_sz, LZ4_UNCOMPRESSED); + if (write < 0) { + LOGW("LZ4HC decompression failure (%d)\n", write); + return -1; + } + bwrite(out_buf, write); + + // Reset + buf_off = 0; + block_sz = 0; + } else { + // Copy to internal buffer + memcpy(buffer + buf_off, inbuf, size); + buf_off += size; + size = 0; + } + } while (size != 0); + return ret; + } + +private: + char *out_buf; + char *buffer; + bool init; + unsigned block_sz; + int buf_off; +}; + +class LZ4_encoder : public cpr_stream { +public: + explicit LZ4_encoder(FILE *fp) + : cpr_stream(fp), outbuf(new char[LZ4_COMPRESSED]), buf(new char[LZ4_UNCOMPRESSED]), + init(false), buf_off(0), in_total(0) {} + + ~LZ4_encoder() override { + destroy(); + delete[] outbuf; + delete[] buf; + } + + int write(const void *in, size_t size) override { + if (!init) { + bwrite("\x02\x21\x4c\x18", 4); + init = true; + } + if (size == 0) + return 0; + in_total += size; + const char *inbuf = (const char *) in; + size_t consumed; + int write; + do { + if (buf_off + size >= LZ4_UNCOMPRESSED) { + consumed = LZ4_UNCOMPRESSED - buf_off; + memcpy(buf + buf_off, inbuf, consumed); + inbuf += consumed; + size -= consumed; + + write = LZ4_compress_HC(buf, outbuf, LZ4_UNCOMPRESSED, LZ4_COMPRESSED, 9); + if (write == 0) { + LOGW("LZ4HC compression failure\n"); + return false; + } + bwrite(&write, sizeof(write)); + bwrite(outbuf, write); + + // Reset buffer + buf_off = 0; + } else { + // Copy to internal buffer + memcpy(buf + buf_off, inbuf, size); + buf_off += size; + size = 0; + } + } while (size != 0); + return true; + } + +protected: + void finish() override { + if (buf_off) { + int write = LZ4_compress_HC(buf, outbuf, buf_off, LZ4_COMPRESSED, 9); + bwrite(&write, sizeof(write)); + bwrite(outbuf, write); + } + bwrite(&in_total, sizeof(in_total)); + } + +private: + char *outbuf; + char *buf; + bool init; + int buf_off; + unsigned in_total; +}; + +filter_stream *get_encoder(format_t type, FILE *fp) { + switch (type) { + case XZ: + return new xz_encoder(fp); + case LZMA: + return new lzma_encoder(fp); + case BZIP2: + return new bz_encoder(fp); + case LZ4: + return new LZ4F_encoder(fp); + case LZ4_LEGACY: + return new LZ4_encoder(fp); + case GZIP: + default: + return new gz_encoder(fp); + } +} + +filter_stream *get_decoder(format_t type, FILE *fp) { + switch (type) { + case XZ: + case LZMA: + return new lzma_decoder(fp); + case BZIP2: + return new bz_decoder(fp); + case LZ4: + return new LZ4F_decoder(fp); + case LZ4_LEGACY: + return new LZ4_decoder(fp); + case GZIP: + default: + return new gz_decoder(fp); + } } void decompress(char *infile, const char *outfile) { - bool in_std = strcmp(infile, "-") == 0; + bool in_std = infile == "-"sv; bool rm_in = false; - FILE *in_file = in_std ? stdin : xfopen(infile, "re"); - int out_fd = -1; - unique_ptr cmp; + FILE *in_fp = in_std ? stdin : xfopen(infile, "re"); + unique_ptr strm; - read_file(in_file, [&](void *buf, size_t len) -> void { - if (out_fd < 0) { + char buf[4096]; + size_t len; + while ((len = fread(buf, 1, sizeof(buf), in_fp))) { + if (!strm) { format_t type = check_fmt(buf, len); - fprintf(stderr, "Detected format: [%s]\n", fmt2name[type]); - if (!COMPRESSED(type)) LOGE("Input file is not a supported compressed type!\n"); - cmp.reset(get_decoder(type)); + fprintf(stderr, "Detected format: [%s]\n", fmt2name[type]); /* If user does not provide outfile, infile has to be either * .[ext], or '-'. Outfile will be either or '-'. @@ -60,18 +600,16 @@ void decompress(char *infile, const char *outfile) { } } - out_fd = strcmp(outfile, "-") == 0 ? - STDOUT_FILENO : xopen(outfile, O_WRONLY | O_CREAT | O_TRUNC, 0644); - cmp->setOut(make_unique(out_fd)); + FILE *out_fp = outfile == "-"sv ? stdout : xfopen(outfile, "we"); + strm.reset(get_decoder(type, out_fp)); if (ext) *ext = '.'; } - if (!cmp->write(buf, len)) + if (strm->write(buf, len) < 0) LOGE("Decompression error!\n"); - }); + } - cmp->finalize(); - fclose(in_file); - close(out_fd); + strm->close(); + fclose(in_fp); if (rm_in) unlink(infile); @@ -80,458 +618,42 @@ void decompress(char *infile, const char *outfile) { void compress(const char *method, const char *infile, const char *outfile) { auto it = name2fmt.find(method); if (it == name2fmt.end()) - LOGE("Unsupported compression method: [%s]\n", method); + LOGE("Unknown compression method: [%s]\n", method); - unique_ptr cmp(get_encoder(it->second)); - - bool in_std = strcmp(infile, "-") == 0; + bool in_std = infile == "-"sv; bool rm_in = false; - FILE *in_file = in_std ? stdin : xfopen(infile, "re"); - int out_fd; + FILE *in_fp = in_std ? stdin : xfopen(infile, "re"); + FILE *out_fp; if (outfile == nullptr) { if (in_std) { - out_fd = STDOUT_FILENO; + out_fp = stdout; } else { /* If user does not provide outfile and infile is not * STDIN, output to .[ext] */ - char *tmp = new char[strlen(infile) + 5]; - sprintf(tmp, "%s%s", infile, fmt2ext[it->second]); - out_fd = xopen(tmp, O_WRONLY | O_CREAT | O_TRUNC, 0644); - fprintf(stderr, "Compressing to [%s]\n", tmp); - delete[] tmp; + string tmp(infile); + tmp += fmt2ext[it->second]; + out_fp = xfopen(tmp.data(), "we"); + fprintf(stderr, "Compressing to [%s]\n", tmp.data()); rm_in = true; } } else { - out_fd = strcmp(outfile, "-") == 0 ? - STDOUT_FILENO : xopen(outfile, O_WRONLY | O_CREAT | O_TRUNC, 0644); + out_fp = outfile == "-"sv ? stdout : xfopen(outfile, "we"); } - cmp->setOut(make_unique(out_fd)); + unique_ptr strm(get_encoder(it->second, out_fp)); - read_file(in_file, [&](void *buf, size_t len) -> void { - if (!cmp->write(buf, len)) + char buf[4096]; + size_t len; + while ((len = fread(buf, 1, sizeof(buf), in_fp))) { + if (strm->write(buf, len) < 0) LOGE("Compression error!\n"); - }); + }; - cmp->finalize(); - fclose(in_file); - close(out_fd); + strm->close(); + fclose(in_fp); if (rm_in) unlink(infile); } - -Compression *get_encoder(format_t type) { - switch (type) { - case XZ: - return new XZEncoder(); - case LZMA: - return new LZMAEncoder(); - case BZIP2: - return new BZEncoder(); - case LZ4: - return new LZ4FEncoder(); - case LZ4_LEGACY: - return new LZ4Encoder(); - case GZIP: - default: - return new GZEncoder(); - } -} - -Compression *get_decoder(format_t type) { - switch (type) { - case XZ: - case LZMA: - return new LZMADecoder(); - case BZIP2: - return new BZDecoder(); - case LZ4: - return new LZ4FDecoder(); - case LZ4_LEGACY: - return new LZ4Decoder(); - case GZIP: - default: - return new GZDecoder(); - } -} - -GZStream::GZStream(int mode) : mode(mode), strm({}) { - switch(mode) { - case 0: - inflateInit2(&strm, 15 | 16); - break; - case 1: - deflateInit2(&strm, 9, Z_DEFLATED, 15 | 16, 8, Z_DEFAULT_STRATEGY); - break; - } -} - -bool GZStream::write(const void *in, size_t size) { - return size ? write(in, size, Z_NO_FLUSH) : true; -} - -uint64_t GZStream::finalize() { - write(nullptr, 0, Z_FINISH); - uint64_t total = strm.total_out; - switch(mode) { - case 0: - inflateEnd(&strm); - break; - case 1: - deflateEnd(&strm); - break; - } - return total; -} - -bool GZStream::write(const void *in, size_t size, int flush) { - int ret; - strm.next_in = (Bytef *) in; - strm.avail_in = size; - do { - strm.next_out = outbuf; - strm.avail_out = sizeof(outbuf); - switch(mode) { - case 0: - ret = inflate(&strm, flush); - break; - case 1: - ret = deflate(&strm, flush); - break; - } - if (ret == Z_STREAM_ERROR) { - LOGW("Gzip %s failed (%d)\n", mode ? "encode" : "decode", ret); - return false; - } - FilterOutStream::write(outbuf, sizeof(outbuf) - strm.avail_out); - } while (strm.avail_out == 0); - return true; -} - -BZStream::BZStream(int mode) : mode(mode), strm({}) { - switch(mode) { - case 0: - BZ2_bzDecompressInit(&strm, 0, 0); - break; - case 1: - BZ2_bzCompressInit(&strm, 9, 0, 0); - break; - } -} - -bool BZStream::write(const void *in, size_t size) { - return size ? write(in, size, BZ_RUN) : true; -} - -uint64_t BZStream::finalize() { - if (mode) - write(nullptr, 0, BZ_FINISH); - uint64_t total = ((uint64_t) strm.total_out_hi32 << 32) + strm.total_out_lo32; - switch(mode) { - case 0: - BZ2_bzDecompressEnd(&strm); - break; - case 1: - BZ2_bzCompressEnd(&strm); - break; - } - return total; -} - -bool BZStream::write(const void *in, size_t size, int flush) { - int ret; - strm.next_in = (char *) in; - strm.avail_in = size; - do { - strm.avail_out = sizeof(outbuf); - strm.next_out = outbuf; - switch(mode) { - case 0: - ret = BZ2_bzDecompress(&strm); - break; - case 1: - ret = BZ2_bzCompress(&strm, flush); - break; - } - if (ret < 0) { - LOGW("Bzip2 %s failed (%d)\n", mode ? "encode" : "decode", ret); - return false; - } - FilterOutStream::write(outbuf, sizeof(outbuf) - strm.avail_out); - } while (strm.avail_out == 0); - return true; -} - -LZMAStream::LZMAStream(int mode) : mode(mode), strm(LZMA_STREAM_INIT) { - lzma_options_lzma opt; - int ret; - - // Initialize preset - lzma_lzma_preset(&opt, 9); - lzma_filter filters[] = { - { .id = LZMA_FILTER_LZMA2, .options = &opt }, - { .id = LZMA_VLI_UNKNOWN, .options = nullptr }, - }; - - switch(mode) { - case 0: - ret = lzma_auto_decoder(&strm, UINT64_MAX, 0); - break; - case 1: - ret = lzma_stream_encoder(&strm, filters, LZMA_CHECK_CRC32); - break; - case 2: - ret = lzma_alone_encoder(&strm, &opt); - break; - } -} - -bool LZMAStream::write(const void *in, size_t size) { - return size ? write(in, size, LZMA_RUN) : true; -} - -uint64_t LZMAStream::finalize() { - write(nullptr, 0, LZMA_FINISH); - uint64_t total = strm.total_out; - lzma_end(&strm); - return total; -} - -bool LZMAStream::write(const void *in, size_t size, lzma_action flush) { - int ret; - strm.next_in = (uint8_t *) in; - strm.avail_in = size; - do { - strm.avail_out = sizeof(outbuf); - strm.next_out = outbuf; - ret = lzma_code(&strm, flush); - if (ret != LZMA_OK && ret != LZMA_STREAM_END) { - LOGW("LZMA %s failed (%d)\n", mode ? "encode" : "decode", ret); - return false; - } - FilterOutStream::write(outbuf, sizeof(outbuf) - strm.avail_out); - } while (strm.avail_out == 0); - return true; -} - -LZ4FDecoder::LZ4FDecoder() : outbuf(nullptr), total(0) { - LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION); -} - -LZ4FDecoder::~LZ4FDecoder() { - LZ4F_freeDecompressionContext(ctx); - delete[] outbuf; -} - -bool LZ4FDecoder::write(const void *in, size_t size) { - auto inbuf = (const uint8_t *) in; - if (!outbuf) - read_header(inbuf, size); - size_t read, write; - LZ4F_errorCode_t ret; - do { - read = size; - write = outCapacity; - ret = LZ4F_decompress(ctx, outbuf, &write, inbuf, &read, nullptr); - if (LZ4F_isError(ret)) { - LOGW("LZ4 decode error: %s\n", LZ4F_getErrorName(ret)); - return false; - } - size -= read; - inbuf += read; - total += write; - FilterOutStream::write(outbuf, write); - } while (size != 0 || write != 0); - return true; -} - -uint64_t LZ4FDecoder::finalize() { - return total; -} - -void LZ4FDecoder::read_header(const uint8_t *&in, size_t &size) { - size_t read = size; - LZ4F_frameInfo_t info; - LZ4F_getFrameInfo(ctx, &info, in, &read); - switch (info.blockSizeID) { - case LZ4F_default: - case LZ4F_max64KB: outCapacity = 1 << 16; break; - case LZ4F_max256KB: outCapacity = 1 << 18; break; - case LZ4F_max1MB: outCapacity = 1 << 20; break; - case LZ4F_max4MB: outCapacity = 1 << 22; break; - } - outbuf = new uint8_t[outCapacity]; - in += read; - size -= read; -} - -LZ4FEncoder::LZ4FEncoder() : outbuf(nullptr), outCapacity(0), total(0) { - LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); -} - -LZ4FEncoder::~LZ4FEncoder() { - LZ4F_freeCompressionContext(ctx); - delete[] outbuf; -} - -bool LZ4FEncoder::write(const void *in, size_t size) { - if (!outbuf) - write_header(); - if (size == 0) - return true; - auto inbuf = (const uint8_t *) in; - size_t read, write; - do { - read = size > BLOCK_SZ ? BLOCK_SZ : size; - write = LZ4F_compressUpdate(ctx, outbuf, outCapacity, inbuf, read, nullptr); - if (LZ4F_isError(write)) { - LOGW("LZ4 encode error: %s\n", LZ4F_getErrorName(write)); - return false; - } - size -= read; - inbuf += read; - total += write; - FilterOutStream::write(outbuf, write); - } while (size != 0); - return true; -} - -uint64_t LZ4FEncoder::finalize() { - size_t write = LZ4F_compressEnd(ctx, outbuf, outCapacity, nullptr); - total += write; - FilterOutStream::write(outbuf, write); - return total; -} - -void LZ4FEncoder::write_header() { - LZ4F_preferences_t prefs { - .autoFlush = 1, - .compressionLevel = 9, - .frameInfo = { - .blockMode = LZ4F_blockIndependent, - .blockSizeID = LZ4F_max4MB, - .blockChecksumFlag = LZ4F_noBlockChecksum, - .contentChecksumFlag = LZ4F_contentChecksumEnabled - } - }; - outCapacity = LZ4F_compressBound(BLOCK_SZ, &prefs); - outbuf = new uint8_t[outCapacity]; - size_t write = LZ4F_compressBegin(ctx, outbuf, outCapacity, &prefs); - total += write; - FilterOutStream::write(outbuf, write); -} - -LZ4Decoder::LZ4Decoder() : outbuf(new char[LZ4_UNCOMPRESSED]), buf(new char[LZ4_COMPRESSED]), -init(false), block_sz(0), buf_off(0), total(0) {} - -LZ4Decoder::~LZ4Decoder() { - delete[] outbuf; - delete[] buf; -} - -bool LZ4Decoder::write(const void *in, size_t size) { - const char *inbuf = (const char *) in; - if (!init) { - // Skip magic - inbuf += 4; - size -= 4; - init = true; - } - int write; - size_t consumed; - do { - if (block_sz == 0) { - block_sz = *((unsigned *) inbuf); - inbuf += sizeof(unsigned); - size -= sizeof(unsigned); - } else if (buf_off + size >= block_sz) { - consumed = block_sz - buf_off; - memcpy(buf + buf_off, inbuf, consumed); - inbuf += consumed; - size -= consumed; - - write = LZ4_decompress_safe(buf, outbuf, block_sz, LZ4_UNCOMPRESSED); - if (write < 0) { - LOGW("LZ4HC decompression failure (%d)\n", write); - return false; - } - FilterOutStream::write(outbuf, write); - total += write; - - // Reset - buf_off = 0; - block_sz = 0; - } else { - // Copy to internal buffer - memcpy(buf + buf_off, inbuf, size); - buf_off += size; - size = 0; - } - } while (size != 0); - return true; -} - -uint64_t LZ4Decoder::finalize() { - return total; -} - -LZ4Encoder::LZ4Encoder() : outbuf(new char[LZ4_COMPRESSED]), buf(new char[LZ4_UNCOMPRESSED]), -init(false), buf_off(0), out_total(0), in_total(0) {} - -LZ4Encoder::~LZ4Encoder() { - delete[] outbuf; - delete[] buf; -} - -bool LZ4Encoder::write(const void *in, size_t size) { - if (!init) { - FilterOutStream::write("\x02\x21\x4c\x18", 4); - init = true; - } - if (size == 0) - return true; - in_total += size; - const char *inbuf = (const char *) in; - size_t consumed; - int write; - do { - if (buf_off + size >= LZ4_UNCOMPRESSED) { - consumed = LZ4_UNCOMPRESSED - buf_off; - memcpy(buf + buf_off, inbuf, consumed); - inbuf += consumed; - size -= consumed; - - write = LZ4_compress_HC(buf, outbuf, LZ4_UNCOMPRESSED, LZ4_COMPRESSED, 9); - if (write == 0) { - LOGW("LZ4HC compression failure\n"); - return false; - } - FilterOutStream::write(&write, sizeof(write)); - FilterOutStream::write(outbuf, write); - out_total += write + sizeof(write); - - // Reset buffer - buf_off = 0; - } else { - // Copy to internal buffer - memcpy(buf + buf_off, inbuf, size); - buf_off += size; - size = 0; - } - } while (size != 0); - return true; -} - -uint64_t LZ4Encoder::finalize() { - if (buf_off) { - int write = LZ4_compress_HC(buf, outbuf, buf_off, LZ4_COMPRESSED, 9); - FilterOutStream::write(&write, sizeof(write)); - FilterOutStream::write(outbuf, write); - out_total += write + sizeof(write); - } - FilterOutStream::write(&in_total, sizeof(in_total)); - return out_total + sizeof(in_total); -} diff --git a/native/jni/magiskboot/compress.h b/native/jni/magiskboot/compress.h index 81b9bead8..d3c92e999 100644 --- a/native/jni/magiskboot/compress.h +++ b/native/jni/magiskboot/compress.h @@ -1,174 +1,10 @@ #pragma once -#include -#include -#include -#include -#include -#include #include #include "format.h" -#define CHUNK 0x40000 - -class Compression : public FilterOutStream { -public: - virtual uint64_t finalize() = 0; -}; - -class GZStream : public Compression { -public: - bool write(const void *in, size_t size) override; - uint64_t finalize() override; - -protected: - explicit GZStream(int mode); - -private: - int mode; - z_stream strm; - uint8_t outbuf[CHUNK]; - - bool write(const void *in, size_t size, int flush); -}; - -class GZDecoder : public GZStream { -public: - GZDecoder() : GZStream(0) {}; -}; - -class GZEncoder : public GZStream { -public: - GZEncoder() : GZStream(1) {}; -}; - -class BZStream : public Compression { -public: - bool write(const void *in, size_t size) override; - uint64_t finalize() override; - -protected: - explicit BZStream(int mode); - -private: - int mode; - bz_stream strm; - char outbuf[CHUNK]; - - bool write(const void *in, size_t size, int flush); -}; - -class BZDecoder : public BZStream { -public: - BZDecoder() : BZStream(0) {}; -}; - -class BZEncoder : public BZStream { -public: - BZEncoder() : BZStream(1) {}; -}; - -class LZMAStream : public Compression { -public: - bool write(const void *in, size_t size) override; - uint64_t finalize() override; - -protected: - explicit LZMAStream(int mode); - -private: - int mode; - lzma_stream strm; - uint8_t outbuf[CHUNK]; - - bool write(const void *in, size_t size, lzma_action flush); -}; - -class LZMADecoder : public LZMAStream { -public: - LZMADecoder() : LZMAStream(0) {} -}; - -class XZEncoder : public LZMAStream { -public: - XZEncoder() : LZMAStream(1) {} -}; - -class LZMAEncoder : public LZMAStream { -public: - LZMAEncoder() : LZMAStream(2) {} -}; - -class LZ4FDecoder : public Compression { -public: - LZ4FDecoder(); - ~LZ4FDecoder() override; - bool write(const void *in, size_t size) override; - uint64_t finalize() override; - -private: - LZ4F_decompressionContext_t ctx; - uint8_t *outbuf; - size_t outCapacity; - uint64_t total; - - void read_header(const uint8_t *&in, size_t &size); -}; - -class LZ4FEncoder : public Compression { -public: - LZ4FEncoder(); - ~LZ4FEncoder() override; - bool write(const void *in, size_t size) override; - uint64_t finalize() override; - -private: - static constexpr size_t BLOCK_SZ = 1 << 22; - LZ4F_compressionContext_t ctx; - uint8_t *outbuf; - size_t outCapacity; - uint64_t total; - - void write_header(); -}; - -#define LZ4_UNCOMPRESSED 0x800000 -#define LZ4_COMPRESSED LZ4_COMPRESSBOUND(LZ4_UNCOMPRESSED) - -class LZ4Decoder : public Compression { -public: - LZ4Decoder(); - ~LZ4Decoder() override; - bool write(const void *in, size_t size) override; - uint64_t finalize() override; - -private: - char *outbuf; - char *buf; - bool init; - unsigned block_sz; - int buf_off; - uint64_t total; -}; - -class LZ4Encoder : public Compression { -public: - LZ4Encoder(); - ~LZ4Encoder() override; - bool write(const void *in, size_t size) override; - uint64_t finalize() override; - -private: - char *outbuf; - char *buf; - bool init; - int buf_off; - uint64_t out_total; - unsigned in_total; -}; - -Compression *get_encoder(format_t type); -Compression *get_decoder(format_t type); +filter_stream *get_encoder(format_t type, FILE *fp = nullptr); +filter_stream *get_decoder(format_t type, FILE *fp = nullptr); void compress(const char *method, const char *infile, const char *outfile); void decompress(char *infile, const char *outfile); diff --git a/native/jni/magiskboot/ramdisk.cpp b/native/jni/magiskboot/ramdisk.cpp index 157ae5ac4..faabde059 100644 --- a/native/jni/magiskboot/ramdisk.cpp +++ b/native/jni/magiskboot/ramdisk.cpp @@ -241,14 +241,17 @@ void magisk_cpio::compress() { return; fprintf(stderr, "Compressing cpio -> [%s]\n", RAMDISK_XZ); auto init = entries.extract("init"); - XZEncoder encoder; - encoder.setOut(make_unique()); - output(encoder); - encoder.finalize(); + + uint8_t *data; + size_t len; + FILE *fp = open_stream(get_encoder(XZ, open_stream(data, len))); + dump(fp); + entries.clear(); entries.insert(std::move(init)); auto xz = new cpio_entry(RAMDISK_XZ, S_IFREG); - static_cast(encoder.getOut())->release(xz->data, xz->filesize); + xz->data = data; + xz->filesize = len; insert(xz); } @@ -257,15 +260,16 @@ void magisk_cpio::decompress() { if (it == entries.end()) return; fprintf(stderr, "Decompressing cpio [%s]\n", RAMDISK_XZ); - LZMADecoder decoder; - decoder.setOut(make_unique()); - decoder.write(it->second->data, it->second->filesize); - decoder.finalize(); + + char *data; + size_t len; + auto strm = get_decoder(XZ, open_stream(data, len)); + strm->write(it->second->data, it->second->filesize); + delete strm; + entries.erase(it); - char *buf; - size_t sz; - static_cast(decoder.getOut())->getbuf(buf, sz); - load_cpio(buf, sz); + load_cpio(data, len); + free(data); } int cpio_commands(int argc, char *argv[]) { @@ -338,4 +342,4 @@ int cpio_commands(int argc, char *argv[]) { cpio.dump(incpio); return 0; -} \ No newline at end of file +} diff --git a/native/jni/utils/cpio.cpp b/native/jni/utils/cpio.cpp index 89840c98d..76e4ccb5e 100644 --- a/native/jni/utils/cpio.cpp +++ b/native/jni/utils/cpio.cpp @@ -49,8 +49,7 @@ cpio_entry_base::cpio_entry_base(const cpio_newc_header *h) void cpio::dump(const char *file) { fprintf(stderr, "Dump cpio: [%s]\n", file); - FDOutStream fd_out(xopen(file, O_WRONLY | O_CREAT | O_TRUNC, 0644), true); - output(fd_out); + dump(xfopen(file, "we")); } void cpio::rm(entry_map::iterator &it) { @@ -110,9 +109,9 @@ bool cpio::exists(const char *name) { return entries.count(name) != 0; } -#define do_out(b, l) out.write(b, l); pos += (l) -#define out_align() out.write(zeros, align_off(pos, 4)); pos = do_align(pos, 4) -void cpio::output(OutStream &out) { +#define do_out(buf, len) pos += fwrite(buf, len, 1, out); +#define out_align() do_out(zeros, align_off(pos, 4)) +void cpio::dump(FILE *out) { size_t pos = 0; unsigned inode = 300000; char header[111]; @@ -147,6 +146,7 @@ void cpio::output(OutStream &out) { do_out(header, 110); do_out("TRAILER!!!\0", 11); out_align(); + fclose(out); } cpio_rw::cpio_rw(const char *file) { @@ -221,12 +221,12 @@ bool cpio_rw::mv(const char *from, const char *to) { #define pos_align(p) p = do_align(p, 4) -void cpio_rw::load_cpio(char *buf, size_t sz) { +void cpio_rw::load_cpio(const char *buf, size_t sz) { size_t pos = 0; - cpio_newc_header *header; + const cpio_newc_header *header; unique_ptr entry; while (pos < sz) { - header = (cpio_newc_header *)(buf + pos); + header = reinterpret_cast(buf + pos); entry = make_unique(header); pos += sizeof(*header); string_view name_view(buf + pos); diff --git a/native/jni/utils/include/cpio.h b/native/jni/utils/include/cpio.h index 4f6f09917..51a2514d3 100644 --- a/native/jni/utils/include/cpio.h +++ b/native/jni/utils/include/cpio.h @@ -30,7 +30,7 @@ struct cpio_entry : public cpio_entry_base { explicit cpio_entry(const char *name, uint32_t mode) : filename(name) { this->mode = mode; } - explicit cpio_entry(cpio_newc_header *h) : cpio_entry_base(h) {} + explicit cpio_entry(const cpio_newc_header *h) : cpio_entry_base(h) {} ~cpio_entry() override { free(data); }; }; @@ -48,7 +48,7 @@ public: protected: entry_map entries; void rm(entry_map::iterator &it); - void output(OutStream &out); + void dump(FILE *out); }; class cpio_rw : public cpio { @@ -64,7 +64,7 @@ public: protected: void insert(cpio_entry *e); void mv(entry_map::iterator &it, const char *to); - void load_cpio(char *buf, size_t sz); + void load_cpio(const char *buf, size_t sz); }; class cpio_mmap : public cpio { diff --git a/native/jni/utils/include/stream.h b/native/jni/utils/include/stream.h index 43d0562d0..b296d72ab 100644 --- a/native/jni/utils/include/stream.h +++ b/native/jni/utils/include/stream.h @@ -15,8 +15,6 @@ FILE *open_stream(Args &&... args) { return open_stream(new T(args...)); } -/* Base classes */ - class stream { public: virtual int read(void *buf, size_t len); @@ -26,17 +24,17 @@ public: virtual ~stream() = default; }; +// Delegates all operations to the base FILE pointer class filter_stream : public stream { public: filter_stream(FILE *fp) : fp(fp) {} - int close() override { return fclose(fp); } - virtual ~filter_stream() { close(); } + ~filter_stream() override { if (fp) close(); } - void set_base(FILE *f) { - if (fp) fclose(fp); - fp = f; - } + int read(void *buf, size_t len) override; + int write(const void *buf, size_t len) override; + int close() override; + void set_base(FILE *f); template void set_base(Args&&... args) { set_base(open_stream(args...)); @@ -46,18 +44,7 @@ protected: FILE *fp; }; -class filter_in_stream : public filter_stream { -public: - filter_in_stream(FILE *fp = nullptr) : filter_stream(fp) {} - int read(void *buf, size_t len) override { return fread(buf, len, 1, fp); } -}; - -class filter_out_stream : public filter_stream { -public: - filter_out_stream(FILE *fp = nullptr) : filter_stream(fp) {} - int write(const void *buf, size_t len) override { return fwrite(buf, len, 1, fp); } -}; - +// Handy interface for classes that need custom seek logic class seekable_stream : public stream { protected: size_t _pos = 0; @@ -66,8 +53,7 @@ protected: virtual size_t end_pos() = 0; }; -/* Concrete classes */ - +// Byte stream that dynamically allocates memory class byte_stream : public seekable_stream { public: byte_stream(uint8_t *&buf, size_t &len); @@ -76,7 +62,6 @@ public: int read(void *buf, size_t len) override; int write(const void *buf, size_t len) override; off_t seek(off_t off, int whence) override; - virtual ~byte_stream() = default; private: uint8_t *&_buf; @@ -87,101 +72,14 @@ private: size_t end_pos() override { return _len; } }; -class fd_stream : stream { +// File stream but does not close the file descriptor at any time +class fd_stream : public stream { public: fd_stream(int fd) : fd(fd) {} int read(void *buf, size_t len) override; int write(const void *buf, size_t len) override; off_t seek(off_t off, int whence) override; - virtual ~fd_stream() = default; private: int fd; }; - -/* TODO: Replace classes below to new implementation */ - -class OutStream { -public: - virtual bool write(const void *buf, size_t len) = 0; - virtual ~OutStream() = default; -}; - -typedef std::unique_ptr strm_ptr; - -class FilterOutStream : public OutStream { -public: - FilterOutStream() = default; - - FilterOutStream(strm_ptr &&ptr) : out(std::move(ptr)) {} - - void setOut(strm_ptr &&ptr) { out = std::move(ptr); } - - OutStream *getOut() { return out.get(); } - - bool write(const void *buf, size_t len) override { - return out ? out->write(buf, len) : false; - } - -protected: - strm_ptr out; -}; - -class FDOutStream : public OutStream { -public: - FDOutStream(int fd, bool close = false) : fd(fd), close(close) {} - - bool write(const void *buf, size_t len) override { - return ::write(fd, buf, len) == len; - } - - ~FDOutStream() override { - if (close) - ::close(fd); - } - -protected: - int fd; - bool close; -}; - -class BufOutStream : public OutStream { -public: - BufOutStream() : buf(nullptr), off(0), cap(0) {}; - - bool write(const void *b, size_t len) override { - bool resize = false; - while (off + len > cap) { - cap = cap ? cap << 1 : 1 << 19; - resize = true; - } - if (resize) - buf = (char *) xrealloc(buf, cap); - memcpy(buf + off, b, len); - off += len; - return true; - } - - template - void release(bytes *&b, length &len) { - b = buf; - len = off; - buf = nullptr; - off = cap = 0; - } - - template - void getbuf(bytes *&b, length &len) const { - b = buf; - len = off; - } - - ~BufOutStream() override { - free(buf); - } - -protected: - char *buf; - size_t off; - size_t cap; -}; diff --git a/native/jni/utils/stream.cpp b/native/jni/utils/stream.cpp index 442373cf6..1aa7d87fd 100644 --- a/native/jni/utils/stream.cpp +++ b/native/jni/utils/stream.cpp @@ -49,6 +49,25 @@ int stream::close() { return 0; } +int filter_stream::read(void *buf, size_t len) { + return fread(buf, len, 1, fp); +} + +int filter_stream::write(const void *buf, size_t len) { + return fwrite(buf, len, 1, fp); +} + +int filter_stream::close() { + int ret = fclose(fp); + fp = nullptr; + return ret; +} + +void filter_stream::set_base(FILE *f) { + if (fp) fclose(fp); + fp = f; +} + off_t seekable_stream::new_pos(off_t off, int whence) { off_t new_pos; switch (whence) {