Migrate to generic stream implementation

This commit is contained in:
topjohnwu 2019-11-21 06:08:02 -05:00
parent 4f9a25ee89
commit d26d804cc2
8 changed files with 648 additions and 770 deletions

View File

@ -21,19 +21,18 @@ using namespace std;
uint32_t dyn_img_hdr::j32 = 0;
uint64_t dyn_img_hdr::j64 = 0;
static int64_t one_step(unique_ptr<Compression> &&ptr, int fd, const void *in, size_t size) {
ptr->setOut(make_unique<FDOutStream>(fd));
if (!ptr->write(in, size))
return -1;
return ptr->finalize();
}
static int64_t decompress(format_t type, int fd, const void *in, size_t size) {
return one_step(unique_ptr<Compression>(get_decoder(type)), fd, in, size);
static void decompress(format_t type, int fd, const void *in, size_t size) {
unique_ptr<stream> ptr(get_decoder(type, open_stream<fd_stream>(fd)));
ptr->write(in, size);
}
static int64_t compress(format_t type, int fd, const void *in, size_t size) {
return one_step(unique_ptr<Compression>(get_encoder(type)), fd, in, size);
auto prev = lseek(fd, 0, SEEK_CUR);
unique_ptr<stream> ptr(get_encoder(type, open_stream<fd_stream>(fd)));
ptr->write(in, size);
ptr->close();
auto now = lseek(fd, 0, SEEK_CUR);
return now - prev;
}
static void dump(void *buf, size_t size, const char *filename) {

File diff suppressed because it is too large Load Diff

View File

@ -1,174 +1,10 @@
#pragma once
#include <zlib.h>
#include <bzlib.h>
#include <lzma.h>
#include <lz4.h>
#include <lz4frame.h>
#include <lz4hc.h>
#include <stream.h>
#include "format.h"
#define CHUNK 0x40000
class Compression : public FilterOutStream {
public:
virtual uint64_t finalize() = 0;
};
class GZStream : public Compression {
public:
bool write(const void *in, size_t size) override;
uint64_t finalize() override;
protected:
explicit GZStream(int mode);
private:
int mode;
z_stream strm;
uint8_t outbuf[CHUNK];
bool write(const void *in, size_t size, int flush);
};
class GZDecoder : public GZStream {
public:
GZDecoder() : GZStream(0) {};
};
class GZEncoder : public GZStream {
public:
GZEncoder() : GZStream(1) {};
};
class BZStream : public Compression {
public:
bool write(const void *in, size_t size) override;
uint64_t finalize() override;
protected:
explicit BZStream(int mode);
private:
int mode;
bz_stream strm;
char outbuf[CHUNK];
bool write(const void *in, size_t size, int flush);
};
class BZDecoder : public BZStream {
public:
BZDecoder() : BZStream(0) {};
};
class BZEncoder : public BZStream {
public:
BZEncoder() : BZStream(1) {};
};
class LZMAStream : public Compression {
public:
bool write(const void *in, size_t size) override;
uint64_t finalize() override;
protected:
explicit LZMAStream(int mode);
private:
int mode;
lzma_stream strm;
uint8_t outbuf[CHUNK];
bool write(const void *in, size_t size, lzma_action flush);
};
class LZMADecoder : public LZMAStream {
public:
LZMADecoder() : LZMAStream(0) {}
};
class XZEncoder : public LZMAStream {
public:
XZEncoder() : LZMAStream(1) {}
};
class LZMAEncoder : public LZMAStream {
public:
LZMAEncoder() : LZMAStream(2) {}
};
class LZ4FDecoder : public Compression {
public:
LZ4FDecoder();
~LZ4FDecoder() override;
bool write(const void *in, size_t size) override;
uint64_t finalize() override;
private:
LZ4F_decompressionContext_t ctx;
uint8_t *outbuf;
size_t outCapacity;
uint64_t total;
void read_header(const uint8_t *&in, size_t &size);
};
class LZ4FEncoder : public Compression {
public:
LZ4FEncoder();
~LZ4FEncoder() override;
bool write(const void *in, size_t size) override;
uint64_t finalize() override;
private:
static constexpr size_t BLOCK_SZ = 1 << 22;
LZ4F_compressionContext_t ctx;
uint8_t *outbuf;
size_t outCapacity;
uint64_t total;
void write_header();
};
#define LZ4_UNCOMPRESSED 0x800000
#define LZ4_COMPRESSED LZ4_COMPRESSBOUND(LZ4_UNCOMPRESSED)
class LZ4Decoder : public Compression {
public:
LZ4Decoder();
~LZ4Decoder() override;
bool write(const void *in, size_t size) override;
uint64_t finalize() override;
private:
char *outbuf;
char *buf;
bool init;
unsigned block_sz;
int buf_off;
uint64_t total;
};
class LZ4Encoder : public Compression {
public:
LZ4Encoder();
~LZ4Encoder() override;
bool write(const void *in, size_t size) override;
uint64_t finalize() override;
private:
char *outbuf;
char *buf;
bool init;
int buf_off;
uint64_t out_total;
unsigned in_total;
};
Compression *get_encoder(format_t type);
Compression *get_decoder(format_t type);
filter_stream *get_encoder(format_t type, FILE *fp = nullptr);
filter_stream *get_decoder(format_t type, FILE *fp = nullptr);
void compress(const char *method, const char *infile, const char *outfile);
void decompress(char *infile, const char *outfile);

View File

@ -241,14 +241,17 @@ void magisk_cpio::compress() {
return;
fprintf(stderr, "Compressing cpio -> [%s]\n", RAMDISK_XZ);
auto init = entries.extract("init");
XZEncoder encoder;
encoder.setOut(make_unique<BufOutStream>());
output(encoder);
encoder.finalize();
uint8_t *data;
size_t len;
FILE *fp = open_stream(get_encoder(XZ, open_stream<byte_stream>(data, len)));
dump(fp);
entries.clear();
entries.insert(std::move(init));
auto xz = new cpio_entry(RAMDISK_XZ, S_IFREG);
static_cast<BufOutStream *>(encoder.getOut())->release(xz->data, xz->filesize);
xz->data = data;
xz->filesize = len;
insert(xz);
}
@ -257,15 +260,16 @@ void magisk_cpio::decompress() {
if (it == entries.end())
return;
fprintf(stderr, "Decompressing cpio [%s]\n", RAMDISK_XZ);
LZMADecoder decoder;
decoder.setOut(make_unique<BufOutStream>());
decoder.write(it->second->data, it->second->filesize);
decoder.finalize();
char *data;
size_t len;
auto strm = get_decoder(XZ, open_stream<byte_stream>(data, len));
strm->write(it->second->data, it->second->filesize);
delete strm;
entries.erase(it);
char *buf;
size_t sz;
static_cast<BufOutStream *>(decoder.getOut())->getbuf(buf, sz);
load_cpio(buf, sz);
load_cpio(data, len);
free(data);
}
int cpio_commands(int argc, char *argv[]) {

View File

@ -49,8 +49,7 @@ cpio_entry_base::cpio_entry_base(const cpio_newc_header *h)
void cpio::dump(const char *file) {
fprintf(stderr, "Dump cpio: [%s]\n", file);
FDOutStream fd_out(xopen(file, O_WRONLY | O_CREAT | O_TRUNC, 0644), true);
output(fd_out);
dump(xfopen(file, "we"));
}
void cpio::rm(entry_map::iterator &it) {
@ -110,9 +109,9 @@ bool cpio::exists(const char *name) {
return entries.count(name) != 0;
}
#define do_out(b, l) out.write(b, l); pos += (l)
#define out_align() out.write(zeros, align_off(pos, 4)); pos = do_align(pos, 4)
void cpio::output(OutStream &out) {
#define do_out(buf, len) pos += fwrite(buf, len, 1, out);
#define out_align() do_out(zeros, align_off(pos, 4))
void cpio::dump(FILE *out) {
size_t pos = 0;
unsigned inode = 300000;
char header[111];
@ -147,6 +146,7 @@ void cpio::output(OutStream &out) {
do_out(header, 110);
do_out("TRAILER!!!\0", 11);
out_align();
fclose(out);
}
cpio_rw::cpio_rw(const char *file) {
@ -221,12 +221,12 @@ bool cpio_rw::mv(const char *from, const char *to) {
#define pos_align(p) p = do_align(p, 4)
void cpio_rw::load_cpio(char *buf, size_t sz) {
void cpio_rw::load_cpio(const char *buf, size_t sz) {
size_t pos = 0;
cpio_newc_header *header;
const cpio_newc_header *header;
unique_ptr<cpio_entry> entry;
while (pos < sz) {
header = (cpio_newc_header *)(buf + pos);
header = reinterpret_cast<const cpio_newc_header *>(buf + pos);
entry = make_unique<cpio_entry>(header);
pos += sizeof(*header);
string_view name_view(buf + pos);

View File

@ -30,7 +30,7 @@ struct cpio_entry : public cpio_entry_base {
explicit cpio_entry(const char *name, uint32_t mode) : filename(name) {
this->mode = mode;
}
explicit cpio_entry(cpio_newc_header *h) : cpio_entry_base(h) {}
explicit cpio_entry(const cpio_newc_header *h) : cpio_entry_base(h) {}
~cpio_entry() override { free(data); };
};
@ -48,7 +48,7 @@ public:
protected:
entry_map entries;
void rm(entry_map::iterator &it);
void output(OutStream &out);
void dump(FILE *out);
};
class cpio_rw : public cpio {
@ -64,7 +64,7 @@ public:
protected:
void insert(cpio_entry *e);
void mv(entry_map::iterator &it, const char *to);
void load_cpio(char *buf, size_t sz);
void load_cpio(const char *buf, size_t sz);
};
class cpio_mmap : public cpio {

View File

@ -15,8 +15,6 @@ FILE *open_stream(Args &&... args) {
return open_stream(new T(args...));
}
/* Base classes */
class stream {
public:
virtual int read(void *buf, size_t len);
@ -26,17 +24,17 @@ public:
virtual ~stream() = default;
};
// Delegates all operations to the base FILE pointer
class filter_stream : public stream {
public:
filter_stream(FILE *fp) : fp(fp) {}
int close() override { return fclose(fp); }
virtual ~filter_stream() { close(); }
~filter_stream() override { if (fp) close(); }
void set_base(FILE *f) {
if (fp) fclose(fp);
fp = f;
}
int read(void *buf, size_t len) override;
int write(const void *buf, size_t len) override;
int close() override;
void set_base(FILE *f);
template <class T, class... Args >
void set_base(Args&&... args) {
set_base(open_stream<T>(args...));
@ -46,18 +44,7 @@ protected:
FILE *fp;
};
class filter_in_stream : public filter_stream {
public:
filter_in_stream(FILE *fp = nullptr) : filter_stream(fp) {}
int read(void *buf, size_t len) override { return fread(buf, len, 1, fp); }
};
class filter_out_stream : public filter_stream {
public:
filter_out_stream(FILE *fp = nullptr) : filter_stream(fp) {}
int write(const void *buf, size_t len) override { return fwrite(buf, len, 1, fp); }
};
// Handy interface for classes that need custom seek logic
class seekable_stream : public stream {
protected:
size_t _pos = 0;
@ -66,8 +53,7 @@ protected:
virtual size_t end_pos() = 0;
};
/* Concrete classes */
// Byte stream that dynamically allocates memory
class byte_stream : public seekable_stream {
public:
byte_stream(uint8_t *&buf, size_t &len);
@ -76,7 +62,6 @@ public:
int read(void *buf, size_t len) override;
int write(const void *buf, size_t len) override;
off_t seek(off_t off, int whence) override;
virtual ~byte_stream() = default;
private:
uint8_t *&_buf;
@ -87,101 +72,14 @@ private:
size_t end_pos() override { return _len; }
};
class fd_stream : stream {
// File stream but does not close the file descriptor at any time
class fd_stream : public stream {
public:
fd_stream(int fd) : fd(fd) {}
int read(void *buf, size_t len) override;
int write(const void *buf, size_t len) override;
off_t seek(off_t off, int whence) override;
virtual ~fd_stream() = default;
private:
int fd;
};
/* TODO: Replace classes below to new implementation */
class OutStream {
public:
virtual bool write(const void *buf, size_t len) = 0;
virtual ~OutStream() = default;
};
typedef std::unique_ptr<OutStream> strm_ptr;
class FilterOutStream : public OutStream {
public:
FilterOutStream() = default;
FilterOutStream(strm_ptr &&ptr) : out(std::move(ptr)) {}
void setOut(strm_ptr &&ptr) { out = std::move(ptr); }
OutStream *getOut() { return out.get(); }
bool write(const void *buf, size_t len) override {
return out ? out->write(buf, len) : false;
}
protected:
strm_ptr out;
};
class FDOutStream : public OutStream {
public:
FDOutStream(int fd, bool close = false) : fd(fd), close(close) {}
bool write(const void *buf, size_t len) override {
return ::write(fd, buf, len) == len;
}
~FDOutStream() override {
if (close)
::close(fd);
}
protected:
int fd;
bool close;
};
class BufOutStream : public OutStream {
public:
BufOutStream() : buf(nullptr), off(0), cap(0) {};
bool write(const void *b, size_t len) override {
bool resize = false;
while (off + len > cap) {
cap = cap ? cap << 1 : 1 << 19;
resize = true;
}
if (resize)
buf = (char *) xrealloc(buf, cap);
memcpy(buf + off, b, len);
off += len;
return true;
}
template <typename bytes, typename length>
void release(bytes *&b, length &len) {
b = buf;
len = off;
buf = nullptr;
off = cap = 0;
}
template <typename bytes, typename length>
void getbuf(bytes *&b, length &len) const {
b = buf;
len = off;
}
~BufOutStream() override {
free(buf);
}
protected:
char *buf;
size_t off;
size_t cap;
};

View File

@ -49,6 +49,25 @@ int stream::close() {
return 0;
}
int filter_stream::read(void *buf, size_t len) {
return fread(buf, len, 1, fp);
}
int filter_stream::write(const void *buf, size_t len) {
return fwrite(buf, len, 1, fp);
}
int filter_stream::close() {
int ret = fclose(fp);
fp = nullptr;
return ret;
}
void filter_stream::set_base(FILE *f) {
if (fp) fclose(fp);
fp = f;
}
off_t seekable_stream::new_pos(off_t off, int whence) {
off_t new_pos;
switch (whence) {