From f43c8262c264b3aa2ff571ff63e4cdb5eb3288cd Mon Sep 17 00:00:00 2001
From: Igor Canadi <icanadi@fb.com>
Date: Mon, 9 Jun 2014 12:26:09 -0700
Subject: [PATCH] Don't compress block bigger than 2GB

Summary: This is a temporary solution to a issue that we have with compression libraries. See task #4453446.

Test Plan: make check doesn't complain :)

Reviewers: haobo, ljin, yhchiang, dhruba, sdong

Reviewed By: sdong

Subscribers: leveldb

Differential Revision: https://reviews.facebook.net/D18975
---
 include/rocksdb/statistics.h       |  2 ++
 table/block_based_table_builder.cc | 13 ++++++++++---
 table/block_based_table_builder.h  |  5 +++++
 3 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/include/rocksdb/statistics.h b/include/rocksdb/statistics.h
index dcd82f663..7d5235f65 100644
--- a/include/rocksdb/statistics.h
+++ b/include/rocksdb/statistics.h
@@ -125,6 +125,7 @@ enum Tickers {
   NUMBER_SUPERVERSION_ACQUIRES,
   NUMBER_SUPERVERSION_RELEASES,
   NUMBER_SUPERVERSION_CLEANUPS,
+  NUMBER_BLOCK_NOT_COMPRESSED,
   TICKER_ENUM_MAX
 };
 
@@ -183,6 +184,7 @@ const std::vector<std::pair<Tickers, std::string>> TickersNameMap = {
     {NUMBER_SUPERVERSION_ACQUIRES, "rocksdb.number.superversion_acquires"},
     {NUMBER_SUPERVERSION_RELEASES, "rocksdb.number.superversion_releases"},
     {NUMBER_SUPERVERSION_CLEANUPS, "rocksdb.number.superversion_cleanups"},
+    {NUMBER_BLOCK_NOT_COMPRESSED, "rocksdb.number.block.not_compressed"},
 };
 
 /**
diff --git a/table/block_based_table_builder.cc b/table/block_based_table_builder.cc
index 2ec6c1174..cf864ef4a 100644
--- a/table/block_based_table_builder.cc
+++ b/table/block_based_table_builder.cc
@@ -541,9 +541,16 @@ void BlockBasedTableBuilder::WriteBlock(const Slice& raw_block_contents,
   Rep* r = rep_;
 
   auto type = r->compression_type;
-  auto block_contents =
-      CompressBlock(raw_block_contents, r->options.compression_opts, &type,
-                    &r->compressed_output);
+  Slice block_contents;
+  if (raw_block_contents.size() < kCompressionSizeLimit) {
+    block_contents =
+        CompressBlock(raw_block_contents, r->options.compression_opts, &type,
+                      &r->compressed_output);
+  } else {
+    RecordTick(r->options.statistics.get(), NUMBER_BLOCK_NOT_COMPRESSED);
+    type = kNoCompression;
+    block_contents = raw_block_contents;
+  }
   WriteRawBlock(block_contents, type, handle);
   r->compressed_output.clear();
 }
diff --git a/table/block_based_table_builder.h b/table/block_based_table_builder.h
index 1fae6d069..72a2f207a 100644
--- a/table/block_based_table_builder.h
+++ b/table/block_based_table_builder.h
@@ -9,6 +9,7 @@
 
 #pragma once
 #include <stdint.h>
+#include <limits>
 
 #include "rocksdb/flush_block_policy.h"
 #include "rocksdb/options.h"
@@ -84,6 +85,10 @@ class BlockBasedTableBuilder : public TableBuilder {
   // REQUIRES: Finish(), Abandon() have not been called
   void Flush();
 
+  // Some compression libraries fail when the raw size is bigger than int. If
+  // uncompressed size is bigger than kCompressionSizeLimit, don't compress it
+  const uint64_t kCompressionSizeLimit = std::numeric_limits<int>::max();
+
   // No copying allowed
   BlockBasedTableBuilder(const BlockBasedTableBuilder&) = delete;
   void operator=(const BlockBasedTableBuilder&) = delete;