From bb42ef068e95e54235b9f960449173c2648b63b6 Mon Sep 17 00:00:00 2001 From: Connor Tumbleson Date: Fri, 6 Apr 2018 14:57:09 -0400 Subject: [PATCH 1/9] Revert "fix: Applications with implicit sdk versioning lose that information" This reverts commit 25a164dd10e2e7b0300295e860b7c2772314ba0f. --- .../apktool-lib/src/main/java/brut/androlib/Androlib.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/brut.apktool/apktool-lib/src/main/java/brut/androlib/Androlib.java b/brut.apktool/apktool-lib/src/main/java/brut/androlib/Androlib.java index 4c46008e..513cc773 100644 --- a/brut.apktool/apktool-lib/src/main/java/brut/androlib/Androlib.java +++ b/brut.apktool/apktool-lib/src/main/java/brut/androlib/Androlib.java @@ -692,10 +692,6 @@ public class Androlib { if (inputFile.isDirectory()) { continue; } - if (! inputFile.isFile()) { - LOGGER.warning(String.format("File could not be located: %s", normalizedPath)); - continue; - } LOGGER.fine(String.format("Copying uncompressed asset: %s", normalizedPath)); ZipEntry newEntry = getStoredZipEntry(normalizedPath, inputFile); From 46ebcb0fab84af5af175e47eae740d9d1a2b074d Mon Sep 17 00:00:00 2001 From: Connor Tumbleson Date: Fri, 6 Apr 2018 14:57:24 -0400 Subject: [PATCH 2/9] Revert "fix: don't use deprecated property "doNotCompress"" This reverts commit 258fb82fe2a0bf9c6dd9b8d433178b20273f5a60. --- .../src/main/java/brut/androlib/res/AndrolibResources.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brut.apktool/apktool-lib/src/main/java/brut/androlib/res/AndrolibResources.java b/brut.apktool/apktool-lib/src/main/java/brut/androlib/res/AndrolibResources.java index 594c6e82..b8ceeec9 100644 --- a/brut.apktool/apktool-lib/src/main/java/brut/androlib/res/AndrolibResources.java +++ b/brut.apktool/apktool-lib/src/main/java/brut/androlib/res/AndrolibResources.java @@ -601,7 +601,7 @@ final public class AndrolibResources { throws AndrolibException { try { - ZipUtils.zipFolders(rawDir, apkFile, assetDir, apkOptions.noCompressAssets); + ZipUtils.zipFolders(rawDir, apkFile, assetDir, apkOptions.doNotCompress); } catch (IOException | BrutException ex) { throw new AndrolibException(ex); } From 72196f3544a02b3d3bdb517a8054ac3bdf5b9195 Mon Sep 17 00:00:00 2001 From: Connor Tumbleson Date: Fri, 6 Apr 2018 14:58:11 -0400 Subject: [PATCH 3/9] Revert "fix: normalize paths for uncompressed assets & unknown files" This reverts commit ca25a2694408ab7989fd107db7f8ff300fa3f779. --- .../src/main/java/brut/androlib/Androlib.java | 10 ++++------ .../java/brut/androlib/res/data/ResUnknownFiles.java | 3 +-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/brut.apktool/apktool-lib/src/main/java/brut/androlib/Androlib.java b/brut.apktool/apktool-lib/src/main/java/brut/androlib/Androlib.java index 513cc773..adf6c994 100644 --- a/brut.apktool/apktool-lib/src/main/java/brut/androlib/Androlib.java +++ b/brut.apktool/apktool-lib/src/main/java/brut/androlib/Androlib.java @@ -653,8 +653,7 @@ public class Androlib { // loop through unknown files for (Map.Entry unknownFileInfo : files.entrySet()) { - String normalizedPath = BrutIO.normalizePath(unknownFileInfo.getKey()); - String cleanedPath = BrutIO.sanitizeUnknownFile(unknownFileDir, normalizedPath); + String cleanedPath = BrutIO.sanitizeUnknownFile(unknownFileDir, unknownFileInfo.getKey()); File inputFile = new File(unknownFileDir, cleanedPath); if (inputFile.isDirectory()) { continue; @@ -685,16 +684,15 @@ public class Androlib { File assetFileDir = new File(appDir, ASSET_DIRNAME); for (String asset : files) { - String normalizedPath = BrutIO.normalizePath(asset); - String cleanedPath = BrutIO.sanitizeUnknownFile(assetFileDir, normalizedPath); + String cleanedPath = BrutIO.sanitizeUnknownFile(assetFileDir, asset); File inputFile = new File(appDir, cleanedPath); if (inputFile.isDirectory()) { continue; } - LOGGER.fine(String.format("Copying uncompressed asset: %s", normalizedPath)); - ZipEntry newEntry = getStoredZipEntry(normalizedPath, inputFile); + LOGGER.fine(String.format("Copying uncompressed asset: %s", asset)); + ZipEntry newEntry = getStoredZipEntry(cleanedPath, inputFile); outputFile.putNextEntry(newEntry); BrutIO.copy(inputFile, outputFile); outputFile.closeEntry(); diff --git a/brut.apktool/apktool-lib/src/main/java/brut/androlib/res/data/ResUnknownFiles.java b/brut.apktool/apktool-lib/src/main/java/brut/androlib/res/data/ResUnknownFiles.java index a73362aa..595c4343 100644 --- a/brut.apktool/apktool-lib/src/main/java/brut/androlib/res/data/ResUnknownFiles.java +++ b/brut.apktool/apktool-lib/src/main/java/brut/androlib/res/data/ResUnknownFiles.java @@ -16,7 +16,6 @@ */ package brut.androlib.res.data; -import brut.util.BrutIO; import java.util.LinkedHashMap; import java.util.Map; @@ -28,7 +27,7 @@ public class ResUnknownFiles { private final Map mUnknownFiles = new LinkedHashMap<>(); public void addUnknownFileInfo(String file, String value) { - mUnknownFiles.put(BrutIO.normalizePath(file), value); + mUnknownFiles.put(file, value); } public Map getUnknownFiles() { From 0700c684b6e1898e9a0272e21d51080ad7c43440 Mon Sep 17 00:00:00 2001 From: Connor Tumbleson Date: Fri, 6 Apr 2018 14:58:35 -0400 Subject: [PATCH 4/9] Revert "build: introduce "buildUnknownAndUncompressedAssets"" This reverts commit e35f66b0e084252a96a800298fd98e3a184ae0e4. --- .../src/main/java/brut/androlib/Androlib.java | 112 +++++------------- 1 file changed, 32 insertions(+), 80 deletions(-) diff --git a/brut.apktool/apktool-lib/src/main/java/brut/androlib/Androlib.java b/brut.apktool/apktool-lib/src/main/java/brut/androlib/Androlib.java index adf6c994..35852f30 100644 --- a/brut.apktool/apktool-lib/src/main/java/brut/androlib/Androlib.java +++ b/brut.apktool/apktool-lib/src/main/java/brut/androlib/Androlib.java @@ -19,6 +19,7 @@ package brut.androlib; import brut.androlib.meta.MetaInfo; import brut.androlib.meta.UsesFramework; import brut.androlib.res.AndrolibResources; +import brut.androlib.res.data.ResConfigFlags; import brut.androlib.res.data.ResPackage; import brut.androlib.res.data.ResTable; import brut.androlib.res.data.ResUnknownFiles; @@ -261,7 +262,6 @@ public class Androlib { apkOptions.isFramework = meta.isFrameworkApk; apkOptions.resourcesAreCompressed = meta.compressionType; apkOptions.doNotCompress = meta.doNotCompress; - apkOptions.noCompressAssets = meta.noCompressAssets; mAndRes.setSdkInfo(meta.sdkInfo); mAndRes.setPackageId(meta.packageInfo); @@ -293,8 +293,8 @@ public class Androlib { buildApk(appDir, outFile); // we must go after the Apk is built, and copy the files in via Zip - // this is because Aapt won't add files it doesn't know (ex unknown files & uncompressed assets) - buildUnknownAndUncompressedAssets(appDir, outFile, meta); + // this is because Aapt won't add files it doesn't know (ex unknown files) + buildUnknownFiles(appDir, outFile, meta); // we copied the AndroidManifest.xml to AndroidManifest.xml.orig so we can edit it // lets restore the unedited one, to not change the original @@ -589,46 +589,38 @@ public class Androlib { } } - public void buildUnknownAndUncompressedAssets(File appDir, File outFile, MetaInfo meta) + public void buildUnknownFiles(File appDir, File outFile, MetaInfo meta) throws AndrolibException { - if (meta.unknownFiles == null && meta.noCompressAssets == null) { - return; - } - if (meta.unknownFiles != null) { LOGGER.info("Copying unknown files/dir..."); - } - File tempFile = new File(outFile.getParent(), outFile.getName() + ".apktool_temp"); - boolean renamed = outFile.renameTo(tempFile); - if (!renamed) { - throw new AndrolibException("Unable to rename temporary file"); - } + Map files = meta.unknownFiles; + File tempFile = new File(outFile.getParent(), outFile.getName() + ".apktool_temp"); + boolean renamed = outFile.renameTo(tempFile); + if (!renamed) { + throw new AndrolibException("Unable to rename temporary file"); + } - try ( - ZipFile inputFile = new ZipFile(tempFile); - ZipOutputStream actualOutput = new ZipOutputStream(new FileOutputStream(outFile)) - ) { - copyExistingFiles(inputFile, actualOutput, meta.noCompressAssets); - copyUncompressedAssetFiles(appDir, actualOutput, meta.noCompressAssets); - copyUnknownFiles(appDir, actualOutput, meta.unknownFiles); - } catch (IOException | BrutException ex) { - throw new AndrolibException(ex); - } + try ( + ZipFile inputFile = new ZipFile(tempFile); + ZipOutputStream actualOutput = new ZipOutputStream(new FileOutputStream(outFile)) + ) { + copyExistingFiles(inputFile, actualOutput); + copyUnknownFiles(appDir, actualOutput, files); + } catch (IOException | BrutException ex) { + throw new AndrolibException(ex); + } - // Remove our temporary file. - tempFile.delete(); + // Remove our temporary file. + tempFile.delete(); + } } - private void copyExistingFiles(ZipFile inputFile, ZipOutputStream outputFile, Collection assets) - throws IOException { + private void copyExistingFiles(ZipFile inputFile, ZipOutputStream outputFile) throws IOException { // First, copy the contents from the existing outFile: Enumeration entries = inputFile.entries(); while (entries.hasMoreElements()) { ZipEntry entry = new ZipEntry(entries.nextElement()); - if (assets != null && assets.contains(entry.getName())) { - continue; - } // We can't reuse the compressed size because it depends on compression sizes. entry.setCompressedSize(-1); @@ -645,25 +637,25 @@ public class Androlib { private void copyUnknownFiles(File appDir, ZipOutputStream outputFile, Map files) throws BrutException, IOException { - File unknownFileDir = new File(appDir, UNK_DIRNAME); - if (files == null || files.isEmpty()) { - return; - } // loop through unknown files for (Map.Entry unknownFileInfo : files.entrySet()) { - String cleanedPath = BrutIO.sanitizeUnknownFile(unknownFileDir, unknownFileInfo.getKey()); - File inputFile = new File(unknownFileDir, cleanedPath); + File inputFile = new File(unknownFileDir, BrutIO.sanitizeUnknownFile(unknownFileDir, unknownFileInfo.getKey())); if (inputFile.isDirectory()) { continue; } - ZipEntry newEntry = new ZipEntry(cleanedPath); + ZipEntry newEntry = new ZipEntry(unknownFileInfo.getKey()); int method = Integer.parseInt(unknownFileInfo.getValue()); LOGGER.fine(String.format("Copying unknown file %s with method %d", unknownFileInfo.getKey(), method)); if (method == ZipEntry.STORED) { - newEntry = getStoredZipEntry(cleanedPath, inputFile); + newEntry.setMethod(ZipEntry.STORED); + newEntry.setSize(inputFile.length()); + newEntry.setCompressedSize(-1); + BufferedInputStream unknownFile = new BufferedInputStream(new FileInputStream(inputFile)); + CRC32 crc = BrutIO.calculateCrc(unknownFile); + newEntry.setCrc(crc.getValue()); } else { newEntry.setMethod(ZipEntry.DEFLATED); } @@ -674,45 +666,6 @@ public class Androlib { } } - private void copyUncompressedAssetFiles(File appDir, ZipOutputStream outputFile, Collection files) - throws BrutException, IOException { - - if (files == null || files.isEmpty()) { - return; - } - - File assetFileDir = new File(appDir, ASSET_DIRNAME); - - for (String asset : files) { - String cleanedPath = BrutIO.sanitizeUnknownFile(assetFileDir, asset); - - File inputFile = new File(appDir, cleanedPath); - if (inputFile.isDirectory()) { - continue; - } - - LOGGER.fine(String.format("Copying uncompressed asset: %s", asset)); - ZipEntry newEntry = getStoredZipEntry(cleanedPath, inputFile); - outputFile.putNextEntry(newEntry); - BrutIO.copy(inputFile, outputFile); - outputFile.closeEntry(); - } - } - - private ZipEntry getStoredZipEntry(String cleanedPath, File inputFile) - throws IOException { - - ZipEntry newEntry = new ZipEntry(cleanedPath); - newEntry.setMethod(ZipEntry.STORED); - newEntry.setSize(inputFile.length()); - newEntry.setCompressedSize(-1); - BufferedInputStream unknownFile = new BufferedInputStream(new FileInputStream(inputFile)); - CRC32 crc = BrutIO.calculateCrc(unknownFile); - newEntry.setCrc(crc.getValue()); - - return newEntry; - } - public void buildApk(File appDir, File outApk) throws AndrolibException { LOGGER.info("Building apk file..."); if (outApk.exists()) { @@ -810,7 +763,6 @@ public class Androlib { private final static String SMALI_DIRNAME = "smali"; private final static String APK_DIRNAME = "build/apk"; private final static String UNK_DIRNAME = "unknown"; - private final static String ASSET_DIRNAME = "assets"; private final static String[] APK_RESOURCES_FILENAMES = new String[] { "resources.arsc", "AndroidManifest.xml", "res" }; private final static String[] APK_RESOURCES_WITHOUT_RES_FILENAMES = new String[] { @@ -825,6 +777,6 @@ public class Androlib { // Taken from AOSP's frameworks/base/tools/aapt/Package.cpp private final static Pattern NO_COMPRESS_PATTERN = Pattern.compile("\\.(" + "jpg|jpeg|png|gif|wav|mp2|mp3|ogg|aac|mpg|mpeg|mid|midi|smf|jet|rtttl|imy|xmf|mp4|" + - "m4a|m4v|3gp|3gpp|3g2|3gpp2|amr|awb|wma|wmv|webm|mkv|arsc)$"); + "m4a|m4v|3gp|3gpp|3g2|3gpp2|amr|awb|wma|wmv|webm|mkv)$"); } From 4d8922868616b85180507f899f95b7edaebb8687 Mon Sep 17 00:00:00 2001 From: Connor Tumbleson Date: Fri, 6 Apr 2018 14:58:42 -0400 Subject: [PATCH 5/9] Revert "decode: store uncompressed assets to new property" This reverts commit 14eedde165623e0f28bad036b3ce8af883a0a635. --- .../apktool-lib/src/main/java/brut/androlib/ApkDecoder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkDecoder.java b/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkDecoder.java index 5188beb7..4bf5b001 100644 --- a/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkDecoder.java +++ b/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkDecoder.java @@ -418,7 +418,7 @@ public class ApkDecoder { private void putFileCompressionInfo(MetaInfo meta) throws AndrolibException { if (mUncompressedFiles != null && !mUncompressedFiles.isEmpty()) { - meta.noCompressAssets = mUncompressedFiles; + meta.doNotCompress = mUncompressedFiles; } } From 923ae6e9b8627ab3d5d1e4dac0d2436d1e9012ca Mon Sep 17 00:00:00 2001 From: Connor Tumbleson Date: Fri, 6 Apr 2018 14:58:48 -0400 Subject: [PATCH 6/9] Revert "tests: refactor tests to use assertEquals instead of hacky assertTrue" This reverts commit d6e3d013e7117d9c41ddb21db1e42859dfec4da2. --- .../apktool-lib/src/test/java/brut/androlib/BaseTest.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/brut.apktool/apktool-lib/src/test/java/brut/androlib/BaseTest.java b/brut.apktool/apktool-lib/src/test/java/brut/androlib/BaseTest.java index 5c9d7a8c..d24bd9d6 100644 --- a/brut.apktool/apktool-lib/src/test/java/brut/androlib/BaseTest.java +++ b/brut.apktool/apktool-lib/src/test/java/brut/androlib/BaseTest.java @@ -31,7 +31,6 @@ import java.util.Map; import java.util.Set; import java.util.logging.Logger; -import static junit.framework.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.custommonkey.xmlunit.XMLAssert.assertXMLEqual; @@ -46,11 +45,11 @@ public class BaseTest { Map controlFiles = control.unknownFiles; Map testFiles = test.unknownFiles; - assertEquals(controlFiles.size(), testFiles.size()); + assertTrue(controlFiles.size() == testFiles.size()); // Make sure that the compression methods are still the same for (Map.Entry controlEntry : controlFiles.entrySet()) { - assertEquals(controlEntry.getValue(), testFiles.get(controlEntry.getKey())); + assertTrue(controlEntry.getValue().equals(testFiles.get(controlEntry.getKey()))); } } From a073bad32dfa79922a72fbfb41ecdaf3210d46ff Mon Sep 17 00:00:00 2001 From: Connor Tumbleson Date: Fri, 6 Apr 2018 14:58:56 -0400 Subject: [PATCH 7/9] Revert "tests: update test to use new "noCompressAssets" property" This reverts commit 653472d006f25d9933fe60a9978cfd65f196a723. --- .../brut/androlib/decode/DoubleExtensionUnknownFileTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brut.apktool/apktool-lib/src/test/java/brut/androlib/decode/DoubleExtensionUnknownFileTest.java b/brut.apktool/apktool-lib/src/test/java/brut/androlib/decode/DoubleExtensionUnknownFileTest.java index a08d5f5b..9290b05b 100644 --- a/brut.apktool/apktool-lib/src/test/java/brut/androlib/decode/DoubleExtensionUnknownFileTest.java +++ b/brut.apktool/apktool-lib/src/test/java/brut/androlib/decode/DoubleExtensionUnknownFileTest.java @@ -62,7 +62,7 @@ public class DoubleExtensionUnknownFileTest extends BaseTest { apkDecoder.decode(); MetaInfo metaInfo = new Androlib().readMetaFile(decodedApk); - for (String string : metaInfo.noCompressAssets) { + for (String string : metaInfo.doNotCompress) { if (StringUtils.countMatches(string, ".") > 1) { assertTrue(string.equalsIgnoreCase("assets/bin/Data/sharedassets1.assets.split0")); } From d755724ed839bfc25f631564cebb89e86abd7041 Mon Sep 17 00:00:00 2001 From: Connor Tumbleson Date: Fri, 6 Apr 2018 14:59:03 -0400 Subject: [PATCH 8/9] Revert "decode: only store full relative path to uncompressed assets" This reverts commit bd9c9c7cec67337518f0f6e986be8634f2a452bb. --- .../src/main/java/brut/androlib/Androlib.java | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/brut.apktool/apktool-lib/src/main/java/brut/androlib/Androlib.java b/brut.apktool/apktool-lib/src/main/java/brut/androlib/Androlib.java index 35852f30..7240554c 100644 --- a/brut.apktool/apktool-lib/src/main/java/brut/androlib/Androlib.java +++ b/brut.apktool/apktool-lib/src/main/java/brut/androlib/Androlib.java @@ -41,6 +41,8 @@ import java.util.zip.ZipFile; import java.util.zip.ZipOutputStream; import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.StringUtils; /** * @author Ryszard Wiśniewski @@ -159,15 +161,28 @@ public class Androlib { } } - public void recordUncompressedFiles(ExtFile apkFile, Collection uncompressedFiles) throws AndrolibException { + public void recordUncompressedFiles(ExtFile apkFile, Collection uncompressedFilesOrExts) throws AndrolibException { try { Directory unk = apkFile.getDirectory(); Set files = unk.getFiles(true); + String ext; for (String file : files) { if (isAPKFileNames(file) && !NO_COMPRESS_PATTERN.matcher(file).find()) { - if (unk.getCompressionLevel(file) == 0 && !uncompressedFiles.contains(file)) { - uncompressedFiles.add(file); + if (unk.getCompressionLevel(file) == 0) { + + if (StringUtils.countMatches(file, ".") > 1) { + ext = file; + } else { + ext = FilenameUtils.getExtension(file); + if (ext.isEmpty()) { + ext = file; + } + } + + if (! uncompressedFilesOrExts.contains(ext)) { + uncompressedFilesOrExts.add(ext); + } } } } From 6554881606e300a44c4b1deeb4fda8acd2d7e2a0 Mon Sep 17 00:00:00 2001 From: Connor Tumbleson Date: Fri, 6 Apr 2018 14:59:10 -0400 Subject: [PATCH 9/9] Revert "options: deprecate "doNotCompress", introduce "noCompressAssets"" This reverts commit cb6b6dfda813c377b4406b12a8b78fbb79afdda5. --- .../apktool-lib/src/main/java/brut/androlib/ApkOptions.java | 3 --- .../apktool-lib/src/main/java/brut/androlib/meta/MetaInfo.java | 3 --- 2 files changed, 6 deletions(-) diff --git a/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkOptions.java b/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkOptions.java index 8eec9205..0a81ef07 100644 --- a/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkOptions.java +++ b/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkOptions.java @@ -28,10 +28,7 @@ public class ApkOptions { public boolean isFramework = false; public boolean resourcesAreCompressed = false; public boolean useAapt2 = false; - - @Deprecated public Collection doNotCompress; - public Collection noCompressAssets; public String frameworkFolderLocation = null; public String frameworkTag = null; diff --git a/brut.apktool/apktool-lib/src/main/java/brut/androlib/meta/MetaInfo.java b/brut.apktool/apktool-lib/src/main/java/brut/androlib/meta/MetaInfo.java index 9921090e..7da11d4d 100644 --- a/brut.apktool/apktool-lib/src/main/java/brut/androlib/meta/MetaInfo.java +++ b/brut.apktool/apktool-lib/src/main/java/brut/androlib/meta/MetaInfo.java @@ -37,10 +37,7 @@ public class MetaInfo { public boolean sharedLibrary; public boolean sparseResources; public Map unknownFiles; - - @Deprecated public Collection doNotCompress; - public Collection noCompressAssets; private static Yaml getYaml() { DumperOptions options = new DumperOptions();