Feat: Introduce "res-mode" options. (#3318)

* Revert "Remove Apktool Dummys. (#3258)"

This reverts commit 0e226928ce.

* feat: properly add dummys

* refactor: shorten ResTypeSpec

* style: remove extra space

* refactor: extract FlagItem into own class

* refactor: notate which type is null

* fix: only add dummys if enabled

* feat: skip unknown (if enabled)

* feat: introduce "res-mode"

* feat: expose config on res table

* feat: add method to base attr for res skips

* fix: ensure autobuild doesn't choke

* refactor: remove java17 enhanced switch

* refactor: rename methods

* refactor: cleanup res-mode param

* test: introduction of test/sample apk

* refactor: make ResXmlPatcher public for loading XML

* test: assertions for dummy|leave|retain

* fix: prevent using `@null` as a name

* refactor: shorten long param for 'resm'

* refactor: leave for preserve
This commit is contained in:
Connor Tumbleson 2023-09-09 07:29:39 -04:00 committed by GitHub
parent f0ca6d11f9
commit a117132e39
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 335 additions and 35 deletions

View File

@ -167,6 +167,30 @@ public class Main {
if (cli.hasOption("m") || cli.hasOption("match-original")) {
config.analysisMode = true;
}
if (cli.hasOption("res-mode") || cli.hasOption("resolve-resources-mode")) {
String mode = cli.getOptionValue("res-mode");
if (mode == null) {
mode = cli.getOptionValue("resolve-resources-mode");
}
switch (mode) {
case "remove":
case "delete":
config.setDecodeResolveMode(Config.DECODE_RES_RESOLVE_REMOVE);
break;
case "dummy":
case "dummies":
config.setDecodeResolveMode(Config.DECODE_RES_RESOLVE_DUMMY);
break;
case "keep":
case "preserve":
config.setDecodeResolveMode(Config.DECODE_RES_RESOLVE_RETAIN);
break;
default:
System.err.println("Unknown resolve resources mode: " + mode);
System.err.println("Expect: 'remove', 'dummy' or 'keep'.");
System.exit(1);
}
}
File outDir;
if (cli.hasOption("o") || cli.hasOption("output")) {
@ -301,8 +325,6 @@ public class Main {
}
private static void _Options() {
// create options
Option versionOption = Option.builder("version")
.longOpt("version")
.desc("Print the version.")
@ -409,6 +431,13 @@ public class Main {
.desc("Skip changes detection and build all files.")
.build();
Option resolveResModeOption = Option.builder("resm")
.longOpt("resource-mode")
.desc("Sets the resolve resources mode. Possible values are: 'remove' (default), 'dummy' or 'keep'.")
.hasArg(true)
.argName("mode")
.build();
Option aaptOption = Option.builder("a")
.longOpt("aapt")
.hasArg(true)
@ -469,6 +498,7 @@ public class Main {
decodeOptions.addOption(apiLevelOption);
decodeOptions.addOption(noAssetOption);
decodeOptions.addOption(forceManOption);
decodeOptions.addOption(resolveResModeOption);
buildOptions.addOption(apiLevelOption);
buildOptions.addOption(debugBuiOption);
@ -525,6 +555,7 @@ public class Main {
allOptions.addOption(debugDecOption);
allOptions.addOption(noDbgOption);
allOptions.addOption(forceManOption);
allOptions.addOption(resolveResModeOption);
allOptions.addOption(noAssetOption);
allOptions.addOption(keepResOption);
allOptions.addOption(debugBuiOption);

View File

@ -23,6 +23,7 @@ import java.io.File;
import java.util.logging.Logger;
public class Config {
private static Config instance = null;
private final static Logger LOGGER = Logger.getLogger(Config.class.getName());
public final static short DECODE_SOURCES_NONE = 0x0000;
@ -38,6 +39,10 @@ public class Config {
public final static short DECODE_ASSETS_NONE = 0x0000;
public final static short DECODE_ASSETS_FULL = 0x0001;
public final static short DECODE_RES_RESOLVE_REMOVE = 0x0000;
public final static short DECODE_RES_RESOLVE_DUMMY = 0x0001;
public final static short DECODE_RES_RESOLVE_RETAIN = 0x0002;
// Build options
public boolean forceBuildAll = false;
public boolean forceDeleteFramework = false;
@ -55,6 +60,7 @@ public class Config {
public short decodeResources = DECODE_RESOURCES_FULL;
public short forceDecodeManifest = FORCE_DECODE_MANIFEST_NONE;
public short decodeAssets = DECODE_ASSETS_FULL;
public short decodeResolveMode = DECODE_RES_RESOLVE_REMOVE;
public int apiLevel = 0;
public boolean analysisMode = false;
public boolean forceDelete = false;
@ -72,8 +78,23 @@ public class Config {
return this.useAapt2 || this.aaptVersion == 2;
}
private Config() {
public boolean isDecodeResolveModeUsingDummies() {
return decodeResolveMode == DECODE_RES_RESOLVE_DUMMY;
}
public boolean isDecodeResolveModeRemoving() {
return decodeResolveMode == DECODE_RES_RESOLVE_REMOVE;
}
private Config() {
instance = this;
}
public static Config getInstance() {
if (instance == null) {
instance = new Config();
}
return instance;
}
private void setDefaultFrameworkDirectory() {
@ -105,6 +126,13 @@ public class Config {
decodeSources = mode;
}
public void setDecodeResolveMode(short mode) throws AndrolibException {
if (mode != DECODE_RES_RESOLVE_REMOVE && mode != DECODE_RES_RESOLVE_DUMMY && mode != DECODE_RES_RESOLVE_RETAIN) {
throw new AndrolibException("Invalid decode resources mode");
}
decodeResolveMode = mode;
}
public void setDecodeResources(short mode) throws AndrolibException {
if (mode != DECODE_RESOURCES_NONE && mode != DECODE_RESOURCES_FULL) {
throw new AndrolibException("Invalid decode resources mode");

View File

@ -105,6 +105,10 @@ public class ResResSpec {
return mType;
}
public boolean isDummyResSpec() {
return getName().startsWith("APKTOOL_DUMMY_");
}
public void addResource(ResResource res) throws AndrolibException {
addResource(res, false);
}

View File

@ -70,6 +70,10 @@ public class ResTable {
return mConfig.analysisMode;
}
public Config getConfig() {
return mConfig;
}
public boolean isMainPkgLoaded() {
return mMainPkgLoaded;
}

View File

@ -0,0 +1,41 @@
/*
* Copyright (C) 2010 Ryszard Wiśniewski <brut.alll@gmail.com>
* Copyright (C) 2010 Connor Tumbleson <connor.tumbleson@gmail.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package brut.androlib.res.data.arsc;
import brut.androlib.exceptions.AndrolibException;
import brut.androlib.res.data.value.ResReferenceValue;
public class FlagItem {
public final ResReferenceValue ref;
public final int flag;
public String value;
public FlagItem(ResReferenceValue ref, int flag) {
this.ref = ref;
this.flag = flag;
}
public String getValue() throws AndrolibException {
if (value == null) {
if (ref.referentIsNull()) {
return String.format("APKTOOL_MISSING_0x%08x", ref.getRawIntValue());
}
value = ref.getReferent().getName();
}
return value;
}
}

View File

@ -25,6 +25,7 @@ import org.xmlpull.v1.XmlSerializer;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.logging.Logger;
public class ResEnumAttr extends ResAttr {
ResEnumAttr(ResReferenceValue parent, int type, Integer min, Integer max,
@ -46,15 +47,21 @@ public class ResEnumAttr extends ResAttr {
}
@Override
protected void serializeBody(XmlSerializer serializer, ResResource res)
throws AndrolibException, IOException {
protected void serializeBody(XmlSerializer serializer, ResResource res) throws AndrolibException, IOException {
for (Duo<ResReferenceValue, ResIntValue> duo : mItems) {
int intVal = duo.m2.getValue();
// #2836 - Support skipping items if the resource cannot be identified.
ResResSpec m1Referent = duo.m1.getReferent();
if (m1Referent == null && shouldRemoveUnknownRes()) {
LOGGER.fine(String.format("null enum reference: m1=0x%08x(%s), m2=0x%08x(%s)",
duo.m1.getRawIntValue(), duo.m1.getType(), duo.m2.getRawIntValue(), duo.m2.getType()));
continue;
}
serializer.startTag(null, "enum");
serializer.attribute(null, "name",
m1Referent != null ? m1Referent.getName() : "@null"
m1Referent != null ? m1Referent.getName() : String.format("APKTOOL_MISSING_0x%08x", duo.m1.getRawIntValue())
);
serializer.attribute(null, "value", String.valueOf(intVal));
serializer.endTag(null, "enum");
@ -81,4 +88,6 @@ public class ResEnumAttr extends ResAttr {
private final Duo<ResReferenceValue, ResIntValue>[] mItems;
private final Map<Integer, String> mItemsCache = new HashMap<>();
private static final Logger LOGGER = Logger.getLogger(ResEnumAttr.class.getName());
}

View File

@ -17,12 +17,15 @@
package brut.androlib.res.data.value;
import brut.androlib.exceptions.AndrolibException;
import brut.androlib.res.data.ResResSpec;
import brut.androlib.res.data.ResResource;
import brut.androlib.res.data.arsc.FlagItem;
import brut.util.Duo;
import org.xmlpull.v1.XmlSerializer;
import java.io.IOException;
import java.util.Arrays;
import java.util.logging.Logger;
public class ResFlagsAttr extends ResAttr {
ResFlagsAttr(ResReferenceValue parent, int type, Integer min, Integer max,
@ -38,7 +41,7 @@ public class ResFlagsAttr extends ResAttr {
@Override
public String convertToResXmlFormat(ResScalarValue value)
throws AndrolibException {
if(value instanceof ResReferenceValue) {
if (value instanceof ResReferenceValue) {
return value.encodeAsResXml();
}
if (!(value instanceof ResIntValue)) {
@ -70,13 +73,19 @@ public class ResFlagsAttr extends ResAttr {
}
@Override
protected void serializeBody(XmlSerializer serializer, ResResource res)
throws AndrolibException, IOException {
protected void serializeBody(XmlSerializer serializer, ResResource res) throws AndrolibException, IOException {
for (FlagItem item : mItems) {
ResResSpec referent = item.ref.getReferent();
// #2836 - Support skipping items if the resource cannot be identified.
if (referent == null && shouldRemoveUnknownRes()) {
LOGGER.fine(String.format("null flag reference: 0x%08x(%s)", item.ref.getValue(), item.ref.getType()));
continue;
}
serializer.startTag(null, "flag");
serializer.attribute(null, "name", item.getValue());
serializer.attribute(null, "value",
String.format("0x%08x", item.flag));
serializer.attribute(null, "value", String.format("0x%08x", item.flag));
serializer.endTag(null, "flag");
}
}
@ -130,24 +139,5 @@ public class ResFlagsAttr extends ResAttr {
private FlagItem[] mZeroFlags;
private FlagItem[] mFlags;
private static class FlagItem {
public final ResReferenceValue ref;
public final int flag;
public String value;
public FlagItem(ResReferenceValue ref, int flag) {
this.ref = ref;
this.flag = flag;
}
public String getValue() throws AndrolibException {
if (value == null) {
if (ref.referentIsNull()) {
return "@null";
}
value = ref.getReferent().getName();
}
return value;
}
}
private static final Logger LOGGER = Logger.getLogger(ResFlagsAttr.class.getName());
}

View File

@ -81,6 +81,11 @@ public abstract class ResScalarValue extends ResIntBasedValue implements
}
}
// Dummy attributes should be <item> with type attribute
if (res.getResSpec().isDummyResSpec()) {
item = true;
}
// Android does not allow values (false) for ids.xml anymore
// https://issuetracker.google.com/issues/80475496
// But it decodes as a ResBoolean, which makes no sense. So force it to empty

View File

@ -53,7 +53,7 @@ public class ResStyleValue extends ResBagValue implements
ResResSpec spec = mItem.m1.getReferent();
if (spec == null) {
LOGGER.fine(String.format("null reference: m1=0x%08x(%s), m2=0x%08x(%s)",
LOGGER.fine(String.format("null style reference: m1=0x%08x(%s), m2=0x%08x(%s)",
mItem.m1.getRawIntValue(), mItem.m1.getType(), mItem.m2.getRawIntValue(), mItem.m2.getType()));
continue;
}

View File

@ -16,6 +16,10 @@
*/
package brut.androlib.res.data.value;
public class ResValue {
import brut.androlib.Config;
public class ResValue {
public boolean shouldRemoveUnknownRes() {
return Config.getInstance().isDecodeResolveModeRemoving();
}
}

View File

@ -59,11 +59,11 @@ public class ARSCDecoder {
mIn = new ExtCountingDataInput(new LittleEndianDataInputStream(arscStream));
mResTable = resTable;
mKeepBroken = keepBroken;
mMissingResSpecMap = new LinkedHashMap<>();
}
private ResPackage[] readResourceTable() throws IOException, AndrolibException {
Set<ResPackage> pkgs = new LinkedHashSet<>();
ResTypeSpec typeSpec;
int chunkNumber = 1;
@ -118,6 +118,10 @@ public class ARSCDecoder {
}
}
if (mResTable.getConfig().isDecodeResolveModeUsingDummies() && mPkg != null && mPkg.getResSpecCount() > 0) {
addMissingResSpecs();
}
return pkgs.toArray(new ResPackage[0]);
}
@ -300,6 +304,7 @@ public class ARSCDecoder {
mResId = (mResId & 0xffff0000) | i;
int offset = entryOffsetMap.get(i);
if (offset == NO_ENTRY) {
mMissingResSpecMap.put(mResId, typeId);
continue;
}
@ -315,6 +320,8 @@ public class ARSCDecoder {
EntryData entryData = readEntryData();
if (entryData != null) {
readEntry(entryData);
} else {
mMissingResSpecMap.put(mResId, typeId);
}
}
@ -589,6 +596,30 @@ public class ARSCDecoder {
mResTypeSpecs.put(resTypeSpec.getId(), resTypeSpec);
}
private void addMissingResSpecs() throws AndrolibException {
for (int resId : mMissingResSpecMap.keySet()) {
int typeId = mMissingResSpecMap.get(resId);
String resName = "APKTOOL_DUMMY_" + Integer.toHexString(resId);
ResID id = new ResID(resId);
ResResSpec spec = new ResResSpec(id, resName, mPkg, mResTypeSpecs.get(typeId));
// If we already have this resID don't add it again.
if (! mPkg.hasResSpec(id)) {
mPkg.addResSpec(spec);
spec.getType().addResSpec(spec);
ResType resType = mPkg.getOrCreateConfig(new ResConfigFlags());
// We are going to make dummy attributes a null reference (@null) now instead of a boolean false.
// This is because aapt2 is stricter when it comes to what we can put in an application.
ResValue value = new ResReferenceValue(mPkg, 0, "");
ResResource res = new ResResource(resType, spec, value);
resType.addResource(res);
spec.addResource(res);
}
}
}
private ARSCHeader nextChunk() throws IOException {
return mHeader = ARSCHeader.read(mIn);
}
@ -614,6 +645,7 @@ public class ARSCDecoder {
private ResType mType;
private int mResId;
private int mTypeIdOffset = 0;
private final HashMap<Integer, Integer> mMissingResSpecMap;
private final HashMap<Integer, ResTypeSpec> mResTypeSpecs = new HashMap<>();
private final static short ENTRY_FLAG_COMPLEX = 0x0001;

View File

@ -365,7 +365,7 @@ public final class ResXmlPatcher {
* @throws SAXException
* @throws ParserConfigurationException
*/
private static Document loadDocument(File file)
public static Document loadDocument(File file)
throws IOException, SAXException, ParserConfigurationException {
DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance();

View File

@ -18,15 +18,19 @@ package brut.androlib;
import brut.androlib.exceptions.AndrolibException;
import brut.androlib.res.Framework;
import brut.androlib.res.xml.ResXmlPatcher;
import brut.common.BrutException;
import brut.directory.DirUtil;
import brut.directory.Directory;
import brut.directory.FileDirectory;
import brut.util.OS;
import org.w3c.dom.Document;
import org.xml.sax.SAXException;
import org.xmlpull.v1.XmlPullParser;
import org.xmlpull.v1.XmlPullParserException;
import org.xmlpull.v1.XmlPullParserFactory;
import javax.xml.parsers.ParserConfigurationException;
import java.io.*;
import java.net.URL;
import java.net.URLDecoder;
@ -77,6 +81,14 @@ public abstract class TestUtils {
}
}
public static Document getDocumentFromFile(File file) throws BrutException {
try {
return ResXmlPatcher.loadDocument(file);
} catch (ParserConfigurationException | SAXException | IOException ex) {
throw new BrutException(ex);
}
}
public static void copyResourceDir(Class<?> class_, String dirPath, File out) throws BrutException {
if (!out.exists()) {
out.mkdirs();

View File

@ -0,0 +1,140 @@
/*
* Copyright (C) 2010 Ryszard Wiśniewski <brut.alll@gmail.com>
* Copyright (C) 2010 Connor Tumbleson <connor.tumbleson@gmail.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package brut.androlib.decode;
import brut.androlib.ApkDecoder;
import brut.androlib.BaseTest;
import brut.androlib.Config;
import brut.androlib.TestUtils;
import brut.directory.ExtFile;
import brut.common.BrutException;
import brut.util.OS;
import java.io.File;
import java.io.IOException;
import org.junit.*;
import org.w3c.dom.Document;
import static org.junit.Assert.*;
public class ResourceModeTest extends BaseTest {
@BeforeClass
public static void beforeClass() throws Exception {
TestUtils.cleanFrameworkFile();
sTmpDir = new ExtFile(OS.createTempDirectory());
TestUtils.copyResourceDir(ResourceModeTest.class, "decode/issue2836/", sTmpDir);
}
@AfterClass
public static void afterClass() throws BrutException {
OS.rmdir(sTmpDir);
}
@Test
public void checkDecodingModeAsRemove() throws BrutException, IOException {
String apk = "issue2836.apk";
Config config = Config.getDefaultConfig();
config.setDecodeResolveMode(Config.DECODE_RES_RESOLVE_REMOVE);
// decode issue2836.apk
ApkDecoder apkDecoder = new ApkDecoder(config, new File(sTmpDir + File.separator + apk));
sTestOrigDir = new ExtFile(sTmpDir + File.separator + apk + "remove.out");
File outDir = new File(sTmpDir + File.separator + apk + "remove.out");
apkDecoder.decode(outDir);
File stringsXml = new File(sTestOrigDir,"res/values/strings.xml");
assertTrue(stringsXml.isFile());
File attrXml = new File(sTestOrigDir,"res/values/attrs.xml");
Document attrDocument = TestUtils.getDocumentFromFile(attrXml);
assertEquals(3, attrDocument.getElementsByTagName("enum").getLength());
File colorXml = new File(sTestOrigDir,"res/values/colors.xml");
Document colorDocument = TestUtils.getDocumentFromFile(colorXml);
assertEquals(8, colorDocument.getElementsByTagName("color").getLength());
assertEquals(0, colorDocument.getElementsByTagName("item").getLength());
File publicXml = new File(sTestOrigDir,"res/values/public.xml");
Document publicDocument = TestUtils.getDocumentFromFile(publicXml);
assertEquals(21, publicDocument.getElementsByTagName("public").getLength());
}
@Test
public void checkDecodingModeAsDummies() throws BrutException, IOException {
String apk = "issue2836.apk";
Config config = Config.getDefaultConfig();
config.setDecodeResolveMode(Config.DECODE_RES_RESOLVE_DUMMY);
// decode issue2836.apk
ApkDecoder apkDecoder = new ApkDecoder(config, new File(sTmpDir + File.separator + apk));
sTestOrigDir = new ExtFile(sTmpDir + File.separator + apk + "dummies.out");
File outDir = new File(sTmpDir + File.separator + apk + "dummies.out");
apkDecoder.decode(outDir);
File stringsXml = new File(sTestOrigDir,"res/values/strings.xml");
assertTrue(stringsXml.isFile());
File attrXml = new File(sTestOrigDir,"res/values/attrs.xml");
Document attrDocument = TestUtils.getDocumentFromFile(attrXml);
assertEquals(4, attrDocument.getElementsByTagName("enum").getLength());
File colorXml = new File(sTestOrigDir,"res/values/colors.xml");
Document colorDocument = TestUtils.getDocumentFromFile(colorXml);
assertEquals(8, colorDocument.getElementsByTagName("color").getLength());
assertEquals(1, colorDocument.getElementsByTagName("item").getLength());
File publicXml = new File(sTestOrigDir,"res/values/public.xml");
Document publicDocument = TestUtils.getDocumentFromFile(publicXml);
assertEquals(22, publicDocument.getElementsByTagName("public").getLength());
}
@Test
public void checkDecodingModeAsLeave() throws BrutException, IOException {
String apk = "issue2836.apk";
Config config = Config.getDefaultConfig();
config.setDecodeResolveMode(Config.DECODE_RES_RESOLVE_RETAIN);
// decode issue2836.apk
ApkDecoder apkDecoder = new ApkDecoder(config, new File(sTmpDir + File.separator + apk));
sTestOrigDir = new ExtFile(sTmpDir + File.separator + apk + "leave.out");
File outDir = new File(sTmpDir + File.separator + apk + "leave.out");
apkDecoder.decode(outDir);
File stringsXml = new File(sTestOrigDir,"res/values/strings.xml");
assertTrue(stringsXml.isFile());
File attrXml = new File(sTestOrigDir,"res/values/attrs.xml");
Document attrDocument = TestUtils.getDocumentFromFile(attrXml);
assertEquals(4, attrDocument.getElementsByTagName("enum").getLength());
File colorXml = new File(sTestOrigDir,"res/values/colors.xml");
Document colorDocument = TestUtils.getDocumentFromFile(colorXml);
assertEquals(8, colorDocument.getElementsByTagName("color").getLength());
assertEquals(0, colorDocument.getElementsByTagName("item").getLength());
File publicXml = new File(sTestOrigDir,"res/values/public.xml");
Document publicDocument = TestUtils.getDocumentFromFile(publicXml);
assertEquals(21, publicDocument.getElementsByTagName("public").getLength());
}
}