added option to include generic/permissive network security config file durin… (#2791)

* added option to include permissive network security config file during build

* added tests for app with existing network config and for app without

* minor fixes for pull 2791

* refactor: slim down test app for network config

* style: remove extra newlines

* refactor: moved network tests to aapt2

* refactor: remove unused exceptions

* test (aapt2): ensure aapt2 is used for net-sec-conf

* fix (cli): block use of net-sec-conf on aapt1

* fix conflict

Co-authored-by: Connor Tumbleson <connor@sourcetoad.com>
Co-authored-by: Connor Tumbleson <connor.tumbleson@gmail.com>

brut.apktool/apktool-cli/src/main/java/brut/apktool/Main.java

@@ -216,6 +216,9 @@ public class Main {
         if (cli.hasOption("d") || cli.hasOption("debug")) {
             buildOptions.debugMode = true;
         }
+        if (cli.hasOption("n") || cli.hasOption("net-sec-conf")) {
+            buildOptions.netSecConf = true;
+        }
         if (cli.hasOption("v") || cli.hasOption("verbose")) {
             buildOptions.verbose = true;
         }
@@ -247,6 +250,11 @@ public class Main {
             outFile = null;
         }
 
+        if (buildOptions.netSecConf && !buildOptions.useAapt2) {
+            System.err.println("-n / --net-sec-conf is only supported with --use-aapt2.");
+            System.exit(1);
+        }
+
         // try and build apk
         try {
             if (cli.hasOption("a") || cli.hasOption("aapt")) {
@@ -366,6 +374,11 @@ public class Main {
                 .desc("Sets android:debuggable to \"true\" in the APK's compiled manifest")
                 .build();
 
+        Option netSecConfOption = Option.builder("n")
+            .longOpt("net-sec-conf")
+            .desc("Adds a generic Network Security Configuration file in the output APK")
+            .build();
+
         Option noDbgOption = Option.builder("b")
                 .longOpt("no-debug-info")
                 .desc("don't write out debug info (.local, .param, .line, etc.)")
@@ -473,6 +486,7 @@ public class Main {
 
             buildOptions.addOption(apiLevelOption);
             buildOptions.addOption(debugBuiOption);
+            buildOptions.addOption(netSecConfOption);
             buildOptions.addOption(aaptOption);
             buildOptions.addOption(originalOption);
             buildOptions.addOption(aapt2Option);
@@ -528,6 +542,7 @@ public class Main {
         allOptions.addOption(noAssetOption);
         allOptions.addOption(keepResOption);
         allOptions.addOption(debugBuiOption);
+        allOptions.addOption(netSecConfOption);
         allOptions.addOption(aaptOption);
         allOptions.addOption(originalOption);
         allOptions.addOption(verboseOption);

brut.apktool/apktool-lib/src/main/java/brut/androlib/Androlib.java

@@ -20,6 +20,7 @@ import brut.androlib.meta.MetaInfo;
 import brut.androlib.meta.UsesFramework;
 import brut.androlib.options.BuildOptions;
 import brut.androlib.res.AndrolibResources;
+import brut.androlib.res.data.ResConfigFlags;
 import brut.androlib.res.data.ResPackage;
 import brut.androlib.res.data.ResTable;
 import brut.androlib.res.data.ResUnknownFiles;
@@ -35,7 +36,10 @@ import brut.util.*;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.FilenameUtils;
 import org.jf.dexlib2.iface.DexFile;
+import org.xml.sax.SAXException;
 
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.TransformerException;
 import java.io.*;
 import java.util.*;
 import java.util.logging.Logger;
@@ -486,6 +490,23 @@ public class Androlib {
                     }
                 }
 
+                if (buildOptions.netSecConf) {
+                    MetaInfo meta = readMetaFile(new ExtFile(appDir));
+                    if (meta.sdkInfo != null && meta.sdkInfo.get("targetSdkVersion") != null) {
+                        if (Integer.parseInt(meta.sdkInfo.get("targetSdkVersion")) < ResConfigFlags.SDK_NOUGAT) {
+                            LOGGER.warning("Target SDK version is lower than 24! Network Security Configuration might be ignored!");
+                        }
+                    }
+                    File netSecConfOrig = new File(appDir, "res/xml/network_security_config.xml");
+                    if (netSecConfOrig.exists()) {
+                        LOGGER.info("Replacing existing network_security_config.xml!");
+                        netSecConfOrig.delete();
+                    }
+                    ResXmlPatcher.modNetworkSecurityConfig(netSecConfOrig);
+                    ResXmlPatcher.setNetworkSecurityConfig(new File(appDir, "AndroidManifest.xml"));
+                    LOGGER.info("Added permissive network security config in manifest");
+                }
+
                 File apkFile = File.createTempFile("APKTOOL", null);
                 apkFile.delete();
                 resourceFile.delete();
@@ -518,7 +539,7 @@ public class Androlib {
                 apkFile.delete();
             }
             return true;
-        } catch (IOException | BrutException ex) {
+        } catch (IOException | BrutException | ParserConfigurationException | TransformerException | SAXException ex) {
             throw new AndrolibException(ex);
         }
     }

brut.apktool/apktool-lib/src/main/java/brut/androlib/options/BuildOptions.java

@@ -22,6 +22,7 @@ public class BuildOptions {
     public boolean forceBuildAll = false;
     public boolean forceDeleteFramework = false;
     public boolean debugMode = false;
+    public boolean netSecConf = false;
     public boolean verbose = false;
     public boolean copyOriginalFiles = false;
     public final boolean updateFiles = false;

brut.apktool/apktool-lib/src/main/java/brut/androlib/res/xml/ResXmlPatcher.java

@@ -17,10 +17,7 @@
 package brut.androlib.res.xml;
 
 import brut.androlib.AndrolibException;
-import org.w3c.dom.Document;
+import org.w3c.dom.*;
-import org.w3c.dom.NamedNodeMap;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
 import org.xml.sax.SAXException;
 
 import javax.xml.parsers.DocumentBuilder;
@@ -97,6 +94,71 @@ public final class ResXmlPatcher {
         }
     }
 
+    /**
+     * Sets the value of the network security config in the AndroidManifest file
+     *
+     * @param file AndroidManifest file
+     */
+    public static void setNetworkSecurityConfig(File file) {
+        if (file.exists()) {
+            try {
+                Document doc = loadDocument(file);
+                Node application = doc.getElementsByTagName("application").item(0);
+
+                // load attr
+                NamedNodeMap attr = application.getAttributes();
+                Node netSecConfAttr = attr.getNamedItem("android:networkSecurityConfig");
+
+                if (netSecConfAttr == null) {
+                    // there is not an already existing network security configuration
+                    netSecConfAttr = doc.createAttribute("android:networkSecurityConfig");
+                    attr.setNamedItem(netSecConfAttr);
+                }
+
+                // whether it already existed or it was created now set it to the proper value
+                netSecConfAttr.setNodeValue("@xml/network_security_config");
+
+                saveDocument(file, doc);
+
+            } catch (SAXException | ParserConfigurationException | IOException | TransformerException ignored) {
+            }
+        }
+    }
+
+    /**
+     * Creates a modified network security config file that is more permissive
+     *
+     * @param file network security config file
+     */
+    public static void modNetworkSecurityConfig(File file)
+        throws ParserConfigurationException, TransformerException, IOException, SAXException {
+
+        DocumentBuilderFactory documentFactory = DocumentBuilderFactory.newInstance();
+        DocumentBuilder documentBuilder = documentFactory.newDocumentBuilder();
+        Document document = documentBuilder.newDocument();
+
+        Element root = document.createElement("network-security-config");
+        document.appendChild(root);
+        Element baseConfig = document.createElement("base-config");
+        root.appendChild(baseConfig);
+        Element trustAnchors = document.createElement("trust-anchors");
+        baseConfig.appendChild(trustAnchors);
+
+        Element certSystem = document.createElement("certificates");
+        Attr attrSystem = document.createAttribute("src");
+        attrSystem.setValue("system");
+        certSystem.setAttributeNode(attrSystem);
+        trustAnchors.appendChild(certSystem);
+
+        Element certUser = document.createElement("certificates");
+        Attr attrUser = document.createAttribute("src");
+        attrUser.setValue("user");
+        certUser.setAttributeNode(attrUser);
+        trustAnchors.appendChild(certUser);
+
+        saveDocument(file, document);
+    }
+
     /**
      * Any @string reference in a provider value in AndroidManifest.xml will break on
      * build, thus preventing the application from installing. This is from a bug/error

brut.apktool/apktool-lib/src/test/java/brut/androlib/aapt2/NetworkConfigTest.java

@@ -0,0 +1,129 @@
+/*
+ *  Copyright (C) 2010 Ryszard Wiśniewski <brut.alll@gmail.com>
+ *  Copyright (C) 2010 Connor Tumbleson <connor.tumbleson@gmail.com>
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *       https://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package brut.androlib.aapt2;
+
+import brut.androlib.*;
+import brut.androlib.options.BuildOptions;
+import brut.common.BrutException;
+import brut.directory.ExtFile;
+import brut.util.OS;
+import org.custommonkey.xmlunit.XMLUnit;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.w3c.dom.Document;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.xml.sax.SAXException;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+
+import static org.custommonkey.xmlunit.XMLAssert.assertXMLEqual;
+import static org.junit.Assert.*;
+
+public class NetworkConfigTest extends BaseTest {
+
+    @BeforeClass
+    public static void beforeClass() throws Exception {
+        TestUtils.cleanFrameworkFile();
+
+        sTmpDir = new ExtFile(OS.createTempDirectory());
+        sTestOrigDir = new ExtFile(sTmpDir, "testapp-orig");
+        sTestNewDir = new ExtFile(sTmpDir, "testapp-new");
+        LOGGER.info("Unpacking testapp...");
+        TestUtils.copyResourceDir(NetworkConfigTest.class, "aapt2/network_config/", sTestOrigDir);
+
+        LOGGER.info("Building testapp.apk...");
+        BuildOptions buildOptions = new BuildOptions();
+        buildOptions.netSecConf = true;
+        buildOptions.useAapt2 = true;
+        File testApk = new File(sTmpDir, "testapp.apk");
+        new Androlib(buildOptions).build(sTestOrigDir, testApk);
+
+        LOGGER.info("Decoding testapp.apk...");
+        ApkDecoder apkDecoder = new ApkDecoder(testApk);
+        apkDecoder.setOutDir(sTestNewDir);
+        apkDecoder.decode();
+    }
+
+    @AfterClass
+    public static void afterClass() throws BrutException {
+        OS.rmdir(sTmpDir);
+    }
+
+    @Test
+    public void buildAndDecodeTest() {
+        assertTrue(sTestNewDir.isDirectory());
+    }
+
+    @Test
+    public void netSecConfGeneric() throws IOException, SAXException {
+        LOGGER.info("Comparing network security configuration file...");
+        String expected = TestUtils.replaceNewlines("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" +
+            "<network-security-config><base-config><trust-anchors><certificates src=\"system\"/><certificates src=\"us" +
+            "er\"/></trust-anchors></base-config></network-security-config>");
+
+        byte[] encoded = Files.readAllBytes(Paths.get(String.valueOf(sTestNewDir), "res/xml/network_security_config.xml"));
+        String obtained = TestUtils.replaceNewlines(new String(encoded));
+
+        XMLUnit.setIgnoreWhitespace(true);
+        XMLUnit.setIgnoreAttributeOrder(true);
+        XMLUnit.setCompareUnmatched(false);
+
+        assertXMLEqual(expected, obtained);
+    }
+
+    @Test
+    public void netSecConfInManifest() throws IOException, ParserConfigurationException, SAXException {
+        LOGGER.info("Validating network security config in Manifest...");
+        Document doc = loadDocument(new File(sTestNewDir + "/AndroidManifest.xml"));
+        Node application = doc.getElementsByTagName("application").item(0);
+        NamedNodeMap attr = application.getAttributes();
+        Node debugAttr = attr.getNamedItem("android:networkSecurityConfig");
+        assertEquals("@xml/network_security_config", debugAttr.getNodeValue());
+    }
+
+    private static Document loadDocument(File file)
+        throws IOException, SAXException, ParserConfigurationException {
+
+        DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance();
+        docFactory.setFeature(FEATURE_DISABLE_DOCTYPE_DECL, true);
+        docFactory.setFeature(FEATURE_LOAD_DTD, false);
+
+        try {
+            docFactory.setAttribute(ACCESS_EXTERNAL_DTD, " ");
+            docFactory.setAttribute(ACCESS_EXTERNAL_SCHEMA, " ");
+        } catch (IllegalArgumentException ex) {
+            LOGGER.warning("JAXP 1.5 Support is required to validate XML");
+        }
+
+        DocumentBuilder docBuilder = docFactory.newDocumentBuilder();
+        try (FileInputStream inputStream = new FileInputStream(file)) {
+            return docBuilder.parse(inputStream);
+        }
+    }
+
+    private static final String ACCESS_EXTERNAL_DTD = "http://javax.xml.XMLConstants/property/accessExternalDTD";
+    private static final String ACCESS_EXTERNAL_SCHEMA = "http://javax.xml.XMLConstants/property/accessExternalSchema";
+    private static final String FEATURE_LOAD_DTD = "http://apache.org/xml/features/nonvalidating/load-external-dtd";
+    private static final String FEATURE_DISABLE_DOCTYPE_DECL = "http://apache.org/xml/features/disallow-doctype-decl";
+}

brut.apktool/apktool-lib/src/test/java/brut/androlib/aapt2/NoNetworkConfigTest.java

@@ -0,0 +1,134 @@
+/*
+ *  Copyright (C) 2010 Ryszard Wiśniewski <brut.alll@gmail.com>
+ *  Copyright (C) 2010 Connor Tumbleson <connor.tumbleson@gmail.com>
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *       https://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package brut.androlib.aapt2;
+
+import brut.androlib.*;
+import brut.androlib.options.BuildOptions;
+import brut.common.BrutException;
+import brut.directory.ExtFile;
+import brut.util.OS;
+import org.custommonkey.xmlunit.XMLUnit;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.w3c.dom.Document;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.xml.sax.SAXException;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+
+import static org.custommonkey.xmlunit.XMLAssert.assertXMLEqual;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class NoNetworkConfigTest extends BaseTest {
+
+    @BeforeClass
+    public static void beforeClass() throws Exception {
+        TestUtils.cleanFrameworkFile();
+
+        sTmpDir = new ExtFile(OS.createTempDirectory());
+        sTestOrigDir = new ExtFile(sTmpDir, "testapp-orig");
+        sTestNewDir = new ExtFile(sTmpDir, "testapp-new");
+        LOGGER.info("Unpacking testapp...");
+        TestUtils.copyResourceDir(NoNetworkConfigTest.class, "aapt2/testapp/", sTestOrigDir);
+
+        LOGGER.info("Building testapp.apk...");
+        BuildOptions buildOptions = new BuildOptions();
+        buildOptions.netSecConf = true;
+        buildOptions.useAapt2 = true;
+        File testApk = new File(sTmpDir, "testapp.apk");
+        new Androlib(buildOptions).build(sTestOrigDir, testApk);
+
+        LOGGER.info("Decoding testapp.apk...");
+        ApkDecoder apkDecoder = new ApkDecoder(testApk);
+        apkDecoder.setOutDir(sTestNewDir);
+        apkDecoder.decode();
+    }
+
+    @AfterClass
+    public static void afterClass() throws BrutException {
+        OS.rmdir(sTmpDir);
+    }
+
+    @Test
+    public void buildAndDecodeTest() {
+        assertTrue(sTestNewDir.isDirectory());
+    }
+
+    @Test
+    public void netSecConfGeneric() throws IOException, SAXException {
+        LOGGER.info("Comparing network security configuration file...");
+        String expected = TestUtils.replaceNewlines("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" +
+            "<network-security-config><base-config><trust-anchors><certificates src=\"system\"/><certificates src=\"us" +
+            "er\"/></trust-anchors></base-config></network-security-config>");
+
+        byte[] encoded = Files.readAllBytes(Paths.get(String.valueOf(sTestNewDir), "res/xml/network_security_config.xml"));
+        String obtained = TestUtils.replaceNewlines(new String(encoded));
+
+        XMLUnit.setIgnoreWhitespace(true);
+        XMLUnit.setIgnoreAttributeOrder(true);
+        XMLUnit.setCompareUnmatched(false);
+
+        assertXMLEqual(expected, obtained);
+    }
+
+    @Test
+    public void netSecConfInManifest() throws IOException, ParserConfigurationException, SAXException {
+        LOGGER.info("Validating network security config in Manifest...");
+        Document doc = loadDocument(new File(sTestNewDir + "/AndroidManifest.xml"));
+        Node application = doc.getElementsByTagName("application").item(0);
+        NamedNodeMap attr = application.getAttributes();
+        Node debugAttr = attr.getNamedItem("android:networkSecurityConfig");
+        assertEquals("@xml/network_security_config", debugAttr.getNodeValue());
+    }
+
+    private static Document loadDocument(File file)
+        throws IOException, SAXException, ParserConfigurationException {
+
+        DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance();
+        docFactory.setFeature(FEATURE_DISABLE_DOCTYPE_DECL, true);
+        docFactory.setFeature(FEATURE_LOAD_DTD, false);
+
+        try {
+            docFactory.setAttribute(ACCESS_EXTERNAL_DTD, " ");
+            docFactory.setAttribute(ACCESS_EXTERNAL_SCHEMA, " ");
+        } catch (IllegalArgumentException ex) {
+            LOGGER.warning("JAXP 1.5 Support is required to validate XML");
+        }
+
+        DocumentBuilder docBuilder = docFactory.newDocumentBuilder();
+        // Not using the parse(File) method on purpose, so that we can control when
+        // to close it. Somehow parse(File) does not seem to close the file in all cases.
+        try (FileInputStream inputStream = new FileInputStream(file)) {
+            return docBuilder.parse(inputStream);
+        }
+    }
+
+    private static final String ACCESS_EXTERNAL_DTD = "http://javax.xml.XMLConstants/property/accessExternalDTD";
+    private static final String ACCESS_EXTERNAL_SCHEMA = "http://javax.xml.XMLConstants/property/accessExternalSchema";
+    private static final String FEATURE_LOAD_DTD = "http://apache.org/xml/features/nonvalidating/load-external-dtd";
+    private static final String FEATURE_DISABLE_DOCTYPE_DECL = "http://apache.org/xml/features/disallow-doctype-decl";
+}

brut.apktool/apktool-lib/src/test/resources/aapt2/network_config/AndroidManifest.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android" android:compileSdkVersion="23" android:compileSdkVersionCodename="6.0-2438415" package="brut.apktool.testapp" platformBuildVersionCode="23" platformBuildVersionName="6.0-2438415">
+    <application>
+        android:networkSecurityConfig="@xml/network_security_config"
+    </application>
+</manifest>

brut.apktool/apktool-lib/src/test/resources/aapt2/network_config/apktool.yml

@@ -0,0 +1,9 @@
+version: 2.0.0
+apkFileName: testapp.apk
+isFrameworkApk: false
+usesFramework:
+  ids:
+  - 1
+versionInfo:
+  versionCode: '1'
+  versionName: '1.0'

brut.apktool/apktool-lib/src/test/resources/aapt2/network_config/res/values/public.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+    <public type="string" name="hello_world" id="0x7f020000" />
+</resources>

brut.apktool/apktool-lib/src/test/resources/aapt2/network_config/res/values/strings.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+    <string name="hello_world">Hello World</string>
+</resources>

brut.apktool/apktool-lib/src/test/resources/aapt2/network_config/res/xml/network_security_config.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<network-security-config>
+    <domain-config cleartextTrafficPermitted="false">
+        <domain includeSubdomains="true">example.com</domain>
+        <pin-set>
+            <pin digest="SHA-256">OEJax6JVAMiUP7wzOiLPU7KW38Cdx3afNZOYR2iOFZ4=</pin>
+        </pin-set>
+    </domain-config>
+</network-security-config>

brut.apktool/apktool-lib/src/test/resources/aapt2/network_config/smali/HelloWorld.smali

@@ -0,0 +1,15 @@
+.class public LHelloWorld;
+
+.super Ljava/lang/Object;
+
+.method public static main([Ljava/lang/String;)V
+    .registers 2
+
+    sget-object v0, Ljava/lang/System;->out:Ljava/io/PrintStream;
+
+    const/high16 v1, 0x7f020000
+
+    invoke-virtual {v0, v1}, Ljava/io/PrintStream;->println(Ljava/lang/String;)V
+
+    return-void
+.end method