feat(YouTube - Custom filter): Custom filtering of the protocol buffer (#562)

app/src/main/java/app/revanced/integrations/youtube/TrieSearch.java

@@ -231,47 +231,57 @@ public abstract class TrieSearch<T> {
         }
 
         /**
+         * This method is static and uses a loop to avoid all recursion.
+         * This is done for performance since the JVM does not do tail recursion optimization.
+         *
+         * @param startNode          Node to start the search from.
          * @param searchText         Text to search for patterns in.
          * @param searchTextLength   Length of the search text.
          * @param searchTextIndex    Current recursive search text index. Also, the end index of the current pattern match.
-         * @param currentMatchLength current search depth, and also the length of the current pattern match.
          * @return If any pattern matches, and it's associated callback halted the search.
          */
-        private boolean matches(T searchText, int searchTextLength, int searchTextIndex, int currentMatchLength,
+        private static <T> boolean matches(final TrieNode<T> startNode, final T searchText, final int searchTextLength,
-                                Object callbackParameter) {
+                                           int searchTextIndex, final Object callbackParameter) {
-            if (leaf != null && leaf.matches(this,
+            TrieNode<T> node = startNode;
-                    searchText, searchTextLength, searchTextIndex, callbackParameter)) {
+            int currentMatchLength = 0;
-                return true; // Leaf exists and it matched the search text.
+
-            }
+            while (true) {
-            if (endOfPatternCallback != null) {
+                TrieCompressedPath<T> leaf = node.leaf;
-                final int matchStartIndex = searchTextIndex - currentMatchLength;
+                if (leaf != null && leaf.matches(node, searchText, searchTextLength, searchTextIndex, callbackParameter)) {
-                for (@Nullable TriePatternMatchedCallback<T> callback : endOfPatternCallback) {
+                    return true; // Leaf exists and it matched the search text.
-                    if (callback == null) {
+                }
-                        return true; // No callback and all matches are valid.
+                List<TriePatternMatchedCallback<T>> endOfPatternCallback = node.endOfPatternCallback;
-                    }
+                if (endOfPatternCallback != null) {
-                    if (callback.patternMatched(searchText, matchStartIndex, currentMatchLength, callbackParameter)) {
+                    final int matchStartIndex = searchTextIndex - currentMatchLength;
-                        return true; // Callback confirmed the match.
+                    for (@Nullable TriePatternMatchedCallback<T> callback : endOfPatternCallback) {
+                        if (callback == null) {
+                            return true; // No callback and all matches are valid.
+                        }
+                        if (callback.patternMatched(searchText, matchStartIndex, currentMatchLength, callbackParameter)) {
+                            return true; // Callback confirmed the match.
+                        }
                     }
                 }
-            }
+                TrieNode<T>[] children = node.children;
-            if (children == null) {
+                if (children == null) {
-                return false; // Reached a graph end point and there's no further patterns to search.
+                    return false; // Reached a graph end point and there's no further patterns to search.
-            }
+                }
-            if (searchTextIndex == searchTextLength) {
+                if (searchTextIndex == searchTextLength) {
-                return false; // Reached end of the search text and found no matches.
+                    return false; // Reached end of the search text and found no matches.
-            }
+                }
 
-            final char character = getCharValue(searchText, searchTextIndex);
+                // Use the start node to reduce VM method lookup, since all nodes are the same class type.
-            if (isInvalidRange(character)) {
+                final char character = startNode.getCharValue(searchText, searchTextIndex);
-                return false; // Not an ASCII letter/number/symbol.
+                final int arrayIndex = hashIndexForTableSize(children.length, character);
+                TrieNode<T> child = children[arrayIndex];
+                if (child == null || child.nodeValue != character) {
+                    return false;
+                }
+
+                node = child;
+                searchTextIndex++;
+                currentMatchLength++;
             }
-            final int arrayIndex = hashIndexForTableSize(children.length, character);
-            TrieNode<T> child = children[arrayIndex];
-            if (child == null || child.nodeValue != character) {
-                return false;
-            }
-            return child.matches(searchText, searchTextLength, searchTextIndex + 1,
-                    currentMatchLength + 1, callbackParameter);
         }
 
         /**
@@ -388,7 +398,7 @@ public abstract class TrieSearch<T> {
             return false; // No patterns were added.
         }
         for (int i = startIndex; i < endIndex; i++) {
-            if (root.matches(textToSearch, endIndex, i, 0, callbackParameter)) return true;
+            if (TrieNode.matches(root, textToSearch, endIndex, i, callbackParameter)) return true;
         }
         return false;
     }

app/src/main/java/app/revanced/integrations/youtube/patches/components/CustomFilter.java

@@ -0,0 +1,178 @@
+package app.revanced.integrations.youtube.patches.components;
+
+import static app.revanced.integrations.shared.StringRef.str;
+
+import androidx.annotation.NonNull;
+import androidx.annotation.Nullable;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import app.revanced.integrations.shared.Logger;
+import app.revanced.integrations.shared.Utils;
+import app.revanced.integrations.youtube.ByteTrieSearch;
+import app.revanced.integrations.youtube.StringTrieSearch;
+import app.revanced.integrations.youtube.settings.Settings;
+
+/**
+ * Allows custom filtering using a path and optionally a proto buffer string.
+ */
+@SuppressWarnings("unused")
+final class CustomFilter extends Filter {
+
+    private static void showInvalidSyntaxToast(@NonNull String expression) {
+        Utils.showToastLong(str("revanced_custom_filter_toast_invalid_syntax", expression));
+    }
+
+    private static void showInvalidCharactersToast(@NonNull String expression) {
+        Utils.showToastLong(str("revanced_custom_filter_toast_invalid_characters", expression));
+    }
+
+    private static class CustomFilterGroup extends StringFilterGroup {
+        /**
+         * Optional character for the path that indicates the custom filter path must match the start.
+         * Must be the first character of the expression.
+         */
+        public static final String SYNTAX_STARTS_WITH = "^";
+
+        /**
+         * Optional character that separates the path from a proto buffer string pattern.
+         */
+        public static final String SYNTAX_BUFFER_SYMBOL = "$";
+
+        /**
+         * @return the parsed objects, or NULL if there was a parse error.
+         */
+        @Nullable
+        @SuppressWarnings("ConstantConditions")
+        static Collection<CustomFilterGroup> parseCustomFilterGroups() {
+            String rawCustomFilterText = Settings.CUSTOM_FILTER_STRINGS.get();
+            if (rawCustomFilterText.isBlank()) {
+                return Collections.emptyList();
+            }
+
+            // Map key is the path including optional special characters (^ and/or $)
+            Map<String, CustomFilterGroup> result = new HashMap<>();
+            Pattern pattern = Pattern.compile(
+                    "(" // map key group
+                            + "(\\Q" + SYNTAX_STARTS_WITH + "\\E?)" // optional starts with
+                            + "([^\\Q" + SYNTAX_BUFFER_SYMBOL + "\\E]*)" // path
+                            + "(\\Q" + SYNTAX_BUFFER_SYMBOL + "\\E?)" // optional buffer symbol
+                            + ")" // end map key group
+                            + "(.*)"); // optional buffer string
+
+            for (String expression : rawCustomFilterText.split("\n")) {
+                if (expression.isBlank()) continue;
+
+                Matcher matcher = pattern.matcher(expression);
+                if (!matcher.find()) {
+                    showInvalidSyntaxToast(expression);
+                    return null;
+                }
+
+                final String mapKey = matcher.group(1);
+                final boolean pathStartsWith = !matcher.group(2).isEmpty();
+                final String path = matcher.group(3);
+                final boolean hasBufferSymbol = !matcher.group(4).isEmpty();
+                final String bufferString = matcher.group(5);
+
+                if (path.isBlank() || (hasBufferSymbol && bufferString.isBlank())) {
+                    showInvalidSyntaxToast(expression);
+                    return null;
+                }
+                if (!StringTrieSearch.isValidPattern(path)
+                        || (hasBufferSymbol && !StringTrieSearch.isValidPattern(bufferString))) {
+                    // Currently only ASCII is allowed.
+                    showInvalidCharactersToast(path);
+                    return null;
+                }
+
+                // Use one group object for all expressions with the same path.
+                // This ensures the buffer is searched exactly once
+                // when multiple paths are used with different buffer strings.
+                CustomFilterGroup group = result.get(mapKey);
+                if (group == null) {
+                    group = new CustomFilterGroup(pathStartsWith, path);
+                    result.put(mapKey, group);
+                }
+                if (hasBufferSymbol) {
+                    group.addBufferString(bufferString);
+                }
+            }
+
+            return result.values();
+        }
+
+        final boolean startsWith;
+        ByteTrieSearch bufferSearch;
+
+        CustomFilterGroup(boolean startsWith, @NonNull String path) {
+            super(Settings.CUSTOM_FILTER, path);
+            this.startsWith = startsWith;
+        }
+
+        void addBufferString(@NonNull String bufferString) {
+            if (bufferSearch == null) {
+                bufferSearch = new ByteTrieSearch();
+            }
+            bufferSearch.addPattern(bufferString.getBytes());
+        }
+
+        @NonNull
+        @Override
+        public String toString() {
+            StringBuilder builder = new StringBuilder();
+            builder.append("CustomFilterGroup{");
+            builder.append("path=");
+            if (startsWith) builder.append(SYNTAX_STARTS_WITH);
+            builder.append(filters[0]);
+
+            if (bufferSearch != null) {
+                String delimitingCharacter = "❙";
+                builder.append(", bufferStrings=");
+                builder.append(delimitingCharacter);
+                for (byte[] bufferString : bufferSearch.getPatterns()) {
+                    builder.append(new String(bufferString));
+                    builder.append(delimitingCharacter);
+                }
+            }
+            builder.append("}");
+            return builder.toString();
+        }
+    }
+
+    public CustomFilter() {
+        Collection<CustomFilterGroup> groups = CustomFilterGroup.parseCustomFilterGroups();
+        if (groups == null) {
+            Settings.CUSTOM_FILTER_STRINGS.resetToDefault();
+            Utils.showToastLong(str("revanced_custom_filter_toast_reset"));
+            groups = Objects.requireNonNull(CustomFilterGroup.parseCustomFilterGroups());
+        }
+
+        if (!groups.isEmpty()) {
+            CustomFilterGroup[] groupsArray = groups.toArray(new CustomFilterGroup[0]);
+            Logger.printDebug(()-> "Using Custom filters: " + Arrays.toString(groupsArray));
+            addPathCallbacks(groupsArray);
+        }
+    }
+
+    @Override
+    public boolean isFiltered(@Nullable String identifier, String path, byte[] protobufBufferArray,
+                              StringFilterGroup matchedGroup, FilterContentType contentType, int contentIndex) {
+        // All callbacks are custom filter groups.
+        CustomFilterGroup custom = (CustomFilterGroup) matchedGroup;
+        if (custom.startsWith && contentIndex != 0) {
+            return false;
+        }
+        if (custom.bufferSearch != null && !custom.bufferSearch.matches(protobufBufferArray)) {
+            return false;
+        }
+        return super.isFiltered(identifier, path, protobufBufferArray, matchedGroup, contentType, contentIndex);
+    }
+}

app/src/main/java/app/revanced/integrations/youtube/patches/components/LayoutComponentsFilter.java

@@ -18,7 +18,6 @@ public final class LayoutComponentsFilter extends Filter {
             null,
             "cell_description_body"
     );
-    private final CustomFilterGroup custom;
 
     private static final ByteArrayFilterGroup mixPlaylists = new ByteArrayFilterGroup(
             Settings.HIDE_MIX_PLAYLISTS,
@@ -68,11 +67,6 @@ public final class LayoutComponentsFilter extends Filter {
 
         // Paths.
 
-        custom = new CustomFilterGroup(
-                Settings.CUSTOM_FILTER,
-                Settings.CUSTOM_FILTER_STRINGS
-        );
-
         final var communityPosts = new StringFilterGroup(
                 Settings.HIDE_COMMUNITY_POSTS,
                 "post_base_wrapper"
@@ -226,7 +220,6 @@ public final class LayoutComponentsFilter extends Filter {
         );
 
         addPathCallbacks(
-                custom,
                 expandableMetadata,
                 inFeedSurvey,
                 notifyMe,
@@ -270,8 +263,7 @@ public final class LayoutComponentsFilter extends Filter {
         if (matchedGroup == notifyMe || matchedGroup == inFeedSurvey || matchedGroup == expandableMetadata)
             return super.isFiltered(identifier, path, protobufBufferArray, matchedGroup, contentType, contentIndex);
 
-        if (matchedGroup != custom && exceptions.matches(path))
+        if (exceptions.matches(path)) return false; // Exceptions are not filtered.
-            return false; // Exceptions are not filtered.
 
         // TODO: This also hides the feed Shorts shelf header
         if (matchedGroup == searchResultShelfHeader && contentIndex != 0) return false;

app/src/main/java/app/revanced/integrations/youtube/patches/components/LithoFilterPatch.java

@@ -18,7 +18,6 @@ import app.revanced.integrations.shared.Logger;
 import app.revanced.integrations.shared.Utils;
 import app.revanced.integrations.shared.settings.BooleanSetting;
 import app.revanced.integrations.shared.settings.BaseSettings;
-import app.revanced.integrations.shared.settings.StringSetting;
 import app.revanced.integrations.youtube.ByteTrieSearch;
 import app.revanced.integrations.youtube.StringTrieSearch;
 import app.revanced.integrations.youtube.TrieSearch;
@@ -138,25 +137,6 @@ class StringFilterGroup extends FilterGroup<String> {
     }
 }
 
-final class CustomFilterGroup extends StringFilterGroup {
-
-    private static String[] getFilterPatterns(StringSetting setting) {
-        String[] patterns = setting.get().split("\\s+");
-        for (String pattern : patterns) {
-            if (!StringTrieSearch.isValidPattern(pattern)) {
-                Utils.showToastLong("Invalid custom filter, resetting to default");
-                setting.resetToDefault();
-                return getFilterPatterns(setting);
-            }
-        }
-        return patterns;
-    }
-
-    public CustomFilterGroup(BooleanSetting setting, StringSetting filter) {
-        super(setting, getFilterPatterns(filter));
-    }
-}
-
 /**
  * If you have more than 1 filter patterns, then all instances of
  * this class should filtered using {@link ByteArrayFilterGroupList#check(byte[])},

app/src/main/java/app/revanced/integrations/youtube/settings/Settings.java

@@ -292,7 +292,7 @@ public class Settings extends BaseSettings {
     static {
         // region Migration
 
-        // region Migrate settings from old Preference categories into replacement "revanced_prefs" category.
+        // Migrate settings from old Preference categories into replacement "revanced_prefs" category.
         // This region must run before all other migration code.
 
         // The YT and RYD migration portion of this can be removed anytime,
@@ -338,7 +338,6 @@ public class Settings extends BaseSettings {
                 migrateFromOldPreferences(ytPrefs, setting, key);
             }
         }
-        // end region
 
 
         // Do _not_ delete this SB private user id migration property until sometime in 2024.
@@ -346,16 +345,6 @@ public class Settings extends BaseSettings {
         // and more time should be given for users who rarely upgrade.
         migrateOldSettingToNew(DEPRECATED_SB_UUID_OLD_MIGRATION_SETTING, SB_PRIVATE_USER_ID);
 
-        // This migration may need to remain here for a while.
-        // Older online guides will still reference using commas,
-        // and this code will automatically convert anything the user enters to newline format,
-        // and also migrate any imported older settings that using commas.
-        String componentsToFilter = Settings.CUSTOM_FILTER_STRINGS.get();
-        if (componentsToFilter.contains(",")) {
-            Logger.printInfo(() -> "Migrating custom filter strings to new line format");
-            Settings.CUSTOM_FILTER_STRINGS.save(componentsToFilter.replace(",", "\n"));
-        }
-
         // endregion
     }
 }