MarcoBuster/Gadgetbridge
1
0
Fork 0
mirror of https://codeberg.org/Freeyourgadget/Gadgetbridge synced 2024-11-29 13:26:50 +01:00
Code Releases Wiki Activity

Fix character mark removal before flattening to ASCII

Browse Source
This commit is contained in:
José Rebelo 2023-07-11 18:15:49 +01:00
parent 12dd9651e7
commit bdb904faf9
2 changed files with 15 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
app/src/main/java/nodomain/freeyourgadget/gadgetbridge/util/language/impl/FlattenToAsciiTransliterator.java
View File

@ -32,8 +32,11 @@ public class FlattenToAsciiTransliterator implements Transliterator {
return txt; return txt;
} }
// Decompose the string into its canonical decomposition (splits base characters from accents/marks)
txt = Normalizer.normalize(txt, Normalizer.Form.NFD); txt = Normalizer.normalize(txt, Normalizer.Form.NFD);
txt = new String(txt.getBytes(StandardCharsets.US_ASCII), StandardCharsets.US_ASCII); // Remove all marks (characters intended to be combined with another character), keeping the base glyphs
return txt.replaceAll("\\p{M}", ""); txt = txt.replaceAll("\\p{M}", "");
// Flatten the resulting string to ASCII
return new String(txt.getBytes(StandardCharsets.US_ASCII), StandardCharsets.US_ASCII);
} }
} }

10
app/src/test/java/nodomain/freeyourgadget/gadgetbridge/util/language/LanguageUtilsTest.java
View File

@ -7,6 +7,7 @@ import org.junit.Test;
import nodomain.freeyourgadget.gadgetbridge.GBApplication; import nodomain.freeyourgadget.gadgetbridge.GBApplication;
import nodomain.freeyourgadget.gadgetbridge.impl.GBDevice; import nodomain.freeyourgadget.gadgetbridge.impl.GBDevice;
import nodomain.freeyourgadget.gadgetbridge.test.TestBase; import nodomain.freeyourgadget.gadgetbridge.test.TestBase;
import nodomain.freeyourgadget.gadgetbridge.util.language.impl.FlattenToAsciiTransliterator;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotNull;
@ -183,6 +184,15 @@ public class LanguageUtilsTest extends TestBase {
assertEquals("croatian transliteration failed", expected, output); assertEquals("croatian transliteration failed", expected, output);
} }
@Test
public void testFlattenToAscii() throws Exception {
final FlattenToAsciiTransliterator transliterator = new FlattenToAsciiTransliterator();
String input = "ä ș ț ă";
String output = transliterator.transliterate(input);
String expected = "a s t a";
assertEquals("flatten to ascii transliteration failed", expected, output);
}
@Test @Test
public void testTransliterateOption() throws Exception { public void testTransliterateOption() throws Exception {
enableTransliteration(false); enableTransliteration(false);