Echoes of trash-test

Remove glyph remap for Switch gender symbols (none actually)
Might be wise to convert StringConverter to singletons...

PKHeX.Core/Legality/Encounters/Templates/Gen8/EncounterStatic8U.cs

@@ -46,4 +46,46 @@ public sealed record EncounterStatic8U : EncounterStatic8Nest<EncounterStatic8U>
     protected override bool IsMatchLocation(PKM pk) => Location == pk.Met_Location;
 
     public bool IsShinyXorValid(ushort pkShinyXor) => pkShinyXor is > 15 or 1;
+
+    public bool ShouldHaveScientistTrash => !SpeciesCategory.IsLegendary(Species)
+                                         && !SpeciesCategory.IsSubLegendary(Species);
+
+    protected override void FinishCorrelation(PK8 pk, ulong seed)
+    {
+        if (!ShouldHaveScientistTrash)
+            return;
+
+        ApplyTrashBytes(pk);
+    }
+
+    public void ApplyTrashBytes(PKM pk)
+    {
+        // Normally we would apply the trash before applying the OT, but we already did.
+        // Just add in the expected trash after the OT.
+        var ot = pk.OT_Trash;
+        var language = pk.Language;
+        var scientist = GetScientistName(language);
+        StringConverter8.ApplyTrashBytes(ot, scientist);
+    }
+
+    public static TrashMatch HasScientistTrash(PKM pk)
+    {
+        var language = pk.Language;
+        var name = GetScientistName(language);
+        return StringConverter8.GetTrashState(pk.OT_Trash, name);
+    }
+
+    private static ReadOnlySpan<char> GetScientistName(int language) => language switch
+    {
+        (int)LanguageID.Japanese => "けんきゅういん",
+        (int)LanguageID.English => "Scientist",
+        (int)LanguageID.French => "Scientifique",
+        (int)LanguageID.Italian => "Scienziata",
+        (int)LanguageID.German => "Forscherin",
+        (int)LanguageID.Spanish => "Científica",
+        (int)LanguageID.Korean => "연구원",
+        (int)LanguageID.ChineseS => "研究员",
+        (int)LanguageID.ChineseT => "研究員",
+        _ => ReadOnlySpan<char>.Empty,
+    };
 }

PKHeX.Core/PKM/Strings/StringConverter12.cs

@@ -29,13 +29,14 @@ public static class StringConverter12
         foreach (var c in data)
         {
             var b = table[c];
-            if (b == G1Terminator && c is not (G1TerminatorCode or 0))
+            if (b == G1Terminator && c is not (G1TerminatorCode or G1TerminatorZero))
                 return false;
         }
         return true;
     }
 
     public const byte G1TerminatorCode = 0x50;
+    public const byte G1TerminatorZero = 0x00;
     public const char G1Terminator = '\0';
     public const byte G1TradeOTCode = 0x5D;
     public const char G1TradeOT = '*';

PKHeX.Core/PKM/Strings/StringConverter8.cs

@@ -1,4 +1,6 @@
 using System;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
 using static System.Buffers.Binary.BinaryPrimitives;
 
 namespace PKHeX.Core;
@@ -33,7 +35,7 @@ public static class StringConverter8
             var value = ReadUInt16LittleEndian(data[i..]);
             if (value == TerminatorNull)
                 break;
-            result[i/2] = StringConverter.SanitizeChar((char)value);
+            result[i/2] = (char)value;
         }
         return i/2;
     }
@@ -53,14 +55,7 @@ public static class StringConverter8
         if (option is StringConverterOption.ClearZero)
             destBuffer.Clear();
 
-        bool isFullWidth = StringConverter.GetIsFullWidthString(value);
-        for (int i = 0; i < value.Length; i++)
-        {
-            char c = value[i];
-            if (!isFullWidth)
-                c = StringConverter.UnSanitizeChar(c);
-            WriteUInt16LittleEndian(destBuffer[(i * 2)..], c);
-        }
+        WriteCharacters(destBuffer, value);
 
         int count = value.Length * 2;
         if (count == destBuffer.Length)
@@ -68,4 +63,62 @@ public static class StringConverter8
         WriteUInt16LittleEndian(destBuffer[count..], TerminatorNull);
         return count + 2;
     }
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    private static void WriteCharacters(Span<byte> destBuffer, ReadOnlySpan<char> value)
+    {
+        if (BitConverter.IsLittleEndian)
+        {
+            var u16 = MemoryMarshal.Cast<char, byte>(value);
+            u16.CopyTo(destBuffer);
+            return;
+        }
+        for (int i = 0; i < value.Length; i++)
+            WriteUInt16LittleEndian(destBuffer[(i * 2)..], value[i]);
+    }
+
+    /// <summary>
+    /// Applies the under string to the top string, if possible.
+    /// </summary>
+    /// <param name="top">Displayed string</param>
+    /// <param name="under">Previous string</param>
+    /// <returns>Indication of the under string's presence.</returns>
+    public static TrashMatch ApplyTrashBytes(Span<byte> top, ReadOnlySpan<char> under)
+    {
+        var index = TrashBytes.GetStringLength(top);
+        if (index == -1)
+            return TrashMatch.Skipped;
+        index++; // hop over the terminator
+        if (index >= under.Length) // Overlapping
+            return TrashMatch.TooLongToTell;
+
+        var src = under[index..];
+        var dest = top[(index * 2)..];
+        SetString(dest, src, src.Length, StringConverterOption.None);
+        return TrashMatch.Present;
+    }
+
+    /// <summary>
+    /// Checks the displayed top string against the under string to see if the under is present.
+    /// </summary>
+    /// <param name="top">Displayed string</param>
+    /// <param name="under">Previous string</param>
+    /// <returns>Indication of the under string's presence.</returns>
+    public static TrashMatch GetTrashState(ReadOnlySpan<byte> top, ReadOnlySpan<char> under)
+    {
+        if (under.Length == 0)
+            return TrashMatch.Skipped;
+
+        var index = TrashBytes.GetStringLength(top);
+        if ((uint)index >= under.Length)
+            return TrashMatch.TooLongToTell;
+        index++; // hop over the terminator
+
+        // Adjust our spans to the relevant sections
+        under = under[index..];
+        var relevantSection = top[(index * 2)..];
+        Span<byte> expect = stackalloc byte[relevantSection.Length];
+        WriteCharacters(expect, under);
+        return relevantSection.SequenceEqual(expect) ? TrashMatch.Present : TrashMatch.NotPresent;
+    }
 }

PKHeX.Core/PKM/Strings/Trash/TrashBytes.cs

@@ -0,0 +1,35 @@
+using System;
+using System.Runtime.InteropServices;
+
+namespace PKHeX.Core;
+
+/// <summary>
+/// 16-bit encoded string utility
+/// </summary>
+public static class TrashBytes
+{
+    /// <summary>
+    /// Gets the length of the string based on the terminator.
+    /// </summary>
+    /// <param name="buffer">Buffer to check the length of.</param>
+    /// <param name="terminator">String terminator to search for.</param>
+    /// <returns>Index of the terminator, or max length if not found.</returns>
+    public static int GetStringLength(ReadOnlySpan<byte> buffer, ushort terminator = 0)
+    {
+        int index = FindTerminatorIndex(buffer, terminator);
+        return index == -1 ? buffer.Length / 2 : index;
+    }
+
+    /// <summary>
+    /// Returns a 16-bit aligned index of the terminator.
+    /// </summary>
+    /// <param name="buffer">Backing buffer of the string.</param>
+    /// <param name="terminator">Terminator character to search for.</param>
+    /// <returns>Index of the terminator, or -1 if not found.</returns>
+    /// <remarks>When used on a raw string, returns the computed length of the string, assuming a terminator is present.</remarks>
+    public static int FindTerminatorIndex(ReadOnlySpan<byte> buffer, ushort terminator = 0)
+    {
+        var u16 = MemoryMarshal.Cast<byte, ushort>(buffer);
+        return u16.IndexOf(terminator);
+    }
+}

PKHeX.Core/PKM/Strings/Trash/TrashBytes8.cs

@@ -0,0 +1,24 @@
+using System;
+
+namespace PKHeX.Core;
+
+/// <summary>
+/// Generation 3 Encoding Utility
+/// </summary>
+public static class TrashBytes8
+{
+    /// <inheritdoc cref="TrashBytes.GetStringLength"/>
+    public static int GetStringLength(ReadOnlySpan<byte> buffer)
+    {
+        int index = FindTerminatorIndex(buffer);
+        return index == -1 ? buffer.Length : index;
+    }
+
+    /// <summary>
+    /// Returns a 16-bit aligned index of the terminator.
+    /// </summary>
+    /// <param name="buffer">Backing buffer of the string.</param>
+    /// <returns>Index of the terminator, or -1 if not found.</returns>
+    public static int FindTerminatorIndex(ReadOnlySpan<byte> buffer)
+        => buffer.IndexOf<byte>(0xFF);
+}

PKHeX.Core/PKM/Strings/Trash/TrashBytesGB.cs

@@ -0,0 +1,25 @@
+using System;
+using static PKHeX.Core.StringConverter12;
+
+namespace PKHeX.Core;
+
+/// <summary>
+/// Generation 1 &amp; 2 Encoding Utility
+/// </summary>
+public static class TrashBytesGB
+{
+    /// <inheritdoc cref="TrashBytes.GetStringLength"/>
+    public static int GetStringLength(ReadOnlySpan<byte> buffer)
+    {
+        int index = FindTerminatorIndex(buffer);
+        return index == -1 ? buffer.Length : index;
+    }
+
+    /// <summary>
+    /// Returns a 16-bit aligned index of the terminator.
+    /// </summary>
+    /// <param name="buffer">Backing buffer of the string.</param>
+    /// <returns>Index of the terminator, or -1 if not found.</returns>
+    public static int FindTerminatorIndex(ReadOnlySpan<byte> buffer)
+        => buffer.IndexOfAny(G1TerminatorZero, G1TerminatorCode);
+}

PKHeX.Core/PKM/Strings/Trash/TrashMatch.cs

@@ -0,0 +1,24 @@
+namespace PKHeX.Core;
+
+public enum TrashMatch
+{
+    /// <summary>
+    /// Expected under-layer of trash was not found.
+    /// </summary>
+    NotPresent,
+
+    /// <summary>
+    /// Expected under-layer of trash was found.
+    /// </summary>
+    Present,
+
+    /// <summary>
+    /// Displayed string is too long, with all bytes covering the initial trash.
+    /// </summary>
+    TooLongToTell,
+
+    /// <summary>
+    /// Ignored due to other issues that would be flagged by other checks.
+    /// </summary>
+    Skipped,
+}