diff --git a/Switch_Toolbox_Library/Compression/CompressionMenus.cs b/Switch_Toolbox_Library/Compression/CompressionMenus.cs index 878c9391..b977c4fe 100644 --- a/Switch_Toolbox_Library/Compression/CompressionMenus.cs +++ b/Switch_Toolbox_Library/Compression/CompressionMenus.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text; using System.Threading.Tasks; using System.Windows.Forms; @@ -21,7 +22,7 @@ namespace Toolbox.Library.IO var items = new List(); items.Add(CreateMenu("Yaz0")); items.Add(CreateMenu("Gzip")); - items.Add(CreateMenu("LZ77 (Wii Type 11)", false)); + items.Add(CreateMenu("LZ77 (Wii Type 11)")); items.Add(CreateMenu("LZSS", false)); items.Add(CreateMenu("lZMA")); items.Add(CreateMenu("lZ4")); @@ -113,7 +114,7 @@ namespace Toolbox.Library.IO if (fileNames.Length == 0) return; - string ext = Compress ? ".comp" : ".dec"; + string ext = Compress ? ".comp" : ""; if (compressionFormat.Extension.Length > 0 && Compress) ext = compressionFormat.Extension[0].Replace("*", string.Empty); @@ -126,6 +127,7 @@ namespace Toolbox.Library.IO foreach (var file in fileNames) { string name = Path.GetFileName(file); + name = name.Count(c => c == '.') > 1 && !Compress ? name.Remove(name.LastIndexOf('.')) : name; using (var data = new FileStream(file, FileMode.Open, FileAccess.Read)) { try diff --git a/Switch_Toolbox_Library/Compression/Formats/LZ77.cs b/Switch_Toolbox_Library/Compression/Formats/LZ77.cs index d6db6208..bf250fed 100644 --- a/Switch_Toolbox_Library/Compression/Formats/LZ77.cs +++ b/Switch_Toolbox_Library/Compression/Formats/LZ77.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using Toolbox.Library.IO; using Toolbox.Library.Forms; using System.Runtime.InteropServices; +using Toolbox.Library.Compression.LZ77_wii_11_compresss.Formats.Nitro; namespace Toolbox.Library { @@ -54,12 +55,18 @@ namespace Toolbox.Library } } } - + // A modified version of dsdecmp for compressing files into the Wii LZ77 type 11 .lz -Adapted by:DanielSvoboda public Stream Compress(Stream stream) { - MemoryStream mem = new MemoryStream(); - - return mem; + using (var reader = new FileReader(stream, true)) + { + using (MemoryStream outstream = new MemoryStream()) + { + LZ11 lz11 = new LZ11(); + int compressedSize = lz11.Compress(stream, stream.Length, outstream); + return new MemoryStream(outstream.ToArray()); + } + } } } } diff --git a/Switch_Toolbox_Library/Compression/LZ77_wii_11_compresss/Exceptions/InputTooLargeException.cs b/Switch_Toolbox_Library/Compression/LZ77_wii_11_compresss/Exceptions/InputTooLargeException.cs new file mode 100644 index 00000000..e13a3074 --- /dev/null +++ b/Switch_Toolbox_Library/Compression/LZ77_wii_11_compresss/Exceptions/InputTooLargeException.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Toolbox.Library.Compression.LZ77_wii_11_compresss.Exceptions +{ + /// + /// An exception indicating that the file cannot be compressed, because the decompressed size + /// cannot be represented in the current compression format. + /// + public class InputTooLargeException : Exception + { + /// + /// Creates a new exception that indicates that the input is too big to be compressed. + /// + public InputTooLargeException() + : base("The compression ratio is not high enough to fit the input " + + "in a single compressed file.") + { } + } +} diff --git a/Switch_Toolbox_Library/Compression/LZ77_wii_11_compresss/Exceptions/NotEnoughDataException.cs b/Switch_Toolbox_Library/Compression/LZ77_wii_11_compresss/Exceptions/NotEnoughDataException.cs new file mode 100644 index 00000000..3d536bda --- /dev/null +++ b/Switch_Toolbox_Library/Compression/LZ77_wii_11_compresss/Exceptions/NotEnoughDataException.cs @@ -0,0 +1,41 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Toolbox.Library.Compression.LZ77_wii_11_compresss.Exceptions +{ + /// + /// An exception that is thrown by the decompression functions when there + /// is not enough data available in order to properly decompress the input. + /// + public class NotEnoughDataException : IOException + { + private long currentOutSize; + private long totalOutSize; + /// + /// Gets the actual number of written bytes. + /// + public long WrittenLength { get { return this.currentOutSize; } } + /// + /// Gets the number of bytes that was supposed to be written. + /// + public long DesiredLength { get { return this.totalOutSize; } } + + /// + /// Creates a new NotEnoughDataException. + /// + /// The actual number of written bytes. + /// The desired number of written bytes. + public NotEnoughDataException(long currentOutSize, long totalOutSize) + : base("Not enough data availble; 0x" + currentOutSize.ToString("X") + + " of " + (totalOutSize < 0 ? "???" : ("0x" + totalOutSize.ToString("X"))) + + " bytes written.") + { + this.currentOutSize = currentOutSize; + this.totalOutSize = totalOutSize; + } + } +} diff --git a/Switch_Toolbox_Library/Compression/LZ77_wii_11_compresss/Exceptions/StreamTooShortException.cs b/Switch_Toolbox_Library/Compression/LZ77_wii_11_compresss/Exceptions/StreamTooShortException.cs new file mode 100644 index 00000000..11b5bfd7 --- /dev/null +++ b/Switch_Toolbox_Library/Compression/LZ77_wii_11_compresss/Exceptions/StreamTooShortException.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Toolbox.Library.Compression.LZ77_wii_11_compresss.Exceptions +{ + /// + /// An exception thrown by the compression or decompression function, indicating that the + /// given input length was too large for the given input stream. + /// + public class StreamTooShortException : EndOfStreamException + { + /// + /// Creates a new exception that indicates that the stream was shorter than the given input length. + /// + public StreamTooShortException() + : base("The end of the stream was reached " + + "before the given amout of data was read.") + { } + } +} diff --git a/Switch_Toolbox_Library/Compression/LZ77_wii_11_compresss/Exceptions/TooMuchInputException.cs b/Switch_Toolbox_Library/Compression/LZ77_wii_11_compresss/Exceptions/TooMuchInputException.cs new file mode 100644 index 00000000..408277cc --- /dev/null +++ b/Switch_Toolbox_Library/Compression/LZ77_wii_11_compresss/Exceptions/TooMuchInputException.cs @@ -0,0 +1,32 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Toolbox.Library.Compression.LZ77_wii_11_compresss.Exceptions +{ + /// + /// An exception indication that the input has more data than required in order + /// to decompress it. This may indicate that more sub-files are present in the file. + /// + public class TooMuchInputException : Exception + { + /// + /// Gets the number of bytes read by the decompressed to decompress the stream. + /// + public long ReadBytes { get; private set; } + + /// + /// Creates a new exception indicating that the input has more data than necessary for + /// decompressing th stream. It may indicate that other data is present after the compressed + /// stream. + /// + /// The number of bytes read by the decompressor. + /// The indicated length of the input stream. + public TooMuchInputException(long readBytes, long totLength) + : base("The input contains more data than necessary. Only used 0x" + + readBytes.ToString("X") + " of 0x" + totLength.ToString("X") + " bytes") + { + this.ReadBytes = readBytes; + } + } +} diff --git a/Switch_Toolbox_Library/Compression/LZ77_wii_11_compresss/Formats/CompositeFormat.cs b/Switch_Toolbox_Library/Compression/LZ77_wii_11_compresss/Formats/CompositeFormat.cs new file mode 100644 index 00000000..f76d8075 --- /dev/null +++ b/Switch_Toolbox_Library/Compression/LZ77_wii_11_compresss/Formats/CompositeFormat.cs @@ -0,0 +1,195 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.IO; +using Toolbox.Library.Compression.LZ77_wii_11_compresss.Exceptions; + +namespace Toolbox.Library.Compression.LZ77_wii_11_compresss.Formats +{ + /// + /// A format that is composed of multiple formats. + /// When compressing, the input is compressed using the best contained format. + /// When decompressing, all contained formats will try to decompress the file, until one succeeds. + /// + public abstract class CompositeFormat : CompressionFormat + { + /// + /// The actual list of formats this format is somposed of. + /// + private List formats; + + + #region Constructors + + /// + /// Creates a new composite format based on the given sequence of formats. + /// + protected CompositeFormat(IEnumerable formats) + { + this.formats = new List(formats); + } + /// + /// Creates a new composite format based on the given formats. + /// + protected CompositeFormat(params CompressionFormat[] formats) + { + this.formats = new List(formats); + } + + #endregion + + + #region Method: Supports + /// + /// Checks if any of the contained formats supports the given input. + /// + public override bool Supports(System.IO.Stream stream, long inLength) + { + foreach (CompositeFormat fmt in this.formats) + { + if (fmt.Supports(stream, inLength)) + return true; + } + return false; + } + #endregion + + #region Method: Decompress + /// + /// Attempts to decompress the given input by letting all contained formats + /// try to decompress the input. + /// + public override long Decompress(System.IO.Stream instream, long inLength, System.IO.Stream outstream) + { + byte[] inputData = new byte[instream.Length]; + instream.Read(inputData, 0, inputData.Length); + + foreach (CompressionFormat format in this.formats) + { + if (!format.SupportsDecompression) + continue; + using (MemoryStream input = new MemoryStream(inputData)) + { + if (!format.Supports(input, inputData.Length)) + continue; + MemoryStream output = new MemoryStream(); + try + { + long decLength = format.Decompress(input, inputData.Length, output); + if (decLength > 0) + { + output.WriteTo(outstream); + return decLength; + } + } + catch (Exception) { continue; } + } + } + + throw new InvalidDataException("Input cannot be decompressed using the " + this.ShortFormatString + " formats."); + } + #endregion + + #region Method: Compress & Field: LastUsedCompressFormatString + /// + /// Gets the ShortFormatString of the last CompressionFormat that was used to compress input. + /// + public string LastUsedCompressFormatString { get; private set; } + /// + /// Compresses the given input using the contained format that yields the best results in terms of + /// size reduction. + /// + public override int Compress(System.IO.Stream instream, long inLength, System.IO.Stream outstream) + { + // only read the input data once from the file. + byte[] inputData = new byte[instream.Length]; + instream.Read(inputData, 0, inputData.Length); + + MemoryStream bestOutput = null; + string bestFormatString = ""; + int minCompSize = int.MaxValue; + foreach (CompressionFormat format in formats) + { + if (!format.SupportsCompression) + continue; + + #region compress the file in each format, and save the best one + + MemoryStream currentOutput = new MemoryStream(); + int currentOutSize; + try + { + using (MemoryStream input = new MemoryStream(inputData)) + { + currentOutSize = format.Compress(input, input.Length, currentOutput); + } + } + catch (InputTooLargeException i) + { + Console.WriteLine(i.Message); + bestFormatString = format.ShortFormatString; + return -1; + } + catch (Exception) + { + continue; + } + if (currentOutSize < minCompSize) + { + bestOutput = currentOutput; + minCompSize = currentOutSize; + bestFormatString = format.ShortFormatString; + } + + #endregion + } + + if (bestOutput == null) + return -1; + bestOutput.WriteTo(outstream); + this.LastUsedCompressFormatString = bestFormatString; + return minCompSize; + } + #endregion + + #region Method: ParseCompressionOptions(args) + /// + /// Handles the compression options for each of the contained compression formats. + /// + public override int ParseCompressionOptions(string[] args) + { + // try each option on each of the formats. + // each pass over the formats lets them try to consume the options. + // if one or more formats consume at least one option, the maximum number + // of consumed options is treated as 'handled'; they are ignored in the + // next pass. This continues until none of the formats consume the next + // value in the options. + + int totalOptionCount = 0; + bool usedOption = true; + while (usedOption) + { + usedOption = false; + if (args.Length <= totalOptionCount) + break; + int maxOptionCount = 0; + string[] subArray = new string[args.Length - totalOptionCount]; + Array.Copy(args, totalOptionCount, subArray, 0, subArray.Length); + foreach (CompressionFormat format in this.formats) + { + int optCount = format.ParseCompressionOptions(subArray); + maxOptionCount = Math.Max(optCount, maxOptionCount); + } + + if (maxOptionCount > 0) + { + totalOptionCount += maxOptionCount; + usedOption = true; + } + } + return totalOptionCount; + } + #endregion + + } +} diff --git a/Switch_Toolbox_Library/Compression/LZ77_wii_11_compresss/Formats/CompressionFormat.cs b/Switch_Toolbox_Library/Compression/LZ77_wii_11_compresss/Formats/CompressionFormat.cs new file mode 100644 index 00000000..66b91069 --- /dev/null +++ b/Switch_Toolbox_Library/Compression/LZ77_wii_11_compresss/Formats/CompressionFormat.cs @@ -0,0 +1,86 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.IO; + +namespace Toolbox.Library.Compression.LZ77_wii_11_compresss.Formats +{ + /// + /// Base class for all compression formats. + /// + public abstract class CompressionFormat + { + /// + /// Checks if the decompressor for this format supports the data from the given stream. + /// Returns false when it is certain that the given data is not supported. + /// False positives may occur, as this method should not do any decompression, and may + /// mis-interpret a similar data format as compressed. + /// + /// The stream that may or may not contain compressed data. The + /// position of this stream may change during this call, but will be returned to its + /// original position when the method returns. + /// The length of the input stream. + /// False if the data can certainly not be decompressed using this decompressor. + /// True if the data may potentially be decompressed using this decompressor. + public abstract bool Supports(Stream stream, long inLength); + + + + /// + /// Decompresses the given stream, writing the decompressed data to the given output stream. + /// Assumes Supports(instream) returns true. + /// After this call, the input stream will be positioned at the end of the compressed stream, + /// or at the initial position + inLength, whichever comes first. + /// + /// The stream to decompress. At the end of this method, the position + /// of this stream is directly after the compressed data. + /// The length of the input data. Not necessarily all of the + /// input data may be read (if there is padding, for example), however never more than + /// this number of bytes is read from the input stream. + /// The stream to write the decompressed data to. + /// The length of the output data. + /// When the given length of the input data + /// is not enough to properly decompress the input. + public abstract long Decompress(Stream instream, long inLength, Stream outstream); + + + + /// + /// Compresses the next inLength bytes from the input stream, + /// and writes the compressed data to the given output stream. + /// + /// The stream to read plaintext data from. + /// The length of the plaintext data. + /// The stream to write the compressed data to. + /// The size of the compressed stream. If -1, the file could not be compressed. + public abstract int Compress(Stream instream, long inLength, Stream outstream); + + /// + /// Gets a short string identifying this compression format. + /// + public abstract string ShortFormatString { get; } + /// + /// Gets a short description of this compression format (used in the program usage). + /// + public abstract string Description { get; } + + /// + /// Gets if this format supports compressing a file. + /// + public abstract bool SupportsCompression { get; } + /// + /// Gets if this format supports decompressing a file. + /// + public virtual bool SupportsDecompression { get { return true; } } + /// + /// Gets the value that must be given on the command line in order to compress using this format. + /// + public abstract string CompressionFlag { get; } + /// + /// Parses any input specific for this format. Does nothing by default. + /// + /// Any arguments that may be used by the format. + /// The number of input arguments consumed by this format. + public virtual int ParseCompressionOptions(string[] args) { return 0; } + } +} diff --git a/Switch_Toolbox_Library/Compression/LZ77_wii_11_compresss/Formats/Nitro/LZ11.cs b/Switch_Toolbox_Library/Compression/LZ77_wii_11_compresss/Formats/Nitro/LZ11.cs new file mode 100644 index 00000000..f26c14a7 --- /dev/null +++ b/Switch_Toolbox_Library/Compression/LZ77_wii_11_compresss/Formats/Nitro/LZ11.cs @@ -0,0 +1,605 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.IO; +using Toolbox.Library.Compression.LZ77_wii_11_compresss.Exceptions; +using Toolbox.Library.Compression.LZ77_wii_11_compresss.Utils; + +namespace Toolbox.Library.Compression.LZ77_wii_11_compresss.Formats.Nitro +{ + /// + /// Compressor and decompressor for the LZ-0x11 format used in many of the games for the + /// newer Nintendo consoles and handhelds. + /// + public sealed class LZ11 : NitroCFormat + { + /// + /// Gets a short string identifying this compression format. + /// + public override string ShortFormatString + { + get { return "LZ-11"; } + } + + /// + /// Gets a short description of this compression format (used in the program usage). + /// + public override string Description + { + get { return "Variant of the LZ-0x10 format to support longer repetitions."; } + } + + /// + /// Gets the value that must be given on the command line in order to compress using this format. + /// + public override string CompressionFlag + { + get { return "lz11"; } + } + + /// + /// Gets if this format supports compressing a file. + /// + public override bool SupportsCompression + { + get { return true; } + } + + private static bool lookAhead = false; + /// + /// Sets the flag that determines if 'look-ahead'/DP should be used when compressing + /// with the LZ-11 format. The default is false, which is what is used in the original + /// implementation. + /// + public static bool LookAhead + { + set { lookAhead = value; } + } + + /// + /// Creates a new instance of the LZ-11 compression format. + /// + public LZ11() : base(0x11) { } + + /// + /// Checks if the given aguments have the '-opt' option, which makes this format + /// compress using (near-)optimal compression instead of the original compression algorithm. + /// + public override int ParseCompressionOptions(string[] args) + { + LookAhead = false; + if (args.Length > 0) + if (args[0] == "-opt") + { + LookAhead = true; + return 1; + } + return 0; + } + + #region Decompression method + /// + /// Decompresses the input using the LZ-11 compression scheme. + /// + public override long Decompress(Stream instream, long inLength, Stream outstream) + { + #region Format definition in NDSTEK style + /* Data header (32bit) + Bit 0-3 Reserved + Bit 4-7 Compressed type (must be 1 for LZ77) + Bit 8-31 Size of decompressed data. if 0, the next 4 bytes are decompressed length + Repeat below. Each Flag Byte followed by eight Blocks. + Flag data (8bit) + Bit 0-7 Type Flags for next 8 Blocks, MSB first + Block Type 0 - Uncompressed - Copy 1 Byte from Source to Dest + Bit 0-7 One data byte to be copied to dest + Block Type 1 - Compressed - Copy LEN Bytes from Dest-Disp-1 to Dest + If Reserved is 0: - Default + Bit 0-3 Disp MSBs + Bit 4-7 LEN - 3 + Bit 8-15 Disp LSBs + If Reserved is 1: - Higher compression rates for files with (lots of) long repetitions + Bit 4-7 Indicator + If Indicator > 1: + Bit 0-3 Disp MSBs + Bit 4-7 LEN - 1 (same bits as Indicator) + Bit 8-15 Disp LSBs + If Indicator is 1: A(B CD E)(F GH) + Bit 0-3 (LEN - 0x111) MSBs + Bit 4-7 Indicator; unused + Bit 8-15 (LEN- 0x111) 'middle'-SBs + Bit 16-19 Disp MSBs + Bit 20-23 (LEN - 0x111) LSBs + Bit 24-31 Disp LSBs + If Indicator is 0: + Bit 0-3 (LEN - 0x11) MSBs + Bit 4-7 Indicator; unused + Bit 8-11 Disp MSBs + Bit 12-15 (LEN - 0x11) LSBs + Bit 16-23 Disp LSBs + */ + #endregion + + long readBytes = 0; + + byte type = (byte)instream.ReadByte(); + if (type != base.magicByte) + throw new InvalidDataException("The provided stream is not a valid LZ-0x11 " + + "compressed stream (invalid type 0x" + type.ToString("X") + ")"); + byte[] sizeBytes = new byte[3]; + instream.Read(sizeBytes, 0, 3); + int decompressedSize = IOUtils.ToNDSu24(sizeBytes, 0); + readBytes += 4; + if (decompressedSize == 0) + { + sizeBytes = new byte[4]; + instream.Read(sizeBytes, 0, 4); + decompressedSize = IOUtils.ToNDSs32(sizeBytes, 0); + readBytes += 4; + } + + // the maximum 'DISP-1' is still 0xFFF. + int bufferLength = 0x1000; + byte[] buffer = new byte[bufferLength]; + int bufferOffset = 0; + + int currentOutSize = 0; + int flags = 0, mask = 1; + while (currentOutSize < decompressedSize) + { + // (throws when requested new flags byte is not available) + #region Update the mask. If all flag bits have been read, get a new set. + // the current mask is the mask used in the previous run. So if it masks the + // last flag bit, get a new flags byte. + if (mask == 1) + { + if (readBytes >= inLength) + throw new NotEnoughDataException(currentOutSize, decompressedSize); + flags = instream.ReadByte(); readBytes++; + if (flags < 0) + throw new StreamTooShortException(); + mask = 0x80; + } + else + { + mask >>= 1; + } + #endregion + + // bit = 1 <=> compressed. + if ((flags & mask) > 0) + { + // (throws when not enough bytes are available) + #region Get length and displacement('disp') values from next 2, 3 or 4 bytes + + // read the first byte first, which also signals the size of the compressed block + if (readBytes >= inLength) + throw new NotEnoughDataException(currentOutSize, decompressedSize); + int byte1 = instream.ReadByte(); readBytes++; + if (byte1 < 0) + throw new StreamTooShortException(); + + int length = byte1 >> 4; + int disp = -1; + if (length == 0) + { + #region case 0; 0(B C)(D EF) + (0x11)(0x1) = (LEN)(DISP) + + // case 0: + // data = AB CD EF (with A=0) + // LEN = ABC + 0x11 == BC + 0x11 + // DISP = DEF + 1 + + // we need two more bytes available + if (readBytes + 1 >= inLength) + throw new NotEnoughDataException(currentOutSize, decompressedSize); + int byte2 = instream.ReadByte(); readBytes++; + int byte3 = instream.ReadByte(); readBytes++; + if (byte3 < 0) + throw new StreamTooShortException(); + + length = (((byte1 & 0x0F) << 4) | (byte2 >> 4)) + 0x11; + disp = (((byte2 & 0x0F) << 8) | byte3) + 0x1; + + #endregion + } + else if (length == 1) + { + #region case 1: 1(B CD E)(F GH) + (0x111)(0x1) = (LEN)(DISP) + + // case 1: + // data = AB CD EF GH (with A=1) + // LEN = BCDE + 0x111 + // DISP = FGH + 1 + + // we need three more bytes available + if (readBytes + 2 >= inLength) + throw new NotEnoughDataException(currentOutSize, decompressedSize); + int byte2 = instream.ReadByte(); readBytes++; + int byte3 = instream.ReadByte(); readBytes++; + int byte4 = instream.ReadByte(); readBytes++; + if (byte4 < 0) + throw new StreamTooShortException(); + + length = (((byte1 & 0x0F) << 12) | (byte2 << 4) | (byte3 >> 4)) + 0x111; + disp = (((byte3 & 0x0F) << 8) | byte4) + 0x1; + + #endregion + } + else + { + #region case > 1: (A)(B CD) + (0x1)(0x1) = (LEN)(DISP) + + // case other: + // data = AB CD + // LEN = A + 1 + // DISP = BCD + 1 + + // we need only one more byte available + if (readBytes >= inLength) + throw new NotEnoughDataException(currentOutSize, decompressedSize); + int byte2 = instream.ReadByte(); readBytes++; + if (byte2 < 0) + throw new StreamTooShortException(); + + length = ((byte1 & 0xF0) >> 4) + 0x1; + disp = (((byte1 & 0x0F) << 8) | byte2) + 0x1; + + #endregion + } + + if (disp > currentOutSize) + throw new InvalidDataException("Cannot go back more than already written. " + + "DISP = " + disp + ", #written bytes = 0x" + currentOutSize.ToString("X") + + " before 0x" + instream.Position.ToString("X") + " with indicator 0x" + + (byte1 >> 4).ToString("X")); + #endregion + + int bufIdx = bufferOffset + bufferLength - disp; + for (int i = 0; i < length; i++) + { + byte next = buffer[bufIdx % bufferLength]; + bufIdx++; + outstream.WriteByte(next); + buffer[bufferOffset] = next; + bufferOffset = (bufferOffset + 1) % bufferLength; + } + currentOutSize += length; + } + else + { + if (readBytes >= inLength) + throw new NotEnoughDataException(currentOutSize, decompressedSize); + int next = instream.ReadByte(); readBytes++; + if (next < 0) + throw new StreamTooShortException(); + + outstream.WriteByte((byte)next); currentOutSize++; + buffer[bufferOffset] = (byte)next; + bufferOffset = (bufferOffset + 1) % bufferLength; + } + } + + if (readBytes < inLength) + { + // the input may be 4-byte aligned. + if ((readBytes ^ (readBytes & 3)) + 4 < inLength) + throw new TooMuchInputException(readBytes, inLength); + } + + return decompressedSize; + } + #endregion + + #region Original compression method + /// + /// Compresses the input using the 'original', unoptimized compression algorithm. + /// This algorithm should yield files that are the same as those found in the games. + /// (delegates to the optimized method if LookAhead is set) + /// + public unsafe override int Compress(Stream instream, long inLength, Stream outstream) + { + // make sure the decompressed size fits in 3 bytes. + // There should be room for four bytes, however I'm not 100% sure if that can be used + // in every game, as it may not be a built-in function. + if (inLength > 0xFFFFFF) + throw new InputTooLargeException(); + + // use the other method if lookahead is enabled + if (lookAhead) + { + return CompressWithLA(instream, inLength, outstream); + } + + // save the input data in an array to prevent having to go back and forth in a file + byte[] indata = new byte[inLength]; + int numReadBytes = instream.Read(indata, 0, (int)inLength); + if (numReadBytes != inLength) + throw new StreamTooShortException(); + + + + // write the compression header first + + //this has been included to work with (Paper Mario: Color Splash for the Wii U), if the test fails in other games it should be removed :) + byte[] byteArray = new byte[] { 0xF0, 0x0E, 0x00, 0x00, 0xF0, 0x0E, 0x00, 0x00, 0x13, 0xF0, 0x0E, 0x00 }; + outstream.Write(byteArray, 0, byteArray.Length); + + outstream.WriteByte(this.magicByte); + outstream.WriteByte((byte)(inLength & 0xFF)); + outstream.WriteByte((byte)((inLength >> 8) & 0xFF)); + outstream.WriteByte((byte)((inLength >> 16) & 0xFF)); + + int compressedLength = 4; + + fixed (byte* instart = &indata[0]) + { + // we do need to buffer the output, as the first byte indicates which blocks are compressed. + // this version does not use a look-ahead, so we do not need to buffer more than 8 blocks at a time. + // (a block is at most 4 bytes long) + byte[] outbuffer = new byte[8 * 4 + 1]; + outbuffer[0] = 0; + int bufferlength = 1, bufferedBlocks = 0; + int readBytes = 0; + while (readBytes < inLength) + { + #region If 8 blocks are bufferd, write them and reset the buffer + // we can only buffer 8 blocks at a time. + if (bufferedBlocks == 8) + { + outstream.Write(outbuffer, 0, bufferlength); + compressedLength += bufferlength; + // reset the buffer + outbuffer[0] = 0; + bufferlength = 1; + bufferedBlocks = 0; + } + #endregion + + // determine if we're dealing with a compressed or raw block. + // it is a compressed block when the next 3 or more bytes can be copied from + // somewhere in the set of already compressed bytes. + int disp; + int oldLength = Math.Min(readBytes, 0x1000); + int length = LZUtil.GetOccurrenceLength(instart + readBytes, (int)Math.Min(inLength - readBytes, 0x10110), + instart + readBytes - oldLength, oldLength, out disp); + + // length not 3 or more? next byte is raw data + if (length < 3) + { + outbuffer[bufferlength++] = *(instart + (readBytes++)); + } + else + { + // 3 or more bytes can be copied? next (length) bytes will be compressed into 2 bytes + readBytes += length; + + // mark the next block as compressed + outbuffer[0] |= (byte)(1 << (7 - bufferedBlocks)); + + if (length > 0x110) + { + // case 1: 1(B CD E)(F GH) + (0x111)(0x1) = (LEN)(DISP) + outbuffer[bufferlength] = 0x10; + outbuffer[bufferlength] |= (byte)(((length - 0x111) >> 12) & 0x0F); + bufferlength++; + outbuffer[bufferlength] = (byte)(((length - 0x111) >> 4) & 0xFF); + bufferlength++; + outbuffer[bufferlength] = (byte)(((length - 0x111) << 4) & 0xF0); + } + else if (length > 0x10) + { + // case 0; 0(B C)(D EF) + (0x11)(0x1) = (LEN)(DISP) + outbuffer[bufferlength] = 0x00; + outbuffer[bufferlength] |= (byte)(((length - 0x111) >> 4) & 0x0F); + bufferlength++; + outbuffer[bufferlength] = (byte)(((length - 0x111) << 4) & 0xF0); + } + else + { + // case > 1: (A)(B CD) + (0x1)(0x1) = (LEN)(DISP) + outbuffer[bufferlength] = (byte)(((length - 1) << 4) & 0xF0); + } + // the last 1.5 bytes are always the disp + outbuffer[bufferlength] |= (byte)(((disp - 1) >> 8) & 0x0F); + bufferlength++; + outbuffer[bufferlength] = (byte)((disp - 1) & 0xFF); + bufferlength++; + } + bufferedBlocks++; + } + + // copy the remaining blocks to the output + if (bufferedBlocks > 0) + { + outstream.Write(outbuffer, 0, bufferlength); + compressedLength += bufferlength; + /*/ make the compressed file 4-byte aligned. + while ((compressedLength % 4) != 0) + { + outstream.WriteByte(0); + compressedLength++; + }/**/ + } + } + + return compressedLength; + } + #endregion + + #region Dynamic Programming compression method + /// + /// Variation of the original compression method, making use of Dynamic Programming to 'look ahead' + /// and determine the optimal 'length' values for the compressed blocks. Is not 100% optimal, + /// as the flag-bytes are not taken into account. + /// + private unsafe int CompressWithLA(Stream instream, long inLength, Stream outstream) + { + // save the input data in an array to prevent having to go back and forth in a file + byte[] indata = new byte[inLength]; + int numReadBytes = instream.Read(indata, 0, (int)inLength); + if (numReadBytes != inLength) + throw new StreamTooShortException(); + + // write the compression header first + outstream.WriteByte(this.magicByte); + outstream.WriteByte((byte)(inLength & 0xFF)); + outstream.WriteByte((byte)((inLength >> 8) & 0xFF)); + outstream.WriteByte((byte)((inLength >> 16) & 0xFF)); + + int compressedLength = 4; + + fixed (byte* instart = &indata[0]) + { + // we do need to buffer the output, as the first byte indicates which blocks are compressed. + // this version does not use a look-ahead, so we do not need to buffer more than 8 blocks at a time. + // blocks are at most 4 bytes long. + byte[] outbuffer = new byte[8 * 4 + 1]; + outbuffer[0] = 0; + int bufferlength = 1, bufferedBlocks = 0; + int readBytes = 0; + + // get the optimal choices for len and disp + int[] lengths, disps; + this.GetOptimalCompressionLengths(instart, indata.Length, out lengths, out disps); + while (readBytes < inLength) + { + // we can only buffer 8 blocks at a time. + if (bufferedBlocks == 8) + { + outstream.Write(outbuffer, 0, bufferlength); + compressedLength += bufferlength; + // reset the buffer + outbuffer[0] = 0; + bufferlength = 1; + bufferedBlocks = 0; + } + + + if (lengths[readBytes] == 1) + { + outbuffer[bufferlength++] = *(instart + (readBytes++)); + } + else + { + // mark the next block as compressed + outbuffer[0] |= (byte)(1 << (7 - bufferedBlocks)); + + if (lengths[readBytes] > 0x110) + { + // case 1: 1(B CD E)(F GH) + (0x111)(0x1) = (LEN)(DISP) + outbuffer[bufferlength] = 0x10; + outbuffer[bufferlength] |= (byte)(((lengths[readBytes] - 0x111) >> 12) & 0x0F); + bufferlength++; + outbuffer[bufferlength] = (byte)(((lengths[readBytes] - 0x111) >> 4) & 0xFF); + bufferlength++; + outbuffer[bufferlength] = (byte)(((lengths[readBytes] - 0x111) << 4) & 0xF0); + } + else if (lengths[readBytes] > 0x10) + { + // case 0; 0(B C)(D EF) + (0x11)(0x1) = (LEN)(DISP) + outbuffer[bufferlength] = 0x00; + outbuffer[bufferlength] |= (byte)(((lengths[readBytes] - 0x111) >> 4) & 0x0F); + bufferlength++; + outbuffer[bufferlength] = (byte)(((lengths[readBytes] - 0x111) << 4) & 0xF0); + } + else + { + // case > 1: (A)(B CD) + (0x1)(0x1) = (LEN)(DISP) + outbuffer[bufferlength] = (byte)(((lengths[readBytes] - 1) << 4) & 0xF0); + } + // the last 1.5 bytes are always the disp + outbuffer[bufferlength] |= (byte)(((disps[readBytes] - 1) >> 8) & 0x0F); + bufferlength++; + outbuffer[bufferlength] = (byte)((disps[readBytes] - 1) & 0xFF); + bufferlength++; + + readBytes += lengths[readBytes]; + } + + + bufferedBlocks++; + } + + // copy the remaining blocks to the output + if (bufferedBlocks > 0) + { + outstream.Write(outbuffer, 0, bufferlength); + compressedLength += bufferlength; + /*/ make the compressed file 4-byte aligned. + while ((compressedLength % 4) != 0) + { + outstream.WriteByte(0); + compressedLength++; + }/**/ + } + } + + return compressedLength; + } + #endregion + + #region DP compression helper method; GetOptimalCompressionLengths + /// + /// Gets the optimal compression lengths for each start of a compressed block using Dynamic Programming. + /// This takes O(n^2) time, although in practice it will often be O(n^3) since one of the constants is 0x10110 + /// (the maximum length of a compressed block) + /// + /// The data to compress. + /// The length of the data to compress. + /// The optimal 'length' of the compressed blocks. For each byte in the input data, + /// this value is the optimal 'length' value. If it is 1, the block should not be compressed. + /// The 'disp' values of the compressed blocks. May be 0, in which case the + /// corresponding length will never be anything other than 1. + private unsafe void GetOptimalCompressionLengths(byte* indata, int inLength, out int[] lengths, out int[] disps) + { + lengths = new int[inLength]; + disps = new int[inLength]; + int[] minLengths = new int[inLength]; + + for (int i = inLength - 1; i >= 0; i--) + { + // first get the compression length when the next byte is not compressed + minLengths[i] = int.MaxValue; + lengths[i] = 1; + if (i + 1 >= inLength) + minLengths[i] = 1; + else + minLengths[i] = 1 + minLengths[i + 1]; + // then the optimal compressed length + int oldLength = Math.Min(0x1000, i); + // get the appropriate disp while at it. Takes at most O(n) time if oldLength is considered O(n) and 0x10110 constant. + // however since a lot of files will not be larger than 0x10110, this will often take ~O(n^2) time. + // be sure to bound the input length with 0x10110, as that's the maximum length for LZ-11 compressed blocks. + int maxLen = LZUtil.GetOccurrenceLength(indata + i, Math.Min(inLength - i, 0x10110), + indata + i - oldLength, oldLength, out disps[i]); + if (disps[i] > i) + throw new Exception("disp is too large"); + for (int j = 3; j <= maxLen; j++) + { + int blocklen; + if (j > 0x110) + blocklen = 4; + else if (j > 0x10) + blocklen = 3; + else + blocklen = 2; + int newCompLen; + if (i + j >= inLength) + newCompLen = blocklen; + else + newCompLen = blocklen + minLengths[i + j]; + if (newCompLen < minLengths[i]) + { + lengths[i] = j; + minLengths[i] = newCompLen; + } + } + } + + // we could optimize this further to also optimize it with regard to the flag-bytes, but that would require 8 times + // more space and time (one for each position in the block) for only a potentially tiny increase in compression ratio. + } + #endregion + } +} diff --git a/Switch_Toolbox_Library/Compression/LZ77_wii_11_compresss/Formats/Nitro/NitroCFormat.cs b/Switch_Toolbox_Library/Compression/LZ77_wii_11_compresss/Formats/Nitro/NitroCFormat.cs new file mode 100644 index 00000000..c5b033db --- /dev/null +++ b/Switch_Toolbox_Library/Compression/LZ77_wii_11_compresss/Formats/Nitro/NitroCFormat.cs @@ -0,0 +1,91 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Toolbox.Library.Compression.LZ77_wii_11_compresss.Exceptions; +using Toolbox.Library.Compression.LZ77_wii_11_compresss.Utils; + +namespace Toolbox.Library.Compression.LZ77_wii_11_compresss.Formats.Nitro +{ + /// + /// Base class for Nitro-based decompressors. Uses the 1-byte magic and 3-byte decompression + /// size format. + /// + public abstract class NitroCFormat : CompressionFormat + { + /// + /// If true, Nitro Decompressors will not decompress files that have a decompressed + /// size (plaintext size) larger than MaxPlaintextSize. + /// + public static bool SkipLargePlaintexts = true; + /// + /// The maximum allowed size of the decompressed file (plaintext size) allowed for Nitro + /// Decompressors. Only used when SkipLargePlaintexts = true. + /// If the expected plaintext size is larger that this, the 'Supports' method will partially + /// decompress the data to check if the file is OK. + /// + public static int MaxPlaintextSize = 0x180000; + + /// + /// The first byte of every file compressed with the format for this particular + /// Nitro Dcompressor instance. + /// + protected byte magicByte; + + /// + /// Creates a new instance of the Nitro Compression Format base class. + /// + /// The expected first byte of the file for this format. + protected NitroCFormat(byte magicByte) + { + this.magicByte = magicByte; + } + + /// + /// Checks if the first four (or eight) bytes match the format used in nitro compression formats. + /// + public override bool Supports(System.IO.Stream stream, long inLength) + { + long startPosition = stream.Position; + try + { + int firstByte = stream.ReadByte(); + if (firstByte != this.magicByte) + return false; + // no need to read the size info as well if it's used anyway. + if (!SkipLargePlaintexts) + return true; + byte[] sizeBytes = new byte[3]; + stream.Read(sizeBytes, 0, 3); + int outSize = IOUtils.ToNDSu24(sizeBytes, 0); + if (outSize == 0) + { + sizeBytes = new byte[4]; + stream.Read(sizeBytes, 0, 4); + outSize = (int)IOUtils.ToNDSu32(sizeBytes, 0); + } + if (outSize <= MaxPlaintextSize) + return true; + + try + { + stream.Position = startPosition; + this.Decompress(stream, Math.Min(Math.Min(inLength, 0x80000), MaxPlaintextSize), new System.IO.MemoryStream()); + // we expect a NotEnoughDataException, since we're giving the decompressor only part of the file. + return false; + } + catch (NotEnoughDataException) + { + return true; + } + catch (Exception) + { + return false; + } + } + finally + { + stream.Position = startPosition; + } + } + } +} diff --git a/Switch_Toolbox_Library/Compression/LZ77_wii_11_compresss/Utils/IOUtils.cs b/Switch_Toolbox_Library/Compression/LZ77_wii_11_compresss/Utils/IOUtils.cs new file mode 100644 index 00000000..f6d66cbe --- /dev/null +++ b/Switch_Toolbox_Library/Compression/LZ77_wii_11_compresss/Utils/IOUtils.cs @@ -0,0 +1,63 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Reflection; +using System.IO; + +namespace Toolbox.Library.Compression.LZ77_wii_11_compresss.Utils +{ + /// + /// Class for I/O-related utility methods. + /// + public static class IOUtils + { + + #region byte[] <-> (u)int + /// + /// Returns a 4-byte unsigned integer as used on the NDS converted from four bytes + /// at a specified position in a byte array. + /// + /// The source of the data. + /// The location of the data in the source. + /// The indicated 4 bytes converted to uint + public static uint ToNDSu32(byte[] buffer, int offset) + { + return (uint)(buffer[offset] + | (buffer[offset + 1] << 8) + | (buffer[offset + 2] << 16) + | (buffer[offset + 3] << 24)); + } + + + /// + /// Returns a 4-byte signed integer as used on the NDS converted from four bytes + /// at a specified position in a byte array. + /// + /// The source of the data. + /// The location of the data in the source. + /// The indicated 4 bytes converted to int + public static int ToNDSs32(byte[] buffer, int offset) + { + return (int)(buffer[offset] + | (buffer[offset + 1] << 8) + | (buffer[offset + 2] << 16) + | (buffer[offset + 3] << 24)); + } + + + /// + /// Returns a 3-byte integer as used in the built-in compression + /// formats in the DS, convrted from three bytes at a specified position in a byte array, + /// + /// The source of the data. + /// The location of the data in the source. + /// The indicated 3 bytes converted to an integer. + public static int ToNDSu24(byte[] buffer, int offset) + { + return (int)(buffer[offset] + | (buffer[offset + 1] << 8) + | (buffer[offset + 2] << 16)); + } + #endregion + } +} diff --git a/Switch_Toolbox_Library/Compression/LZ77_wii_11_compresss/Utils/LZUtil.cs b/Switch_Toolbox_Library/Compression/LZ77_wii_11_compresss/Utils/LZUtil.cs new file mode 100644 index 00000000..f5a1ca84 --- /dev/null +++ b/Switch_Toolbox_Library/Compression/LZ77_wii_11_compresss/Utils/LZUtil.cs @@ -0,0 +1,62 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Toolbox.Library.Compression.LZ77_wii_11_compresss.Utils +{ + /// + /// Utility class for compression using LZ-like compression schemes. + /// + public static class LZUtil + { + /// + /// Determine the maximum size of a LZ-compressed block starting at newPtr, using the already compressed data + /// starting at oldPtr. Takes O(inLength * oldLength) = O(n^2) time. + /// + /// The start of the data that needs to be compressed. + /// The number of bytes that still need to be compressed. + /// (or: the maximum number of bytes that _may_ be compressed into one block) + /// The start of the raw file. + /// The number of bytes already compressed. + /// The offset of the start of the longest block to refer to. + /// The minimum allowed value for 'disp'. + /// The length of the longest sequence of bytes that can be copied from the already decompressed data. + public static unsafe int GetOccurrenceLength(byte* newPtr, int newLength, byte* oldPtr, int oldLength, out int disp, int minDisp = 1) + { + disp = 0; + if (newLength == 0) + return 0; + int maxLength = 0; + // try every possible 'disp' value (disp = oldLength - i) + for (int i = 0; i < oldLength - minDisp; i++) + { + // work from the start of the old data to the end, to mimic the original implementation's behaviour + // (and going from start to end or from end to start does not influence the compression ratio anyway) + byte* currentOldStart = oldPtr + i; + int currentLength = 0; + // determine the length we can copy if we go back (oldLength - i) bytes + // always check the next 'newLength' bytes, and not just the available 'old' bytes, + // as the copied data can also originate from what we're currently trying to compress. + for (int j = 0; j < newLength; j++) + { + // stop when the bytes are no longer the same + if (*(currentOldStart + j) != *(newPtr + j)) + break; + currentLength++; + } + + // update the optimal value + if (currentLength > maxLength) + { + maxLength = currentLength; + disp = oldLength - i; + + // if we cannot do better anyway, stop trying. + if (maxLength == newLength) + break; + } + } + return maxLength; + } + } +} diff --git a/Switch_Toolbox_Library/Toolbox_Library.csproj b/Switch_Toolbox_Library/Toolbox_Library.csproj index 7b12796c..9fcb0979 100644 --- a/Switch_Toolbox_Library/Toolbox_Library.csproj +++ b/Switch_Toolbox_Library/Toolbox_Library.csproj @@ -240,6 +240,16 @@ + + + + + + + + + +