Add support for v4 PACx files

This commit is contained in:
KillzXGaming 2019-08-15 17:20:14 -04:00
parent c48c9d57d0
commit 2264c8117b
11 changed files with 435 additions and 329 deletions

Binary file not shown.

View file

@ -0,0 +1,424 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Toolbox;
using System.Windows.Forms;
using Toolbox.Library;
using Toolbox.Library.IO;
using System.Runtime.InteropServices;
namespace FirstPlugin
{
public class PACx : IArchiveFile, IFileFormat
{
public FileType FileType { get; set; } = FileType.Archive;
public bool CanSave { get; set; }
public string[] Description { get; set; } = new string[] { "Sonic Forces / Tokyo Olympics 2020 Archive" };
public string[] Extension { get; set; } = new string[] { "*.pac" };
public string FileName { get; set; }
public string FilePath { get; set; }
public IFileInfo IFileInfo { get; set; }
public bool CanAddFiles { get; set; }
public bool CanRenameFiles { get; set; }
public bool CanReplaceFiles { get; set; }
public bool CanDeleteFiles { get; set; }
public bool Identify(System.IO.Stream stream)
{
using (var reader = new Toolbox.Library.IO.FileReader(stream, true))
{
return reader.CheckSignature(8, "PACx301L") ||
reader.CheckSignature(8, "PACx302L") ||
reader.CheckSignature(8, "PACx402L");
}
}
public Type[] Types
{
get
{
List<Type> types = new List<Type>();
return types.ToArray();
}
}
public List<FileEntry> files = new List<FileEntry>();
public IEnumerable<ArchiveFileInfo> Files => files;
public string Name { get; set; }
public void ClearFiles() { files.Clear(); }
public uint Checksum;
public static bool IsVersion4;
public List<SplitEntry> SplitEntries = new List<SplitEntry>();
public void Load(System.IO.Stream stream)
{
using (var reader = new FileReader(stream))
{
reader.ByteOrder = Syroot.BinaryData.ByteOrder.LittleEndian;
IsVersion4 = reader.CheckSignature(8, "PACx402L");
if (IsVersion4)
{
var header = reader.ReadStruct<HeaderV4>();
var chunks = reader.ReadMultipleStructs<Chunk>(header.ChunkCount);
Checksum = header.PacID;
//Decompress each chunk now
reader.SeekBegin(header.RootOffset);
ReadRootPac(DecompressChunks(reader, chunks));
//Read splits from root pac
if (SplitEntries.Count > 0)
{
foreach (var pacSplit in SplitEntries)
ReadSplitPac(reader, pacSplit);
}
}
else
{
var header3 = reader.ReadStruct<HeaderV3>();
PacNodeTree tree = new PacNodeTree();
tree.Read(reader, header3);
var rootNode = tree.RootNode;
LoadTree(rootNode);
}
}
}
public void ReadSplitPac(FileReader reader, SplitEntry entry)
{
reader.SeekBegin(entry.SplitOffset);
ReadRootPac(DecompressChunks(reader, entry.Chunks), entry.Name);
}
public byte[] DecompressChunks(FileReader reader, List<Chunk> chunks)
{
List<byte[]> PacChunks = new List<byte[]>();
for (int i = 0; i < chunks.Count; i++)
{
if (chunks[i].CompressedSize == chunks[i].UncompressedSize)
{
PacChunks.Add(reader.ReadBytes((int)chunks[i].UncompressedSize));
}
else
{
PacChunks.Add(STLibraryCompression.Type_LZ4.Decompress(
reader.ReadBytes((int)chunks[i].CompressedSize), 0,
(int)chunks[i].CompressedSize, (int)chunks[i].UncompressedSize));
}
}
return Utils.CombineByteArray(PacChunks.ToArray());
}
public void ReadRootPac(byte[] buffer, string splitName = "")
{
using (var reader = new FileReader(new System.IO.MemoryStream(buffer)))
{
var header3 = reader.ReadStruct<HeaderV3>();
PacNodeTree tree = new PacNodeTree();
tree.Read(reader, header3);
var rootNode = tree.RootNode;
LoadTree(rootNode, splitName);
if (header3.SplitCount != 0)
{
//Read the split data if present
reader.SeekBegin(48 + header3.NodesSize);
ulong splitCount = reader.ReadUInt64();
ulong splitEntriesOffset = reader.ReadUInt64();
reader.SeekBegin(splitEntriesOffset);
for (int i = 0; i < (int)splitCount; i++)
{
SplitEntry entry = new SplitEntry();
entry.SplitNameOffset = reader.ReadUInt64();
entry.SplitCompressedSize = reader.ReadUInt32();
entry.SplitUncompressedSize = reader.ReadUInt32();
entry.SplitOffset = reader.ReadUInt32();
entry.SplitChunkCount = reader.ReadUInt32();
entry.SplitChunksOffset = reader.ReadUInt32();
using (reader.TemporarySeek((long)entry.SplitNameOffset, System.IO.SeekOrigin.Begin))
{
entry.Name = reader.ReadZeroTerminatedString();
}
using (reader.TemporarySeek((long)entry.SplitChunksOffset, System.IO.SeekOrigin.Begin))
{
entry.Chunks = reader.ReadMultipleStructs<Chunk>(entry.SplitChunkCount);
}
Console.WriteLine("SplitName " + entry.Name);
Console.WriteLine("SplitCompressedSize " + entry.SplitCompressedSize);
Console.WriteLine("SplitUncompressedSize " + entry.SplitUncompressedSize);
Console.WriteLine("SplitOffset " + entry.SplitOffset);
Console.WriteLine("SplitChunkCount " + entry.SplitChunkCount);
Console.WriteLine("SplitChunksOffset " + entry.SplitChunksOffset);
SplitEntries.Add(entry);
}
}
}
}
//Documentation from https://pastebin.com/RSAbK46c
[StructLayout(LayoutKind.Sequential, Pack = 1)]
public class HeaderV4
{
public Magic8 Magic = "PACx402L";
public uint PacID;
public uint FileSize;
public uint RootOffset;
public uint RootCompressedSize;
public uint RootUncompressedSize;
public PacType Type = PacType.HasNoSplit;
public ushort Constant = 0x208;
public uint ChunkCount;
}
[StructLayout(LayoutKind.Sequential, Pack = 1)]
public class Chunk
{
// When decompressing the root pac allocate a buffer of
// size Header.RootUncompressedSize, then loop through these
// chunks and decompress each one, one-by-one, into that buffer.
// If you try to decompress all at once instead the data can be corrupted.
public uint CompressedSize; //Compressed as LZ4
public uint UncompressedSize;
}
[StructLayout(LayoutKind.Sequential, Pack = 1)]
public class HeaderV3
{
public Magic8 Magic = "PACx402L";
public uint PacID;
public uint FileSize;
public uint NodesSize;
public uint SplitsInfoSize;
public uint DataEntriesSize;
public uint StringTableSize;
public uint DataSize;
public uint OffsetTableSize;
public PacType Type = PacType.HasNoSplit;
public ushort Constant = 0x108;
public uint SplitCount;
}
public class SplitEntry
{
public ulong SplitNameOffset;
public uint SplitCompressedSize;
public uint SplitUncompressedSize;
public uint SplitOffset;
public uint SplitChunkCount;
public ulong SplitChunksOffset;
public string Name;
public List<Chunk> Chunks;
}
public class EmbeddedPAC
{
}
public enum PacType : ushort
{
// PAC has no splits
HasNoSplit = 1,
// PAC is a split
IsSplit = 2,
// PAC has splits
HasSplit = 5
}
public void LoadTree(PacNode node, string fullPath = "")
{
if (node.HasData && node.Data != null)
{
FileEntry newNode = new FileEntry(node);
newNode.FileName = $"{fullPath}/{newNode.Name}";
files.Add(newNode);
}
if (node.Name != "Node" && node.Name != null)
fullPath += $"/{node.Name}";
for (int i = 0; i < node.Children.Count; i++)
LoadTree(node.Children[i], fullPath);
}
public class FileEntry : ArchiveFileInfo
{
public FileEntry(PacNode node)
{
Name = node.Name;
FileData = node.Data;
if (node.Name == null) Name = "File Node";
if (FileData == null) FileData = new byte[0];
}
}
public class PacNodeTree
{
public PacNode RootNode { get; set; }
public ulong rootNodeOffset;
public void Read(FileReader reader, HeaderV3 pac)
{
uint nodeCount = reader.ReadUInt32();
uint dataNodeCount = reader.ReadUInt32();
rootNodeOffset = reader.ReadUInt64();
ulong dataNodeIndicesOffset = reader.ReadUInt64();
reader.SeekBegin(rootNodeOffset);
RootNode = new PacNode(pac, this);
RootNode.Read(reader);
}
}
public class PacNode
{
public HeaderV3 PacFile;
public byte[] Data;
public PacNodeTree ParentTree;
public string Name { get; set; }
public bool HasData { get; set; }
public List<PacNode> Children = new List<PacNode>();
public DataType DataType;
public PacNode(HeaderV3 pac, PacNodeTree tree)
{
PacFile = pac;
ParentTree = tree;
}
public void Read(FileReader reader)
{
ulong nameOffset = reader.ReadUInt64();
ulong dataOffset = reader.ReadUInt64();
ulong childIndicesOffset = reader.ReadUInt64();
int parentIndex = reader.ReadInt32();
int index = reader.ReadInt32();
int dataIndex = reader.ReadInt32();
ushort childCount = reader.ReadUInt16();
HasData = reader.ReadBoolean();
byte fullPathSize = reader.ReadByte();
Console.WriteLine($"nameOffset {nameOffset}");
Console.WriteLine($"dataOffset {dataOffset}");
Console.WriteLine($"childIndicesOffset {childIndicesOffset}");
Console.WriteLine($"parentIndex {parentIndex}");
Console.WriteLine($"index {index}");
Console.WriteLine($"dataIndex {dataIndex}");
Console.WriteLine($"childCount {childCount}");
Console.WriteLine($"HasData {HasData}");
if (nameOffset == 5490503897632162128)
return;
if (nameOffset != 0)
{
reader.SeekBegin((long)nameOffset);
Name = reader.ReadZeroTerminatedString();
}
if (dataOffset != 0)
{
reader.SeekBegin((long)dataOffset);
uint PacID = reader.ReadUInt32();
//Detecting data in v4 is a pain
//Check if the offset is set within the data section
var dataPos = 48 + PacFile.NodesSize + PacFile.SplitsInfoSize;
if (PacID == PacFile.PacID || IsVersion4 && dataOffset >= dataPos)
{
ulong dataSize = reader.ReadUInt64();
uint padding = reader.ReadUInt32();
ulong dataBlockOffset = reader.ReadUInt64();
ulong padding2 = reader.ReadUInt64();
ulong extensionOffset = reader.ReadUInt64();
DataType = reader.ReadEnum<DataType>(false);
if (extensionOffset != 0)
{
reader.SeekBegin((long)extensionOffset);
string extension = reader.ReadZeroTerminatedString();
Name += extension;
}
if (dataBlockOffset != 0)
{
reader.SeekBegin((long)dataBlockOffset);
Data = reader.ReadBytes((int)dataSize);
}
}
else
{
reader.SeekBegin((long)dataOffset);
PacNodeTree tree = new PacNodeTree();
tree.Read(reader, PacFile);
Children.Add(tree.RootNode);
}
}
if (childIndicesOffset != 0)
{
reader.SeekBegin((long)childIndicesOffset);
int[] Indices = reader.ReadInt32s(childCount);
for (int i = 0; i < childCount; i++)
{
int childIndex = Indices[i];
reader.SeekBegin((uint)ParentTree.rootNodeOffset + (childIndex * 40));
PacNode node = new PacNode(PacFile, ParentTree);
node.Read(reader);
Children.Add(node);
}
}
}
}
public enum DataType : ulong
{
RegularFile = 0,
NotHere = 1,
BINAFile = 2
}
public void Unload()
{
}
public void Save(System.IO.Stream stream)
{
}
public bool AddFile(ArchiveFileInfo archiveFileInfo)
{
return false;
}
public bool DeleteFile(ArchiveFileInfo archiveFileInfo)
{
return false;
}
}
}

View file

@ -1,254 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Toolbox;
using System.Windows.Forms;
using Toolbox.Library;
using Toolbox.Library.IO;
namespace FirstPlugin
{
public class PACx30XL : IArchiveFile, IFileFormat, IDirectoryContainer
{
public FileType FileType { get; set; } = FileType.Archive;
public bool CanSave { get; set; }
public string[] Description { get; set; } = new string[] { "Sonic Forces PAC" };
public string[] Extension { get; set; } = new string[] { "*.pac" };
public string FileName { get; set; }
public string FilePath { get; set; }
public IFileInfo IFileInfo { get; set; }
public bool CanAddFiles { get; set; }
public bool CanRenameFiles { get; set; }
public bool CanReplaceFiles { get; set; }
public bool CanDeleteFiles { get; set; }
public bool Identify(System.IO.Stream stream)
{
using (var reader = new Toolbox.Library.IO.FileReader(stream, true))
{
return reader.CheckSignature(8, "PACx301L") || reader.CheckSignature(8, "PACx302L");
}
}
public Type[] Types
{
get
{
List<Type> types = new List<Type>();
return types.ToArray();
}
}
public List<FileEntry> files = new List<FileEntry>();
public IEnumerable<ArchiveFileInfo> Files => files;
public List<INode> nodes = new List<INode>();
public IEnumerable<INode> Nodes => nodes;
public string Name { get; set; }
public void ClearFiles() { files.Clear(); }
public uint Checksum;
public void Load(System.IO.Stream stream)
{
using (var reader = new FileReader(stream))
{
reader.ByteOrder = Syroot.BinaryData.ByteOrder.LittleEndian;
bool IsVersion2 = reader.CheckSignature(8, "PACx402L");
reader.SeekBegin(8);
Checksum = reader.ReadUInt32();
uint FileSize = reader.ReadUInt32();
if (IsVersion2)
{
}
else
{
uint NodeSectionSize = reader.ReadUInt32();
uint PacDependsSectionSize = reader.ReadUInt32();
uint EntrySectionSize = reader.ReadUInt32();
uint StringTableSize = reader.ReadUInt32();
uint DataSectionSize = reader.ReadUInt32();
uint OffsetTableSize = reader.ReadUInt32();
ushort PacType = reader.ReadUInt16();
ushort constant = reader.ReadUInt16();
uint dependPacCount = reader.ReadUInt32();
PacNodeTree tree = new PacNodeTree();
tree.Read(reader, this);
var rootNode = tree.RootNode;
var dirRoot = new DirectoryEntry(rootNode);
LoadTree(rootNode, dirRoot);
nodes.Add(dirRoot);
}
}
}
public void LoadTree(PacNode node, DirectoryEntry parentNode)
{
INode newNode = null;
if (node.HasData && node.Data != null)
newNode = new FileEntry(node);
else
newNode = new DirectoryEntry(node);
parentNode.nodes.Add(newNode);
for (int i = 0; i < node.Children.Count; i++)
{
LoadTree(node.Children[i], (DirectoryEntry)newNode);
}
}
public class DirectoryEntry : IDirectoryContainer
{
public string Name { get; set; }
public List<INode> nodes = new List<INode>();
public IEnumerable<INode> Nodes => nodes;
public DirectoryEntry(PacNode node)
{
Name = node.Name;
if (node.Name == null) Name = "Node";
}
}
public class FileEntry : ArchiveFileInfo
{
public FileEntry(PacNode node)
{
Name = node.Name;
FileData = node.Data;
if (node.Name == null) Name = "File Node";
if (FileData == null) FileData = new byte[0];
}
}
public class PacNodeTree
{
public PacNode RootNode { get; set; }
public uint rootNodeOffset;
public void Read(FileReader reader, PACx30XL pac)
{
uint nodeCount = reader.ReadUInt32();
uint dataNodeCount = reader.ReadUInt32();
rootNodeOffset = reader.ReadUInt32();
uint dataNodeIndicesOffset = reader.ReadUInt32();
reader.SeekBegin(rootNodeOffset);
RootNode = new PacNode(pac, this);
RootNode.Read(reader);
}
}
public class PacNode
{
public PACx30XL PacFile;
public byte[] Data;
public PacNodeTree ParentTree;
public string Name { get; set; }
public bool HasData { get; set; }
public List<PacNode> Children = new List<PacNode>();
public PacNode(PACx30XL pac, PacNodeTree tree)
{
PacFile = pac;
ParentTree = tree;
}
public void Read(FileReader reader)
{
ulong nameOffset = reader.ReadUInt64();
ulong dataOffset = reader.ReadUInt64();
ulong childIndicesOffset = reader.ReadUInt64();
int parentIndex = reader.ReadInt32();
int index = reader.ReadInt32();
int dataIndex = reader.ReadInt32();
ushort childCount = reader.ReadUInt16();
HasData = reader.ReadBoolean();
byte fullPathSize = reader.ReadByte();
if (nameOffset != 0)
{
reader.SeekBegin((long)nameOffset);
Name = reader.ReadZeroTerminatedString();
}
if (dataOffset != 0)
{
reader.SeekBegin((long)dataOffset);
if (reader.ReadUInt32() == PacFile.Checksum)
{
uint dataSize = reader.ReadUInt32();
ulong padding = reader.ReadUInt64();
ulong dataBlockOffset = reader.ReadUInt64();
ulong padding2 = reader.ReadUInt64();
ulong extensionOffset = reader.ReadUInt64();
ulong dataType = reader.ReadUInt32();
reader.SeekBegin((long)extensionOffset);
string extension = reader.ReadZeroTerminatedString();
Name += extension;
reader.SeekBegin((long)dataBlockOffset);
Data = reader.ReadBytes((int)dataSize);
}
else
{
reader.SeekBegin((long)dataOffset);
PacNodeTree tree = new PacNodeTree();
tree.Read(reader, PacFile);
Children.Add(tree.RootNode);
}
}
if (childIndicesOffset != 0)
{
reader.SeekBegin((long)childIndicesOffset);
int[] Indices = reader.ReadInt32s(childCount);
for (int i =0; i < childCount; i++)
{
int childIndex = Indices[i];
reader.SeekBegin(ParentTree.rootNodeOffset + (childIndex * 40));
PacNode node = new PacNode(PacFile, ParentTree);
node.Read(reader);
Children.Add(node);
}
}
}
}
public void Unload()
{
}
public void Save(System.IO.Stream stream)
{
}
public bool AddFile(ArchiveFileInfo archiveFileInfo)
{
return false;
}
public bool DeleteFile(ArchiveFileInfo archiveFileInfo)
{
return false;
}
}
}

View file

@ -208,6 +208,7 @@
<Compile Include="FileFormats\Archives\ARC.cs" />
<Compile Include="FileFormats\Archives\GFA.cs" />
<Compile Include="FileFormats\Archives\LM2\LM2_Material.cs" />
<Compile Include="FileFormats\Archives\Sonic Forces\PACx.cs" />
<Compile Include="FileFormats\CrashBandicoot\IGZ_Structure.cs" />
<Compile Include="FileFormats\Grezzo\CMB_Enums.cs" />
<Compile Include="FileFormats\Grezzo\CSAB.cs" />
@ -227,7 +228,6 @@
<Compile Include="FileFormats\Archives\ME01.cs" />
<Compile Include="FileFormats\Archives\MKGPDX_PAC.cs" />
<Compile Include="FileFormats\Archives\NXARC.cs" />
<Compile Include="FileFormats\Archives\Sonic Forces\PACx30XL.cs" />
<Compile Include="FileFormats\Archives\RARC.cs" />
<Compile Include="FileFormats\Archives\Sonic Racing\GameDataToc.cs" />
<Compile Include="FileFormats\Archives\Sonic Racing\SPC.cs" />

View file

@ -359,7 +359,7 @@ namespace FirstPlugin
Formats.Add(typeof(GCDisk));
Formats.Add(typeof(TPL));
Formats.Add(typeof(BFTTF));
Formats.Add(typeof(PACx30XL));
Formats.Add(typeof(PACx));
Formats.Add(typeof(BinGzArchive));
Formats.Add(typeof(GAR));
Formats.Add(typeof(CTXB));

View file

@ -48,6 +48,7 @@ namespace Toolbox.Library.IO
//From kuriimu https://github.com/IcySon55/Kuriimu/blob/master/src/Kontract/IO/BinaryReaderX.cs#L40
public T ReadStruct<T>() => ReadBytes(Marshal.SizeOf<T>()).BytesToStruct<T>(ByteOrder == ByteOrder.BigEndian);
public List<T> ReadMultipleStructs<T>(int count) => Enumerable.Range(0, count).Select(_ => ReadStruct<T>()).ToList();
public List<T> ReadMultipleStructs<T>(uint count) => Enumerable.Range(0, (int)count).Select(_ => ReadStruct<T>()).ToList();
public bool CheckSignature(uint Identifier, long position = 0)
{
@ -223,6 +224,7 @@ namespace Toolbox.Library.IO
public void SeekBegin(uint Offset) { Seek(Offset, SeekOrigin.Begin); }
public void SeekBegin(int Offset) { Seek(Offset, SeekOrigin.Begin); }
public void SeekBegin(long Offset) { Seek(Offset, SeekOrigin.Begin); }
public void SeekBegin(ulong Offset) { Seek((long)Offset, SeekOrigin.Begin); }
public long ReadOffset(bool IsRelative, Type OffsetType)
{

View file

@ -15,4 +15,11 @@ namespace Toolbox.Library.IO
public static implicit operator Magic(string s) => new Magic { value = BitConverter.ToInt32(Encoding.ASCII.GetBytes(s), 0) };
}
[StructLayout(LayoutKind.Sequential, Pack = 1)]
public struct Magic8
{
long value;
public static implicit operator string(Magic8 magic) => Encoding.ASCII.GetString(BitConverter.GetBytes(magic.value));
public static implicit operator Magic8(string s) => new Magic8 { value = BitConverter.ToInt64(Encoding.ASCII.GetBytes(s), 0) };
}
}

View file

@ -812,7 +812,6 @@
<Compile Include="Texture Decoding\Wii U\GX2.cs" />
<Compile Include="Texture Decoding\Switch\TegraX1Swizzle.cs" />
<Compile Include="ThemeConfig.cs" />
<Compile Include="UpdateProgram.cs" />
<Compile Include="Util\ColorUtility.cs" />
<Compile Include="Util\ImageUtilty.cs" />
<Compile Include="Util\OpenGLUtils.cs" />

View file

@ -1,72 +0,0 @@
using System;
using System.Linq;
using System.Collections.Generic;
using System.Threading.Tasks;
using Octokit;
using System.IO;
using System.Diagnostics;
using System.Security.Cryptography;
using Toolbox.Library;
namespace Toolbox
{
public class UpdateProgram
{
static Release[] releases;
public static bool CanUpdate = false;
public static Release LatestRelease;
public static void CheckLatest()
{
try
{
var client = new GitHubClient(new ProductHeaderValue("ST_UpdateTool"));
GetReleases(client).Wait();
foreach (Release latest in releases)
{
Console.WriteLine(
"The latest release is tagged at {0} and is named {1} commit {2} date {3}",
latest.TagName,
latest.Name,
latest.TargetCommitish,
latest.Assets[0].UpdatedAt.ToString());
ParseVersion(latest.TagName);
if (Runtime.ProgramVersion != latest.TagName && Major >= 1)
{
CanUpdate = true;
LatestRelease = latest;
}
break;
}
}
catch (Exception ex)
{
Console.WriteLine($"Failed to get latest update\n{ex.ToString()}");
}
}
public static int Major;
public static int Minor;
public static int Revision;
static void ParseVersion(string TagName)
{
char[] chars = TagName.ToCharArray();
Major = int.Parse(chars[1].ToString());
}
static async Task GetReleases(GitHubClient client)
{
List<Release> Releases = new List<Release>();
foreach (Release r in await client.Repository.Release.GetAll("KillzXGaming", "Switch-Toolbox"))
Releases.Add(r);
releases = Releases.ToArray();
}
}
}