diff --git a/coilsnake/assets/ips/Earthbound/gas_station_pack_fix.ips b/coilsnake/assets/ips/Earthbound/gas_station_pack_fix.ips new file mode 100644 index 00000000..899f6df5 Binary files /dev/null and b/coilsnake/assets/ips/Earthbound/gas_station_pack_fix.ips differ diff --git a/coilsnake/assets/ips/Earthbound/gas_station_pack_fix.yml b/coilsnake/assets/ips/Earthbound/gas_station_pack_fix.yml new file mode 100644 index 00000000..824b8e91 --- /dev/null +++ b/coilsnake/assets/ips/Earthbound/gas_station_pack_fix.yml @@ -0,0 +1,4 @@ +Title: Fix pack variable initialization to allow for custom instrument packs on the Gas Station (Part 1) song +Author: cooprocks123e +Hidden: True +Ranges: [] diff --git a/coilsnake/assets/modulelist.txt b/coilsnake/assets/modulelist.txt index 0134b038..b0487247 100644 --- a/coilsnake/assets/modulelist.txt +++ b/coilsnake/assets/modulelist.txt @@ -1,5 +1,6 @@ common.UsedRangeModule common.PatchModule +eb.MusicModule eb.CharacterSubstitutionsModule eb.CccInterfaceModule eb.SkipNamingModule diff --git a/coilsnake/model/common/blocks.py b/coilsnake/model/common/blocks.py index 7cc36e9c..5973f1ec 100644 --- a/coilsnake/model/common/blocks.py +++ b/coilsnake/model/common/blocks.py @@ -16,6 +16,19 @@ def check_range_validity(range, size): elif (begin < 0) or (end >= size): raise OutOfBoundsError("Invalid range[(%#x,%#x)] provided" % (begin, end)) +def fix_slice(key, size): + if not (key.step == 1 or key.step is None): + raise InvalidArgumentError("Slice step must be 1 or None, but is {}".format(key.step)) + start, stop = key.start, key.stop + def fix_single_index(x, default): + if x is None: + x = default + elif x < 0: + x += size + return x + start = fix_single_index(start, 0) + stop = fix_single_index(stop, size) + return slice(start, stop) class Block(object): def __init__(self, size=0): @@ -105,6 +118,7 @@ def write_multi(self, key, item, size): def __getitem__(self, key): if isinstance(key, slice): + key = fix_slice(key, self.size) if key.start > key.stop: raise InvalidArgumentError("Second argument of slice %s must be greater than the first" % key) elif (key.start < 0) or (key.stop - 1 >= self.size): @@ -132,6 +146,7 @@ def __setitem__(self, key, item): self.data[key] = item elif isinstance(key, slice) and \ (isinstance(item, list) or isinstance(item, array.array) or isinstance(item, Block)): + key = fix_slice(key, self.size) if key.start > key.stop: raise InvalidArgumentError("Second argument of slice %s must be greater than the first" % key) elif (key.start < 0) or (key.stop - 1 >= self.size): @@ -164,9 +179,13 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) - def __hash__(self): + def crc32(self): return crc32(self.data) + # Don't rely on this having the same result on different machines! + # Use self.crc32() if you're comparing with a known CRC value. + def __hash__(self): + return self.crc32() class AllocatableBlock(Block): def reset(self, size=0): diff --git a/coilsnake/model/eb/musicpack.py b/coilsnake/model/eb/musicpack.py new file mode 100644 index 00000000..2da5cb77 --- /dev/null +++ b/coilsnake/model/eb/musicpack.py @@ -0,0 +1,979 @@ +from array import array +from dataclasses import dataclass +from functools import lru_cache +import logging +import re +from typing import Any, Dict, List, TextIO, Tuple, Union + +from coilsnake.exceptions.common.exceptions import \ + CoilSnakeInternalError, InvalidUserDataError, OutOfBoundsError +from coilsnake.model.common.blocks import Block +from coilsnake.util.common.yml import yml_load + +CONFIG_TXT_FILENAME = "config.txt" + +YML_INST_PACK_1 = "Instrument Pack 1" +YML_INST_PACK_2 = "Instrument Pack 2" + +YML_SONG_PACK = "Song Pack" +YML_SONG_FILENAME = "Song File" +YML_SONG_TO_REFERENCE = "Song to Reference" +YML_SONG_OFFSET = "Offset" +YML_SONG_ADDRESS = "Address" +YML_SONG_PACK_BUILTIN = "in-engine" + +INST_OVERWRITE = 0x00 +INST_DEFAULT = 0x1a +SAMPLE_OFFSET_OVERWRITE = 0x7000 +SAMPLE_OFFSET_DEFAULT = 0x95b0 + +DYNAMIC_SONG_DATA_START = 0x4800 +DYNAMIC_SONG_DATA_END = 0x6C00 + +log = logging.getLogger(__name__) + +@dataclass +class EBInstrument: + '''Class that keeps track of all of an instrument's data.''' + adsr1: int + adsr2: int + gain: int + multiplier: int + submultiplier: int + sample: Union[Block, int, None] + sample_loop_offset: Union[int, None] + +def extract_pack_parts(rom: Block, pack_ptr: int) -> List[Tuple[int, int, Block]]: + parts = [] + start_bank = pack_ptr & 0xff0000 + while pack_ptr < rom.size and pack_ptr & 0xff0000 == start_bank: + length = rom.read_multi(pack_ptr, 2) + if length == 0: + pack_ptr += 2 + break + addr = rom.read_multi(pack_ptr + 2, 2) + pack_ptr += 4 + data = rom[pack_ptr:pack_ptr+length] + parts.append((addr, length, data)) + pack_ptr += length + # Ensure the bank of the last byte read doesn't exceed the current bank. + if (pack_ptr - 1) & 0xff0000 > start_bank: + raise InvalidUserDataError("Music pack did not end before bank boundary.") + return parts + +@lru_cache(20) +def extract_brr_chunk(brr_start_addr: int, brr_sample_addr: int, brr_block: Block) -> Block: + start = brr_sample_addr - brr_start_addr + end = start + while end < len(brr_block): + brr_header = brr_block[end] + end += 9 + if brr_header & 1: + return brr_block[start:end] + raise InvalidUserDataError("BRR data at ARAM address ${:04X} is missing a terminator".format(brr_sample_addr)) + +def read_hex_or_default_or_overwrite(s: str, default=None, overwrite=None): + match = re.match("^.*(default|overwrite).*$", s, flags=re.IGNORECASE) + if match: + s2 = match.group(1).lower() + if s2 == 'default': + return default + if s2 == 'overwrite': + return overwrite + return int(s.strip(), base=16) + +def parse_config_txt(config_txt: Union[TextIO, str]) -> Tuple[int, int, int, List[EBInstrument], List[EBInstrument]]: + instrument_keyword_hit = False + in_instruments = False + pack_num, base_inst, brr_offset = None, None, None + instruments: List[EBInstrument] = [] + instrument_files: List[EBInstrument] = [] + if isinstance(config_txt, str): + line_iter = enumerate(config_txt.splitlines(), start=1) + else: + line_iter = enumerate(config_txt, start=1) + for line_num, line in line_iter: + # Strip comments out of line + if ';' in line: + line = line.split(';')[0] + # Strip whitespace from line + line = line.strip() + if len(line) <= 0: + continue + if in_instruments: + if line == '}': + # That's it! No more config.txt + break + match = re.match(r'^"([^"]*)"' + r'\s+\$([0-9a-f]{2})' * 5 + r'$', + line, flags=re.IGNORECASE) + if match: + filename = match.group(1) + instinfo = [int(val, base=16) for val in match.groups()[1:]] + instrument = EBInstrument(*(instinfo + [None, None])) + instruments.append(instrument) + instrument_files.append(filename) + continue + raise InvalidUserDataError("Error at line {} in config.txt: expected instrument definition".format(line_num)) + if instrument_keyword_hit: + if line != '{': + raise InvalidUserDataError("Error at line {} in config.txt: expecting '{{'".format(line_num)) + in_instruments = True + continue + # Handle the header text + match = re.match(r"^.*pack num.*: (\w+)\s*$", line, flags=re.IGNORECASE) + if match: + if line_num > 3: + raise InvalidUserDataError("Error at line {} in config.txt: pack number directive can only be within first 3 lines".format(line_num)) + pack_num = read_hex_or_default_or_overwrite(match.group(1), default=None, overwrite="ow") + if pack_num == "ow": + raise InvalidUserDataError("Error at line {} in config.txt: pack number cannot be 'override'".format(line_num)) + continue + match = re.match(r"^.*base inst.*: (\w+)\s*$", line, flags=re.IGNORECASE) + if match: + if line_num > 3: + raise InvalidUserDataError("Error at line {} in config.txt: base instrument directive can only be within first 3 lines".format(line_num)) + base_inst = read_hex_or_default_or_overwrite(match.group(1), default=INST_DEFAULT, overwrite=INST_OVERWRITE) + continue + match = re.match(r"^.*offset.*: (\w+)\s*$", line, flags=re.IGNORECASE) + if match: + if line_num > 3: + raise InvalidUserDataError("Error at line {} in config.txt: BRR sample offset directive can only be within first 3 lines".format(line_num)) + brr_offset = read_hex_or_default_or_overwrite(match.group(1), default=SAMPLE_OFFSET_DEFAULT, overwrite=SAMPLE_OFFSET_OVERWRITE) + continue + match = re.match(r"^#instruments$", line, flags=re.IGNORECASE) + if match: + instrument_keyword_hit = True + continue + raise InvalidUserDataError("Error at line {} in config.txt: Unexpected text '{}'".format(line_num, line)) + + return pack_num, base_inst, brr_offset, instruments, instrument_files + +class GenericMusicPack: + def __init__(self, pack_num: int) -> None: + self.pack_num = pack_num + self.parts: List[Tuple[int, int, Block]] = None + def get_aram_byte(self, addr: int) -> Union[int, None]: + assert isinstance(self.parts, list) + for paddr, plen, pdata in self.parts: + if paddr <= addr < paddr + plen: + return pdata[addr - paddr] + return None + def set_aram_byte(self, addr: int, value: int) -> bool: + assert isinstance(self.parts, list) + for paddr, plen, pdata in self.parts: + if paddr <= addr < paddr + plen: + pdata[addr - paddr] = value + return True + return False + def get_aram_region(self, addr: int, length: int) -> Block: + assert isinstance(self.parts, list) + for paddr, plen, pdata in self.parts: + if paddr <= addr < paddr + plen: + return pdata[addr - paddr : addr - paddr + length] + return None + def set_aram_region(self, addr: int, length: int, value: Any) -> bool: + assert isinstance(self.parts, list) + for paddr, plen, pdata in self.parts: + if paddr <= addr < paddr + plen: + if isinstance(value, int): + value = [value for _ in range(length)] + pdata[addr - paddr : addr - paddr + length] = value + return True + return False + + def load_from_parts(self, parts: List[Tuple[int, int, Block]]) -> None: + ''' + Converts data from the pack part form into the internal data structure. + + This function should be overridden for each different pack type. + ''' + + def save_to_parts(self) -> None: + ''' + Converts data from the internal data structure to pack part form. + + This function should be overridden for each different pack type. + ''' + + def get_pack_binary_data(self) -> Block: + # Ensure we called save_to_parts previously + assert isinstance(self.parts, list) + # Size = (4 header bytes + length) for each block + 2 bytes for 00 00 at the end + total_size = sum((4 + plen for _, plen, _ in self.parts)) + 2 + ret_block = Block(total_size) + dptr = 0 + for paddr, plen, pdata in self.parts: + # Write length, addr + ret_block.write_multi(dptr, plen, 2) + ret_block.write_multi(dptr + 2, paddr, 2) + dptr += 4 + # Write data chunk + ret_block[dptr:dptr+plen] = pdata + dptr += plen + # Write length of 0 to mark end + ret_block.write_multi(dptr, 0, 2) + dptr += 2 + if total_size != dptr: + log.error('Check these parts: %s', self.parts) + raise CoilSnakeInternalError("Coding error: data format miscalculation in musicpack.combine_parts") + return ret_block + def convert_to_files(self) -> List[Tuple[str, Union[Block, str]]]: + ''' + Converts a pack into a set of files to store in a directory. + + This function should be overridden for each different pack type. + ''' + return [("pack.bin", self.get_pack_binary_data())] + + def load_from_files(self, file_loader): + raise NotImplementedError('Please use a subclass of GenericMusicPack') + +class EmptyPack(GenericMusicPack): + def get_pack_binary_data(self) -> Block: + return Block(2) + def load_from_files(self, file_loader): + raise CoilSnakeInternalError("Empty pack cannot be loaded from a file.") + +class InstrumentMusicPack(GenericMusicPack): + def __init__(self, pack_num: int) -> None: + super().__init__(pack_num) + self.instruments: List[EBInstrument] = [] + self.base_instrument: int = 0 + self.brr_sample_dump_offset: int = 0 + + def load_from_parts(self, parts: List[Tuple[int, int, Block]]) -> GenericMusicPack: + if len(parts) != 3: + raise InvalidUserDataError("Invalid number of parts for instrument pack, must be 3") + + # Check for sample dir. part + cand_parts = [p for p in parts if 0x6c00 <= p[0] < 0x6e00] + if len(cand_parts) != 1: + raise InvalidUserDataError("Invalid number of sample directory parts for instrument pack, must be 1") + sample_dir_part = cand_parts[0] + if (sample_dir_part[0] - 0x6c00) % 4 != 0: + raise InvalidUserDataError("Invalid sample directory part for instrument pack, (address - 0x6c00) must be divisible by 4") + if sample_dir_part[1] % 4 != 0: + raise InvalidUserDataError("Invalid sample directory part for instrument pack, length must be divisible by 4") + + # Check for instrument dir. part + cand_parts = [p for p in parts if 0x6e00 <= p[0] < 0x6f80] + if len(cand_parts) != 1: + raise InvalidUserDataError("Invalid number of instrument directory parts for instrument pack, must be 1") + instrument_dir_part = cand_parts[0] + if (instrument_dir_part[0] - 0x6e00) % 6 != 0: + raise InvalidUserDataError("Invalid instrument directory part for instrument pack, (address - 0x6e00) must be divisible by 6") + if instrument_dir_part[1] % 6 != 0: + raise InvalidUserDataError("Invalid instrument directory part for instrument pack, length must be divisible by 6") + + # Check for BRR part + cand_parts = [p for p in parts if 0x7000 <= p[0] <= 0xe800] + if len(cand_parts) != 1: + raise InvalidUserDataError("Invalid number of BRR parts for instrument pack, must be 1") + brr_part = cand_parts[0] + + inst_base = (instrument_dir_part[0] - 0x6e00) // 6 + inst_count = instrument_dir_part[1] // 6 + + # Double-check sample directory + samp_base = (sample_dir_part[0] - 0x6c00) // 4 + samp_count = sample_dir_part[1] // 4 + if inst_base != samp_base: + raise InvalidUserDataError("Invalid start address of instrument or sample directory part, must start at same instrument number") + if samp_count < inst_count: + raise InvalidUserDataError("Invalid number of samples, must be greater than or equal to instrument count, sample count = {}, instrument count = {}".format(samp_count, inst_count)) + + # Let's start unpacking the samples! + self.base_instrument = inst_base + self.brr_sample_dump_offset = brr_part[0] + self.instruments = [] + for inst_rel_num in range(inst_count): + if inst_rel_num + inst_base != instrument_dir_part[2][inst_rel_num*6]: + raise InvalidUserDataError("Invalid instrument directory, sample number must match instrument number") + inst_data = tuple(instrument_dir_part[2][inst_rel_num*6+1 : inst_rel_num*6+6].to_list()) + samp_start, samp_loop = sample_dir_part[2].read_multi(inst_rel_num*4, 2), sample_dir_part[2].read_multi(inst_rel_num*4+2, 2) + # If the sample start is >= the start of BRR data provided by this instrument pack + if samp_start >= brr_part[0]: + # then extract the sample data from the BRR part + sample = extract_brr_chunk(self.brr_sample_dump_offset, samp_start, brr_part[2]) + else: + # otherwise, just use the address + sample = samp_start + inst = EBInstrument(*inst_data, sample, samp_loop - samp_start) + self.instruments.append(inst) + # We've constructed the full list of instruments. We're done! B) + + def save_to_parts(self) -> None: + # Determine the set of BRRs used + brr_set = {inst.sample for inst in self.instruments if isinstance(inst.sample, Block)} + # Determine the addresses of the BRRs + brr_ptr = 0 + brr_addrs: Dict[Block, int] = {} + for brr in brr_set: + brr_addrs[brr] = brr_ptr + brr_ptr += len(brr) + brr_part_size = brr_ptr + # Create the BRR part + brr_part = (self.brr_sample_dump_offset, brr_part_size, Block(brr_part_size)) + for brr in brr_set: + brr_ptr = brr_addrs[brr] + brr_part[2][brr_ptr:brr_ptr + len(brr)] = brr + + # Create the sample and inst. directory parts + inst_count = len(self.instruments) + inst_base = self.base_instrument + sample_dir_part = (0x6c00 + 4 * inst_base, inst_count * 4, Block(inst_count * 4)) + inst_dir_part = (0x6e00 + 6 * inst_base, inst_count * 6, Block(inst_count * 6)) # pylint: disable=C0326 + # Populate sample and inst. directories + for inst_rel_num, inst in enumerate(self.instruments): + # Populate sample directory + if isinstance(inst.sample, Block): + sample_addr = brr_addrs[inst.sample] + self.brr_sample_dump_offset + else: + sample_addr = inst.sample + sample_loop = sample_addr + inst.sample_loop_offset + sample_dir_part[2].write_multi(inst_rel_num * 4 + 0, sample_addr, 2) + sample_dir_part[2].write_multi(inst_rel_num * 4 + 2, sample_loop, 2) + # Populate instrument directory + inst_dir_part[2][inst_rel_num * 6 + 0] = inst_base + inst_rel_num + inst_dir_part[2][inst_rel_num * 6 + 1] = inst.adsr1 + inst_dir_part[2][inst_rel_num * 6 + 2] = inst.adsr2 + inst_dir_part[2][inst_rel_num * 6 + 3] = inst.gain + inst_dir_part[2][inst_rel_num * 6 + 4] = inst.multiplier + inst_dir_part[2][inst_rel_num * 6 + 5] = inst.submultiplier + + # Set our parts + self.parts = [sample_dir_part, inst_dir_part, brr_part] + + # For writing to project + def convert_to_files(self) -> List[Tuple[str, Union[Block, str]]]: + files_to_output: List[Tuple[str, Union[Block, str]]] = [] + # Build BRR use list + brr_use_list: Dict[Block, List[int]] = {} + brr_loop_addr: Dict[Block, int] = {} + brr_filenames: Dict[Block, str] = {} + for inst_num, inst in enumerate(self.instruments, self.base_instrument): + if not isinstance(inst.sample, Block): + continue + if inst.sample not in brr_use_list: + brr_use_list[inst.sample] = [] + brr_loop_addr[inst.sample] = inst.sample_loop_offset + elif brr_loop_addr[inst.sample] != inst.sample_loop_offset: + raise InvalidUserDataError("Sample is shared but with different loop points, cannot save to file.") + brr_use_list[inst.sample].append(inst_num) + # Add BRR files to output list + for brr, use_list in brr_use_list.items(): + # Prepend loop address to BRR + brr_with_loop = Block(len(brr)+2) + brr_with_loop.write_multi(0, brr_loop_addr[brr], 2) + brr_with_loop[2:] = brr + # Generate file name + brr_file_name = "sample-{}.brr".format("-".join(("{:02X}".format(x) for x in use_list))) + brr_filenames[brr] = brr_file_name + # Add to output files + files_to_output.append((brr_file_name, brr_with_loop)) + + # Create "config.txt" + config_lines: List[str] = [] + config_lines.append("Pack number: {:02X}".format(self.pack_num)) + base_instrument_str = "default" if self.base_instrument == 0x1A else "{:02X}".format(self.base_instrument) + config_lines.append("Base instrument: " + base_instrument_str) + brr_sample_dump_offset_str = "default" if self.brr_sample_dump_offset == 0x95B0 else "{:04X}".format(self.brr_sample_dump_offset) + config_lines.append("BRR sample dump offset: " + brr_sample_dump_offset_str) + config_lines.append("") + config_lines.append("#instruments") + config_lines.append("{") + for inst in self.instruments: + if isinstance(inst.sample, Block): + inst_sample = brr_filenames[inst.sample] + else: + inst_sample = '0x{:04X} 0x{:04X}'.format(inst.sample, inst.sample + inst.sample_loop_offset) + line_parts: List[str] = [] + line_parts.append(' "{}"'.format(inst_sample)) + line_parts.append(''.ljust(31-len(line_parts[0]))) + line_parts.append(' ${:02X}'.format(inst.adsr1)) + line_parts.append(' ${:02X}'.format(inst.adsr2)) + line_parts.append(' ${:02X}'.format(inst.gain)) + line_parts.append(' ${:02X}'.format(inst.multiplier)) + line_parts.append(' ${:02X}'.format(inst.submultiplier)) + config_lines.append(''.join(line_parts)) + config_lines.append("}\n") + config_txt = "\n".join(config_lines) + files_to_output.append((CONFIG_TXT_FILENAME, config_txt)) + return files_to_output + + # For loading from project + def load_from_files(self, file_loader): + # Check for config.txt + try: + config_txt = file_loader('config.txt', astext=True) + except FileNotFoundError: + raise InvalidUserDataError("config doesn't exist in instrument pack directory") + # Parse config.txt + pack_num, base_instrument, brr_sample_dump_offset, instruments, instrument_files = parse_config_txt(config_txt) + if pack_num != self.pack_num: + raise InvalidUserDataError("instrument pack {:02X}'s config.txt has pack number {:02X}".format(self.pack_num, pack_num)) + # Set our class's data + self.base_instrument = base_instrument + self.brr_sample_dump_offset = brr_sample_dump_offset + # Load BRR data + for inst, filename in zip(instruments, instrument_files): + try: + brr_raw_data: bytes = file_loader(filename, astext=False).read() + brr_data = Block() + brr_data.from_array(array('B', brr_raw_data)) + except FileNotFoundError: + raise InvalidUserDataError("instrument BRR '{}' doesn't exist in instrument pack directory".format(filename)) + inst.sample = brr_data[2:] + inst.sample_loop_offset = brr_data.read_multi(0, 2) + self.instruments = instruments + +def relocate_song_data(src: int, dst: int, data: Block) -> Block: + # Deep copy so we won't be modifying the original data + out = Block() + out.from_block(data) + # Let's parse a song :) + data_ptr = 0 + try: + # Helper functions + relocation_count = 0 + def consume_byte() -> int: + nonlocal data_ptr + val = out.read_multi(data_ptr, 1) + data_ptr += 1 + return val + def consume_word() -> int: + nonlocal data_ptr + val = out.read_multi(data_ptr, 2) + data_ptr += 2 + return val + def change_last_word(val=None): + nonlocal relocation_count + relocation_count += 1 + if val is None: + val = out.read_multi(data_ptr - 2, 2) + dst - src + out.write_multi(data_ptr - 2, val, 2) + def check_rel_ptr_in_bounds(rel_ptr: int, description: str): + if not 0 <= rel_ptr < len(data): + raise OutOfBoundsError("{} out of bounds (address={}), corrupted song".format(description, rel_ptr)) + + # Parse all phrases / "block list" + pattern_set = set() + phrases_done = set() + phrase_relocations = set() + while True: + phrases_done.add(data_ptr) + pattern_ptr = consume_word() + if pattern_ptr & 0xff00 == 0: + # Special meaning - not a phrase address + if pattern_ptr == 0: + # End of phrases + # - Stop processing + break + if pattern_ptr == 0x80: + # Debug: Fast forward on + # - Skip this, it's not a phrase address + continue + if pattern_ptr == 0x81: + # Debug: Fast forward off + # - Skip this, it's not a phrase address + pass + else: + # Loop (0x01-0x7f) / Jump (0x82-0xff) + # Note that we have to fix the loop address later + # Since we call change_last_word later, we need to store the + # address after the word to change. + phrase_relocations.add(data_ptr + 2) + rel_jump_ptr = consume_word() - src + check_rel_ptr_in_bounds(rel_jump_ptr, "Phrase loop address") + if rel_jump_ptr not in phrases_done: + # Take jump if we haven't yet + data_ptr = rel_jump_ptr + elif pattern_ptr >= 0x80: + # Don't fallthrough on unconditional jump - stop processing + break + else: + # Actually a phrase pointer - add to list + phrase_relocations.add(data_ptr) + rel_pattern_ptr = pattern_ptr - src + check_rel_ptr_in_bounds(rel_pattern_ptr, "Pattern address") + pattern_set.add(rel_pattern_ptr) + # Relocate phrase pointers + for phrase_ptr in phrase_relocations: + data_ptr = phrase_ptr + change_last_word() + + # Parse all patterns + track_list = [] + for pattern_table_addr in pattern_set: + for pattern_idx in range(8): + data_ptr = pattern_table_addr + pattern_idx * 2 + track_ptr = consume_word() + if track_ptr == 0: + continue + rel_track_ptr = track_ptr - src + check_rel_ptr_in_bounds(rel_track_ptr, "Track address") + if rel_track_ptr not in track_list: + track_list.append(rel_track_ptr) + change_last_word() + + # Parse all tracks + tracks_done = set() + cmds_done = set() + for track_ptr in track_list: + data_ptr = track_ptr + if track_ptr in tracks_done: + continue + tracks_done.add(track_ptr) + while True: + cmd = consume_byte() + if cmd == 0: + break + elif cmd < 0xe0: + # 1-byte note duration / parameter / note command + continue + elif cmd == 0xef: + # VCMD $EF: subroutine + # EF aa aa nn + # a = address of subroutine + # n = number of times to repeat + if data_ptr in cmds_done: + consume_word() + consume_byte() + continue + cmds_done.add(data_ptr) + # Deal with subroutine address + rel_sub_ptr = consume_word() - src + check_rel_ptr_in_bounds(rel_sub_ptr, "Subroutine address") + if rel_sub_ptr not in track_list: + track_list.append(rel_sub_ptr) + change_last_word() + # Deal with repeat count + consume_byte() # We don't care what it is, we just skip it + else: + # Other VCMD + # Skip the correct number of bytes + additional_byte_counts = [ + 1, 1, 2, 3, 0, 1, 2, 1, 2, 1, 1, 3, 0, 1, 2, 3, + 1, 3, 3, 0, 1, 3, 0, 3, 3, 3, 1, 2, 0, 0, 0, 0 + ] + for _ in range(additional_byte_counts[cmd-0xe0]): + consume_byte() + + # We should be done! :) + log.debug("Moved {}-byte song from {:04X} to {:04X}, {} relocations".format(len(data), src, dst, relocation_count)) + # Return the adjusted song + return out + except OutOfBoundsError as err: + raise InvalidUserDataError("Error at addr ${:04X}: {}".format(data_ptr + src, err)) + +@dataclass +class Song: + song_number: int + @classmethod + def from_yml_data(cls, song_num: int, yml_data: Dict[str, Union[int, str]]): + raise NotImplementedError('Please use a subclass of Song') + def get_song_packs(self) -> Tuple[int, int, int]: + raise NotImplementedError('Please use a subclass of Song') + def get_song_aram_address(self) -> int: + raise NotImplementedError('Please use a subclass of Song') + def to_yml_lines(self) -> List[str]: + raise NotImplementedError('Please use a subclass of Song') + +@dataclass +class SongWithData(Song): + instrument_pack_1: int + instrument_pack_2: int + + pack_number: int + data_address: int + data: Block + + song_path: str + + @classmethod + def from_yml_data(cls, song_num: int, yml_data: Dict[str, Union[int, str]]): + required_fields = (YML_SONG_PACK, YML_SONG_FILENAME) + for field in required_fields: + if field not in yml_data: + raise InvalidUserDataError("Expected field {} in YAML".format(field)) + song_pack = yml_data[YML_SONG_PACK] + if song_pack == YML_SONG_PACK_BUILTIN: + song_pack = 0xFF + return cls(song_num, None, None, + song_pack, None, None, + yml_data[YML_SONG_FILENAME]) + def get_song_packs(self) -> Tuple[int, int, int]: + return (self.instrument_pack_1, self.instrument_pack_2, self.pack_number) + def get_song_aram_address(self) -> int: + return self.data_address + def to_yml_lines(self) -> List[str]: + if self.is_always_loaded(): + song_pack_str = YML_SONG_PACK_BUILTIN + else: + song_pack_str = '0x{:02X}'.format(self.pack_number) + yml_lines = [ + '{}: {}'.format(YML_SONG_PACK, song_pack_str), + '{}: {}'.format(YML_SONG_FILENAME, self.song_path), + ] + return yml_lines + def is_always_loaded(self): + return self.pack_number == 0xFF + +@dataclass +class SongThatIsPartOfAnother(Song): + parent_song: Union[int, SongWithData] + offset: int + instrument_pack_1: Union[int, None] = None + instrument_pack_2: Union[int, None] = None + + @classmethod + def from_yml_data(cls, song_num: int, yml_data: Dict[str, Union[int, str]]): + required_fields = (YML_SONG_TO_REFERENCE, YML_SONG_OFFSET) + for field in required_fields: + if field not in yml_data: + raise InvalidUserDataError("Expected field {} in YAML") + # Allow for the instrument packs to be overridden + ip1, ip2 = None, None + if YML_INST_PACK_1 in yml_data: + ip1 = yml_data[YML_INST_PACK_1] + if YML_INST_PACK_2 in yml_data: + ip2 = yml_data[YML_INST_PACK_2] + return cls(song_num, yml_data[YML_SONG_TO_REFERENCE], yml_data[YML_SONG_OFFSET], ip1, ip2) + def get_song_packs(self) -> Tuple[int, int, int]: + packs = list(self.parent_song.get_song_packs()) + if self.instrument_pack_1 is not None: + packs[0] = self.instrument_pack_1 + if self.instrument_pack_2 is not None: + packs[1] = self.instrument_pack_2 + return tuple(packs) + def get_song_aram_address(self) -> int: + return self.parent_song.get_song_aram_address() + self.offset + def to_yml_lines(self) -> List[str]: + yml_lines = [ + '{}: 0x{:02X}'.format(YML_SONG_TO_REFERENCE, self.parent_song.song_number), + '{}: {}'.format(YML_SONG_OFFSET, self.offset), + ] + if self.instrument_pack_1 is not None: + yml_lines.append('{}: 0x{:02X}'.format(YML_INST_PACK_1, self.instrument_pack_1)) + if self.instrument_pack_2 is not None: + yml_lines.append('{}: 0x{:02X}'.format(YML_INST_PACK_2, self.instrument_pack_2)) + + return yml_lines + +def song_obj_from_yml(song_num: int, yml_data: Dict[str, Union[str, int]]) -> Song: + if YML_SONG_FILENAME in yml_data: + song_type = SongWithData + elif YML_SONG_TO_REFERENCE in yml_data: + song_type = SongThatIsPartOfAnother + else: + raise InvalidUserDataError("Unable to deduce song type from YAML data in songs.yml") + return song_type.from_yml_data(song_num, yml_data) + +class SongMusicPack(GenericMusicPack): + def __init__(self, pack_num: int) -> None: + super().__init__(pack_num) + self.songs: List[SongWithData] = [] + + def set_data_from_yaml(self, song_metadata: Dict[int, Dict]) -> None: + for song_num, yml_data in song_metadata.items(): + song_obj = song_obj_from_yml(song_num, yml_data) + self.songs.append(song_obj) + + def load_from_parts(self, parts: List[Tuple[int, int, Block]]) -> None: + self.songs = [] + for part_addr, _, part_data in parts: + always_loaded = part_addr < DYNAMIC_SONG_DATA_START + effective_song_pack = self.pack_num + if always_loaded and self.pack_num != 0x01: + raise InvalidUserDataError("Song is in-engine due to start address ${:04X} < $4800, but is in pack ${:02X} instead of $01".format( + part_addr, self.pack_num + )) + if always_loaded: + effective_song_pack = 0xFF + song = SongWithData(None, None, None, + effective_song_pack, part_addr, part_data, + None) + self.songs.append(song) + + def save_to_parts(self) -> None: + # Always-loaded songs are handled by the EngineMusicPack type + filtered_songs = [s for s in self.songs if isinstance(s, SongWithData) and not s.is_always_loaded()] + self.parts, song_output_ptr = songs_to_parts(DYNAMIC_SONG_DATA_START, filtered_songs) + # Ensure song data is in bounds + if song_output_ptr > DYNAMIC_SONG_DATA_END: + raise InvalidUserDataError("Data for song pack ${:02X} extends past data area. " + "Remove songs from the pack.".format(self.pack_num)) + + # For writing to project + def convert_to_files(self) -> List[Tuple[str, Union[Block, str]]]: + files_to_output: List[Tuple[str, Union[Block, str]]] = [] + + for song in self.songs: + if song.song_number is None: + log.error("Issue with song pack ${:02X} at address ${:04X}".format(self.pack_num, song.get_song_aram_address())) + raise CoilSnakeInternalError("Metadata is not set when converting songs to files") + ebm_path = "song-{:02X}.ebm".format(song.song_number) + song.song_path = ebm_path + + ### Write out EBM file + ebm_data = Block(len(song.data) + 4) + # Write length, addr + ebm_data.write_multi(0, len(song.data), 2) + ebm_data.write_multi(2, song.data_address, 2) + # Write data chunk + ebm_data[4:len(ebm_data)] = song.data + # Add to file list + files_to_output.append((ebm_path, ebm_data)) + + ### Write out EBM.yml file + yml_lines = [] + yml_lines.append("{}: 0x{:02X}\n".format(YML_INST_PACK_1, song.instrument_pack_1)) + yml_lines.append("{}: 0x{:02X}\n".format(YML_INST_PACK_2, song.instrument_pack_2)) + yml_str = ''.join(yml_lines) + files_to_output.append((ebm_path + '.yml', yml_str)) + + return files_to_output + + # For loading from project + def load_from_files(self, file_loader): + songs_with_data = {s.song_number: s for s in self.songs if isinstance(s, SongWithData)} + for song in self.songs: + if song.song_number in songs_with_data: + # Read YML + try: + ebm_yml_data = yml_load(file_loader(song.song_path+'.yml', astext=True)) + except FileNotFoundError: + raise InvalidUserDataError("'{}' missing".format(song.song_path+'.yml')) + required_fields = (YML_INST_PACK_1, YML_INST_PACK_2) + for field in required_fields: + if field not in ebm_yml_data: + raise InvalidUserDataError("Expected field {} in YAML".format(field)) + song.instrument_pack_1 = ebm_yml_data[YML_INST_PACK_1] + song.instrument_pack_2 = ebm_yml_data[YML_INST_PACK_2] + # Read EBM + try: + ebm_raw_data = file_loader(song.song_path, astext=False).read() + except FileNotFoundError: + raise InvalidUserDataError("'{}' missing".format(song.song_path)) + ebm_data = Block(len(ebm_raw_data)) + ebm_data.from_array(array('B',ebm_raw_data)) + song.data_address = ebm_data.read_multi(2, 2) + song.data = ebm_data[4:] + elif isinstance(song, SongThatIsPartOfAnother) and isinstance(song.parent_song, int): + try: + song.parent_song = songs_with_data[song.parent_song] + except KeyError: + raise CoilSnakeInternalError("Dependent song is not in same pack as parent song") + +class EngineMusicPack(SongMusicPack): + SONG_ADDRESS_TABLE_ADDR = 0x2E4A + ENGINE_FIXED_PARTS = {0x6E00: 'data-6E00.bin', 0x6F80: 'data-6F80.bin', 0x0500: 'engine.bin'} + # These values are for the part starting at $0500, containing the main SPC program + MAIN_PART_ADDR = 0x0500 + MAIN_PART_LEN = 0x2FDD - 0x0500 + MAIN_PART_LEN_WITH_SONGS = 0x418B + MAIN_PART_SONGS_CRC = 0x0C4F739B + MAIN_PART_SONG_LIST = [ + 0x2FDD, + 0x301C, + 0x31FA, + 0x342A, + 0x36AA, + 0x3A52, + 0x3B81, + 0x3C7B, + 0x3DA1, + 0x4064, + 0x41A8, + 0x4298, + 0x43FB, + 0x44FC, + 0x455D + ] + + def __init__(self, pack_num: int) -> None: + if pack_num != 1: + raise InvalidUserDataError('Engine pack must have pack number $01, has pack ${:02X}'.format(pack_num)) + super().__init__(pack_num) + self.engine_parts: Dict[int, Block] = {} + + def extract_in_engine_songs(self, part: Block): + cls = EngineMusicPack + start_bounds = [x - cls.MAIN_PART_ADDR for x in cls.MAIN_PART_SONG_LIST] + end_bounds = start_bounds[1:] + [cls.MAIN_PART_LEN_WITH_SONGS] + for song_start, song_end in zip(start_bounds, end_bounds): + song_block = part[song_start:song_end] + song = SongWithData(None, None, None, + 0xFF, song_start + cls.MAIN_PART_ADDR, song_block, + None) + self.songs.append(song) + log.info('Separated in-engine songs from main engine part') + + def load_from_parts(self, parts: List[Tuple[int, int, Block]]) -> None: + part_dict = {p[0]: p[2] for p in parts} + + # Ensure we have all the parts we need for the engine to function + for addr in EngineMusicPack.ENGINE_FIXED_PARTS: + if addr not in part_dict: + raise InvalidUserDataError("Expected part at address ${:04X} in pack $01".format(addr)) + self.engine_parts[addr] = part_dict[addr] + + # Split Gas Station into two separate parts (if applicable) + split_gas_station(parts) + + # Use SongMusicPack function to load the songs that are already in their own parts + super().load_from_parts(p for p in parts if p[0] not in self.engine_parts) + + # Get the main part and load the in-engine songs from it (if needed) + main_part = self.engine_parts[EngineMusicPack.MAIN_PART_ADDR] + main_part_songs_crc = main_part[EngineMusicPack.MAIN_PART_LEN:].crc32() + main_part_has_songs = ( + len(main_part) == EngineMusicPack.MAIN_PART_LEN_WITH_SONGS and + main_part_songs_crc == EngineMusicPack.MAIN_PART_SONGS_CRC + ) + log.debug('Engine main part CRC: ROM=%#x Clean=%#x', main_part_songs_crc, EngineMusicPack.MAIN_PART_SONGS_CRC) + if main_part_has_songs: + # Extract songs + self.extract_in_engine_songs(main_part) + # Truncate song data out of main part + main_part = main_part[:EngineMusicPack.MAIN_PART_LEN] + self.engine_parts[EngineMusicPack.MAIN_PART_ADDR] = main_part + + def save_to_parts(self) -> None: + # Start with the engine parts we loaded from the various .bin files + output_parts = [(addr, len(self.engine_parts[addr]), self.engine_parts[addr]) + for addr in EngineMusicPack.ENGINE_FIXED_PARTS] + + # Get in-engine / always-loaded song data + filtered_songs = (s for s in self.songs if isinstance(s, SongWithData) and s.is_always_loaded()) + # Get main engine part so we know where to put the always-loaded songs + main_part_block = self.engine_parts[EngineMusicPack.MAIN_PART_ADDR] + always_loaded_song_parts, song_output_ptr = songs_to_parts(EngineMusicPack.MAIN_PART_ADDR + len(main_part_block), filtered_songs) + # Ensure song data is in bounds + if song_output_ptr > DYNAMIC_SONG_DATA_START: + overage = song_output_ptr - DYNAMIC_SONG_DATA_START + raise InvalidUserDataError("Data for engine pack ${:02X} is too long by {} bytes. " + "Maybe your \"in-engine\" songs are too large.".format(self.pack_num, overage)) + # Have a helpful debug output for the user + log.debug("Engine pack has %d bytes of free space available.", DYNAMIC_SONG_DATA_START - song_output_ptr) + output_parts += always_loaded_song_parts + + # Get dynamically loaded song data that is in this pack (Gas Station 1 in vanilla) + super().save_to_parts() + output_parts += self.parts + + # Set self.parts + self.parts = output_parts + + def convert_to_files(self) -> List[Tuple[str, Union[Block, str]]]: + files = [] + + # Fixed engine parts + for addr, data in self.engine_parts.items(): + files.append((EngineMusicPack.ENGINE_FIXED_PARTS[addr], data)) + + # Song parts + files += super().convert_to_files() + + return files + + def load_from_files(self, file_loader): + for addr, name in EngineMusicPack.ENGINE_FIXED_PARTS.items(): + try: + with file_loader(name) as file: + data = Block() + data.from_array(array('B', file.read())) + self.engine_parts[addr] = data + except FileNotFoundError: + raise InvalidUserDataError("Pack $01 required file '{}' doesn't exist".format( + name + )) + # Load song data + super().load_from_files(file_loader) + + def get_song_address_table_data(self, size: int) -> Block: + if self.parts: + return self.get_aram_region(EngineMusicPack.SONG_ADDRESS_TABLE_ADDR, size) + block = self.engine_parts[EngineMusicPack.MAIN_PART_ADDR] + start_addr = EngineMusicPack.SONG_ADDRESS_TABLE_ADDR - EngineMusicPack.MAIN_PART_ADDR + return block[start_addr:start_addr + size] + + def set_song_address_table_data(self, block: Block) -> None: + assert self.parts + self.set_aram_region(EngineMusicPack.SONG_ADDRESS_TABLE_ADDR, block.size, block) + +def check_if_song_is_part_of_another(song_num: int, song_pack: SongMusicPack, song_addr: int) -> Union[None, SongThatIsPartOfAnother]: + for song in song_pack.songs: + if song.data_address <= song_addr < song.data_address + len(song.data): + return SongThatIsPartOfAnother(song_num, song, song_addr - song.data_address) + return None + +def create_pack_object_from_parts(pack_num: int, parts: List[Tuple[int, int, Block]]) -> GenericMusicPack: + formats = [ + ("instrument", InstrumentMusicPack), + ("engine", EngineMusicPack), + ("song", SongMusicPack), + ] + fmt_fail_msg = {} + for fmt_name, fmt_cls in formats: + try: + pack = fmt_cls(pack_num) + pack.load_from_parts(parts) + except InvalidUserDataError as err: + fmt_fail_msg[fmt_name] = err.message + else: + return pack + fmt_list = ", ".join((fmt[0] for fmt in formats)) + for fmt_name, msg in fmt_fail_msg.items(): + log.debug("Unable to process pack ${:02X} as {} pack, encountered error: \"{}\"".format(pack_num, fmt_name, msg)) + raise InvalidUserDataError("Invalid pack format, must be one of the following: {}".format(fmt_list)) + + +def split_gas_station(parts: List[Tuple[int, int, Block]]) -> None: + # pylint: disable=C0103 + START_ADDR = 0x4800 + COMBINED_SIZE = 0x405 + PT_2_OFFSET = 0x23D + PT_2_CRC = 0xF5E81DDE + PT_2_SIZE = COMBINED_SIZE - PT_2_OFFSET + # pylint: enable=C0103 + # Look for gas station part + for p_idx, part in enumerate(parts): + p_addr, p_size, p_block = part + if not (p_addr == START_ADDR and p_size == COMBINED_SIZE): + # Haven't found gas station part yet - keep looking. + continue + + pt2_crc_rom = p_block[PT_2_OFFSET:].crc32() + log.debug('Gas Station Pt. 2 CRC: ROM=%#x Clean=%#x', pt2_crc_rom, PT_2_CRC) + if pt2_crc_rom != PT_2_CRC: + # We've found the gas station part, but it doesn't have the data we expected. + # We're done here. + log.info("Found Gas Station part - not splitting because it has already been " + "modified.") + return + # This looks like it is the gas station part. Split it in two. + # Remove old combined part from the list. + del parts[p_idx] + # Add the two parts to the list. + parts.append((p_addr, PT_2_OFFSET, p_block[:PT_2_OFFSET])) + parts.append((p_addr + PT_2_OFFSET, PT_2_SIZE, p_block[PT_2_OFFSET:])) + # We're done here. + log.info("Found Gas Station part - split into two separate tracks.") + return + log.info("Did not find Gas Station part - unable to split.") + +def songs_to_parts(start_addr: int, songs: List[SongWithData]): + parts: List[Tuple[int, int, Block]] = [] + song_output_ptr = start_addr + for song in sorted(songs, key=lambda s: s.data_address): + if song.data_address != song_output_ptr: + # Relocate song + log.debug("Relocating song $%02X to address $%04X", song.song_number, song_output_ptr) + new_data = relocate_song_data(song.data_address, song_output_ptr, song.data) + song.data = new_data + size = len(song.data) + song.data_address = song_output_ptr + parts.append((song.data_address, size, song.data)) + song_output_ptr += size + return parts, song_output_ptr diff --git a/coilsnake/model/eb/table.py b/coilsnake/model/eb/table.py index 6f208a13..d6f63918 100644 --- a/coilsnake/model/eb/table.py +++ b/coilsnake/model/eb/table.py @@ -46,6 +46,49 @@ def from_yml_rep(cls, yml_rep): def to_yml_rep(cls, value): return "${:x}".format(value) +class EbHiLoMidPointerTableEntry(LittleEndianIntegerTableEntry): + @staticmethod + def create(size): + return type("EbHiLoMidPointerTableEntry_subclass", + (EbHiLoMidPointerTableEntry,), + {"size": size}) + + @classmethod + def from_block(cls, block, offset): + bank = block[offset] + addr = block.read_multi(offset + 1, 2) + return addr | bank << 16 + + @classmethod + def to_block(cls, block, offset, value): + block[offset] = (value & 0xff0000) >> 16 + block.write_multi(offset + 1, value & 0x00ffff, 2) + + @classmethod + def from_yml_rep(cls, yml_rep): + if not isinstance(yml_rep, str): + raise TableEntryInvalidYmlRepresentationError("Could not parse value[{}] as pointer".format(yml_rep)) + elif not yml_rep: + raise TableEntryInvalidYmlRepresentationError("Could not parse empty string as pointer") + elif yml_rep[0] == "$": + try: + value = int(yml_rep[1:], 16) + except ValueError: + raise TableEntryInvalidYmlRepresentationError("Could not parse value[{}] as pointer".format(yml_rep)) + + return super(EbHiLoMidPointerTableEntry, cls).from_yml_rep(value) + else: + try: + value = EbPointer.label_address_map[yml_rep] + except KeyError: + raise TableEntryInvalidYmlRepresentationError("Unknown pointer label[{}]".format(yml_rep)) + + return super(EbHiLoMidPointerTableEntry, cls).from_yml_rep(value) + + @classmethod + def to_yml_rep(cls, value): + return "${:x}".format(value) + class EbPaletteTableEntry(TableEntry): @classmethod @@ -209,6 +252,7 @@ class EbRowTableEntry(GenericLittleEndianRowTableEntry): TABLE_ENTRY_CLASS_MAP = dict( GenericLittleEndianRowTableEntry.TABLE_ENTRY_CLASS_MAP, **{"pointer": (EbPointerTableEntry, ["name", "size"]), + "hilomid pointer": (EbHiLoMidPointerTableEntry, ["name", "size"]), "palette": (EbPaletteTableEntry, ["name", "size"]), "standardtext": (EbStandardTextTableEntry, ["name", "size"]), "standardtext null-terminated": (EbStandardNullTerminatedTextTableEntry, ["name", "size"])}) diff --git a/coilsnake/modules/eb/MusicModule.py b/coilsnake/modules/eb/MusicModule.py new file mode 100644 index 00000000..5c416edb --- /dev/null +++ b/coilsnake/modules/eb/MusicModule.py @@ -0,0 +1,303 @@ +import logging +from typing import Dict, List + +from coilsnake.exceptions.common.exceptions import InvalidUserDataError +from coilsnake.model.common.ips import IpsPatch +from coilsnake.model.common.table import LittleEndianHexIntegerTableEntry, Table +from coilsnake.model.common.blocks import Block +import coilsnake.model.eb.musicpack as mp +from coilsnake.model.eb.table import eb_table_from_offset +from coilsnake.modules.common.PatchModule import get_ips_filename +from coilsnake.modules.eb.EbModule import EbModule +from coilsnake.util.common.yml import yml_load +from coilsnake.util.eb.pointer import from_snes_address, to_snes_address + +log = logging.getLogger(__name__) + +class MusicModule(EbModule): + NAME = "Music" + FREE_RANGES = [ # Every single vanilla pack location + (0x0BE02A,0x0BFFFF), + (0x0CF617,0x0CFFFF), + (0x0EF8C6,0x0EFFFF), + (0x0FF2B5,0x0FFFFF), + (0x10DFB4,0x10FFFF), + (0x18F6B7,0x18FFFF), + (0x19FC18,0x19FFFF), + (0x1AFB07,0x1AFFFF), + (0x1BF2EB,0x1BFFFF), + (0x1CE037,0x1CFFFF), + (0x1DFECE,0x1DFFFF), + (0x1EFCDD,0x1EFFFF), + (0x1FEC46,0x1FFFFF), + (0x20ED03,0x20FFFF), + (0x21F581,0x21FFFF), + (0x220000,0x22FFFF), + (0x230000,0x23FFFF), + (0x240000,0x24FFFF), + (0x250000,0x25FFFF), + (0x260000,0x26FFFF), + (0x270000,0x27FFFF), + (0x280000,0x28FFFF), + (0x290000,0x29FFFF), + (0x2A0000,0x2AFFFF), + (0x2B0000,0x2BFFFF), + (0x2C0000,0x2CFFFF), + (0x2D0000,0x2DFFFF), + ] + SONG_PACK_TABLE_ROM_ADDR = 0xC4F70A + PACK_POINTER_TABLE_ROM_ADDR = 0xC4F947 + + SONG_ADDRESS_TABLE_SCHEMA = LittleEndianHexIntegerTableEntry.create("Song Address", 2) + + MUSIC_PACK_PATH_FORMAT_STRING = 'Music/Packs/{:02X}/' + + def __init__(self): + super(EbModule, self).__init__() + self.song_pack_table = eb_table_from_offset(MusicModule.SONG_PACK_TABLE_ROM_ADDR) + self.pack_pointer_table = eb_table_from_offset(MusicModule.PACK_POINTER_TABLE_ROM_ADDR) + self.song_address_table = Table(MusicModule.SONG_ADDRESS_TABLE_SCHEMA, name="Song Address Table", num_rows=self.song_pack_table.num_rows) + self.packs: List[mp.GenericMusicPack] = [] + self.songs: Dict[int, mp.Song] = {} + + def read_single_pack(self, rom: Block, pack_num: int) -> mp.GenericMusicPack: + pack_ptr = from_snes_address(self.pack_pointer_table[pack_num][0]) + pack_parts = mp.extract_pack_parts(rom, pack_ptr) + pack = mp.create_pack_object_from_parts(pack_num, pack_parts) + return pack + + def read_from_rom(self, rom): + self.song_pack_table.from_block(rom, from_snes_address(MusicModule.SONG_PACK_TABLE_ROM_ADDR)) + self.pack_pointer_table.from_block(rom, from_snes_address(MusicModule.PACK_POINTER_TABLE_ROM_ADDR)) + # Read out all packs + self.packs = [] + for pack_num in range(self.pack_pointer_table.num_rows): + pack = self.read_single_pack(rom, pack_num) + self.packs.append(pack) + # Read song address table from engine + if not isinstance(self.packs[1], mp.EngineMusicPack): + raise InvalidUserDataError("Expected pack 1 to be the music engine pack, instead it is of type {}".format( + type(self.packs[1]).__name__)) + song_address_table_data = self.packs[1].get_song_address_table_data(self.song_address_table.size) + self.song_address_table.from_block(song_address_table_data, 0) + # Update song metadata + for song_table_ind in range(self.song_pack_table.num_rows): + song_num = song_table_ind + 1 + inst_pack_1, inst_pack_2, song_pack_num = tuple(self.song_pack_table[song_table_ind]) + song_addr = self.song_address_table[song_table_ind] + storage_song_pack_num = 0x01 if song_pack_num == 0xFF else song_pack_num + pack_obj_of_song = self.packs[storage_song_pack_num] + if not isinstance(pack_obj_of_song, mp.SongMusicPack): + raise InvalidUserDataError("Invalid type {} of song pack ${:02X} when processing song ${:02X}, expected SongMusicPack".format(type(pack_obj_of_song).__name__, song_pack_num, song_num)) + matching_songs_in_pack = [x for x in pack_obj_of_song.songs if x.data_address == song_addr] + if len(matching_songs_in_pack) < 1: + log.debug('Unable to find pack part with address ${:04X}'.format(song_addr)) + log.debug('Song addresses in pack: ' + ', '.join('${:04X}'.format(song.data_address) for song in pack_obj_of_song.songs)) + song_obj = mp.check_if_song_is_part_of_another(song_num, pack_obj_of_song, song_addr) + if song_obj is None: + raise InvalidUserDataError("Song pack ${:02X} missing song at address ${:04X} when processing song ${:02X}".format(song_pack_num, song_addr, song_num)) + else: + if len(matching_songs_in_pack) > 1: + raise InvalidUserDataError("Song pack ${:02X} has multiple songs at address ${:04X} when processing song ${:02X}".format(song_pack_num, song_addr, song_num)) + # We've found the song part in the pack. Use that. + song_obj = matching_songs_in_pack[0] + # Handle multiple songs pointing to the same data + if song_obj.song_number: + log.debug("Song pack ${:02X} has multiple references in the song table to address ${:04X} - references include songs ${:02X}, ${:02X}".format(song_pack_num, song_addr, song_obj.song_number, song_num)) + song_obj = mp.SongThatIsPartOfAnother(song_num, song_obj, 0) + else: + # Update data in song object + song_obj.song_number = song_num + # Set the song object's packs now - for SongThatIsPartOfAnother, we will clear it to None later if it's redundant + song_obj.instrument_pack_1 = inst_pack_1 + song_obj.instrument_pack_2 = inst_pack_2 + # Store song object into our lookup table + self.songs[song_num] = song_obj + # Swap any referenced songs if applicable (so the .ebm.yml has the most complete instrument list) + for song_num, song_obj in self.songs.items(): + # Only swap songs that use the exact same song data + if not (isinstance(song_obj, mp.SongThatIsPartOfAnother) and song_obj.offset == 0): + continue + # Get parent info and song number + parent_obj = song_obj.parent_song + assert isinstance(parent_obj, mp.SongWithData), "Internal coding error in music module" + parent_num = parent_obj.song_number + # Get song packs + song_packs = song_obj.get_song_packs() + parent_packs = song_obj.parent_song.get_song_packs() + # Check if there are unspecified packs on the parent and not on the current song + should_swap = any(p == 0xFF for p in parent_packs[:2]) and all(p != 0xFF for p in song_packs[:2]) + # If so, then we should swap the two songs + if not should_swap: + continue + # We are going to keep the same two objects, but swap the song number and packs inside (and also in self.songs) + log.debug('Swapping song $%02X to be the dependent of $%02X', parent_num, song_num) + # Swap song numbers and packs using getattr/setattr trickery + for attr in ('song_number', 'instrument_pack_1', 'instrument_pack_2'): + sa, pa = getattr(song_obj, attr), getattr(parent_obj, attr) + setattr(song_obj, attr, pa) + setattr(parent_obj, attr, sa) + # Update references to song objects in self.songs + self.songs[song_num], self.songs[parent_num] = parent_obj, song_obj + # For all referenced songs, clear the instrument pack if it's redundant. + for song_obj in (obj for obj in self.songs.values() if isinstance(obj, mp.SongThatIsPartOfAnother)): + song_packs = song_obj.get_song_packs() + parent_packs = song_obj.parent_song.get_song_packs() + if song_packs[0] == parent_packs[0]: + song_obj.instrument_pack_1 = None + if song_packs[1] == parent_packs[1]: + song_obj.instrument_pack_2 = None + + def write_to_project(self, resourceOpener): + # Write out each pack to its folder. + for pack in self.packs: + # Each pack knows how to turn itself into files. + # Write those files into the pack folder. + pack_base_fname = self.MUSIC_PACK_PATH_FORMAT_STRING.format(pack.pack_num) + for fname, data in pack.convert_to_files(): + fname_spl = fname.split('.') + assert len(fname_spl) > 1, "Internal error in music module, can't write file '{}'".format(fname) + fname_no_ext = '.'.join(fname_spl[:-1]) + is_text = isinstance(data,str) + with resourceOpener(pack_base_fname+fname_no_ext,fname_spl[-1],is_text) as f: + if is_text: + f.write(data) + else: + f.write(data.data) + # Create and write songs.yml + yml_song_lines = [] + for song_num in sorted(self.songs.keys()): + yml_song_lines.append("0x{:02X}:\n".format(song_num)) + yml_song_lines += (' {}\n'.format(line) for line in self.songs[song_num].to_yml_lines()) + with resourceOpener('Music/songs','yml',True) as f: + f.writelines(yml_song_lines) + + def read_from_project(self, resourceOpener): + self.packs = [] + self.songs = {} + # Read songs.yml + songs_yml_data = yml_load(resourceOpener('Music/songs','yml',True)) + # Look up with song pack, then song number. Result is data from YAML. + pack_song_metadata: Dict[int, Dict[int, Dict]] = {} + for song_num, song_yml in songs_yml_data.items(): + if mp.YML_SONG_PACK in song_yml: + pack = song_yml[mp.YML_SONG_PACK] + if pack == mp.YML_SONG_PACK_BUILTIN: + pack = 0xFF + else: + # This must be a dependent song + parent_song = song_yml[mp.YML_SONG_TO_REFERENCE] + pack = songs_yml_data[parent_song][mp.YML_SONG_PACK] + # Do an extra check to make in-engine songs work + storage_song_pack_num = 0x01 if pack == 0xFF else pack + d = pack_song_metadata.get(storage_song_pack_num, dict()) + d[song_num] = song_yml + pack_song_metadata[storage_song_pack_num] = d + # For each pack: + for pack_num in range(self.pack_pointer_table.num_rows): + pack_base_path = self.MUSIC_PACK_PATH_FORMAT_STRING.format(pack_num) + def file_loader(fname, astext=False): + fname_spl = fname.split('.') + assert len(fname_spl) > 1, "Internal error in music module, can't read file '{}'".format(fname) + fname_no_ext = '.'.join(fname_spl[:-1]) + return resourceOpener(pack_base_path + '/' + fname_no_ext, fname_spl[-1], astext=astext) + eligible_types = [] + # Check for engine.bin + try: + with file_loader('engine.bin'): + eligible_types.append(mp.EngineMusicPack) + except FileNotFoundError: + pass + # Check for config.txt + try: + with file_loader('config.txt'): + eligible_types.append(mp.InstrumentMusicPack) + except FileNotFoundError: + pass + # Check if there are songs in this pack + if pack_num in pack_song_metadata: + eligible_types.append(mp.SongMusicPack) + # Ensure we don't have incompatible pack types + if mp.InstrumentMusicPack in eligible_types and len(eligible_types) > 1: + explanation = None + if mp.SongMusicPack in eligible_types: + explanation = "references in 'songs.yml'" + elif mp.EngineMusicPack in eligible_types: + explanation = "engine.bin" + assert explanation + raise InvalidUserDataError("pack {:02X} has both config.txt and {} - please remove one of these".format( + pack_num, explanation)) + # Try out the pack type - this will be in the order that we check them above. + # We will try Engine, then Instrument, then Song pack type in that order. + if eligible_types: + pack_type = eligible_types[0] + try: + pack_obj = pack_type(pack_num) + if isinstance(pack_obj, mp.SongMusicPack): + pack_obj.set_data_from_yaml(pack_song_metadata[pack_num]) + pack_obj.load_from_files(file_loader) + if isinstance(pack_obj, mp.SongMusicPack): + for song in pack_obj.songs: + self.songs[song.song_number] = song + except InvalidUserDataError as e: + raise InvalidUserDataError("Error reading pack ${:02X} as {}: {}".format(pack_num, pack_type.__name__, e.message)) + else: + log.warn("Music pack ${:02X} contains no data.".format(pack_num)) + pack_obj = mp.EmptyPack(pack_num) + self.packs.append(pack_obj) + return + + def write_to_rom(self, rom): + # Apply Gas Station instrument pack patch + # In the vanilla game, there is a bug in the INITIALIZE_SPC700 function where it loads the + # engine pack by looking at the sequence pack of song 00, but then stores the loaded pack + # value into the secondary instrument pack variable. This means when we play the + # Gas Station song, it will load the whole engine again, which includes some instrument + # data. This instrument data will be loaded after the instrument packs, so this can result + # in some corrupted instruments when changing the instrument packs used by Gas Station. + self.get_patch(rom).apply(rom) + # Prepare packs for writing out by converting them to parts + for pack in self.packs: + pack.save_to_parts() + # Build song address table and song pack table + for song_num, song in self.songs.items(): + song_ind = song_num - 1 + self.song_address_table[song_ind] = song.get_song_aram_address() + self.song_pack_table[song_ind] = song.get_song_packs() + # Write song address table into engine pack + if not isinstance(self.packs[1], mp.EngineMusicPack): + raise InvalidUserDataError("Expected pack 1 to be the music engine pack, instead it is of type {}".format( + type(self.packs[1]).__name__)) + song_address_table_data = Block(self.song_address_table.size) + self.song_address_table.to_block(song_address_table_data, 0) + self.packs[0x01].set_song_address_table_data(song_address_table_data) + # Write out packs + for i, pack in enumerate(self.packs): + data = pack.get_pack_binary_data() + pack_offset = rom.allocate(data=data) + self.pack_pointer_table[i] = [to_snes_address(pack_offset)] + # Write out pack pointer table + self.pack_pointer_table.to_block(block=rom, offset=from_snes_address(self.PACK_POINTER_TABLE_ROM_ADDR)) + # Build song pack table + self.song_pack_table.to_block(block=rom, offset=from_snes_address(self.SONG_PACK_TABLE_ROM_ADDR)) + return + + def get_patch(self, rom): + ips = IpsPatch() + ips.load(get_ips_filename(rom.type, 'gas_station_pack_fix'), 0) + return ips + + def upgrade_project(self, old_version, new_version, rom, resource_open_r, resource_open_w, resource_delete): + if old_version == new_version: + return + if old_version == 11: + # Upgrade 11 to 12 + self.read_from_rom(rom) + self.write_to_project(resource_open_w) + self.upgrade_project( + 11 if old_version < 11 else old_version + 1, + new_version, + rom, + resource_open_r, + resource_open_w, + resource_delete) diff --git a/coilsnake/ui/information.py b/coilsnake/ui/information.py index e7973a21..f0de3670 100644 --- a/coilsnake/ui/information.py +++ b/coilsnake/ui/information.py @@ -1,14 +1,14 @@ from coilsnake.util.common import project VERSION = project.VERSION_NAMES[project.FORMAT_VERSION] -RELEASE_DATE = "March 8, 2021" +RELEASE_DATE = "March 19, 2023" WEBSITE = "http://pk-hack.github.io/CoilSnake" AUTHOR = "the PK Hack community" ADDITIONAL_CREDITS = """- Some portions based on JHack, created by AnyoneEB - Contributions by H.S, Michael1, John Soklaski, João Silva, ShadowOne333, stochaztic, Catador, - and many others""" + cooprocks123e, and many others""" DEPENDENCIES = [ {"name": "CoilSnake logo and icon", "author": "Rydel"}, diff --git a/coilsnake/util/common/project.py b/coilsnake/util/common/project.py index 89d7f2f2..6ad0acc9 100644 --- a/coilsnake/util/common/project.py +++ b/coilsnake/util/common/project.py @@ -12,7 +12,7 @@ # format. Version numbers are necessary because the format of data files may # change between versions of CoilSnake. -FORMAT_VERSION = 11 +FORMAT_VERSION = 12 # Names for each version, corresponding the the CS version VERSION_NAMES = { @@ -26,7 +26,8 @@ 8: "2.3.1", 9: "3.33", 10: "4.0", - 11: "4.1" + 11: "4.1", + 12: "4.2", } # The default project filename diff --git a/setup.py b/setup.py index aea34357..5e5ff532 100755 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ "Pillow>=3.0.0", "PyYAML>=3.11", "CCScriptWriter>=1.2", - "ccscript>=1.339" + "ccscript>=1.500" ] if platform.system() == "Darwin": @@ -21,7 +21,7 @@ setup( name="coilsnake", - version="4.1", + version="4.2", description="CoilSnake", url="https://pk-hack.github.io/CoilSnake", packages=find_packages(), @@ -30,7 +30,7 @@ install_requires=install_requires, dependency_links=[ "https://github.com/Lyrositor/CCScriptWriter/tarball/master#egg=CCScriptWriter-1.2", - "https://github.com/stochaztic/ccscript_legacy/tarball/master#egg=ccscript-1.339" + "https://github.com/charasyn/ccscript_legacy/archive/refs/tags/v1.500.tar.gz#egg=ccscript-1.500" ], ext_modules=[ Extension(