Wiki exporter fixes & Changed cache path_or_ggpk to path_or_file_system

PyPoE/cli/exporter/dat/handler.py

@@ -39,8 +39,8 @@
 from PyPoE.poe.file import dat
 from PyPoE.cli.core import console, Msg
 from PyPoE.cli.exporter import config
-from PyPoE.cli.exporter.util import get_content_ggpk, get_content_ggpk_path
-from PyPoE.poe.file.bundle import Index, Bundle
+from PyPoE.cli.exporter.util import get_content_path
+from PyPoE.poe.file.file_system import FileSystem
 
 # =============================================================================
 # Globals
@@ -106,21 +106,15 @@ def handle(self, args):
         args.spec = spec
 
     def _read_dat_files(self, args, prefix=''):
-        path = get_content_ggpk_path()
+        path = get_content_path()
 
-        console(prefix + 'Reading "%s"...' % path)
+        console(prefix + 'Loading file system...')
 
-        ggpk = get_content_ggpk(path)
-
-        index = Index()
-        print ("Extracting index record...")
-        extracted_record = ggpk[Index.PATH].record.extract()
-        index.read(extracted_record)
+        file_system = FileSystem(root_path=path)
 
         console(prefix + 'Reading .dat files')
 
         dat_files = {}
-        ggpk_data = index.get_dir_record('Data')
         lang = args.language or config.get_option('language')
         dir_path = "Data/"
         if lang != 'English':
@@ -130,17 +124,15 @@ def _read_dat_files(self, args, prefix=''):
         for name in tqdm(args.files):
             file_path = dir_path + name
             try:
-                node = index.get_file_record(file_path)
+                data = file_system.get_file(file_path)
             except FileNotFoundError:
                 console('Skipping "%s" (missing)' % file_path, msg=Msg.warning)
                 remove.append(name)
                 continue
 
             df = dat.DatFile(name)
-
-            print("Reading %s" % file_path)
-            node.bundle.read(ggpk[node.bundle.ggpk_path].record.extract())#contents.decompress()
-            df.read(file_path_or_raw=node.get_file(), use_dat_value=False)
+
+            df.read(file_path_or_raw=data, use_dat_value=False)
 
             dat_files[name] = df
 

PyPoE/cli/exporter/util.py

@@ -44,9 +44,8 @@
 # =============================================================================
 
 __all__ = [
-    'get_content_ggpk_path',
-    'get_content_ggpk_hash',
-    'get_content_ggpk',
+    'get_content_path',
+    'get_content_hash',
     'check_hash',
 ]
 
@@ -55,7 +54,7 @@
 # =============================================================================
 
 
-def get_content_ggpk_path():
+def get_content_path():
     """
     Returns the path to the current content.ggpk based on the specified
     config variables for the version & distributor.
@@ -73,46 +72,26 @@ def get_content_ggpk_path():
         if not paths:
             raise SetupError('No PoE Installation found.')
 
-        return os.path.join(paths[0], 'content.ggpk')
+        return paths[0]
     else:
         return path
 
 
-def get_content_ggpk_hash():
+def get_content_hash():
     """
     Gets the content ggpk based on the stored config variables and returns
     the calculated hash.
 
     :return: Hash of content.ggpk
     :rtype: str
     """
-    ggpk = get_content_ggpk_path()
+    ggpk = get_content_path()
     with open(ggpk, 'rb') as f:
         data = f.read(2**16)
 
     return hashlib.md5(data).hexdigest()
 
 
-def get_content_ggpk(path=None):
-    """
-    Gets the GGPKFile instance based on the stored config variables.
-
-    :param path: path to use if not None, otherwise determine path automatically
-    :type path: str or None
-
-    :return: Parsed GGPKFile instance
-    :rtype: GGPKFile()
-    """
-    if path is None:
-        path = get_content_ggpk_path()
-
-    ggpk = GGPKFile()
-    ggpk.read(path)
-    ggpk.directory_build()
-
-    return ggpk
-
-
 def check_hash():
     """
     Checks the stored hash against the current hash and returns the result
@@ -124,7 +103,7 @@ def check_hash():
         return True
 
     hash_old = config.get_setup_variable('temp_dir', 'hash')
-    hash_new = get_content_ggpk_hash()
+    hash_new = get_content_hash()
 
     if hash_old == hash_new:
         return True

PyPoE/cli/exporter/wiki/core.py

@@ -30,14 +30,11 @@
 # =============================================================================
 
 # Python
-import os
 
 # self
-from PyPoE.poe.file.ggpk import GGPKFile
 from PyPoE.cli.core import console, Msg
 from PyPoE.cli.handler import BaseHandler
 from PyPoE.cli.exporter import config
-from PyPoE.cli.exporter.util import get_content_ggpk_hash, get_content_ggpk_path
 from PyPoE.cli.exporter.wiki.parsers import WIKI_HANDLERS
 from PyPoE.cli.exporter.wiki.admin import ADMIN_HANDLERS
 
@@ -85,24 +82,5 @@ def _setup(self, args):
         :param args: argparse args passed on
         :return:
         """
-        temp_dir = config.get_option('temp_dir', safe=False)
-
-        content_ggpk = get_content_ggpk_path()
-
-        console('Reading "%s"...' % content_ggpk)
-        ggpk = GGPKFile()
-        ggpk.read(content_ggpk)
-
-        console('Building directory...')
-        ggpk.directory_build()
-
-        console('Extracting data files to "%s"...' % temp_dir)
-        ggpk['Data'].extract_to(temp_dir)
-        ggpk['Metadata'].extract_to(temp_dir)
-
-        console('Hashing...')
-
-        config.set_setup_variable('temp_dir', 'hash', get_content_ggpk_hash())
-
         console('Done.')
 

PyPoE/cli/exporter/wiki/parser.py

@@ -62,7 +62,7 @@
 # self
 from PyPoE.cli.core import console, Msg
 from PyPoE.cli.exporter import config
-from PyPoE.cli.exporter.util import get_content_ggpk_path
+from PyPoE.cli.exporter.util import get_content_path
 from PyPoE.poe.constants import MOD_DOMAIN, WORDLISTS, MOD_STATS_RANGE
 from PyPoE.poe.text import parse_description_tags
 from PyPoE.poe.file.dat import RelationalReader, set_default_spec
@@ -72,7 +72,7 @@
     get_custom_translation_file,
     install_data_dependant_quantifiers,
 )
-from PyPoE.poe.file.ggpk import GGPKFile, extract_dds
+from PyPoE.poe.file.file_system import FileSystem
 from PyPoE.poe.file.ot import OTFileCache
 from PyPoE.poe.sim.mods import get_translation_file_from_domain
 
@@ -1455,6 +1455,7 @@ def __init__(self, base_path, parsed_args):
         set_default_spec(version=config.get_option('version'))
 
         self.base_path = base_path
+        self.file_system = FileSystem(root_path=get_content_path())
 
         opt = {
             'use_dat_value': False,
@@ -1464,30 +1465,26 @@ def __init__(self, base_path, parsed_args):
         # Load rr and translations which will be undoubtedly be needed for
         # parsing
         self.rr = RelationalReader(
-            path_or_ggpk=base_path,
+            path_or_file_system=self.file_system,
             files=self._files,
             read_options=opt,
             raise_error_on_missing_relation=False,
             language=config.get_option('language'),
-            load_index=False,
         )
         install_data_dependant_quantifiers(self.rr)
         self.tc = TranslationFileCache(
-            path_or_ggpk=base_path,
-            load_index=False,
+            path_or_file_system=self.file_system,
             **self._TC_KWARGS
         )
         for file_name in self._translations:
             self.tc[file_name]
 
         self.ot = OTFileCache(
-            path_or_ggpk=base_path,
-            load_index=False,
+            path_or_file_system=self.file_system,
         )
 
         self.custom = get_custom_translation_file()
 
-        self.ggpk = None
         self._img_path = None
         self.lang = config.get_option('language')
 
@@ -1538,10 +1535,7 @@ def _format_detailed(self, custom, ingame):
 
     def _write_dds(self, data, out_path, parsed_args):
         with open(out_path, 'wb') as f:
-            f.write(extract_dds(
-                data,
-                path_or_ggpk=self.ggpk,
-            ))
+            f.write(self.file_system.extract_dds(data))
 
             console('Wrote "%s"' % out_path)
 
@@ -1555,21 +1549,8 @@ def _write_dds(self, data, out_path, parsed_args):
 
         console('Converted "%s" to png' % out_path)
 
-    def _load_ggpk(self):
-        if self.ggpk is None:
-            self.ggpk = GGPKFile()
-            self.ggpk.read(get_content_ggpk_path())
-            self.ggpk.directory_build()
-            console('content.ggpk has been loaded.')
-
     def _image_init(self, parsed_args):
         if parsed_args.store_images:
-            console(
-                'Images are flagged for extraction. Loading content.ggpk '
-                '...'
-            )
-            self._load_ggpk()
-
             self._img_path = os.path.join(self.base_path, 'img')
             if not os.path.exists(self._img_path):
                 os.makedirs(self._img_path)

PyPoE/cli/exporter/wiki/parsers/incursion.py

@@ -48,7 +48,6 @@
 from PyPoE.cli.exporter import config
 from PyPoE.cli.exporter.wiki import parser
 from PyPoE.cli.exporter.wiki.handler import ExporterHandler, ExporterResult
-from PyPoE.poe.file.ggpk import extract_dds
 from PyPoE.poe.file.idl import IDLFile
 
 # =============================================================================
@@ -192,8 +191,9 @@ def export(self, parsed_args, incursion_rooms):
         idl_sources = set()
         if parsed_args.store_images:
             idl = IDLFile()
-            idl.read(file_path_or_raw=
-                     self.ggpk['Art/UIImages1.txt'].record.extract())
+            idl.read(
+                file_path_or_raw=self.file_system.get_file('Art/UIImages1.txt')
+            )
             idl_lookup = idl.as_dict()
 
         console('Parsing data into templates...')
@@ -249,10 +249,8 @@ def export(self, parsed_args, incursion_rooms):
                         'Writing source file "%s" to images' % src
                     )
                     with open(src, 'wb') as f:
-                        img_data = extract_dds(
-                            self.ggpk[
-                            idl_record.source].record.extract().read(),
-                            path_or_ggpk=self.ggpk,
+                        img_data = self.file_system.extract_dds(
+                            self.file_system.get_file(idl_record.source)
                         )
                         f.write(img_data[:84])
                         if img_data[84:88].decode('ascii') == 'DXT4':

PyPoE/cli/exporter/wiki/parsers/item.py

@@ -1560,7 +1560,7 @@ def __init__(self, *args, **kwargs):
         self._language = config.get_option('language')
         if self._language != 'English':
             self.rr2 = RelationalReader(
-                path_or_ggpk=self.base_path,
+                path_or_file_system=self.base_path,
                 files=['BaseItemTypes.dat', 'Prophecies.dat'],
                 read_options={
                     'use_dat_value': False,
@@ -2608,7 +2608,7 @@ def _process_base_item_type(self, base_item_type, infobox,
                     m_id not in self._IGNORE_DROP_LEVEL_ITEMS_BY_ID:
                 infobox['drop_level'] = base_item_type['DropLevel']
 
-            base_ot = OTFile(parent_or_base_dir_or_ggpk=self.base_path)
+            base_ot = OTFile(parent_or_file_system=self.file_system)
             base_ot.read(
                 self.base_path + '/' + base_item_type['InheritsFrom'] + '.ot')
             try:
@@ -2780,7 +2780,7 @@ def _export(self, parsed_args, items):
                 wiki_message='Item exporter',
             )
 
-            if parsed_args.store_images and self.ggpk:
+            if parsed_args.store_images:
                 if not base_item_type['ItemVisualIdentityKey']['DDSFile']:
                     warnings.warn(
                         'Missing 2d art inventory icon for item "%s"' %
@@ -2789,8 +2789,8 @@ def _export(self, parsed_args, items):
                     continue
 
                 self._write_dds(
-                    data=self.ggpk[base_item_type['ItemVisualIdentityKey'][
-                        'DDSFile']].record.extract().read(),
+                    data=self.file_system.get_file(
+                        base_item_type['ItemVisualIdentityKey']['DDSFile']),
                     out_path=os.path.join(self._img_path, (
                         infobox.get('inventory_icon') or page) +
                         ' inventory icon.dds',
@@ -2865,7 +2865,7 @@ def export_map_icons(self, parsed_args):
         base_ico = os.path.join(self._img_path, 'Base.dds')
 
         self._write_dds(
-            data=self.ggpk[map_series['BaseIcon_DDSFile']].record.extract().read(),
+            data=self.file_system.get_file(map_series['BaseIcon_DDSFile']),
             out_path=base_ico,
             parsed_args=parsed_args,
         )
@@ -2883,8 +2883,8 @@ def export_map_icons(self, parsed_args):
             ico = os.path.join(self._img_path, name + '.dds')
 
             self._write_dds(
-                data=self.ggpk[atlas_node['ItemVisualIdentityKey'][
-                    'DDSFile']].record.extract().read(),
+                data=self.file_system.get_file(
+                    atlas_node['ItemVisualIdentityKey']['DDSFile']),
                 out_path=ico,
                 parsed_args=parsed_args,
             )
@@ -2948,7 +2948,7 @@ def export_map(self, parsed_args):
             base_ico = os.path.join(self._img_path, 'Map base icon.dds')
 
             self._write_dds(
-                data=self.ggpk[map_series['BaseIcon_DDSFile']].record.extract().read(),
+                data=self.file_system.get_file(map_series['BaseIcon_DDSFile']),
                 out_path=base_ico,
                 parsed_args=parsed_args,
             )
@@ -3074,7 +3074,7 @@ def export_map(self, parsed_args):
                 wiki_message='Map exporter',
             )
 
-            if parsed_args.store_images and self.ggpk:
+            if parsed_args.store_images:
                 if atlas_node is None or \
                         not atlas_node['ItemVisualIdentityKey']['DDSFile']:
                     warnings.warn(
@@ -3086,8 +3086,8 @@ def export_map(self, parsed_args):
                 ico = os.path.join(self._img_path, name + ' inventory icon.dds')
 
                 self._write_dds(
-                    data=self.ggpk[atlas_node['ItemVisualIdentityKey'][
-                        'DDSFile']].record.extract().read(),
+                    data=self.file_system.get_file(
+                        atlas_node['ItemVisualIdentityKey']['DDSFile']),
                     out_path=ico,
                     parsed_args=parsed_args,
                 )