Skip to content

Commit

Permalink
Added a FileSystem class that serves as interface between ggpk/bundle…
Browse files Browse the repository at this point in the history 
…/disk io

- some more type hints
- changed version requirement to 3.6 as attribute hints require 3.6+
- since the cache instances now use the FileSystem, the load_index and bundle_cache options have been removed
  • Loading branch information
@OmegaK2
OmegaK2 committed Sep 29, 2020
1 parent 54b0570 commit 135569b
Show file tree
Hide file tree
Showing 16 changed files with 266 additions and 246 deletions.
77 changes: 31 additions & 46 deletions PyPoE/cli/exporter/wiki/admin/unique.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,9 @@
import traceback
from collections import defaultdict

import time
import random

# 3rd-party
import mwclient
import mwparserfromhell
Expand Down Expand Up @@ -103,6 +106,14 @@ def __init__(self, sub_parser):
default=False,
)

copy.add_argument(
'--ignore-drop-text',
dest='ignore_drop_text',
help='ingore drop text and do not prompt for translation',
action='store_true',
default=False,
)

copy.add_argument(
'page',
help='page of the unique item to process',
Expand Down Expand Up @@ -163,7 +174,7 @@ def fuzzy_find_text(self, text, file_name, key, source_list=None, fuzzy_func=fuz
if key not in self.rr_english[file_name].index:
self.rr_english[file_name].build_index(key)
results = self.rr_english[file_name].index[key][text]
if len(results) == 1:
if isinstance(results, str) or len(results) == 1:
return self.rr[file_name][results[0].rowid][key]

# Try to find translation for the name using fuzzy search
Expand All @@ -178,14 +189,14 @@ def fuzzy_find_text(self, text, file_name, key, source_list=None, fuzzy_func=fuz
})

if len(results) == 0:
console('No matching text found.')
console('No matching text found for %s.' % key)
text = input('Enter translated text.\n')
if text == '':
console('No text specified - skipping search for "%s".' % text)
return
return text
elif len(results) >= 2:
console('Multiple matching names found.\n')
console('Multiple matching values found for %s\n' % key)
for i, row in enumerate(results):
row['i'] = i
console('%(i)s: %(ratio)s\n%(text)s\n----------------' % row)
Expand All @@ -199,7 +210,7 @@ def fuzzy_find_text(self, text, file_name, key, source_list=None, fuzzy_func=fuz
else:
return self.rr[file_name][results[0]['id']][key]

def copy(self, pn):
def copy(self, parsed_args, pn):
console('Processing %s' % pn)
page = self.site_english.pages[pn]
if not page.exists:
Expand All @@ -213,52 +224,24 @@ def copy(self, pn):
raise Exception('Item template not found')

console('Finding flavour text...')
ftext = None
if not mwtemplate.has('flavour_text_id') and mwtemplate.has('flavour_text'):
console('Missing flavour_text_id. Trying to find flavour text in FlavourText.dat')

ftext = self.fuzzy_find_text(
mwtemplate.get('flavour_text'),
'FlavourText.dat',
'Text',
fuzzy_func=fuzz.partial_token_set_ratio
fuzzy_func=fuzz.partial_ratio
)

results = []
for row in self.rr_english['FlavourText.dat']:
ratio = fuzz.partial_token_set_ratio(row['Text'], ftext)
if ratio > 90:
results.append({
'id': row['Id'],
'text': row['Text'],
'ratio': ratio,
})

if len(results) == 0:
console('No matching flavour text found.')
text = input('Enter translated flavour text. Type None to skip item entirely.\n')
if text == 'None':
console('Skipping item %s.' % pn)
return
mwtemplate.get('flavour_text').value = text
elif len(results) >= 2:
console('Multiple matching flavour text entries found.\n')
for i, row in enumerate(results):
row['i'] = i
console('%(i)s %(id)s: %(ratio)s\n%(text)s\n----------------' % row)

try:
correct = results[int(input('Enter index of correct translation.'))]
except Exception as e:
traceback.print_exc()

mwtemplate.get('flavour_text_id').value = correct['id']
else:
mwtemplate.get('flavour_text_id').value = results[0]['id']

mwtemplate.get('flavour_text').value = ftext
# Grab flavour text from other language
if mwtemplate.has('flavour_text_id'):
mwtemplate.get('flavour_text').value = ' %s\n' % self.rr['FlavourText.dat'].index['Id'][
mwtemplate.get('flavour_text_id').value.strip()]['Text'].replace('\r', '').replace('\n', '<br>')
elif mwtemplate.has('flavour_text_id'):
ftext = self.rr['FlavourText.dat'].index['Id'][
mwtemplate.get('flavour_text_id').value.strip()]['Text']

if ftext:
mwtemplate.get('flavour_text').value = ' %s\n' % ftext.replace('\r', '').replace('\n', '<br>')

# Need this for multiple things
name = mwtemplate.get('name').value.strip()
Expand Down Expand Up @@ -286,7 +269,7 @@ def copy(self, pn):
pass
elif mwtemplate.has('base_item'):
base = self.fuzzy_find_text(
mwtemplate.get('base_item'),
mwtemplate.get('base_item').value,
'BaseItemTypes.dat',
'Name',
source_list=self.cache[mwtemplate.get('class_id').value.strip()]
Expand All @@ -304,18 +287,18 @@ def copy(self, pn):
# Need to copy the list or it won't be deleted properly as it deletes from itself during iteration
for mwparam in list(mwtemplate.params):
pname = mwparam.name.strip()
if pname.startswith('upgraded_from'):
if pname.startswith('upgraded_from') and \
pname != 'upgraded_from_disabled':
mwtemplate.remove(mwparam.name)
elif pname in ('class'):
mwtemplate.remove(mwparam.name)

if mwtemplate.has('drop_text'):
if mwtemplate.has('drop_text') and not parsed_args.ignore_drop_text:
console('Drop text might need a translation. Current text:\n\n%s' % mwtemplate.get('drop_text').value.strip())
text = input('\nNew text (leave empty to copy old):\n')
if text:
mwtemplate.get('drop_text').value = ' %s\n' % text

console('Saving on other wiki...')
if pn == name:
page = self.site_other.pages[new]
else:
Expand All @@ -327,13 +310,15 @@ def copy(self, pn):
cont = input('y/n?\n') != 'y'
page = self.site_other.pages[t]

console('Saving to "%s" on other wiki...' % page.name)
page.save('%s\n\n[[en:%s]]' % (str(mwtemplate), pn))
console('Done.')

def run(self, parsed_args, **kwargs):
console('Parsing...')
for item in parsed_args.page:
self.copy(item)
self.copy(parsed_args, item)
time.sleep(random.randint(180, 300))


class BaseItemCacheInstance(list):
Expand Down
71 changes: 38 additions & 33 deletions PyPoE/poe/file/bundle.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@
from enum import IntEnum
from io import BytesIO
from tempfile import TemporaryDirectory
from typing import List, Union
from typing import List, Union, Dict, Tuple

# 3rd party
from fnvhash import fnv1a_64
Expand Down Expand Up @@ -147,21 +147,25 @@ class ENCODE_TYPES(IntEnum):
class Bundle(AbstractFileReadOnly):
def __init__(self, *args, **kwargs):
super().__init__(*args, *kwargs)
self.encoder = None
self.unknown = None
self.size_decompressed = None
self.size_compressed = None
self.entry_count = None
self.chunk_size = None
self.unknown3 = None
self.unknown4 = None
self.unknown5 = None
self.unknown6 = None
self.chunks = None
self.data = {}
self.encoder: Union[ENCODE_TYPES, None] = None
self.unknown: Union[int, None] = None
self.size_decompressed: Union[int, None] = None
self.size_compressed: Union[int, None] = None
self.entry_count: Union[int, None] = None
self.chunk_size: Union[int, None] = None
self.unknown3: Union[int, None] = None
self.unknown4: Union[int, None] = None
self.unknown5: Union[int, None] = None
self.unknown6: Union[int, None] = None
self.chunks: Union[Tuple[int, ...], None] = None
self.data: Union[Dict[int, bytes], bytes] = {}

@property
def is_decompressed(self) -> bool:
return isinstance(self.data, bytes)

def _read(self, buffer: BytesIO):
if isinstance(self.data, bytes):
if self.is_decompressed:
raise ValueError('Bundle has been decompressed already')

raw = buffer.read()
Expand Down Expand Up @@ -289,18 +293,19 @@ class BundleRecord(IndexRecord):
_REPR_EXTRA_ATTRIBUTES = {x: None for x in __slots__}

def __init__(self, raw: bytes, parent: 'Index', offset: int):
self.parent = parent
self.parent: Index = parent

name_length = struct.unpack_from('<I', raw, offset=offset)[0]

self.name = struct.unpack_from(
self.name: str = struct.unpack_from(
'%ss' % name_length, raw, offset=offset+4)[0].decode()

self.size = struct.unpack_from('<I', raw, offset=offset+4+name_length)[0]
self.size: int = \
struct.unpack_from('<I', raw, offset=offset+4+name_length)[0]

self.BYTES = name_length + 8
self.BYTES: int = name_length + 8

self.contents = None
self.contents: Union[Bundle, None] = None

@property
def file_name(self) -> str:
Expand Down Expand Up @@ -353,11 +358,11 @@ class FileRecord(IndexRecord):
def __init__(self, raw: bytes, parent: 'Index', offset: int):
data = struct.unpack_from('<QIII', raw, offset=offset)

self.parent = parent
self.hash = data[0]
self.bundle = parent.bundles[data[1]]
self.file_offset = data[2]
self.file_size = data[3]
self.parent: Index = parent
self.hash: int = data[0]
self.bundle: BundleRecord = parent.bundles[data[1]]
self.file_offset: int = data[2]
self.file_size: int = data[3]

def get_file(self) -> bytes:
"""
Expand Down Expand Up @@ -387,14 +392,14 @@ class DirectoryRecord(IndexRecord):
_REPR_EXTRA_ATTRIBUTES = {x: None for x in __slots__}
SIZE = 20

def __init__(self, raw: bytes, parent: 'Index', offset:int):
self.parent = parent
def __init__(self, raw: bytes, parent: 'Index', offset: int):
self.parent: Index = parent
data = struct.unpack_from('<QIII', raw, offset=offset)

self.hash = data[0]
self.offset = data[1]
self.size = data[2]
self.unknown = data[3]
self.hash: int = data[0]
self.offset: int = data[1]
self.size: int = data[2]
self.unknown: int = data[3]
self._paths = None

@property
Expand Down Expand Up @@ -422,9 +427,9 @@ class Index(Bundle):

def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.bundles = {}
self.files = {}
self.directories = {}
self.bundles: Dict[int, BundleRecord] = {}
self.files: Dict[int, FileRecord] = {}
self.directories: Dict[int, DirectoryRecord] = {}

def get_dir_record(self, path: Union[str, bytes]) -> DirectoryRecord:
"""
Expand Down
Loading

0 comments on commit 135569b

Please sign in to comment.