From 789cf6f428c81e67f627d0719085adde3951eade Mon Sep 17 00:00:00 2001 From: jonnylamb Date: Sat, 2 Feb 2008 15:50:30 +0000 Subject: [PATCH] Cleaned up the po2json script. This is now more compact and fixes the ImportError that some experience. --- src/www/l10n/po2json | 729 ++++++++++--------------------------------- 1 file changed, 166 insertions(+), 563 deletions(-) diff --git a/src/www/l10n/po2json b/src/www/l10n/po2json index e169129..485000d 100755 --- a/src/www/l10n/po2json +++ b/src/www/l10n/po2json @@ -1,566 +1,163 @@ #!/usr/bin/env python -# License: MIT (see LICENSE file provided) -__author__ = 'David JEAN LOUIS ' -__version__ = '0.1.0' - - -try: - import struct - import textwrap -except ImportError, exc: - raise ImportError('polib requires python 2.3 or later with the ' \ - 'standard modules "struct" and "textwrap" (details: %s)' % exc) - -### shortcuts for performance improvement and misc variables {{{ -# yes, yes, this is quite ugly but *very* efficient -_dictget = dict.get -_listappend = list.append -_listpop = list.pop -_strjoin = str.join -_strsplit = str.split -_strstrip = str.strip -_strstartsw = str.startswith -_strendsw = str.endswith -_strreplace = str.replace -_textwrap = textwrap.wrap -### }}} - - -def pofile(fpath, wrapwidth=78): - try: - fhandle = open(fpath, 'r+') - except IOError, err: - raise IOError('Unable to open the po file "%s": %s' % (fpath, err)) - parser = _POFileParser(fhandle) - instance = parser.parse() - instance.wrapwidth = wrapwidth - return instance - - -class _BaseFile(list): - def __init__(self, fhandle=None, wrapwidth=78): - """_BaseFile constructor.""" - list.__init__(self) - # the opened file handle - self.fhandle = fhandle - # the width at which lines should be wrapped - self.wrapwidth = wrapwidth - # both po and mo files have metadata - self.metadata = {} - self.metadata_is_fuzzy = 0 - - def __del__(self): - """ - Destructor: close the file handle when the object is - destroyed. - """ - if self.fhandle is not None: - self.fhandle.close() - - def __str__(self): - """Common string representation for po and mo files.""" - ret = '' - if self.metadata_is_fuzzy: - ret = '#, fuzzy\n' - ret += 'msgid ""\n' - ret += 'msgstr ""\n' - for name, value in self.ordered_metadata(): - values = _strsplit(value, '\n') - for i, value in enumerate(values): # handle multiline metadata - if i == 0: - ret += '"%s: %s"\n' % (name, _strstrip(value)) - else: - ret += '"%s"\n' % _strstrip(value) - ret += '\n' - padding = '' - for entry in self: - ret += '%s%s' % (padding, entry.__str__(self.wrapwidth)) - padding = '\n' - return ret.rstrip() - - def __repr__(self): - """Return the official string representation of the object.""" - return '<%s instance at %d>' % (self.__class__.__name__, id(self)) - - def ordered_metadata(self): - """ - Convenience method that return the metadata ordered. The return - value is list of tuples (metadata name, metadata_value). - """ - # copy the dict first - metadata = self.metadata.copy() - data_order = [ - 'Project-Id-Version', - 'Report-Msgid-Bugs-To', - 'POT-Creation-Date', - 'PO-Revision-Date', - 'Last-Translator', - 'Language-Team', - 'MIME-Version', - 'Content-Type', - 'Content-Transfer-Encoding' - ] - ordered_data = [] - for data in data_order: - try: - value = metadata.pop(data) - _listappend(ordered_data, (data, value)) - except KeyError: - pass - # the rest of the metadata won't be order there's no specs for this - for data in sorted(metadata.keys()): - value = metadata[data] - _listappend(ordered_data, (data, value)) - return ordered_data - -class POFile(_BaseFile): - def __init__(self, fhandle=None, wrapwidth=78): - """ - POFile constructor. - """ - # the po file header - self.header = '' - _BaseFile.__init__(self, fhandle, wrapwidth) - - def __str__(self): - """Return the string representation of the po file""" - ret, headers = '', _strsplit(self.header, '\n') - for header in headers: - if _strstartsw(header, ',') or _strstartsw(header, ':'): - ret += '#%s\n' % header - else: - ret += '# %s\n' % header - return ret + _BaseFile.__str__(self) - -class _BaseEntry: - """ - Base class for POEntry or MOEntry objects. - This class must *not* be instanciated directly. - """ - ### class _BaseEntry {{{ - def __init__(self, msgid='', msgstr=''): - """Base Entry constructor.""" - self.msgid = msgid - self.msgstr = msgstr - self.msgid_plural = '' - self.msgstr_plural = {} - self.obsolete = 0 - self.has_plural = 0 - # private vars - self._is_multiline_msgid = 0 - self._is_multiline_msgid_plural = 0 - self._is_multiline_msgstr = 0 - self._is_multiline_msgstr_plural = {} - - def __repr__(self): - """Return the official string representation of the object.""" - return '<%s instance at %d>' % (self.__class__.__name__, id(self)) - - def __str__(self, wrapwidth=78): - """ - Common string representation of the POEntry and MOEntry - objects. - """ - if self.obsolete: - delflag = '#~ ' - else: - delflag = '' - # write the msgid - lines = _strsplit(self.msgid, '\n') - if self._is_multiline_msgid: - ret = '%smsgid ""\n' % delflag - for msgid in lines: - ret += '%s"%s"\n' % (delflag, msgid) - else: - ret = '%smsgid "%s"\n' % (delflag, lines[0]) - # write the msgid_plural if any - if self.msgid_plural: - lines = _strsplit(self.msgid_plural, '\n') - if self._is_multiline_msgid_plural: - ret += '%smsgid_plural ""\n' % delflag - for msgid in lines: - ret += '%s"%s"\n' % (delflag, msgid) - else: - ret += '%smsgid_plural "%s"\n' % (delflag, lines[0]) - if self.msgstr_plural: - msgstrs = self.msgstr_plural - else: - msgstrs = {0:self.msgstr} - for index in sorted(msgstrs.keys()): - msgstr = msgstrs[index] - lines = _strsplit(msgstr, '\n') - plural_index = '' - is_multiline_msgstr = self._is_multiline_msgstr - if self.msgstr_plural: - plural_index = '[%s]' % index - try: - is_multiline_msgstr = \ - self._is_multiline_msgstr_plural[index] - except: - pass - if is_multiline_msgstr: - ret += '%smsgstr%s ""\n' % (delflag, plural_index) - for mstr in lines: - ret += '%s"%s"\n' % (delflag, mstr) - else: - ret += '%smsgstr%s "%s"\n' % (delflag, plural_index, lines[0]) - return ret - ### }}} - - -class POEntry(_BaseEntry): - ### class POEntry {{{ - - def __init__(self, msgid='', msgstr=''): - """POEntry constructor.""" - _BaseEntry.__init__(self, msgid, msgstr) - self.occurences = [] - self.comment = '' - self.tcomment = '' - self.flags = [] - - def __str__(self, wrapwidth=78): - """ - Return the string representation of the entry. - """ - ret = '' - # comment first, if any (with text wrapping as xgettext does) - if self.comment != '': - comments = _strsplit(self.comment, '\n') - for comment in comments: - if wrapwidth > 0 and len(comment) > wrapwidth-3: - lines = _textwrap(comment, wrapwidth, - initial_indent='#. ', - subsequent_indent='#. ', - break_long_words=False) - ret += _strjoin('\n', lines) + '\n' - else: - ret += '#. %s\n' % comment - # translator comment, if any (with text wrapping as xgettext does) - if self.tcomment != '': - tcomments = _strsplit(self.tcomment, '\n') - for tcomment in tcomments: - if wrapwidth > 0 and len(tcomment) > wrapwidth-2: - lines = _textwrap(tcomment, wrapwidth, - initial_indent='# ', - subsequent_indent='# ', - break_long_words=False) - ret += _strjoin('\n', lines) + '\n' - else: - ret += '# %s\n' % tcomment - # occurences (with text wrapping as xgettext does) - if self.occurences: - if wrapwidth > 0: - # XXX textwrap split words that contain hyphen, this is not - # what we want for filenames, so the dirty hack is to - # temporally replace hyphens with a char that a file cannot - # contain, like "*" - filestr, pad = '', '' - for filepath, lineno in self.occurences: - filestr += '%s%s:%s' % \ - (pad, _strreplace(filepath, '-', '*'), lineno) - pad = ' ' - lines = _textwrap(filestr, wrapwidth, - initial_indent='#: ', - subsequent_indent='#: ', - break_long_words=False) - ret += _strjoin('\n', lines) + '\n' - # end of the replace hack - ret = _strreplace(ret, '*', '-') - else: - try: - ret += '#:' - for filepath, lineno in self.occurences: - ret += ' %s:%s' % (filepath, lineno) - except IndexError: - pass - # flags - if self.flags: - ret += '#, %s\n' % _strjoin(', ', self.flags) - return ret + _BaseEntry.__str__(self) - - def translated(self): - """Return True if the entry has been translated or False""" - return self.msgstr != '' or self.msgstr_plural - ### }}} - -class _POFileParser: - """ - A finite state machine to parse efficiently and correctly po - file format. - """ - ### class _POFileParser {{{ - def __init__(self, fhandle): - """ - Constructor. - Keyword argument: - fhandle -- a opened file object. - """ - self.instance = POFile() - self.instance.fhandle = fhandle - self.transitions = {} - self.current_entry = POEntry() - self.current_state = 'ST' - self.current_token = None - # two memo flags used in handlers - self.msgstr_index = 0 - self.entry_obsolete = 0 - # Configure the state machine, by adding transitions. - # Signification of symbols: - # * ST: Beginning of the file (start) - # * HE: Header - # * TC: a translation comment - # * GC: a generated comment - # * OC: a file/line occurence - # * FL: a flags line - # * MI: a msgid - # * MP: a msgid plural - # * MS: a msgstr - # * MX: a msgstr plural - # * MC: a msgid or msgstr continuation line - all_ = ['ST', 'HE', 'GC', 'OC', 'FL', 'TC', 'MS', 'MP', 'MX', 'MI'] - - self.add('TC', ['ST', 'HE'], 'HE') - self.add('TC', ['GC', 'OC', 'FL', 'TC', 'MS', 'MP', 'MX', 'MI'], 'TC') - self.add('GC', all_, 'GC') - self.add('OC', all_, 'OC') - self.add('FL', all_, 'FL') - self.add('MI', ['ST', 'HE', 'GC', 'OC', 'FL', 'TC', 'MS', 'MX'], 'MI') - self.add('MP', ['TC', 'GC', 'MI'], 'MP') - self.add('MS', ['MI', 'MP', 'TC'], 'MS') - self.add('MX', ['MI', 'MX', 'MP', 'TC'], 'MX') - self.add('MC', ['MI', 'MP', 'MS', 'MX'], 'MC') - - def parse(self): - """ - Run the state machine, parse the file line by line and call process() - with the current matched symbol. - """ - lines, i, lastlen = self.instance.fhandle.readlines(), 0, 0 - for line in lines: - line = _strstrip(line) - if line == '': - i = i+1 - continue - if _strstartsw(line, '#~ '): - line = line[3:] - self.entry_obsolete = 1 - else: - self.entry_obsolete = 0 - self.current_token = line - if _strstartsw(line, 'msgid_plural "'): - # we are on a msgid plural - self.process('MP', i+1) - elif _strstartsw(line, 'msgid "'): - # we are on a msgid - self.process('MI', i+1) - elif _strstartsw(line, 'msgstr['): - # we are on a msgstr plural - self.process('MX', i+1) - elif _strstartsw(line, 'msgstr "'): - # we are on a msgstr - self.process('MS', i+1) - elif _strstartsw(line, '#:'): - # we are on a occurences line - self.process('OC', i+1) - elif _strstartsw(line, '"'): - # we are on a continuation line or some metadata - self.process('MC', i+1) - elif _strstartsw(line, '#, '): - # we are on a flags line - self.process('FL', i+1) - elif _strstartsw(line, '# ') or line == '#': - if line == '#': line = line + ' ' - # we are on a translator comment line - self.process('TC', i+1) - elif _strstartsw(line, '#.'): - # we are on a generated comment line - self.process('GC', i+1) - i = i+1 - - if lines and self.current_entry: - # since entries are added when another entry is found, we must add - # the last entry here (only if there are lines) - _listappend(self.instance, self.current_entry) - # before returning the instance, check if there's metadata and if - # so extract it in a dict - firstentry = self.instance[0] - if firstentry.msgid == '': # metadata found - # remove the entry - firstentry = self.instance.pop(0) - self.instance.metadata_is_fuzzy = firstentry.flags - multiline_metadata = 0 - for msg in _strsplit(firstentry.msgstr, '\n'): - if msg != '': - if multiline_metadata: - self.instance.metadata[key] += '\n' + msg - else: - try: - key, val = _strsplit(msg, ':', 1) - self.instance.metadata[key] = val - except: - pass - multiline_metadata = not _strendsw(msg, '\\n') - return self.instance - - def add(self, symbol, states, next_state): - """ - Add a transition to the state machine. - Keywords arguments: - - symbol -- string, the matched token (two chars symbol) - states -- list, a list of states (two chars symbols) - next_state -- the next state the fsm will have after the action - """ - for state in states: - action = getattr(self, 'handle_%s' % next_state.lower()) - self.transitions[(symbol, state)] = (action, next_state) - - def process(self, symbol, linenum): - """ - Process the transition corresponding to the current state and the - symbol provided. - - Keywords arguments: - symbol -- string, the matched token (two chars symbol) - linenum -- integer, the current line number of the parsed file - """ - try: - (action, state) = self.transitions[(symbol, self.current_state)] - except LookupError: - raise IOError('Invalid token in po file on line %s !' % linenum) - if action(): - self.current_state = state - - # state handlers - - def handle_he(self): - """Handle a header comment.""" - token = self.current_token[2:] - if self.instance.header != '': - token = '\n' + token - self.instance.header += token - return 1 - - def handle_tc(self): - """Handle a translator comment.""" - if self.current_state in ['MC', 'MS', 'MX']: - _listappend(self.instance, self.current_entry) - self.current_entry = POEntry() - token = self.current_token[2:] - if self.current_entry.tcomment != '': - token = '\n' + token - self.current_entry.tcomment += token - return 1 - - def handle_gc(self): - """Handle a generated comment.""" - if self.current_state in ['MC', 'MS', 'MX']: - _listappend(self.instance, self.current_entry) - self.current_entry = POEntry() - token = self.current_token[3:] - if self.current_entry.comment != '': - token = '\n' + token - self.current_entry.comment += token - return 1 - - def handle_oc(self): - """Handle a file:num occurence.""" - if self.current_state in ['MC', 'MS', 'MX']: - _listappend(self.instance, self.current_entry) - self.current_entry = POEntry() - try: - occurences = _strsplit(self.current_token[3:]) - for occurence in occurences: - if occurence != '': - fil, line = _strsplit(occurence, ':') - _listappend(self.current_entry.occurences, (fil, line)) - except: - pass - return 1 - - def handle_fl(self): - """Handle a flags line.""" - if self.current_state in ['MC', 'MS', 'MX']: - _listappend(self.instance, self.current_entry) - self.current_entry = POEntry() - try: - self.current_entry.flags += _strsplit(self.current_token[3:], ', ') - except: - pass - return 1 - - def handle_mi(self): - """Handle a msgid.""" - if self.current_state in ['MC', 'MS', 'MX']: - _listappend(self.instance, self.current_entry) - self.current_entry = POEntry() - try: - self.current_entry.obsolete = self.entry_obsolete - self.current_entry.msgid = self.current_token[7:-1] - except: - pass - return 1 - - def handle_mp(self): - """Handle a msgid plural.""" - try: - self.current_entry.has_plural = 1 - self.current_entry.msgid_plural = self.current_token[14:-1] - except: - pass - return 1 - - def handle_ms(self): - """Handle a msgstr.""" - self.current_entry.msgstr = self.current_token[8:-1] - return 1 - - def handle_mx(self): - """Handle a msgstr plural.""" - try: - index, value = self.current_token[7], self.current_token[11:-1] - self.current_entry.msgstr_plural[index] = value - self.msgstr_index = index - self.current_entry._is_multiline_msgstr_plural[index] = 0 - except: - pass - return 1 - - def handle_mc(self): - """Handle a msgid or msgstr continuation line.""" - try: - if self.current_state == 'MI': - pad = self.current_entry.msgid != '' and '\n' or '' - self.current_entry.msgid += pad + self.current_token[1:-1] - self.current_entry._is_multiline_msgid = 1 - elif self.current_state == 'MP': - pad = self.current_entry.msgid_plural != '' and '\n' or '' - self.current_entry.msgid_plural += pad + \ - self.current_token[1:-1] - self.current_entry._is_multiline_msgid_plural = 1 - elif self.current_state == 'MS': - pad = self.current_entry.msgstr != '' and '\n' or '' - self.current_entry.msgstr += pad + self.current_token[1:-1] - self.current_entry._is_multiline_msgstr = 1 - elif self.current_state == 'MX': - msgstr = self.current_entry.msgstr_plural[self.msgstr_index] - pad = msgstr != '' and '\n' or '' - msgstr += pad + self.current_token[1:-1] - self.current_entry.msgstr_plural[self.msgstr_index] = msgstr - self.current_entry._is_multiline_msgstr_plural\ - [self.msgstr_index] = 1 - except: - pass - return 0 - # }}} - import os, sys -temp = sys.path.append(os.path.abspath(os.path.dirname(sys.argv[0])) + '/../../libs/python/') -from bongo.external import simplejson +sys.path.append(os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])), '..', '..', 'libs', 'python', 'bongo', 'external')) +import simplejson from optparse import OptionParser +class POParser(object): + "parse a .po file extracting msgids and msgstrs" + def __init__(self, filename=""): + self.status = 0 + self.files = [] + self.msgid = "" + self.msgstr = "" + self.line = 1 + self.f = None + self.esc = { "n": "\n", "r": "\r", "t": "\t" } + if filename: + self.f = open(filename) + + def open(self, filename): + self.f = open(filename) + + def close(self): + self.f.close() + + def parse(self): + """States table for the finite-states-machine parser: + 0 idle + 1 filename-or-comment + 2 msgid + 3 msgstr + 4 end + """ + # each time we can safely re-initialize those vars + self.files = [] + self.msgid = "" + self.msgstr = "" + + # can't continue if status == 4, this is a dead status + if self.status == 4: + return 0 + + while 1: + # continue scanning, char-by-char + c = self.f.read(1) + if not c: + # EOF -> maybe we have a msgstr to save? + self.status = 4 + if self.msgstr: + return 1 + else: + return 0 + + # keep the line count up-to-date + if c == "\n": + self.line += 1 + + # a pound was detected the previous char... + if self.status == 1: + if c == ":": + # was a line of filenames + row = self.f.readline() + self.files += row.split() + self.line += 1 + elif c == "\n": + # was a single pount on the line + pass + else: + # was a comment... discard + self.f.readline() + self.line += 1 + # in every case, we switch to idle status + self.status = 0; + continue + + # in idle status we search for a '#' or for a 'm' + if self.status == 0: + if c == "#": + # this could be a comment or a filename + self.status = 1; + continue + elif c == "m": + # this should be a msgid start... + s = self.f.read(4) + assert s == "sgid" + # so now we search for a '"' + self.status = 2 + continue + # in idle only those other chars are possibile + assert c in [ "\n", " ", "\t" ] + + # searching for the msgid string + if self.status == 2: + if c == "\n": + # a double LF is not possible here + c = self.f.read(1) + assert c != "\n" + if c == "\"": + # ok, this is the start of the string, + # now search for the end + while 1: + c = self.f.read(1) + if not c: + # EOF, bailout + self.status = 4 + return 0 + if c == "\\": + # a quoted char... + c = self.f.read(1) + if self.esc.has_key(c): + self.msgid += self.esc[c] + else: + self.msgid += c + continue + if c == "\"": + # end of string found + break + # a normal char, add it + self.msgid += c + if c == "m": + # this should be a msgstr identifier + s = self.f.read(5) + assert s == "sgstr" + # ok, now search for the msgstr string + self.status = 3 + + # searching for the msgstr string + if self.status == 3: + if c == "\n": + # a double LF is the end of the msgstr! + c = self.f.read(1) + if c == "\n": + # ok, time to go idle and return + self.status = 0 + self.line += 1 + return 1 + if c == "\"": + # start of string found + while 1: + c = self.f.read(1) + if not c: + # EOF, bail out + self.status = 4 + return 1 + if c == "\\": + # a quoted char... + c = self.f.read(1) + if self.esc.has_key(c): + self.msgid += self.esc[c] + else: + self.msgid += c + continue + if c == "\"": + # end of string + break + # a normal char, add it + self.msgstr += c + + if __name__ == '__main__': parser = OptionParser() parser.add_option("-p", "--pofile", dest="pofile", help="Path to PO file", metavar="FILE") @@ -572,14 +169,20 @@ if __name__ == '__main__': print "You must specifiy an input PO file." sys.exit(1) - po = pofile(options.pofile) + po = POParser(options.pofile) output = {} + + # Skip the first line containing the pofile info like Last-Translator: + po.parse() + if options.en: - for entry in po: - output[entry.msgid] = entry.msgid + while po.parse(): + output[po.msgid] = po.msgid else: - for entry in po: - output[entry.msgid] = entry.msgstr + while po.parse(): + output[po.msgid] = po.msgstr + + po.close() json = simplejson.dumps(output)