diff --git a/contrib/testing/python-runtime-check.sh b/contrib/testing/python-runtime-check.sh new file mode 100755 index 0000000..9c9fd15 --- /dev/null +++ b/contrib/testing/python-runtime-check.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# /**************************************************************************** +# * +# * Copyright (c) 2001 Novell, Inc. All Rights Reserved. +# * +# * This program is free software; you can redistribute it and/or +# * modify it under the terms of version 2 of the GNU General Public License +# * as published by the Free Software Foundation. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, contact Novell, Inc. +# * +# * To contact Novell about this file by physical or electronic mail, you +# * may find current contact information at www.novell.com. +# * +# ****************************************************************************/ + +set -euo pipefail + +temporary=$(mktemp -d "${TMPDIR:-/tmp}/bongo-python-runtime.XXXXXX") +trap 'rm -rf -- "$temporary"' EXIT +export PYTHONPYCACHEPREFIX="${temporary}/pycache" + +python3 -m compileall -q \ + "$(python3 -c 'import sysconfig; print(sysconfig.get_path("purelib"))')/bongo" \ + "$(python3 -c 'import sysconfig; print(sysconfig.get_path("purelib"))')/bongo_web" + +python3 - <<'PY' +import io + +from bongo import table +from bongo.Console import wrap +from bongo.external.simpletal import simpleTAL, simpleTALES +from bongo.external.simpletal.simpleElementTree import parseFile +from bongo.storetool import BackupCommands, CalendarCommands, MailCommands + +assert isinstance(wrap("Grüße 世界", width=8), str) +assert "Müller" in table.format_table(["Name"], [["Müller"]]) + +context = simpleTALES.Context() +context.addGlobal("name", "A&B") +template = simpleTAL.compileHTMLTemplate( + '

placeholder

') +output = io.StringIO() +template.expand(context, output) +assert output.getvalue() == "

A&B

" + +root = parseFile(io.StringIO("value")) +assert root.find("child").text == "value" +PY + +bongo-admin --help >"${temporary}/bongo-admin.help" +bongo-storetool --help >"${temporary}/bongo-storetool.help" +bongo-queuetool --help >"${temporary}/bongo-queuetool.help" +bongo-web --help >"${temporary}/bongo-web.help" +/usr/libexec/bongo/bongo-sa-update --help \ + >"${temporary}/bongo-sa-update.help" + +set +e +/usr/libexec/bongo/bongo-acme-dns-provider \ + --config "${temporary}/missing.json" --provider audit --type nsupdate \ + --operation present --record _acme-challenge.example.test \ + --value audit --ttl 60 >"${temporary}/bongo-acme.json" +status=$? +set -e +test "${status}" -eq 1 +grep -q '"ok":false' "${temporary}/bongo-acme.json" + +echo "PYTHON-3 PASS compileall=installed imports=storetool,template,xml cli=6" diff --git a/src/apps/admin/bongo/admintool/AgentCommands.py b/src/apps/admin/bongo/admintool/AgentCommands.py index d87301a..b8d85cf 100644 --- a/src/apps/admin/bongo/admintool/AgentCommands.py +++ b/src/apps/admin/bongo/admintool/AgentCommands.py @@ -63,8 +63,7 @@ class AgentInfo(Command): attrs = Util.GetAgentAttributes(mdb, dn) keys = list(attrs.keys()) - keys.sort(lambda x,y: cmp(Util.PropertiesPrintable.get(x), - Util.PropertiesPrintable.get(y))) + keys.sort(key=lambda key: Util.PropertiesPrintable.get(key, key)) for k in keys: print(" |->(%s)" % Util.PropertiesPrintable[k]) diff --git a/src/apps/admin/bongo/admintool/DemoCommands.py b/src/apps/admin/bongo/admintool/DemoCommands.py index a9eac0c..06be230 100644 --- a/src/apps/admin/bongo/admintool/DemoCommands.py +++ b/src/apps/admin/bongo/admintool/DemoCommands.py @@ -27,6 +27,11 @@ from bongo.admin import Util from . import MdbUtil + +def _smtp_eols(value): + return value.replace("\r\n", "\n").replace("\r", "\n").replace( + "\n", "\r\n") + if os.path.isfile("demo/contacts/list.vcf") : demopath = "demo" else : @@ -84,7 +89,7 @@ class DemoCommand(Command): data = data.replace("@ADDRESS@", address) data = data.replace("@NAME@", name) - uid = store.Write("/mail/INBOX", DocTypes.Mail, email.Utils.fix_eols(data)) + uid = store.Write("/mail/INBOX", DocTypes.Mail, _smtp_eols(data)) if filename in demo["starMail"] : conversation = store.PropGet(uid, "nmap.mail.conversation") print("starring conversation %s" % (conversation)) diff --git a/src/apps/admin/bongo/admintool/ServerCommands.py b/src/apps/admin/bongo/admintool/ServerCommands.py index 1308976..db76289 100644 --- a/src/apps/admin/bongo/admintool/ServerCommands.py +++ b/src/apps/admin/bongo/admintool/ServerCommands.py @@ -38,8 +38,7 @@ class ServerInfo(Command): attrs = Util.GetServerAttributes(mdb, server) keys = list(attrs.keys()) - keys.sort(lambda x,y: cmp(Util.PropertiesPrintable.get(x), - Util.PropertiesPrintable.get(y))) + keys.sort(key=lambda key: Util.PropertiesPrintable.get(key, key)) for key in keys: print(" |->(%s)" % Util.PropertiesPrintable[key]) diff --git a/src/apps/admin/bongo/admintool/UserCommands.py b/src/apps/admin/bongo/admintool/UserCommands.py index 5365d06..77282c7 100644 --- a/src/apps/admin/bongo/admintool/UserCommands.py +++ b/src/apps/admin/bongo/admintool/UserCommands.py @@ -111,8 +111,7 @@ class UserInfo(Command): attrs = Util.GetUserAttributes(mdb, context, username) keys = list(attrs.keys()) - keys.sort(lambda x,y: cmp(Util.PropertiesPrintable.get(x), - Util.PropertiesPrintable.get(y))) + keys.sort(key=lambda key: Util.PropertiesPrintable.get(key, key)) for k in keys: print(" |->(%s)" % Util.PropertiesPrintable[k]) diff --git a/src/apps/storetool/CMakeLists.txt b/src/apps/storetool/CMakeLists.txt index 99b621b..9a6db54 100644 --- a/src/apps/storetool/CMakeLists.txt +++ b/src/apps/storetool/CMakeLists.txt @@ -13,7 +13,7 @@ ${SBIN_INSTALL_DIR}) if(BUILD_TESTING) add_test(NAME storetool-document-type COMMAND ${CMAKE_COMMAND} -E env - "PYTHONPATH=${CMAKE_SOURCE_DIR}/src/libs/python:${CMAKE_CURRENT_SOURCE_DIR}" + "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}:${CMAKE_SOURCE_DIR}/src/libs/python" "PYTHONPYCACHEPREFIX=${CMAKE_CURRENT_BINARY_DIR}/pycache" ${Python3_EXECUTABLE} -m unittest discover -s ${CMAKE_CURRENT_SOURCE_DIR}/tests -v) diff --git a/src/apps/storetool/bongo/__init__.py b/src/apps/storetool/bongo/__init__.py index f907027..b36383a 100644 --- a/src/apps/storetool/bongo/__init__.py +++ b/src/apps/storetool/bongo/__init__.py @@ -1,7 +1,3 @@ -# THIS FILE SHOULD NOT BE INSTALLED! +from pkgutil import extend_path -# It exists only to merge the src/[...]/storetool/bongo and src/libs/python/bongo -# directories in the Bongo source tree together. In an installed -# system, they're going to be in the same place. - -__path__.append(__path__[0] + "/../../../libs/python/bongo") +__path__ = extend_path(__path__, __name__) diff --git a/src/apps/storetool/bongo/storetool/BackupCommands.py b/src/apps/storetool/bongo/storetool/BackupCommands.py index 8bf38ff..cf65043 100644 --- a/src/apps/storetool/bongo/storetool/BackupCommands.py +++ b/src/apps/storetool/bongo/storetool/BackupCommands.py @@ -1,309 +1,281 @@ -import logging -import string -import sys +# /**************************************************************************** +# * +# * Copyright (c) 2001 Novell, Inc. All Rights Reserved. +# * +# * This program is free software; you can redistribute it and/or +# * modify it under the terms of version 2 of the GNU General Public License +# * as published by the Free Software Foundation. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, contact Novell, Inc. +# * +# * To contact Novell about this file by physical or electronic mail, you +# * may find current contact information at www.novell.com. +# * +# ****************************************************************************/ -from gettext import gettext as _ +"""Portable PAX backups for a Bongo user Store.""" + +from __future__ import annotations + +import io +import posixpath import tarfile from bongo.cmdparse import Command -from bongo.BongoError import BongoError from bongo.store.StoreClient import DocTypes, StoreClient -# import/export store contents as tar files -# ideally we want to farm the tar format stuff itself out into a separate lib. -# Sadly, the TarFile implementation that comes with Python is mostly useless -class PAXHeader: - def __init__(self, filename): - self.keywords = {} - self.filename = filename +_METADATA_PREFIX = "BONGO." +_GENERATED_PROPERTIES = { + "nmap.collection", + "nmap.flags", + "nmap.guid", + "nmap.index", + "nmap.type", +} - def SetKey(self, key, value): - self.keywords[key] = value - def Keys(self): - return self.keywords +def _text(value): + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + return str(value) - def ToString(self): - header = tarfile.TarInfo(self.filename + ".metadata") - header.type = "x" - content = "" - for key in list(self.keywords.keys()): - if self.keywords[key] != None: - content += "%d %s=%s\n" % (len(key) + len(self.keywords[key]) + 5, key, self.keywords[key]) - header.size = len(content) +def _path(collection, filename): + filename = _text(filename) + if filename.startswith("/"): + return filename + if collection == "/": + return "/" + filename + return collection.rstrip("/") + "/" + filename - result = header.tobuf() - result += content - remainder = len(content) % 512 - if remainder != 0: - result += "\0" * (512 - remainder) - return result +def _metadata(item, properties): + result = { + "BONGO.uid": _text(item.uid), + "BONGO.type": str(int(item.type)), + "BONGO.flags": str(int(item.flags)), + } + imapuid = getattr(item, "imapuid", None) + if imapuid not in (None, "", "-", 0, "0"): + result["BONGO.imapuid"] = _text(imapuid) + for key, value in properties.items(): + if value is not None: + result[_METADATA_PREFIX + _text(key)] = _text(value) + return result + + +def _properties(metadata): + result = {} + for key, value in metadata.items(): + if not key.startswith(_METADATA_PREFIX): + continue + name = key[len(_METADATA_PREFIX):] + if name in {"uid", "type", "flags", "imapuid"}: + continue + if name in _GENERATED_PROPERTIES: + continue + result[name] = value + return result + + +def _safe_member(member): + name = member.name + if (not name.startswith("/") or "\0" in name or + posixpath.normpath(name) != name or + member.issym() or member.islnk() or + not (member.isdir() or member.isfile())): + raise ValueError("unsafe or unsupported backup member %r" % name) + metadata = member.pax_headers + for required in ("BONGO.uid", "BONGO.type", "BONGO.flags"): + if required not in metadata: + raise ValueError("%s has no %s metadata" % (name, required)) + int(metadata["BONGO.type"]) + int(metadata["BONGO.flags"]) + + +def validate_archive(filename): + """Return validated members without extracting anything.""" + with tarfile.open(filename, "r:*") as archive: + members = archive.getmembers() + if not members: + raise ValueError("backup archive is empty") + names = set() + for member in members: + _safe_member(member) + if member.name in names: + raise ValueError("duplicate backup member %s" % member.name) + names.add(member.name) + return members - def FromString(self, content): - for line in string.split(content, "\n"): - space = string.find(line, " ") - equals = string.find(line, "=") - key = line[space+1:equals] - value = line[equals+1:] - self.keywords[key] = value class StoreBackupCommand(Command): - log = logging.getLogger("Bongo.StoreTool") - user = "admin" - store = "_system" - def __init__(self): - Command.__init__(self, "store-backup", aliases=["sb"], - summary="Create a backup of a store", - usage="%prog %cmd []") + Command.__init__( + self, "store-backup", aliases=["sb"], + summary="Create a PAX backup of a user store", + usage="%prog %cmd []") - def GetStoreConnection(self): - return StoreClient(self.user, self.store, host=self.host, port=int(self.port), authPassword=self.password) - - def BackupStore(self): - # create a backup file which contains all the contents of the given store - store = self.GetStoreConnection() - docstore = self.GetStoreConnection() + def _connection(self, options): + return StoreClient( + options.user, options.store, host=options.host, + port=int(options.port), authPassword=options.password) + def BackupStore(self, options, output): + store = self._connection(options) + docstore = self._connection(options) try: - backup_file = open("%s.backup" % self.store, "wb") - except IOError as e: - print(str(e)) - return + store.Store(options.store) + docstore.Store(options.store) + collections = list(store.Collections()) + collection_names = ["/"] + [ + collection.name for collection in collections] - collection_list = ['/'] - for collection in store.Collections(): - # write out entries for collections - tar_head = PAXHeader(collection.name) - tar_head.SetKey("BONGO.uid", collection.uid) - tar_head.SetKey("BONGO.type", "%d" % collection.type) - tar_head.SetKey("BONGO.flags", "%d" % collection.flags) - props = docstore.PropGet(collection.uid) - for key in list(props.keys()): - tar_head.SetKey("BONGO.%s" % key, props[key]) - backup_file.write(tar_head.ToString()) + with tarfile.open( + output, "w", format=tarfile.PAX_FORMAT, + encoding="utf-8") as archive: + for collection in collections: + info = tarfile.TarInfo(collection.name.rstrip("/") + "/") + info.type = tarfile.DIRTYPE + info.mode = 0o755 + info.pax_headers = _metadata( + collection, docstore.PropGet(collection.uid)) + archive.addfile(info) - tar_dir = tarfile.TarInfo(collection.name) - tar_dir.size = 0 - tar_dir.type = tarfile.DIRTYPE - tar_dir.mode = 0o755 - backup_file.write(tar_dir.tobuf()) - collection_list.append(collection.name) - - for collection in collection_list: - for document in store.List(collection): - # write out the documents in the collection - if document.type & DocTypes.Folder: - # this is a collection - continue - if document.type == DocTypes.Conversation or document.type == DocTypes.Calendar: - # DB-only file type, don't attempt to back this up - continue - if collection == '/conversations': - # this is a hack; Bongo 0.3 doesn't seem to mark all conversations - # as such correctly :( - continue - - try: - content = docstore.Read(document.filename) - except IOError as e: - print(str(e)) - continue - - # write out meta data associated with document in a PAX extended header - headerx = PAXHeader(document.filename) - headerx.SetKey("BONGO.uid", document.uid) - headerx.SetKey("BONGO.imapuid", document.imapuid) - headerx.SetKey("BONGO.type", "%d" % document.type) - headerx.SetKey("BONGO.flags", "%d" % document.flags) - props = docstore.PropGet(document.uid) - for key in list(props.keys()): - value = props[key] - if value != None: - headerx.SetKey("BONGO.%s" % key, value.encode("utf-8")) - header_content = headerx.ToString() - - backup_file.write(header_content) - - # tar files have a 512 byte header (tar_info.tobuf()), and then the content - # content is padded to a multiple of 512 bytes - tar_info = tarfile.TarInfo(document.filename) - tar_info.size = len(content) - tar_info.mode = 0o644 - tar_info.mtime = int(document.created) - backup_file.write(tar_info.tobuf()) - backup_file.write(content) - space = len(content) % 512 - if space != 0: - backup_file.write("\0" * (512 - space)) - - store.Quit() - docstore.Quit() - - backup_file.write("\0" * 1024) # end of archive marker - backup_file.close() + for collection_name in collection_names: + documents = list(store.List(collection_name)) + for document in documents: + if document.type & DocTypes.Folder: + continue + if (document.type in + (DocTypes.Conversation, DocTypes.Calendar) or + collection_name == "/conversations"): + continue + content = docstore.Read(document.uid) + if isinstance(content, str): + content = content.encode("utf-8") + info = tarfile.TarInfo( + _path(collection_name, document.filename)) + info.size = len(content) + info.mode = 0o600 + info.mtime = int(document.created) + info.pax_headers = _metadata( + document, docstore.PropGet(document.uid)) + archive.addfile(info, io.BytesIO(content)) + finally: + try: + store.Quit() + finally: + docstore.Quit() def Run(self, options, args): - if len(args) == 1 and args[0] == "help": + if len(args) > 1: self.print_help() self.exit() + output = args[0] if args else "%s.backup" % options.store + self.BackupStore(options, output) + print("Backup written to %s" % output) - self.host = options.host - self.port = options.port - self.user = options.user - self.password = options.password - self.store = options.store - self.BackupStore() class StoreRestoreCommand(Command): - log = logging.getLogger("Bongo.StoreTool") - def __init__(self): - Command.__init__(self, "store-restore", aliases=["sr"], - summary="Restore a backup of a user store", - usage="%prog %cmd []") + Command.__init__( + self, "store-restore", aliases=["sr"], + summary="Replace a user store from a validated PAX backup", + usage="%prog %cmd []") - def RestoreStore(self, store, backupfile): - # restore the contents of a backup file to the store we're set to - while True: - # read the tar header, check it's sane - header = backupfile.read(512) - if len(header) != 512: - return + def _connection(self, options): + return StoreClient( + options.user, options.store, host=options.host, + port=int(options.port), authPassword=options.password) - # look at the file metadata, check it's sane - try: - extheader = tarfile.TarInfo.frombuf(header) - except: - # probably the end of the file - return - - if extheader.type != "x": - print("No metadata header found (%s)" % extheader.name) - continue - - metadata = PAXHeader(None) - metadata.FromString(backupfile.read(extheader.size)) - remainder = extheader.size % 512 - if remainder != 0: - backupfile.read(512 - remainder) - - header = backupfile.read(512) - if len(header) != 512: - return - - document = tarfile.TarInfo.frombuf(header) - if document.name == None: - return # done reading from archive? - - if document.size == 0: - if document.type == tarfile.DIRTYPE: - # this is a collection - self.RestoreCollection(store, document.name, metadata.Keys()) - continue - - # pull out file contents, and any padding - content = backupfile.read(document.size) - remainder = document.size % 512 - if remainder != 0: - backupfile.read(512 - remainder) - - # restore the file - self.RestoreFile(store, document.name, content, metadata.Keys()) - - def RestoreProps(self, store, guid, metadata): - del metadata["BONGO.nmap.guid"] - del metadata["BONGO.nmap.type"] - del metadata["BONGO.nmap.flags"] - del metadata["BONGO.nmap.collection"] - del metadata["BONGO.nmap.index"] - if "BONGO.uid" in metadata: - del metadata["BONGO.uid"] - if "BONGO.type" in metadata: - del metadata["BONGO.type"] - if "BONGO.flags" in metadata: - del metadata["BONGO.flags"] - if "BONGO.imapuid" in metadata: - del metadata["BONGO.imapuid"] - for key, value in list(metadata.items()): - if key[0:6] == "BONGO.": - prop = key[6:] - try: - store.PropSet(guid, prop, value) - except: - print("Failed to set property %s to %s on %s" % (prop, value, guid)) - - def RestoreFile(self, store, name, content, metadata): - dir_sep = name.rfind("/") - if dir_sep < 3: - return # invalid name - collection = name[0:dir_sep] - filename = name[dir_sep+1:] - file_guid = None - - if "BONGO.uid" in metadata: - file_guid = metadata["BONGO.uid"] - del metadata["BONGO.uid"] - - file_type = 0 - if "BONGO.type" in metadata: - file_type = int(metadata["BONGO.type"]) - del metadata["BONGO.type"] - - try: - new_guid = store.Write(collection, file_type, content, filename=filename, guid=file_guid, noProcess=True) - except: - print("Failed to restore %s/%s" % (collection, filename)) - return - if "BONGO.flags" in metadata: - try: - store.Flag(new_guid, flags=int(metadata["BONGO.flags"])) - del metadata["BONGO.flags"] - except: - print("Couldn't restore mail flag on %s/%s" % (collection, filename)) - self.RestoreProps(store, new_guid, metadata) - - def RestoreCollection(self, store, name, metadata): - collection = name[0:-1] - try: - if "BONGO.uid" in metadata: - store.Create(collection, guid=metadata["BONGO.uid"]) - del metadata["BONGO.uid"] - else: - store.Create(collection) - except: - print("Failed to create collection %s" % collection) - self.RestoreProps(store, collection, metadata) + def RestoreProps(self, store, target, metadata): + for name, value in _properties(metadata).items(): + store.PropSet(target, name, value) def ClearStore(self, store): - collections = [collection.name for collection in store.Collections()] - collections.sort(lambda x, y: len(x)-len(y), reverse=True) - for collection in collections: - store.Remove(collection) + collections = list(store.Collections()) + paths = ["/"] + [collection.name for collection in collections] + for path in paths: + for document in list(store.List(path)): + if not document.type & DocTypes.Folder: + store.Delete(document.uid) + for collection in sorted( + collections, key=lambda item: len(item.name), reverse=True): + store.Remove(collection.name) + + def RestoreStore(self, store, filename): + members = validate_archive(filename) + self.ClearStore(store) + + with tarfile.open(filename, "r:*") as archive: + directories = sorted( + (member for member in members if member.isdir()), + key=lambda member: len(member.name)) + files = [member for member in members if member.isfile()] + + for member in directories: + name = member.name.rstrip("/") + if name == "": + continue + metadata = member.pax_headers + store.Create( + name, guid=metadata.get("BONGO.uid", ""), + existingOk=True) + self.RestoreProps(store, name, metadata) + + pending_links = [] + for member in files: + source = archive.extractfile(member) + if source is None: + raise ValueError("cannot read backup member %s" % member.name) + content = source.read() + collection, separator, filename_part = member.name.rpartition("/") + if not separator or not filename_part: + raise ValueError("invalid document path %s" % member.name) + if collection == "": + collection = "/" + + metadata = member.pax_headers + document_type = int(metadata["BONGO.type"]) + imapuid = metadata.get("BONGO.imapuid") + guid = store.Write( + collection, document_type, content, + filename=filename_part, + index=imapuid if imapuid not in (None, "", "0", "-") else None, + guid=metadata.get("BONGO.uid"), + timeCreated=str(member.mtime), + flags=int(metadata["BONGO.flags"]), + noProcess=True) + properties = _properties(metadata) + calendars = properties.pop("nmap.event.calendars", None) + for name, value in properties.items(): + store.PropSet(guid, name, value) + if document_type == DocTypes.Event and calendars: + pending_links.append((guid, calendars.splitlines())) + + for guid, calendars in pending_links: + for calendar in calendars: + calendar = calendar.strip() + if calendar: + store.Link(calendar, guid) def Run(self, options, args): - if len(args) == 0: + if len(args) > 1: self.print_help() self.exit() - - backup_file = "%s.backup" % options.store - if len(args) == 1: - # user specified a backup file to restore - backup_file = args[0] - - store = StoreClient(options.user, options.store) - - f = None + filename = args[0] if args else "%s.backup" % options.store + validate_archive(filename) + store = self._connection(options) try: - f = open(backup_file, "r") store.Store(options.store) - self.ClearStore(store) - self.RestoreStore(store, f) - except IOError as e: - print(str(e)) - if f: - f.close() - store.Quit() + self.RestoreStore(store, filename) + finally: + store.Quit() + print("Store restored from %s" % filename) diff --git a/src/apps/storetool/bongo/storetool/CalendarCommands.py b/src/apps/storetool/bongo/storetool/CalendarCommands.py index 7c8bee5..621c5d7 100644 --- a/src/apps/storetool/bongo/storetool/CalendarCommands.py +++ b/src/apps/storetool/bongo/storetool/CalendarCommands.py @@ -18,10 +18,14 @@ import bongo.table as table from bongo.cmdparse import Command from bongo.Contact import Contact from bongo.BongoError import BongoError -from libbongo.libs import bongojson, msgapi from bongo.store.StoreClient import DocTypes, StoreClient, CalendarACL from bongo.store.QueueClient import QueueClient + +def _smtp_eols(value): + return value.replace("\r\n", "\n").replace("\r", "\n").replace( + "\n", "\r\n") + class CalendarsCommand(Command): log = logging.getLogger("Bongo.StoreTool") @@ -190,6 +194,8 @@ class CalendarSubscribeCommand(Command): usage="%prog %cmd ") def Run(self, options, args): + from libbongo.libs import msgapi + if len(args) < 2: self.print_usage() self.exit() @@ -207,6 +213,8 @@ class CalendarImportCommand(Command): usage="%prog %cmd ") def Run(self, options, args): + from libbongo.libs import msgapi + if len(args) < 2: self.print_usage() self.exit() @@ -289,7 +297,7 @@ class CalendarShareCommand(Command): queue.Create() queue.StoreFrom(options.user + "@localhost") queue.StoreTo(address, address) - queue.StoreMessage(email.Utils.fix_eols(msg.as_string())) + queue.StoreMessage(_smtp_eols(msg.as_string())) queue.Run() queue.Quit() diff --git a/src/apps/storetool/bongo/storetool/ExportMailbox.py b/src/apps/storetool/bongo/storetool/ExportMailbox.py index 54c8409..8453b2d 100644 --- a/src/apps/storetool/bongo/storetool/ExportMailbox.py +++ b/src/apps/storetool/bongo/storetool/ExportMailbox.py @@ -1,113 +1,38 @@ -import os -import re -import socket -import errno -import io import mailbox import email -from email.generator import Generator -from email.utils import parsedate_tz, mktime_tz -import sys -from time import time, mktime, strptime, ctime, sleep + +def _message(raw_message): + if isinstance(raw_message, str): + raw_message = raw_message.encode("utf-8") + return email.message_from_bytes(raw_message) + + class MboxMailbox: - def __init__(self,file): - self.boxname = file - self.content = "" + def __init__(self, filename): + self.mailbox = mailbox.mbox(filename, create=True) + self.mailbox.lock() + self.mailbox.clear() - def add(self, msg): - fp = io.StringIO() - g = Generator(fp, mangle_from_=False) - envelope = email.message_from_string(msg) - g.flatten(envelope, unixfrom=True) - self.content += "%s" % fp.getvalue() + def add(self, message): + self.mailbox.add(_message(message)) def write(self): - with open(self.boxname, "w", encoding="utf-8", newline="") as outfile: - outfile.write("%s\n" % self.content) - -class MaildirMailbox(mailbox.Maildir): - def __init__(self,file): - if file[-1] == "/": - self.boxname = file - else: - self.boxname = file + "/" - - for directory in ("cur", "new", "tmp"): - os.makedirs(os.path.join(self.boxname, directory), exist_ok=True) - mailbox.Maildir.__init__(self, file, email.message_from_file) - - def tmp_open(self): - hostname = socket.gethostname() - pid = os.getpid() - while 1: - name = "tmp/%.6f%05d.%s" % (time(), pid, hostname) - try: - os.stat(name) - except OSError as err: - if err.errno == errno.ENOENT: - break - - fd = os.open(name, os.O_WRONLY|os.O_EXCL|os.O_CREAT, 0o600) - return (name, fd) - - - def status(self, message): - status = message.get("Status") - if status == "O": - dir = "cur" - info = "" - elif status == "RO": - dir = "cur" - info = ":2,S" - else: - dir = "new" - info = "" - - return (dir, info) - - def get_delivery_time(self, message): - dtime = None - if "Date" in message: - dtime = mktime_tz(parsedate_tz(message["Date"])) - - return dtime - - def write_message(self, raw_msg, fd, tmp): try: - if isinstance(raw_msg, str): - raw_msg = raw_msg.encode("utf-8") - os.write(fd, raw_msg) - os.fsync(fd) - os.close(fd) - except OSError as err: - os.unlink(tmp) - raise OSError("unable to fsync() or close() temp file: %s" % err) - - - def finish_message(self, tmp, dir, info, dtime): - base_name = os.path.basename(tmp) - dst_name = os.path.join(dir, base_name + info) - os.link(tmp, dst_name) - - if dtime is not None: - atime = os.stat(dst_name).st_atime - os.utime(dst_name, (atime, dtime)) - - return dst_name - - def add(self, raw_msg): - start_dir = os.getcwd() - os.chdir(self.boxname) - (tmp, fd) = self.tmp_open() - - try: - fp = io.StringIO(raw_msg) - message = email.message_from_file(fp) - (dir, info) = self.status(message) - dtime = self.get_delivery_time(message) - self.write_message(raw_msg, fd, tmp) - dst_name = self.finish_message(tmp, dir, info, dtime) + self.mailbox.flush() finally: - os.unlink(tmp) - os.chdir(start_dir) + self.mailbox.unlock() + self.mailbox.close() + + +class MaildirMailbox: + def __init__(self, filename): + self.mailbox = mailbox.Maildir(filename, create=True) + self.mailbox.clear() + + def add(self, message): + self.mailbox.add(_message(message)) + + def write(self): + self.mailbox.flush() + self.mailbox.close() diff --git a/src/apps/storetool/bongo/storetool/MailCommands.py b/src/apps/storetool/bongo/storetool/MailCommands.py index d2074f0..e6bf8ef 100644 --- a/src/apps/storetool/bongo/storetool/MailCommands.py +++ b/src/apps/storetool/bongo/storetool/MailCommands.py @@ -1,13 +1,14 @@ import email import logging import mailbox +import os import sys +import tempfile from gettext import gettext as _ from bongo.cmdparse import Command from bongo.BongoError import BongoError -from libbongo.libs import bongojson, msgapi from bongo.store.StoreClient import DocTypes, StoreClient from bongo.storetool.ExportMailbox import MboxMailbox, MaildirMailbox @@ -26,26 +27,29 @@ class MailImportCommand(Command): collection = "/mail/" + folder store.Create(collection, existingOk=True) + temporary_name = None if type == "mbox": if file == "-": - import tempfile - fd = tempfile.TemporaryFile(mode="w+b") - try: - for line in sys.stdin: fd.write(line) - except StopIteration: pass - fd.seek(0) + with tempfile.NamedTemporaryFile(delete=False) as temporary: + temporary.write(sys.stdin.buffer.read()) + temporary_name = temporary.name + source = temporary_name else: - fd = open(file) - - mailstore = mailbox.PortableUnixMailbox(fd, email.message_from_file) + source = file + mailstore = mailbox.mbox(source, create=False) elif type == "maildir": - mailstore = mailbox.Maildir(file, email.message_from_file) + mailstore = mailbox.Maildir(file, create=False) + else: + raise ValueError("mail source type must be mbox or maildir") - for msg in mailstore: - store.Write(collection, DocTypes.Mail, msg.as_string(True)) - - if fd: - fd.close() + try: + for key in mailstore.iterkeys(): + content = mailstore.get_bytes(key, from_=False) + store.Write(collection, DocTypes.Mail, content) + finally: + mailstore.close() + if temporary_name is not None: + os.unlink(temporary_name) def Run(self, options, args): if len(args) == 0: @@ -94,6 +98,9 @@ class MailExportCommand(Command): continue msg = store.Read(mail.props["nmap.guid"]) maildir.add(msg) + maildir.write() + else: + raise ValueError("mail destination type must be mbox or maildir") def Run(self, options, args): diff --git a/src/apps/storetool/bongo/storetool/StoreCommands.py b/src/apps/storetool/bongo/storetool/StoreCommands.py index d2adf79..8035c74 100644 --- a/src/apps/storetool/bongo/storetool/StoreCommands.py +++ b/src/apps/storetool/bongo/storetool/StoreCommands.py @@ -1,12 +1,7 @@ import logging -import string import sys -from gettext import gettext as _ -import tarfile - from bongo.cmdparse import Command -from bongo.BongoError import BongoError from bongo.store.StoreClient import DocTypes, StoreClient @@ -112,7 +107,7 @@ class DocumentPutCommand(Command): try: store.Store(options.store) - f = open(filename, "r") + f = open(filename, "rb") content = f.read() try: store.Info(document) diff --git a/src/apps/storetool/tests/test_backup_commands.py b/src/apps/storetool/tests/test_backup_commands.py new file mode 100644 index 0000000..b2805e5 --- /dev/null +++ b/src/apps/storetool/tests/test_backup_commands.py @@ -0,0 +1,80 @@ +# /**************************************************************************** +# * +# * Copyright (c) 2001 Novell, Inc. All Rights Reserved. +# * +# * This program is free software; you can redistribute it and/or +# * modify it under the terms of version 2 of the GNU General Public License +# * as published by the Free Software Foundation. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, contact Novell, Inc. +# * +# * To contact Novell about this file by physical or electronic mail, you +# * may find current contact information at www.novell.com. +# * +# ****************************************************************************/ + +import io +import tarfile +from pathlib import Path +import tempfile +import unittest + +from bongo.storetool.BackupCommands import ( + _metadata, _properties, validate_archive) + + +class _Item: + uid = "0000000000000042" + type = 2 + flags = 10 + imapuid = "7" + + +class BackupArchiveTests(unittest.TestCase): + def test_metadata_round_trip_keeps_user_properties(self): + metadata = _metadata( + _Item(), {"nmap.guid": "generated", "display": "Grüße"}) + self.assertEqual(metadata["BONGO.imapuid"], "7") + self.assertEqual(_properties(metadata), {"display": "Grüße"}) + + def test_standard_pax_archive_is_validated_without_extraction(self): + with tempfile.TemporaryDirectory() as directory: + filename = Path(directory) / "store.backup" + with tarfile.open(filename, "w", format=tarfile.PAX_FORMAT) as archive: + info = tarfile.TarInfo("/mail/INBOX/message.eml") + payload = b"Subject: backup\r\n\r\nbody\r\n" + info.size = len(payload) + info.pax_headers = { + "BONGO.uid": "0000000000000042", + "BONGO.type": "2", + "BONGO.flags": "0", + } + archive.addfile(info, io.BytesIO(payload)) + members = validate_archive(filename) + self.assertEqual([member.name for member in members], + ["/mail/INBOX/message.eml"]) + + def test_unsafe_archive_is_rejected(self): + with tempfile.TemporaryDirectory() as directory: + filename = Path(directory) / "unsafe.backup" + with tarfile.open(filename, "w", format=tarfile.PAX_FORMAT) as archive: + info = tarfile.TarInfo("../escape") + info.size = 0 + info.pax_headers = { + "BONGO.uid": "1", + "BONGO.type": "2", + "BONGO.flags": "0", + } + archive.addfile(info, io.BytesIO()) + with self.assertRaisesRegex(ValueError, "unsafe"): + validate_archive(filename) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/apps/storetool/tests/test_python3_runtime.py b/src/apps/storetool/tests/test_python3_runtime.py new file mode 100644 index 0000000..2931802 --- /dev/null +++ b/src/apps/storetool/tests/test_python3_runtime.py @@ -0,0 +1,120 @@ +# /**************************************************************************** +# * +# * Copyright (c) 2001 Novell, Inc. All Rights Reserved. +# * +# * This program is free software; you can redistribute it and/or +# * modify it under the terms of version 2 of the GNU General Public License +# * as published by the Free Software Foundation. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, contact Novell, Inc. +# * +# * To contact Novell about this file by physical or electronic mail, you +# * may find current contact information at www.novell.com. +# * +# ****************************************************************************/ + +import io +from pathlib import Path +import tempfile +import unittest + +from bongo import table +from bongo.Console import wrap +from bongo.external.simpletal import simpleTAL, simpleTALES +from bongo.store.StoreClient import CalendarACL, StoreClient +from bongo.storetool.ExportMailbox import MaildirMailbox, MboxMailbox + + +class _Response: + def __init__(self, code, message=""): + self.code = code + self.message = message + + +class _Stream: + def __init__(self, responses): + self.responses = list(responses) + self.lines = [] + self.raw = [] + + def Write(self, value): + self.lines.append(value) + + def WriteRaw(self, value): + self.raw.append(value) + + def GetResponse(self): + return self.responses.pop(0) + + +class Python3RuntimeTests(unittest.TestCase): + def test_console_table_and_unicode_template_are_text(self): + self.assertIsInstance(wrap("Grüße 世界", width=8), str) + self.assertEqual(table.format_table(["Name"], [["Müller"]]), + "Name\n------\nMüller") + + context = simpleTALES.Context() + context.addGlobal("name", "A&B") + template = simpleTAL.compileHTMLTemplate( + '

placeholder

') + output = io.StringIO() + template.expand(context, output) + self.assertEqual(output.getvalue(), "

A&B

") + + def test_store_wire_lengths_are_encoded_byte_lengths(self): + client = object.__new__(StoreClient) + client.stream = _Stream([ + _Response(2002), _Response(1000, "guid ok"), + _Response(2002), _Response(1000), + _Response(2002), _Response(1000, "guid ok"), + ]) + + client.Write("/mail/INBOX", 2, "Grüße") + client.PropSet("guid", "display", "Grüße") + client.Replace("guid", "Grüße") + + expected = "Grüße".encode("utf-8") + self.assertIn(" 7", client.stream.lines[0]) + self.assertTrue(client.stream.lines[1].endswith(" 7")) + self.assertTrue(client.stream.lines[2].endswith(" 7")) + self.assertEqual(client.stream.raw, [expected, expected, expected]) + + def test_calendar_share_token_uses_python3_hashlib(self): + token = CalendarACL(None)._GetUid("calendar") + self.assertEqual(len(token), 32) + int(token, 16) + + def test_mailbox_writers_accept_store_bytes(self): + message = ( + b"From: sender@example.test\r\n" + b"To: user@example.test\r\n" + b"Subject: Python 3\r\n\r\nBody\r\n") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + mbox_path = root / "mailbox" + writer = MboxMailbox(str(mbox_path)) + writer.add(message) + writer.write() + self.assertIn(b"Subject: Python 3", mbox_path.read_bytes()) + + maildir_path = root / "maildir" + writer = MaildirMailbox(str(maildir_path)) + writer.add(message) + writer.write() + payloads = [ + path.read_bytes() + for folder in ("new", "cur") + for path in (maildir_path / folder).iterdir() + ] + self.assertEqual(len(payloads), 1) + self.assertIn(b"Subject: Python 3", payloads[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/libs/python/bongo/Console.py b/src/libs/python/bongo/Console.py index 5739eb9..b4be649 100644 --- a/src/libs/python/bongo/Console.py +++ b/src/libs/python/bongo/Console.py @@ -1,4 +1,6 @@ import re +from functools import reduce + rx=re.compile("([\u2e80-\uffff])", re.UNICODE) # find out the terminal width @@ -7,7 +9,7 @@ try: (lines, cols) = struct.unpack('hh', fcntl.ioctl(pty.STDOUT_FILENO, termios.TIOCGWINSZ, - '1234')) + b'1234')) if cols == 0: raise Exception("Terminal doesn't support TIOCGWINSZ") @@ -22,11 +24,16 @@ except: # Unicode-safe line wrapping function # from http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/358117 def wrap(text, width=termwidth, encoding="utf8"): + if isinstance(text, bytes): + text = text.decode(encoding, errors="replace") + else: + text = str(text) + return reduce(lambda line, word, width=width: '%s%s%s' % (line, [' ','\n',''][(len(line)-line.rfind('\n')-1 + len(word.split('\n',1)[0] ) >= width) or line[-1:] == '\0' and 2], word), - rx.sub(r'\1\0 ', str(text,encoding)).split(' ') - ).replace('\0', '').encode(encoding) + rx.sub(r'\1\0 ', text).split(' '), + '').replace('\0', '') diff --git a/src/libs/python/bongo/Privs.py b/src/libs/python/bongo/Privs.py index 21304f0..6d27f5d 100644 --- a/src/libs/python/bongo/Privs.py +++ b/src/libs/python/bongo/Privs.py @@ -21,7 +21,7 @@ ############################################################################# import os, pwd -import Xpl +from bongo import Xpl import logging log = logging.getLogger("bongo.Privs") diff --git a/src/libs/python/bongo/admin/Mdb.py b/src/libs/python/bongo/admin/Mdb.py index 3ed2b5d..fe6bf9d 100644 --- a/src/libs/python/bongo/admin/Mdb.py +++ b/src/libs/python/bongo/admin/Mdb.py @@ -1,5 +1,8 @@ -import Handler, Security, Template, Util, sys, traceback +import sys +import traceback + from bongo import MDB +from . import Handler, Security, Template, Util def GetObjectAttribute(req): objdn = req.fields.getfirst('objdn').value diff --git a/src/libs/python/bongo/external/simpletal/simpleElementTree.py b/src/libs/python/bongo/external/simpletal/simpleElementTree.py index f60de3c..a208032 100644 --- a/src/libs/python/bongo/external/simpletal/simpleElementTree.py +++ b/src/libs/python/bongo/external/simpletal/simpleElementTree.py @@ -40,12 +40,13 @@ Module Dependencies: simpleTALES, elementtree """ -from bongo.external.elementtree import ElementTree -import bongo.external.simpleTALES +from xml.etree import ElementTree -class SimpleElementTreeVar (ElementTree._ElementInterface, simpleTALES.ContextVariable): +from bongo.external.simpletal import simpleTALES + +class SimpleElementTreeVar (ElementTree.Element, simpleTALES.ContextVariable): def __init__(self, tag, attrib): - ElementTree._ElementInterface.__init__(self, tag, attrib) + ElementTree.Element.__init__(self, tag, attrib) simpleTALES.ContextVariable.__init__(self) def value (self, pathInfo = None): @@ -68,20 +69,20 @@ class SimpleElementTreeVar (ElementTree._ElementInterface, simpleTALES.ContextVa activeElement = self.find ("/".join (ourParams [1:])) elif (ourParams [0] == 'findall'): # Short cut this - raise simpleTALES.ContextVariable (self.findall ("/".join (ourParams[1:]))) + return simpleTALES.ContextVariable (self.findall ("/".join (ourParams[1:]))) else: # Assume that we wanted to use find activeElement = self.find ("/".join (ourParams)) # Did we find an element and are we looking for an attribute? if (attributeName is not None and activeElement is not None): attrValue = activeElement.attrib.get (attributeName, None) - raise simpleTALES.ContextVariable (attrValue) + return simpleTALES.ContextVariable (attrValue) # Just return the element if (activeElement is None): # Wrap it - raise simpleTALES.ContextVariable (None) - raise activeElement + return simpleTALES.ContextVariable (None) + return activeElement else: return self @@ -93,7 +94,7 @@ class SimpleElementTreeVar (ElementTree._ElementInterface, simpleTALES.ContextVa def parseFile (file): treeBuilder = ElementTree.TreeBuilder (element_factory = SimpleElementTreeVar) - xmlTreeBuilder = ElementTree.XMLTreeBuilder (target=treeBuilder) + xmlTreeBuilder = ElementTree.XMLParser (target=treeBuilder) if (not hasattr (file, 'read')): ourFile = open (file) diff --git a/src/libs/python/bongo/external/simpletal/simpleTAL.py b/src/libs/python/bongo/external/simpletal/simpleTAL.py index 536adfc..1b8c60d 100644 --- a/src/libs/python/bongo/external/simpletal/simpleTAL.py +++ b/src/libs/python/bongo/external/simpletal/simpleTAL.py @@ -39,7 +39,7 @@ try: except: import bongo.external.simpletal.DummyLogger as logging -import xml.sax, cgi, io, codecs, re, types, copy, sys +import xml.sax, html, io, codecs, re, types, copy, sys import bongo.external.simpletal.sgmlentitynames as sgmlentitynames import bongo.external.simpletal as simpletal @@ -156,7 +156,7 @@ class TemplateInterpreter: result.append (' ') result.append (attName) result.append ('="') - result.append (cgi.escape (attValue, quote=1)) + result.append (html.escape (attValue, quote=True)) result.append ('"') if (singletonFlag): result.append (" />") @@ -450,15 +450,15 @@ class TemplateInterpreter: self.file.write (str (resultVal)) else: if (isinstance (resultVal, str)): - self.file.write (cgi.escape (resultVal)) + self.file.write (html.escape (resultVal)) elif (isinstance (resultVal, bytes)): # THIS IS NOT A BUG! # Use Unicode in the Context object if you are not using Ascii - self.file.write (cgi.escape (str (resultVal, 'ascii'))) + self.file.write (html.escape (str (resultVal, 'ascii'))) else: # THIS IS NOT A BUG! # Use Unicode in the Context object if you are not using Ascii - self.file.write (cgi.escape (str (resultVal))) + self.file.write (html.escape (str (resultVal))) if (self.outputTag and not args[1]): # Do NOT output end tag if a singleton with no content @@ -574,7 +574,7 @@ class HTMLTemplateInterpreter (TemplateInterpreter): result.append (' ') result.append (attName) result.append ('="') - result.append (cgi.escape (attValue, quote=1)) + result.append (html.escape (attValue, quote=True)) result.append ('"') if (singletonFlag): result.append (" />") @@ -692,7 +692,10 @@ class HTMLTemplate (Template): # This method must wrap outputFile if required by the encoding, and write out # any template pre-amble (DTD, Encoding, etc) - encodingFile = codecs.lookup (outputEncoding)[3](outputFile, 'replace') + if isinstance (outputFile, io.TextIOBase): + encodingFile = outputFile + else: + encodingFile = codecs.lookup (outputEncoding)[3](outputFile, 'replace') self.expandInline (context, encodingFile, interpreter) def expandInline (self, context, outputFile, interpreter=None): @@ -721,7 +724,10 @@ class XMLTemplate (Template): # any template pre-amble (DTD, Encoding, etc) # Write out the XML prolog - encodingFile = codecs.lookup (outputEncoding)[3](outputFile, 'replace') + if isinstance (outputFile, io.TextIOBase): + encodingFile = outputFile + else: + encodingFile = codecs.lookup (outputEncoding)[3](outputFile, 'replace') if (not suppressXMLDeclaration): if (outputEncoding.lower() != "utf-8"): encodingFile.write ('\n' % outputEncoding.lower()) @@ -807,7 +813,7 @@ class TemplateCompiler: result.append (' ') result.append (attName) result.append ('="') - result.append (cgi.escape (attValue, quote=1)) + result.append (html.escape (attValue, quote=True)) result.append ('"') if (singletonFlag): result.append (" />") @@ -1279,10 +1285,12 @@ class HTMLTemplateCompiler (TemplateCompiler, FixedHTMLParser.HTMLParser): self.log = logging.getLogger ("simpleTAL.HTMLTemplateCompiler") def parseTemplate (self, file, encoding="iso-8859-1", minimizeBooleanAtts = 0): - encodedFile = codecs.lookup (encoding)[2](file, 'replace') self.encoding = encoding self.minimizeBooleanAtts = minimizeBooleanAtts - self.feed (encodedFile.read()) + content = file.read() + if isinstance (content, bytes): + content = content.decode (encoding, errors='replace') + self.feed (content) self.close() def tagAsText (self, xxx_todo_changeme3, singletonFlag=0): @@ -1301,7 +1309,7 @@ class HTMLTemplateCompiler (TemplateCompiler, FixedHTMLParser.HTMLParser): result.append (' ') result.append (attName) result.append ('="') - result.append (cgi.escape (attValue, quote=1)) + result.append (html.escape (attValue, quote=True)) result.append ('"') if (singletonFlag): result.append (" />") @@ -1365,7 +1373,7 @@ class HTMLTemplateCompiler (TemplateCompiler, FixedHTMLParser.HTMLParser): self.popTag ((tag, None)) def handle_data (self, data): - self.parseData (cgi.escape (data)) + self.parseData (html.escape (data)) # These two methods are required so that we expand all character and entity references prior to parsing the template. def handle_charref (self, ref): @@ -1465,7 +1473,7 @@ class XMLTemplateCompiler (TemplateCompiler, xml.sax.handler.ContentHandler, xml def characters (self, data): #self.log.debug ("Recieved Real Data: " + data) # Escape any data we recieve - we don't want any: <&> in there. - self.parseData (cgi.escape (data)) + self.parseData (html.escape (data)) def processingInstruction (self, target, data): self.log.debug ("Recieved processing instruction.") @@ -1484,8 +1492,10 @@ def compileHTMLTemplate (template, inputEncoding="ISO-8859-1", minimizeBooleanAt To use the resulting template object call: template.expand (context, outputFile) """ - if (isinstance (template, bytes) or isinstance (template, str)): + if (isinstance (template, bytes)): # It's a string! + templateFile = io.BytesIO (template) + elif (isinstance (template, str)): templateFile = io.StringIO (template) else: templateFile = template @@ -1500,6 +1510,8 @@ def compileXMLTemplate (template): """ if (isinstance (template, bytes)): # It's a string! + templateFile = io.BytesIO (template) + elif (isinstance (template, str)): templateFile = io.StringIO (template) else: templateFile = template diff --git a/src/libs/python/bongo/external/simpletal/simpleTALUtils.py b/src/libs/python/bongo/external/simpletal/simpleTALUtils.py index 7940f1a..0db8889 100644 --- a/src/libs/python/bongo/external/simpletal/simpleTALUtils.py +++ b/src/libs/python/bongo/external/simpletal/simpleTALUtils.py @@ -34,8 +34,9 @@ Module Dependencies: None """ -import io, os, stat, threading, sys, codecs, sgmllib, cgi, re, types -import bongo.external.simpletal, bongo.external.simpleTAL +import io, os, stat, threading, sys, codecs, sgmllib, html, re, types +from bongo.external import simpletal +from bongo.external.simpletal import simpleTAL __version__ = simpletal.__version__ @@ -82,7 +83,7 @@ class HTMLStructureCleaner (sgmllib.SGMLParser): self.outputFile.write ('') def handle_data (self, data): - self.outputFile.write (cgi.escape (data)) + self.outputFile.write (html.escape (data)) def handle_charref (self, ref): self.outputFile.write ('&#%s;' % ref) @@ -178,7 +179,7 @@ def tagAsText (tag,atts): # We already have some escaped characters in here, so assume it's all valid result += ' %s="%s"' % (name, value) else: - result += ' %s="%s"' % (name, cgi.escape (value)) + result += ' %s="%s"' % (name, html.escape (value)) result += ">" return result @@ -256,14 +257,14 @@ class MacroExpansionInterpreter (simpleTAL.TemplateInterpreter): elif (isinstance (resultVal, bytes)): self.file.write (str (resultVal, 'ascii')) else: - self.file.write (str (str (resultVal), 'ascii')) + self.file.write (str (resultVal)) else: if (isinstance (resultVal, str)): - self.file.write (cgi.escape (resultVal)) + self.file.write (html.escape (resultVal)) elif (isinstance (resultVal, bytes)): - self.file.write (cgi.escape (str (resultVal, 'ascii'))) + self.file.write (html.escape (str (resultVal, 'ascii'))) else: - self.file.write (cgi.escape (str (str (resultVal), 'ascii'))) + self.file.write (html.escape (str (resultVal))) if (self.outputTag and not args[1]): self.file.write ('') @@ -287,4 +288,3 @@ def ExpandMacros (context, template, outputEncoding="ISO-8859-1"): result = out.getvalue() reencoder = codecs.lookup (outputEncoding)[0] return reencoder (result)[0] - diff --git a/src/libs/python/bongo/nmap/NmapClient.py b/src/libs/python/bongo/nmap/NmapClient.py index 78d8bd5..a5604b9 100755 --- a/src/libs/python/bongo/nmap/NmapClient.py +++ b/src/libs/python/bongo/nmap/NmapClient.py @@ -2,7 +2,6 @@ import logging from libbongo.libs import msgapi from .CommandStream import * from .NmapConnection import NmapConnection -from io import StringIO import bongo __all__ = ["DocTypes", @@ -261,4 +260,3 @@ class NmapClient: if r.code != 1000: raise CommandError(r) return r - diff --git a/src/libs/python/bongo/nmap/QueueClient.py b/src/libs/python/bongo/nmap/QueueClient.py index 0c4d0b2..a69078e 100755 --- a/src/libs/python/bongo/nmap/QueueClient.py +++ b/src/libs/python/bongo/nmap/QueueClient.py @@ -1,7 +1,7 @@ import logging import bongo, libbongo.libs from .CommandStream import * -from StoreConnection import StoreConnection +from bongo.store.StoreConnection import StoreConnection class RoutingFlags: NoFlags = 0 diff --git a/src/libs/python/bongo/store/StoreClient.py b/src/libs/python/bongo/store/StoreClient.py index d8f7720..3e59fbe 100644 --- a/src/libs/python/bongo/store/StoreClient.py +++ b/src/libs/python/bongo/store/StoreClient.py @@ -1,14 +1,13 @@ import logging from .CommandStream import * from .StoreConnection import StoreConnection -from io import StringIO +from io import BytesIO import bongo import re import time import random import hashlib import socket -import string class DocTypes: Unknown = 0x0001 @@ -56,28 +55,28 @@ class DocFlags: revNames[flag] = name # some helpful functions + @staticmethod def GetName(flag): return DocFlags.names[flag] - GetName = staticmethod(GetName) + @staticmethod def ByName(name): return DocFlags.revNames[name] - ByName = staticmethod(ByName) + @staticmethod def AllFlags(): return list(DocFlags.names.keys()) - AllFlags = staticmethod(AllFlags) - def GetMask(hash): + @staticmethod + def GetMask(values): flags = mask = 0 - for flag, status in list(hash.items()): + for flag, status in list(values.items()): if status: flags = flags | flag mask = mask | flag return flags, mask - GetMask = staticmethod(GetMask) class FlagMode: Show = 0 @@ -165,7 +164,7 @@ class CalendarACL : # if we can't get a network address, just imagine one a = random.random() * 100000000000000000 data = str(t) + ' ' + str (r) + ' ' + str(a) + ' ' + str(args) - data = hashlib.md5(data).hexdigest() + data = hashlib.md5(data.encode("utf-8")).hexdigest() return data @@ -680,15 +679,10 @@ class StoreClient: # into something approaching utf-8. This is obviously lossy, # and we really don't want to have to do this - need to get the # store as utf-8 clean as possible, really. - def force_utf8(self, str): - out = [] - append = out.append - for ch in str: - if ch < "\200": - append(ch) - else: - append('?') - return string.join(out, "") + def force_utf8(self, value): + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + return str(value) def PropGet(self, doc, name=None): command = "PROPGET %s" % doc @@ -723,7 +717,7 @@ class StoreClient: try: props[key] = str(raw_data, "utf-8") except UnicodeDecodeError as e: - data = self.force_utf8(raw_data) + props[key] = self.force_utf8(raw_data) # eat the \r\n afterward self.stream.Read(2) r = self.stream.GetResponse() @@ -737,20 +731,22 @@ class StoreClient: (key, length) = r.message.split(" ", 2) raw_data = self.stream.Read(int(length)) self.stream.Read(2) - props["nmap.mail.imapuid"] = raw_data + props["nmap.mail.imapuid"] = self.force_utf8(raw_data) return props def PropSet(self, doc, name, value=None): if value is None: value = "" + wire_value = value if isinstance(value, bytes) else value.encode("utf-8") - self.stream.Write("PROPSET %s %s %d" % (doc, name, len(value))) + self.stream.Write("PROPSET %s %s %d" % + (doc, name, len(wire_value))) r = self.stream.GetResponse() if r.code != 2002: raise CommandError(r) - self.stream.WriteRaw(value) + self.stream.WriteRaw(wire_value) r = self.stream.GetResponse() if r.code != 1000: raise CommandError(r) @@ -780,7 +776,7 @@ class StoreClient: s = self.stream.Read(int(length)) # eat the \r\n afterward self.stream.Read(2) - return StringIO(s) + return BytesIO(s) def Remove(self, collection): self.stream.Write("REMOVE %s" % collection) @@ -808,13 +804,14 @@ class StoreClient: raise CommandError(r) def Replace(self, doc, data): - self.stream.Write("REPLACE %s %d" % (doc, len(data))) + wire_data = data if isinstance(data, bytes) else data.encode("utf-8") + self.stream.Write("REPLACE %s %d" % (doc, len(wire_data))) r = self.stream.GetResponse() if r.code != 2002: raise CommandError(r) - self.stream.WriteRaw(data) + self.stream.WriteRaw(wire_data) r = self.stream.GetResponse() (guid, junk) = r.message.split(" ", 1) @@ -877,7 +874,7 @@ class StoreClient: command = command + " G%s" % guid if timeCreated is not None: - command += " T" + timeCreated + command += " T" + str(timeCreated) if flags is not None: command = command + " Z%d" % flags @@ -904,8 +901,10 @@ class StoreClient: def Write(self, collection, type, data, filename=None, index=None, guid=None, timeCreated=None, flags=None, link=None, noProcess=None): if data is None: data = "" + wire_data = data if isinstance(data, bytes) else data.encode("utf-8") - command = "WRITE \"%s\" %d %d" % (collection, type, len(data)) + command = "WRITE \"%s\" %d %d" % ( + collection, type, len(wire_data)) if index is not None: command = command + " I%s" % index @@ -917,7 +916,7 @@ class StoreClient: command = command + " G%s" % guid if timeCreated is not None: - command += " T" + timeCreated + command += " T" + str(timeCreated) if flags is not None: command = command + " Z%d" % flags @@ -935,7 +934,7 @@ class StoreClient: if r.code != 2002: raise CommandError(r) - self.stream.WriteRaw(data) + self.stream.WriteRaw(wire_data) r = self.stream.GetResponse() if r.code != 1000: @@ -943,7 +942,3 @@ class StoreClient: (guid, junk) = r.message.split(" ", 1) return guid - - def Reset(self) : - self.stream.Write("RESET\r\n") - self.stream.GetResponse() diff --git a/src/libs/python/bongo/table.py b/src/libs/python/bongo/table.py index 9d1e062..8f7e8db 100644 --- a/src/libs/python/bongo/table.py +++ b/src/libs/python/bongo/table.py @@ -1,8 +1,6 @@ # some simple code for formatting text tables def format_row(elts, widths, sep=" | "): - import string - row = [] for i in range(len(elts)): if elts[i] is None: @@ -11,7 +9,7 @@ def format_row(elts, widths, sep=" | "): elt = elts[i] row.append("%-*s" % (widths[i], elt)) - return string.join(row, sep) + return sep.join(row) def format_separator(widths): elts = [ "-" * width for width in widths ] @@ -41,5 +39,4 @@ def format_table(labels, rows): else: table.append(format_row(row, widths).rstrip()) - import string - return string.join(table, "\n") + return "\n".join(table)