Complete Python 3 runtime port of installed tools

contrib/testing/python-runtime-check.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+# /****************************************************************************
+#  * <Novell-copyright>
+#  * Copyright (c) 2001 Novell, Inc. All Rights Reserved.
+#  *
+#  * This program is free software; you can redistribute it and/or
+#  * modify it under the terms of version 2 of the GNU General Public License
+#  * as published by the Free Software Foundation.
+#  *
+#  * This program is distributed in the hope that it will be useful,
+#  * but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  * GNU General Public License for more details.
+#  *
+#  * You should have received a copy of the GNU General Public License
+#  * along with this program; if not, contact Novell, Inc.
+#  *
+#  * To contact Novell about this file by physical or electronic mail, you
+#  * may find current contact information at www.novell.com.
+#  * </Novell-copyright>
+#  ****************************************************************************/
+
+set -euo pipefail
+
+temporary=$(mktemp -d "${TMPDIR:-/tmp}/bongo-python-runtime.XXXXXX")
+trap 'rm -rf -- "$temporary"' EXIT
+export PYTHONPYCACHEPREFIX="${temporary}/pycache"
+
+python3 -m compileall -q \
+    "$(python3 -c 'import sysconfig; print(sysconfig.get_path("purelib"))')/bongo" \
+    "$(python3 -c 'import sysconfig; print(sysconfig.get_path("purelib"))')/bongo_web"
+
+python3 - <<'PY'
+import io
+
+from bongo import table
+from bongo.Console import wrap
+from bongo.external.simpletal import simpleTAL, simpleTALES
+from bongo.external.simpletal.simpleElementTree import parseFile
+from bongo.storetool import BackupCommands, CalendarCommands, MailCommands
+
+assert isinstance(wrap("Grüße 世界", width=8), str)
+assert "Müller" in table.format_table(["Name"], [["Müller"]])
+
+context = simpleTALES.Context()
+context.addGlobal("name", "A&B")
+template = simpleTAL.compileHTMLTemplate(
+    '<p tal:content="name">placeholder</p>')
+output = io.StringIO()
+template.expand(context, output)
+assert output.getvalue() == "<p>A&amp;B</p>"
+
+root = parseFile(io.StringIO("<root><child>value</child></root>"))
+assert root.find("child").text == "value"
+PY
+
+bongo-admin --help >"${temporary}/bongo-admin.help"
+bongo-storetool --help >"${temporary}/bongo-storetool.help"
+bongo-queuetool --help >"${temporary}/bongo-queuetool.help"
+bongo-web --help >"${temporary}/bongo-web.help"
+/usr/libexec/bongo/bongo-sa-update --help \
+    >"${temporary}/bongo-sa-update.help"
+
+set +e
+/usr/libexec/bongo/bongo-acme-dns-provider \
+    --config "${temporary}/missing.json" --provider audit --type nsupdate \
+    --operation present --record _acme-challenge.example.test \
+    --value audit --ttl 60 >"${temporary}/bongo-acme.json"
+status=$?
+set -e
+test "${status}" -eq 1
+grep -q '"ok":false' "${temporary}/bongo-acme.json"
+
+echo "PYTHON-3 PASS compileall=installed imports=storetool,template,xml cli=6"

src/apps/admin/bongo/admintool/AgentCommands.py

@@ -63,8 +63,7 @@ class AgentInfo(Command):
             attrs = Util.GetAgentAttributes(mdb, dn)
 
             keys = list(attrs.keys())
-            keys.sort(lambda x,y: cmp(Util.PropertiesPrintable.get(x),
-                                      Util.PropertiesPrintable.get(y)))
+            keys.sort(key=lambda key: Util.PropertiesPrintable.get(key, key))
 
             for k in keys:
                 print(" |->(%s)" % Util.PropertiesPrintable[k])

src/apps/admin/bongo/admintool/DemoCommands.py

@@ -27,6 +27,11 @@ from bongo.admin import Util
 
 from . import MdbUtil
 
+
+def _smtp_eols(value):
+    return value.replace("\r\n", "\n").replace("\r", "\n").replace(
+        "\n", "\r\n")
+
 if os.path.isfile("demo/contacts/list.vcf") :
     demopath = "demo"
 else :
@@ -84,7 +89,7 @@ class DemoCommand(Command):
             data = data.replace("@ADDRESS@", address)
             data = data.replace("@NAME@", name)
             
-            uid = store.Write("/mail/INBOX", DocTypes.Mail, email.Utils.fix_eols(data))
+            uid = store.Write("/mail/INBOX", DocTypes.Mail, _smtp_eols(data))
             if filename in demo["starMail"] :
                 conversation = store.PropGet(uid, "nmap.mail.conversation")
                 print("starring conversation %s" % (conversation))

src/apps/admin/bongo/admintool/ServerCommands.py

@@ -38,8 +38,7 @@ class ServerInfo(Command):
 
             attrs = Util.GetServerAttributes(mdb, server)
             keys = list(attrs.keys())
-            keys.sort(lambda x,y: cmp(Util.PropertiesPrintable.get(x),
-                                      Util.PropertiesPrintable.get(y)))
+            keys.sort(key=lambda key: Util.PropertiesPrintable.get(key, key))
 
             for key in keys:
                 print(" |->(%s)" % Util.PropertiesPrintable[key])

src/apps/admin/bongo/admintool/UserCommands.py

@@ -111,8 +111,7 @@ class UserInfo(Command):
             attrs = Util.GetUserAttributes(mdb, context, username)
 
             keys = list(attrs.keys())
-            keys.sort(lambda x,y: cmp(Util.PropertiesPrintable.get(x),
-                                      Util.PropertiesPrintable.get(y)))
+            keys.sort(key=lambda key: Util.PropertiesPrintable.get(key, key))
 
             for k in keys:
                 print(" |->(%s)" % Util.PropertiesPrintable[k])

src/apps/storetool/CMakeLists.txt

@@ -13,7 +13,7 @@ ${SBIN_INSTALL_DIR})
 if(BUILD_TESTING)
 	add_test(NAME storetool-document-type
 		COMMAND ${CMAKE_COMMAND} -E env
-			"PYTHONPATH=${CMAKE_SOURCE_DIR}/src/libs/python:${CMAKE_CURRENT_SOURCE_DIR}"
+			"PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}:${CMAKE_SOURCE_DIR}/src/libs/python"
 			"PYTHONPYCACHEPREFIX=${CMAKE_CURRENT_BINARY_DIR}/pycache"
 			${Python3_EXECUTABLE} -m unittest discover
 			-s ${CMAKE_CURRENT_SOURCE_DIR}/tests -v)

src/apps/storetool/bongo/__init__.py

@@ -1,7 +1,3 @@
-# THIS FILE SHOULD NOT BE INSTALLED!
+from pkgutil import extend_path
 
-# It exists only to merge the src/[...]/storetool/bongo and src/libs/python/bongo
-# directories in the Bongo source tree together.  In an installed
-# system, they're going to be in the same place.
-
-__path__.append(__path__[0] + "/../../../libs/python/bongo")
+__path__ = extend_path(__path__, __name__)

src/apps/storetool/bongo/storetool/BackupCommands.py

@@ -1,309 +1,281 @@
-import logging
-import string
-import sys
+# /****************************************************************************
+#  * <Novell-copyright>
+#  * Copyright (c) 2001 Novell, Inc. All Rights Reserved.
+#  *
+#  * This program is free software; you can redistribute it and/or
+#  * modify it under the terms of version 2 of the GNU General Public License
+#  * as published by the Free Software Foundation.
+#  *
+#  * This program is distributed in the hope that it will be useful,
+#  * but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  * GNU General Public License for more details.
+#  *
+#  * You should have received a copy of the GNU General Public License
+#  * along with this program; if not, contact Novell, Inc.
+#  *
+#  * To contact Novell about this file by physical or electronic mail, you
+#  * may find current contact information at www.novell.com.
+#  * </Novell-copyright>
+#  ****************************************************************************/
 
-from gettext import gettext as _
+"""Portable PAX backups for a Bongo user Store."""
+
+from __future__ import annotations
+
+import io
+import posixpath
 import tarfile
 
 from bongo.cmdparse import Command
-from bongo.BongoError import BongoError
 from bongo.store.StoreClient import DocTypes, StoreClient
 
-# import/export store contents as tar files
-# ideally we want to farm the tar format stuff itself out into a separate lib.
-# Sadly, the TarFile implementation that comes with Python is mostly useless 
 
-class PAXHeader:
-    def __init__(self, filename):
-        self.keywords = {}
-        self.filename = filename
+_METADATA_PREFIX = "BONGO."
+_GENERATED_PROPERTIES = {
+    "nmap.collection",
+    "nmap.flags",
+    "nmap.guid",
+    "nmap.index",
+    "nmap.type",
+}
 
-    def SetKey(self, key, value):
-        self.keywords[key] = value
 
-    def Keys(self):
-        return self.keywords
+def _text(value):
+    if isinstance(value, bytes):
+        return value.decode("utf-8", errors="replace")
+    return str(value)
 
-    def ToString(self):
-        header = tarfile.TarInfo(self.filename + ".metadata")
-        header.type = "x"
 
-        content = ""
-        for key in list(self.keywords.keys()):
-            if self.keywords[key] != None:
-                content += "%d %s=%s\n" % (len(key) + len(self.keywords[key]) + 5, key, self.keywords[key])
-        header.size = len(content)
+def _path(collection, filename):
+    filename = _text(filename)
+    if filename.startswith("/"):
+        return filename
+    if collection == "/":
+        return "/" + filename
+    return collection.rstrip("/") + "/" + filename
 
-        result = header.tobuf()
-        result += content
-        remainder = len(content) % 512
-        if remainder != 0:
-            result += "\0" * (512 - remainder)
 
-        return result
+def _metadata(item, properties):
+    result = {
+        "BONGO.uid": _text(item.uid),
+        "BONGO.type": str(int(item.type)),
+        "BONGO.flags": str(int(item.flags)),
+    }
+    imapuid = getattr(item, "imapuid", None)
+    if imapuid not in (None, "", "-", 0, "0"):
+        result["BONGO.imapuid"] = _text(imapuid)
+    for key, value in properties.items():
+        if value is not None:
+            result[_METADATA_PREFIX + _text(key)] = _text(value)
+    return result
+
+
+def _properties(metadata):
+    result = {}
+    for key, value in metadata.items():
+        if not key.startswith(_METADATA_PREFIX):
+            continue
+        name = key[len(_METADATA_PREFIX):]
+        if name in {"uid", "type", "flags", "imapuid"}:
+            continue
+        if name in _GENERATED_PROPERTIES:
+            continue
+        result[name] = value
+    return result
+
+
+def _safe_member(member):
+    name = member.name
+    if (not name.startswith("/") or "\0" in name or
+            posixpath.normpath(name) != name or
+            member.issym() or member.islnk() or
+            not (member.isdir() or member.isfile())):
+        raise ValueError("unsafe or unsupported backup member %r" % name)
+    metadata = member.pax_headers
+    for required in ("BONGO.uid", "BONGO.type", "BONGO.flags"):
+        if required not in metadata:
+            raise ValueError("%s has no %s metadata" % (name, required))
+    int(metadata["BONGO.type"])
+    int(metadata["BONGO.flags"])
+
+
+def validate_archive(filename):
+    """Return validated members without extracting anything."""
+    with tarfile.open(filename, "r:*") as archive:
+        members = archive.getmembers()
+        if not members:
+            raise ValueError("backup archive is empty")
+        names = set()
+        for member in members:
+            _safe_member(member)
+            if member.name in names:
+                raise ValueError("duplicate backup member %s" % member.name)
+            names.add(member.name)
+        return members
 
-    def FromString(self, content):
-        for line in string.split(content, "\n"):
-            space = string.find(line, " ")
-            equals = string.find(line, "=")
-            key = line[space+1:equals]
-            value = line[equals+1:]
-            self.keywords[key] = value
 
 class StoreBackupCommand(Command):
-    log = logging.getLogger("Bongo.StoreTool")
-    user = "admin"
-    store = "_system"
-
     def __init__(self):
-        Command.__init__(self, "store-backup", aliases=["sb"],
-                         summary="Create a backup of a store",
-                         usage="%prog %cmd [<output>]")
+        Command.__init__(
+            self, "store-backup", aliases=["sb"],
+            summary="Create a PAX backup of a user store",
+            usage="%prog %cmd [<output>]")
 
-    def GetStoreConnection(self):
-        return StoreClient(self.user, self.store, host=self.host, port=int(self.port), authPassword=self.password)
-
-    def BackupStore(self):
-        # create a backup file which contains all the contents of the given store
-        store = self.GetStoreConnection()
-        docstore = self.GetStoreConnection()
+    def _connection(self, options):
+        return StoreClient(
+            options.user, options.store, host=options.host,
+            port=int(options.port), authPassword=options.password)
 
+    def BackupStore(self, options, output):
+        store = self._connection(options)
+        docstore = self._connection(options)
         try:
-            backup_file = open("%s.backup" % self.store, "wb")
-        except IOError as e:
-            print(str(e))
-            return
+            store.Store(options.store)
+            docstore.Store(options.store)
+            collections = list(store.Collections())
+            collection_names = ["/"] + [
+                collection.name for collection in collections]
 
-        collection_list = ['/']
-        for collection in store.Collections():
-            # write out entries for collections
-            tar_head = PAXHeader(collection.name)
-            tar_head.SetKey("BONGO.uid", collection.uid)
-            tar_head.SetKey("BONGO.type", "%d" % collection.type)
-            tar_head.SetKey("BONGO.flags", "%d" % collection.flags)
-            props = docstore.PropGet(collection.uid)
-            for key in list(props.keys()):
-                tar_head.SetKey("BONGO.%s" % key, props[key])
-            backup_file.write(tar_head.ToString())
+            with tarfile.open(
+                    output, "w", format=tarfile.PAX_FORMAT,
+                    encoding="utf-8") as archive:
+                for collection in collections:
+                    info = tarfile.TarInfo(collection.name.rstrip("/") + "/")
+                    info.type = tarfile.DIRTYPE
+                    info.mode = 0o755
+                    info.pax_headers = _metadata(
+                        collection, docstore.PropGet(collection.uid))
+                    archive.addfile(info)
 
-            tar_dir = tarfile.TarInfo(collection.name)
-            tar_dir.size = 0
-            tar_dir.type = tarfile.DIRTYPE
-            tar_dir.mode = 0o755
-            backup_file.write(tar_dir.tobuf())
-            collection_list.append(collection.name)
-
-        for collection in collection_list:
-            for document in store.List(collection):
-                # write out the documents in the collection
-                if document.type & DocTypes.Folder:
-                    # this is a collection
-                    continue
-                if document.type == DocTypes.Conversation or document.type == DocTypes.Calendar:
-                    # DB-only file type, don't attempt to back this up
-                    continue
-                if collection == '/conversations':
-                    # this is a hack; Bongo 0.3 doesn't seem to mark all conversations
-                    # as such correctly :(
-                    continue
-                
-                try:
-                    content = docstore.Read(document.filename)
-                except IOError as e:
-                    print(str(e))
-                    continue
-               
-                # write out meta data associated with document in a PAX extended header
-                headerx = PAXHeader(document.filename)
-                headerx.SetKey("BONGO.uid", document.uid)
-                headerx.SetKey("BONGO.imapuid", document.imapuid)
-                headerx.SetKey("BONGO.type", "%d" % document.type)
-                headerx.SetKey("BONGO.flags", "%d" % document.flags)
-                props = docstore.PropGet(document.uid)
-                for key in list(props.keys()):
-                    value = props[key]
-                    if value != None:
-                        headerx.SetKey("BONGO.%s" % key, value.encode("utf-8"))
-                header_content = headerx.ToString()
-                
-                backup_file.write(header_content)
-
-                # tar files have a 512 byte header (tar_info.tobuf()), and then the content
-                # content is padded to a multiple of 512 bytes
-                tar_info = tarfile.TarInfo(document.filename)
-                tar_info.size = len(content)
-                tar_info.mode = 0o644
-                tar_info.mtime = int(document.created)
-                backup_file.write(tar_info.tobuf())
-                backup_file.write(content)
-                space = len(content) % 512
-                if space != 0:
-                    backup_file.write("\0" * (512 - space))
-
-        store.Quit()
-        docstore.Quit()
-
-        backup_file.write("\0" * 1024) # end of archive marker
-        backup_file.close()
+                for collection_name in collection_names:
+                    documents = list(store.List(collection_name))
+                    for document in documents:
+                        if document.type & DocTypes.Folder:
+                            continue
+                        if (document.type in
+                                (DocTypes.Conversation, DocTypes.Calendar) or
+                                collection_name == "/conversations"):
+                            continue
+                        content = docstore.Read(document.uid)
+                        if isinstance(content, str):
+                            content = content.encode("utf-8")
+                        info = tarfile.TarInfo(
+                            _path(collection_name, document.filename))
+                        info.size = len(content)
+                        info.mode = 0o600
+                        info.mtime = int(document.created)
+                        info.pax_headers = _metadata(
+                            document, docstore.PropGet(document.uid))
+                        archive.addfile(info, io.BytesIO(content))
+        finally:
+            try:
+                store.Quit()
+            finally:
+                docstore.Quit()
 
     def Run(self, options, args):
-        if len(args) == 1 and args[0] == "help":
+        if len(args) > 1:
             self.print_help()
             self.exit()
+        output = args[0] if args else "%s.backup" % options.store
+        self.BackupStore(options, output)
+        print("Backup written to %s" % output)
 
-        self.host = options.host
-        self.port = options.port
-        self.user = options.user
-        self.password = options.password
-        self.store = options.store
-        self.BackupStore()
 
 class StoreRestoreCommand(Command):
-    log = logging.getLogger("Bongo.StoreTool")
-
     def __init__(self):
-        Command.__init__(self, "store-restore", aliases=["sr"],
-                         summary="Restore a backup of a user store",
-                         usage="%prog %cmd <store> [<backup>]")
+        Command.__init__(
+            self, "store-restore", aliases=["sr"],
+            summary="Replace a user store from a validated PAX backup",
+            usage="%prog %cmd [<backup>]")
 
-    def RestoreStore(self, store, backupfile):
-        # restore the contents of a backup file to the store we're set to
-        while True:
-            # read the tar header, check it's sane
-            header = backupfile.read(512)
-            if len(header) != 512:
-                return
+    def _connection(self, options):
+        return StoreClient(
+            options.user, options.store, host=options.host,
+            port=int(options.port), authPassword=options.password)
 
-            # look at the file metadata, check it's sane
-            try:
-                extheader = tarfile.TarInfo.frombuf(header)
-            except:
-                # probably the end of the file
-                return
-
-            if extheader.type != "x": 
-                print("No metadata header found (%s)" % extheader.name)
-                continue
-
-            metadata = PAXHeader(None)
-            metadata.FromString(backupfile.read(extheader.size))
-            remainder = extheader.size % 512
-            if remainder != 0:
-                backupfile.read(512 - remainder)
-
-            header = backupfile.read(512)
-            if len(header) != 512:
-                return
-
-            document = tarfile.TarInfo.frombuf(header)
-            if document.name == None:
-                return # done reading from archive?
-
-            if document.size == 0:
-                if document.type == tarfile.DIRTYPE:
-                    # this is a collection
-                    self.RestoreCollection(store, document.name, metadata.Keys())
-                continue
-
-            # pull out file contents, and any padding
-            content = backupfile.read(document.size)
-            remainder = document.size % 512
-            if remainder != 0:
-                backupfile.read(512 - remainder) 
-
-            # restore the file
-            self.RestoreFile(store, document.name, content, metadata.Keys())
-
-    def RestoreProps(self, store, guid, metadata):
-        del metadata["BONGO.nmap.guid"]
-        del metadata["BONGO.nmap.type"]
-        del metadata["BONGO.nmap.flags"]
-        del metadata["BONGO.nmap.collection"]
-        del metadata["BONGO.nmap.index"]
-        if "BONGO.uid" in metadata:
-            del metadata["BONGO.uid"]
-        if "BONGO.type" in metadata:
-            del metadata["BONGO.type"]
-        if "BONGO.flags" in metadata:
-            del metadata["BONGO.flags"]
-        if "BONGO.imapuid" in metadata:
-            del metadata["BONGO.imapuid"]
-        for key, value in list(metadata.items()):
-            if key[0:6] == "BONGO.":
-                prop = key[6:]
-                try:
-                    store.PropSet(guid, prop, value)
-                except:
-                    print("Failed to set property %s to %s on %s" % (prop, value, guid))
-
-    def RestoreFile(self, store, name, content, metadata):
-        dir_sep = name.rfind("/")
-        if dir_sep < 3:
-            return # invalid name
-        collection = name[0:dir_sep]
-        filename = name[dir_sep+1:]
-        file_guid = None
-
-        if "BONGO.uid" in metadata:
-            file_guid = metadata["BONGO.uid"]
-            del metadata["BONGO.uid"]
-            
-        file_type = 0
-        if "BONGO.type" in metadata:
-            file_type = int(metadata["BONGO.type"])
-            del metadata["BONGO.type"]
-
-        try:
-            new_guid = store.Write(collection, file_type, content, filename=filename, guid=file_guid, noProcess=True)
-        except: 
-            print("Failed to restore %s/%s" % (collection, filename))
-            return
-        if "BONGO.flags" in metadata:
-            try:
-                store.Flag(new_guid, flags=int(metadata["BONGO.flags"]))
-                del metadata["BONGO.flags"]
-            except:
-                print("Couldn't restore mail flag on %s/%s" % (collection, filename))
-        self.RestoreProps(store, new_guid, metadata)
-
-    def RestoreCollection(self, store, name, metadata):
-        collection = name[0:-1]
-        try:
-            if "BONGO.uid" in metadata:
-                store.Create(collection, guid=metadata["BONGO.uid"])
-                del metadata["BONGO.uid"]
-            else:
-                store.Create(collection)
-        except:
-            print("Failed to create collection %s" % collection)
-        self.RestoreProps(store, collection, metadata)
+    def RestoreProps(self, store, target, metadata):
+        for name, value in _properties(metadata).items():
+            store.PropSet(target, name, value)
 
     def ClearStore(self, store):
-        collections = [collection.name for collection in store.Collections()]
-        collections.sort(lambda x, y: len(x)-len(y), reverse=True)
-        for collection in collections:
-             store.Remove(collection)
+        collections = list(store.Collections())
+        paths = ["/"] + [collection.name for collection in collections]
+        for path in paths:
+            for document in list(store.List(path)):
+                if not document.type & DocTypes.Folder:
+                    store.Delete(document.uid)
+        for collection in sorted(
+                collections, key=lambda item: len(item.name), reverse=True):
+            store.Remove(collection.name)
+
+    def RestoreStore(self, store, filename):
+        members = validate_archive(filename)
+        self.ClearStore(store)
+
+        with tarfile.open(filename, "r:*") as archive:
+            directories = sorted(
+                (member for member in members if member.isdir()),
+                key=lambda member: len(member.name))
+            files = [member for member in members if member.isfile()]
+
+            for member in directories:
+                name = member.name.rstrip("/")
+                if name == "":
+                    continue
+                metadata = member.pax_headers
+                store.Create(
+                    name, guid=metadata.get("BONGO.uid", ""),
+                    existingOk=True)
+                self.RestoreProps(store, name, metadata)
+
+            pending_links = []
+            for member in files:
+                source = archive.extractfile(member)
+                if source is None:
+                    raise ValueError("cannot read backup member %s" % member.name)
+                content = source.read()
+                collection, separator, filename_part = member.name.rpartition("/")
+                if not separator or not filename_part:
+                    raise ValueError("invalid document path %s" % member.name)
+                if collection == "":
+                    collection = "/"
+
+                metadata = member.pax_headers
+                document_type = int(metadata["BONGO.type"])
+                imapuid = metadata.get("BONGO.imapuid")
+                guid = store.Write(
+                    collection, document_type, content,
+                    filename=filename_part,
+                    index=imapuid if imapuid not in (None, "", "0", "-") else None,
+                    guid=metadata.get("BONGO.uid"),
+                    timeCreated=str(member.mtime),
+                    flags=int(metadata["BONGO.flags"]),
+                    noProcess=True)
+                properties = _properties(metadata)
+                calendars = properties.pop("nmap.event.calendars", None)
+                for name, value in properties.items():
+                    store.PropSet(guid, name, value)
+                if document_type == DocTypes.Event and calendars:
+                    pending_links.append((guid, calendars.splitlines()))
+
+            for guid, calendars in pending_links:
+                for calendar in calendars:
+                    calendar = calendar.strip()
+                    if calendar:
+                        store.Link(calendar, guid)
 
     def Run(self, options, args):
-        if len(args) == 0:
+        if len(args) > 1:
             self.print_help()
             self.exit()
-
-        backup_file = "%s.backup" % options.store
-        if len(args) == 1:
-            # user specified a backup file to restore
-            backup_file = args[0]
-
-        store = StoreClient(options.user, options.store)
-
-        f = None
+        filename = args[0] if args else "%s.backup" % options.store
+        validate_archive(filename)
+        store = self._connection(options)
         try:
-            f = open(backup_file, "r")
             store.Store(options.store)
-            self.ClearStore(store)
-            self.RestoreStore(store, f)
-        except IOError as e:
-            print(str(e))
-        if f:
-            f.close()
-        store.Quit()
+            self.RestoreStore(store, filename)
+        finally:
+            store.Quit()
+        print("Store restored from %s" % filename)

src/apps/storetool/bongo/storetool/CalendarCommands.py

@@ -18,10 +18,14 @@ import bongo.table as table
 from bongo.cmdparse import Command
 from bongo.Contact import Contact
 from bongo.BongoError import BongoError
-from libbongo.libs import bongojson, msgapi
 from bongo.store.StoreClient import DocTypes, StoreClient, CalendarACL
 from bongo.store.QueueClient import QueueClient
 
+
+def _smtp_eols(value):
+    return value.replace("\r\n", "\n").replace("\r", "\n").replace(
+        "\n", "\r\n")
+
 class CalendarsCommand(Command):
     log = logging.getLogger("Bongo.StoreTool")
 
@@ -190,6 +194,8 @@ class CalendarSubscribeCommand(Command):
                          usage="%prog %cmd <name> <url>")
 
     def Run(self, options, args):
+        from libbongo.libs import msgapi
+
         if len(args) < 2:
             self.print_usage()
             self.exit()
@@ -207,6 +213,8 @@ class CalendarImportCommand(Command):
                          usage="%prog %cmd <name> <file>")
 
     def Run(self, options, args):
+        from libbongo.libs import msgapi
+
         if len(args) < 2:
             self.print_usage()
             self.exit()
@@ -289,7 +297,7 @@ class CalendarShareCommand(Command):
         queue.Create()
         queue.StoreFrom(options.user + "@localhost")
         queue.StoreTo(address, address)
-        queue.StoreMessage(email.Utils.fix_eols(msg.as_string()))
+        queue.StoreMessage(_smtp_eols(msg.as_string()))
         queue.Run()
         queue.Quit()
 

src/apps/storetool/bongo/storetool/ExportMailbox.py

@@ -1,113 +1,38 @@
-import os
-import re
-import socket
-import errno
-import io
 import mailbox
 import email
-from email.generator import Generator
-from email.utils import parsedate_tz, mktime_tz
-import sys
 
-from time import time, mktime, strptime, ctime, sleep
+
+def _message(raw_message):
+    if isinstance(raw_message, str):
+        raw_message = raw_message.encode("utf-8")
+    return email.message_from_bytes(raw_message)
+
+
 class MboxMailbox:
-    def __init__(self,file):
-        self.boxname = file
-        self.content = ""
+    def __init__(self, filename):
+        self.mailbox = mailbox.mbox(filename, create=True)
+        self.mailbox.lock()
+        self.mailbox.clear()
 
-    def add(self, msg):
-        fp = io.StringIO()
-        g = Generator(fp, mangle_from_=False)
-        envelope = email.message_from_string(msg)
-        g.flatten(envelope, unixfrom=True)
-        self.content += "%s" % fp.getvalue()
+    def add(self, message):
+        self.mailbox.add(_message(message))
 
     def write(self):
-        with open(self.boxname, "w", encoding="utf-8", newline="") as outfile:
-            outfile.write("%s\n" % self.content)
-
-class MaildirMailbox(mailbox.Maildir):
-    def __init__(self,file):
-        if file[-1] == "/":
-            self.boxname = file
-        else:
-            self.boxname = file + "/"
-
-        for directory in ("cur", "new", "tmp"):
-            os.makedirs(os.path.join(self.boxname, directory), exist_ok=True)
-        mailbox.Maildir.__init__(self, file, email.message_from_file)
-
-    def tmp_open(self):
-        hostname = socket.gethostname()
-        pid = os.getpid()
-        while 1:
-            name = "tmp/%.6f%05d.%s" % (time(), pid, hostname)
-            try:
-                os.stat(name)
-            except OSError as err:
-                if err.errno == errno.ENOENT:
-                    break
-
-        fd = os.open(name, os.O_WRONLY|os.O_EXCL|os.O_CREAT, 0o600)
-        return (name, fd)
-
-
-    def status(self, message):
-        status = message.get("Status")
-        if status == "O":
-            dir = "cur"
-            info = ""
-        elif status == "RO":
-            dir = "cur"
-            info = ":2,S"
-        else:
-            dir = "new"
-            info = ""
-
-        return (dir, info)
-
-    def get_delivery_time(self, message):
-        dtime = None
-        if "Date" in message:
-            dtime = mktime_tz(parsedate_tz(message["Date"]))
-
-        return dtime
-
-    def write_message(self, raw_msg, fd, tmp):
         try:
-            if isinstance(raw_msg, str):
-                raw_msg = raw_msg.encode("utf-8")
-            os.write(fd, raw_msg)
-            os.fsync(fd)
-            os.close(fd)
-        except OSError as err:
-            os.unlink(tmp)
-            raise OSError("unable to fsync() or close() temp file: %s" % err)
-
-
-    def finish_message(self, tmp, dir, info, dtime):
-        base_name = os.path.basename(tmp)
-        dst_name = os.path.join(dir, base_name + info)
-        os.link(tmp, dst_name)
-
-        if dtime is not None:
-            atime = os.stat(dst_name).st_atime
-            os.utime(dst_name, (atime, dtime))
-
-        return dst_name
-
-    def add(self, raw_msg):
-        start_dir = os.getcwd()
-        os.chdir(self.boxname)
-        (tmp, fd) = self.tmp_open()
-
-        try:
-            fp = io.StringIO(raw_msg)
-            message = email.message_from_file(fp)
-            (dir, info) = self.status(message)
-            dtime = self.get_delivery_time(message)
-            self.write_message(raw_msg, fd, tmp)
-            dst_name = self.finish_message(tmp, dir, info, dtime)
+            self.mailbox.flush()
         finally:
-            os.unlink(tmp)
-            os.chdir(start_dir)
+            self.mailbox.unlock()
+            self.mailbox.close()
+
+
+class MaildirMailbox:
+    def __init__(self, filename):
+        self.mailbox = mailbox.Maildir(filename, create=True)
+        self.mailbox.clear()
+
+    def add(self, message):
+        self.mailbox.add(_message(message))
+
+    def write(self):
+        self.mailbox.flush()
+        self.mailbox.close()

src/apps/storetool/bongo/storetool/MailCommands.py

@@ -1,13 +1,14 @@
 import email
 import logging
 import mailbox
+import os
 import sys
+import tempfile
 
 from gettext import gettext as _
 
 from bongo.cmdparse import Command
 from bongo.BongoError import BongoError
-from libbongo.libs import bongojson, msgapi
 from bongo.store.StoreClient import DocTypes, StoreClient
 from bongo.storetool.ExportMailbox import MboxMailbox, MaildirMailbox
 
@@ -26,26 +27,29 @@ class MailImportCommand(Command):
         collection = "/mail/" + folder
         store.Create(collection, existingOk=True)
 
+        temporary_name = None
         if type == "mbox":
             if file == "-":
-                import tempfile
-                fd = tempfile.TemporaryFile(mode="w+b")
-                try:
-                    for line in sys.stdin: fd.write(line)
-                except StopIteration: pass
-                fd.seek(0)
+                with tempfile.NamedTemporaryFile(delete=False) as temporary:
+                    temporary.write(sys.stdin.buffer.read())
+                    temporary_name = temporary.name
+                source = temporary_name
             else:
-                fd = open(file)
-
-            mailstore = mailbox.PortableUnixMailbox(fd, email.message_from_file)
+                source = file
+            mailstore = mailbox.mbox(source, create=False)
         elif type == "maildir":
-            mailstore = mailbox.Maildir(file, email.message_from_file)
+            mailstore = mailbox.Maildir(file, create=False)
+        else:
+            raise ValueError("mail source type must be mbox or maildir")
 
-        for msg in mailstore:
-            store.Write(collection, DocTypes.Mail, msg.as_string(True))
-
-        if fd:
-            fd.close()
+        try:
+            for key in mailstore.iterkeys():
+                content = mailstore.get_bytes(key, from_=False)
+                store.Write(collection, DocTypes.Mail, content)
+        finally:
+            mailstore.close()
+            if temporary_name is not None:
+                os.unlink(temporary_name)
 
     def Run(self, options, args):
         if len(args) == 0:
@@ -94,6 +98,9 @@ class MailExportCommand(Command):
                     continue
                 msg = store.Read(mail.props["nmap.guid"])
                 maildir.add(msg)
+            maildir.write()
+        else:
+            raise ValueError("mail destination type must be mbox or maildir")
 
 
     def Run(self, options, args):

src/apps/storetool/bongo/storetool/StoreCommands.py

@@ -1,12 +1,7 @@
 import logging
-import string
 import sys
 
-from gettext import gettext as _
-import tarfile
-
 from bongo.cmdparse import Command
-from bongo.BongoError import BongoError
 from bongo.store.StoreClient import DocTypes, StoreClient
 
 
@@ -112,7 +107,7 @@ class DocumentPutCommand(Command):
         
         try:
             store.Store(options.store)
-            f = open(filename, "r")
+            f = open(filename, "rb")
             content = f.read()
             try:
                 store.Info(document)

src/apps/storetool/tests/test_backup_commands.py

@@ -0,0 +1,80 @@
+# /****************************************************************************
+#  * <Novell-copyright>
+#  * Copyright (c) 2001 Novell, Inc. All Rights Reserved.
+#  *
+#  * This program is free software; you can redistribute it and/or
+#  * modify it under the terms of version 2 of the GNU General Public License
+#  * as published by the Free Software Foundation.
+#  *
+#  * This program is distributed in the hope that it will be useful,
+#  * but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  * GNU General Public License for more details.
+#  *
+#  * You should have received a copy of the GNU General Public License
+#  * along with this program; if not, contact Novell, Inc.
+#  *
+#  * To contact Novell about this file by physical or electronic mail, you
+#  * may find current contact information at www.novell.com.
+#  * </Novell-copyright>
+#  ****************************************************************************/
+
+import io
+import tarfile
+from pathlib import Path
+import tempfile
+import unittest
+
+from bongo.storetool.BackupCommands import (
+    _metadata, _properties, validate_archive)
+
+
+class _Item:
+    uid = "0000000000000042"
+    type = 2
+    flags = 10
+    imapuid = "7"
+
+
+class BackupArchiveTests(unittest.TestCase):
+    def test_metadata_round_trip_keeps_user_properties(self):
+        metadata = _metadata(
+            _Item(), {"nmap.guid": "generated", "display": "Grüße"})
+        self.assertEqual(metadata["BONGO.imapuid"], "7")
+        self.assertEqual(_properties(metadata), {"display": "Grüße"})
+
+    def test_standard_pax_archive_is_validated_without_extraction(self):
+        with tempfile.TemporaryDirectory() as directory:
+            filename = Path(directory) / "store.backup"
+            with tarfile.open(filename, "w", format=tarfile.PAX_FORMAT) as archive:
+                info = tarfile.TarInfo("/mail/INBOX/message.eml")
+                payload = b"Subject: backup\r\n\r\nbody\r\n"
+                info.size = len(payload)
+                info.pax_headers = {
+                    "BONGO.uid": "0000000000000042",
+                    "BONGO.type": "2",
+                    "BONGO.flags": "0",
+                }
+                archive.addfile(info, io.BytesIO(payload))
+            members = validate_archive(filename)
+            self.assertEqual([member.name for member in members],
+                             ["/mail/INBOX/message.eml"])
+
+    def test_unsafe_archive_is_rejected(self):
+        with tempfile.TemporaryDirectory() as directory:
+            filename = Path(directory) / "unsafe.backup"
+            with tarfile.open(filename, "w", format=tarfile.PAX_FORMAT) as archive:
+                info = tarfile.TarInfo("../escape")
+                info.size = 0
+                info.pax_headers = {
+                    "BONGO.uid": "1",
+                    "BONGO.type": "2",
+                    "BONGO.flags": "0",
+                }
+                archive.addfile(info, io.BytesIO())
+            with self.assertRaisesRegex(ValueError, "unsafe"):
+                validate_archive(filename)
+
+
+if __name__ == "__main__":
+    unittest.main()

src/apps/storetool/tests/test_python3_runtime.py

@@ -0,0 +1,120 @@
+# /****************************************************************************
+#  * <Novell-copyright>
+#  * Copyright (c) 2001 Novell, Inc. All Rights Reserved.
+#  *
+#  * This program is free software; you can redistribute it and/or
+#  * modify it under the terms of version 2 of the GNU General Public License
+#  * as published by the Free Software Foundation.
+#  *
+#  * This program is distributed in the hope that it will be useful,
+#  * but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  * GNU General Public License for more details.
+#  *
+#  * You should have received a copy of the GNU General Public License
+#  * along with this program; if not, contact Novell, Inc.
+#  *
+#  * To contact Novell about this file by physical or electronic mail, you
+#  * may find current contact information at www.novell.com.
+#  * </Novell-copyright>
+#  ****************************************************************************/
+
+import io
+from pathlib import Path
+import tempfile
+import unittest
+
+from bongo import table
+from bongo.Console import wrap
+from bongo.external.simpletal import simpleTAL, simpleTALES
+from bongo.store.StoreClient import CalendarACL, StoreClient
+from bongo.storetool.ExportMailbox import MaildirMailbox, MboxMailbox
+
+
+class _Response:
+    def __init__(self, code, message=""):
+        self.code = code
+        self.message = message
+
+
+class _Stream:
+    def __init__(self, responses):
+        self.responses = list(responses)
+        self.lines = []
+        self.raw = []
+
+    def Write(self, value):
+        self.lines.append(value)
+
+    def WriteRaw(self, value):
+        self.raw.append(value)
+
+    def GetResponse(self):
+        return self.responses.pop(0)
+
+
+class Python3RuntimeTests(unittest.TestCase):
+    def test_console_table_and_unicode_template_are_text(self):
+        self.assertIsInstance(wrap("Grüße 世界", width=8), str)
+        self.assertEqual(table.format_table(["Name"], [["Müller"]]),
+                         "Name\n------\nMüller")
+
+        context = simpleTALES.Context()
+        context.addGlobal("name", "A&B")
+        template = simpleTAL.compileHTMLTemplate(
+            '<p tal:content="name">placeholder</p>')
+        output = io.StringIO()
+        template.expand(context, output)
+        self.assertEqual(output.getvalue(), "<p>A&amp;B</p>")
+
+    def test_store_wire_lengths_are_encoded_byte_lengths(self):
+        client = object.__new__(StoreClient)
+        client.stream = _Stream([
+            _Response(2002), _Response(1000, "guid ok"),
+            _Response(2002), _Response(1000),
+            _Response(2002), _Response(1000, "guid ok"),
+        ])
+
+        client.Write("/mail/INBOX", 2, "Grüße")
+        client.PropSet("guid", "display", "Grüße")
+        client.Replace("guid", "Grüße")
+
+        expected = "Grüße".encode("utf-8")
+        self.assertIn(" 7", client.stream.lines[0])
+        self.assertTrue(client.stream.lines[1].endswith(" 7"))
+        self.assertTrue(client.stream.lines[2].endswith(" 7"))
+        self.assertEqual(client.stream.raw, [expected, expected, expected])
+
+    def test_calendar_share_token_uses_python3_hashlib(self):
+        token = CalendarACL(None)._GetUid("calendar")
+        self.assertEqual(len(token), 32)
+        int(token, 16)
+
+    def test_mailbox_writers_accept_store_bytes(self):
+        message = (
+            b"From: sender@example.test\r\n"
+            b"To: user@example.test\r\n"
+            b"Subject: Python 3\r\n\r\nBody\r\n")
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            mbox_path = root / "mailbox"
+            writer = MboxMailbox(str(mbox_path))
+            writer.add(message)
+            writer.write()
+            self.assertIn(b"Subject: Python 3", mbox_path.read_bytes())
+
+            maildir_path = root / "maildir"
+            writer = MaildirMailbox(str(maildir_path))
+            writer.add(message)
+            writer.write()
+            payloads = [
+                path.read_bytes()
+                for folder in ("new", "cur")
+                for path in (maildir_path / folder).iterdir()
+            ]
+            self.assertEqual(len(payloads), 1)
+            self.assertIn(b"Subject: Python 3", payloads[0])
+
+
+if __name__ == "__main__":
+    unittest.main()

src/libs/python/bongo/Console.py

@@ -1,4 +1,6 @@
 import re
+from functools import reduce
+
 rx=re.compile("([\u2e80-\uffff])", re.UNICODE)
 
 # find out the terminal width
@@ -7,7 +9,7 @@ try:
     (lines, cols) = struct.unpack('hh',
                                   fcntl.ioctl(pty.STDOUT_FILENO,
                                               termios.TIOCGWINSZ,
-                                              '1234'))
+                                              b'1234'))
     if cols == 0:
         raise Exception("Terminal doesn't support TIOCGWINSZ")
 
@@ -22,11 +24,16 @@ except:
 # Unicode-safe line wrapping function
 # from http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/358117
 def wrap(text, width=termwidth, encoding="utf8"):
+    if isinstance(text, bytes):
+        text = text.decode(encoding, errors="replace")
+    else:
+        text = str(text)
+
     return reduce(lambda line, word, width=width: '%s%s%s' %
                   (line,
                    [' ','\n',''][(len(line)-line.rfind('\n')-1
                                   + len(word.split('\n',1)[0] ) >= width) or
                                  line[-1:] == '\0' and 2],
                    word),
-                  rx.sub(r'\1\0 ', str(text,encoding)).split(' ')
-                  ).replace('\0', '').encode(encoding)
+                  rx.sub(r'\1\0 ', text).split(' '),
+                  '').replace('\0', '')

src/libs/python/bongo/Privs.py

@@ -21,7 +21,7 @@
 #############################################################################
 
 import os, pwd
-import Xpl
+from bongo import Xpl
 
 import logging
 log = logging.getLogger("bongo.Privs")

src/libs/python/bongo/admin/Mdb.py

@@ -1,5 +1,8 @@
-import Handler, Security, Template, Util, sys, traceback
+import sys
+import traceback
+
 from bongo import MDB
+from . import Handler, Security, Template, Util
 
 def GetObjectAttribute(req):
     objdn = req.fields.getfirst('objdn').value

src/libs/python/bongo/external/simpletal/simpleElementTree.py

@@ -40,12 +40,13 @@
 		Module Dependencies: simpleTALES, elementtree
 """
 
-from bongo.external.elementtree import ElementTree
-import bongo.external.simpleTALES
+from xml.etree import ElementTree
 
-class SimpleElementTreeVar (ElementTree._ElementInterface, simpleTALES.ContextVariable):
+from bongo.external.simpletal import simpleTALES
+
+class SimpleElementTreeVar (ElementTree.Element, simpleTALES.ContextVariable):
 	def __init__(self, tag, attrib):
-		ElementTree._ElementInterface.__init__(self, tag, attrib)
+		ElementTree.Element.__init__(self, tag, attrib)
 		simpleTALES.ContextVariable.__init__(self)
 		
 	def value (self, pathInfo = None):
@@ -68,20 +69,20 @@ class SimpleElementTreeVar (ElementTree._ElementInterface, simpleTALES.ContextVa
 					activeElement = self.find ("/".join (ourParams [1:]))
 				elif (ourParams [0] == 'findall'):
 					# Short cut this
-					raise simpleTALES.ContextVariable (self.findall ("/".join (ourParams[1:])))
+					return simpleTALES.ContextVariable (self.findall ("/".join (ourParams[1:])))
 				else:
 					# Assume that we wanted to use find
 					activeElement = self.find ("/".join (ourParams))
 			# Did we find an element and are we looking for an attribute?
 			if (attributeName is not None and activeElement is not None):
 				attrValue = activeElement.attrib.get (attributeName, None)
-				raise simpleTALES.ContextVariable (attrValue)
+				return simpleTALES.ContextVariable (attrValue)
 			
 			# Just return the element
 			if (activeElement is None):
 				# Wrap it
-				raise simpleTALES.ContextVariable (None)
-			raise activeElement
+				return simpleTALES.ContextVariable (None)
+			return activeElement
 		else:
 			return self
 			
@@ -93,7 +94,7 @@ class SimpleElementTreeVar (ElementTree._ElementInterface, simpleTALES.ContextVa
 
 def parseFile (file):
 	treeBuilder = ElementTree.TreeBuilder (element_factory = SimpleElementTreeVar)
-	xmlTreeBuilder = ElementTree.XMLTreeBuilder (target=treeBuilder)
+	xmlTreeBuilder = ElementTree.XMLParser (target=treeBuilder)
 	
 	if (not hasattr (file, 'read')):
 		ourFile = open (file)

src/libs/python/bongo/external/simpletal/simpleTAL.py

@@ -39,7 +39,7 @@ try:
 except:
 	import bongo.external.simpletal.DummyLogger as logging
 	
-import xml.sax, cgi, io, codecs, re, types, copy, sys
+import xml.sax, html, io, codecs, re, types, copy, sys
 
 import bongo.external.simpletal.sgmlentitynames as sgmlentitynames
 import bongo.external.simpletal as simpletal
@@ -156,7 +156,7 @@ class TemplateInterpreter:
 			result.append (' ')
 			result.append (attName)
 			result.append ('="')
-			result.append (cgi.escape (attValue, quote=1))
+			result.append (html.escape (attValue, quote=True))
 			result.append ('"')
 		if (singletonFlag):
 			result.append (" />")
@@ -450,15 +450,15 @@ class TemplateInterpreter:
 						self.file.write (str (resultVal))
 			else:
 				if (isinstance (resultVal, str)):
-					self.file.write (cgi.escape (resultVal))
+					self.file.write (html.escape (resultVal))
 				elif (isinstance (resultVal, bytes)):
 					# THIS IS NOT A BUG!
 					# Use Unicode in the Context object if you are not using Ascii
-					self.file.write (cgi.escape (str (resultVal, 'ascii')))
+					self.file.write (html.escape (str (resultVal, 'ascii')))
 				else:
 					# THIS IS NOT A BUG!
 					# Use Unicode in the Context object if you are not using Ascii
-					self.file.write (cgi.escape (str (resultVal)))
+					self.file.write (html.escape (str (resultVal)))
 					
 		if (self.outputTag and not args[1]):
 			# Do NOT output end tag if a singleton with no content
@@ -574,7 +574,7 @@ class HTMLTemplateInterpreter (TemplateInterpreter):
 				result.append (' ')
 				result.append (attName)
 				result.append ('="')
-				result.append (cgi.escape (attValue, quote=1))
+				result.append (html.escape (attValue, quote=True))
 				result.append ('"')
 		if (singletonFlag):
 			result.append (" />")
@@ -692,7 +692,10 @@ class HTMLTemplate (Template):
 		# This method must wrap outputFile if required by the encoding, and write out
 		# any template pre-amble (DTD, Encoding, etc)
 		
-		encodingFile = codecs.lookup (outputEncoding)[3](outputFile, 'replace')
+		if isinstance (outputFile, io.TextIOBase):
+			encodingFile = outputFile
+		else:
+			encodingFile = codecs.lookup (outputEncoding)[3](outputFile, 'replace')
 		self.expandInline (context, encodingFile, interpreter)
 		
 	def expandInline (self, context, outputFile, interpreter=None):
@@ -721,7 +724,10 @@ class XMLTemplate (Template):
 		# any template pre-amble (DTD, Encoding, etc)
 		
 		# Write out the XML prolog
-		encodingFile = codecs.lookup (outputEncoding)[3](outputFile, 'replace')
+		if isinstance (outputFile, io.TextIOBase):
+			encodingFile = outputFile
+		else:
+			encodingFile = codecs.lookup (outputEncoding)[3](outputFile, 'replace')
 		if (not suppressXMLDeclaration):
 			if (outputEncoding.lower() != "utf-8"):
 				encodingFile.write ('<?xml version="1.0" encoding="%s"?>\n' % outputEncoding.lower())
@@ -807,7 +813,7 @@ class TemplateCompiler:
 			result.append (' ')
 			result.append (attName)
 			result.append ('="')
-			result.append (cgi.escape (attValue, quote=1))
+			result.append (html.escape (attValue, quote=True))
 			result.append ('"')
 		if (singletonFlag):
 			result.append (" />")
@@ -1279,10 +1285,12 @@ class HTMLTemplateCompiler (TemplateCompiler, FixedHTMLParser.HTMLParser):
 		self.log = logging.getLogger ("simpleTAL.HTMLTemplateCompiler")
 		
 	def parseTemplate (self, file, encoding="iso-8859-1", minimizeBooleanAtts = 0):
-		encodedFile = codecs.lookup (encoding)[2](file, 'replace')
 		self.encoding = encoding
 		self.minimizeBooleanAtts = minimizeBooleanAtts
-		self.feed (encodedFile.read())
+		content = file.read()
+		if isinstance (content, bytes):
+			content = content.decode (encoding, errors='replace')
+		self.feed (content)
 		self.close()
 		
 	def tagAsText (self, xxx_todo_changeme3, singletonFlag=0):
@@ -1301,7 +1309,7 @@ class HTMLTemplateCompiler (TemplateCompiler, FixedHTMLParser.HTMLParser):
 				result.append (' ')
 				result.append (attName)
 				result.append ('="')
-				result.append (cgi.escape (attValue, quote=1))
+				result.append (html.escape (attValue, quote=True))
 				result.append ('"')
 		if (singletonFlag):
 			result.append (" />")
@@ -1365,7 +1373,7 @@ class HTMLTemplateCompiler (TemplateCompiler, FixedHTMLParser.HTMLParser):
 			self.popTag ((tag, None))
 			
 	def handle_data (self, data):
-		self.parseData (cgi.escape (data))
+		self.parseData (html.escape (data))
 		
 	# These two methods are required so that we expand all character and entity references prior to parsing the template.
 	def handle_charref (self, ref):
@@ -1465,7 +1473,7 @@ class XMLTemplateCompiler (TemplateCompiler, xml.sax.handler.ContentHandler, xml
 	def characters (self, data):
 		#self.log.debug ("Recieved Real Data: " + data)
 		# Escape any data we recieve - we don't want any: <&> in there.
-		self.parseData (cgi.escape (data))
+		self.parseData (html.escape (data))
 		
 	def processingInstruction (self, target, data):
 		self.log.debug ("Recieved processing instruction.")
@@ -1484,8 +1492,10 @@ def compileHTMLTemplate (template, inputEncoding="ISO-8859-1", minimizeBooleanAt
 			To use the resulting template object call:
 				template.expand (context, outputFile)
 	"""
-	if (isinstance (template, bytes) or isinstance (template, str)):
+	if (isinstance (template, bytes)):
 		# It's a string!
+		templateFile = io.BytesIO (template)
+	elif (isinstance (template, str)):
 		templateFile = io.StringIO (template)
 	else:
 		templateFile = template
@@ -1500,6 +1510,8 @@ def compileXMLTemplate (template):
 	"""
 	if (isinstance (template, bytes)):
 		# It's a string!
+		templateFile = io.BytesIO (template)
+	elif (isinstance (template, str)):
 		templateFile = io.StringIO (template)
 	else:
 		templateFile = template

src/libs/python/bongo/external/simpletal/simpleTALUtils.py

@@ -34,8 +34,9 @@
 		Module Dependencies: None
 """
 
-import io, os, stat, threading, sys, codecs, sgmllib, cgi, re, types
-import bongo.external.simpletal, bongo.external.simpleTAL
+import io, os, stat, threading, sys, codecs, sgmllib, html, re, types
+from bongo.external import simpletal
+from bongo.external.simpletal import simpleTAL
 
 __version__ = simpletal.__version__
 
@@ -82,7 +83,7 @@ class HTMLStructureCleaner (sgmllib.SGMLParser):
 		self.outputFile.write ('</' + tag + '>')
 			
 	def handle_data (self, data):
-		self.outputFile.write (cgi.escape (data))
+		self.outputFile.write (html.escape (data))
 		
 	def handle_charref (self, ref):
 		self.outputFile.write ('&#%s;' % ref)
@@ -178,7 +179,7 @@ def tagAsText (tag,atts):
 			# We already have some escaped characters in here, so assume it's all valid
 			result += ' %s="%s"' % (name, value)
 		else:
-			result += ' %s="%s"' % (name, cgi.escape (value))
+			result += ' %s="%s"' % (name, html.escape (value))
 	result += ">"
 	return result
 
@@ -256,14 +257,14 @@ class MacroExpansionInterpreter (simpleTAL.TemplateInterpreter):
 					elif (isinstance (resultVal, bytes)):
 						self.file.write (str (resultVal, 'ascii'))
 					else:
-						self.file.write (str (str (resultVal), 'ascii'))
+						self.file.write (str (resultVal))
 			else:
 				if (isinstance (resultVal, str)):
-					self.file.write (cgi.escape (resultVal))
+					self.file.write (html.escape (resultVal))
 				elif (isinstance (resultVal, bytes)):
-					self.file.write (cgi.escape (str (resultVal, 'ascii')))
+					self.file.write (html.escape (str (resultVal, 'ascii')))
 				else:
-					self.file.write (cgi.escape (str (str (resultVal), 'ascii')))
+					self.file.write (html.escape (str (resultVal)))
 					
 		if (self.outputTag and not args[1]):
 			self.file.write ('</' + args[0] + '>')
@@ -287,4 +288,3 @@ def ExpandMacros (context, template, outputEncoding="ISO-8859-1"):
 	result = out.getvalue()
 	reencoder = codecs.lookup (outputEncoding)[0]
 	return reencoder (result)[0]
-

src/libs/python/bongo/nmap/NmapClient.py

@@ -2,7 +2,6 @@ import logging
 from libbongo.libs import msgapi
 from .CommandStream import *
 from .NmapConnection import NmapConnection
-from io import StringIO
 import bongo
 
 __all__ = ["DocTypes",
@@ -261,4 +260,3 @@ class NmapClient:
         if r.code != 1000:
             raise CommandError(r)
         return r
-

src/libs/python/bongo/nmap/QueueClient.py

@@ -1,7 +1,7 @@
 import logging
 import bongo, libbongo.libs
 from .CommandStream import *
-from StoreConnection import StoreConnection
+from bongo.store.StoreConnection import StoreConnection
 
 class RoutingFlags:
     NoFlags = 0

src/libs/python/bongo/store/StoreClient.py

@@ -1,14 +1,13 @@
 import logging
 from .CommandStream import *
 from .StoreConnection import StoreConnection
-from io import StringIO
+from io import BytesIO
 import bongo
 import re
 import time
 import random
 import hashlib
 import socket
-import string
 
 class DocTypes:
     Unknown = 0x0001
@@ -56,28 +55,28 @@ class DocFlags:
         revNames[flag] = name
 
     # some helpful functions
+    @staticmethod
     def GetName(flag):
         return DocFlags.names[flag]
-    GetName = staticmethod(GetName)
 
+    @staticmethod
     def ByName(name):
         return DocFlags.revNames[name]
-    ByName = staticmethod(ByName)
 
+    @staticmethod
     def AllFlags():
         return list(DocFlags.names.keys())
-    AllFlags = staticmethod(AllFlags)
 
-    def GetMask(hash):
+    @staticmethod
+    def GetMask(values):
         flags = mask = 0
 
-        for flag, status in list(hash.items()):
+        for flag, status in list(values.items()):
             if status:
                 flags = flags | flag
             mask = mask | flag
 
         return flags, mask
-    GetMask = staticmethod(GetMask)
 
 class FlagMode:
     Show = 0
@@ -165,7 +164,7 @@ class CalendarACL :
             # if we can't get a network address, just imagine one
             a = random.random() * 100000000000000000
         data = str(t) + ' ' + str (r) + ' ' + str(a) + ' ' + str(args)
-        data = hashlib.md5(data).hexdigest()
+        data = hashlib.md5(data.encode("utf-8")).hexdigest()
         return data
 
     
@@ -680,15 +679,10 @@ class StoreClient:
     # into something approaching utf-8. This is obviously lossy,
     # and we really don't want to have to do this - need to get the
     # store as utf-8 clean as possible, really.
-    def force_utf8(self, str):
-        out = []
-        append = out.append
-        for ch in str:
-            if ch < "\200":
-                append(ch)
-            else:
-                append('?')
-        return string.join(out, "")
+    def force_utf8(self, value):
+        if isinstance(value, bytes):
+            return value.decode("utf-8", errors="replace")
+        return str(value)
 
     def PropGet(self, doc, name=None):
         command = "PROPGET %s" % doc
@@ -723,7 +717,7 @@ class StoreClient:
             try:
                 props[key] = str(raw_data, "utf-8")
             except UnicodeDecodeError as e:
-                data = self.force_utf8(raw_data)
+                props[key] = self.force_utf8(raw_data)
             # eat the \r\n afterward
             self.stream.Read(2)
             r = self.stream.GetResponse()
@@ -737,20 +731,22 @@ class StoreClient:
             (key, length) = r.message.split(" ", 2)
             raw_data = self.stream.Read(int(length))
             self.stream.Read(2)
-            props["nmap.mail.imapuid"] = raw_data
+            props["nmap.mail.imapuid"] = self.force_utf8(raw_data)
         return props
 
     def PropSet(self, doc, name, value=None):
         if value is None:
             value = ""
+        wire_value = value if isinstance(value, bytes) else value.encode("utf-8")
 
-        self.stream.Write("PROPSET %s %s %d" % (doc, name, len(value)))
+        self.stream.Write("PROPSET %s %s %d" %
+                          (doc, name, len(wire_value)))
 
         r = self.stream.GetResponse()
         if r.code != 2002:
             raise CommandError(r)
 
-        self.stream.WriteRaw(value)
+        self.stream.WriteRaw(wire_value)
         r = self.stream.GetResponse()
         if r.code != 1000:
             raise CommandError(r)
@@ -780,7 +776,7 @@ class StoreClient:
         s = self.stream.Read(int(length))
         # eat the \r\n afterward
         self.stream.Read(2)
-        return StringIO(s)
+        return BytesIO(s)
 
     def Remove(self, collection):
         self.stream.Write("REMOVE %s" % collection)
@@ -808,13 +804,14 @@ class StoreClient:
             raise CommandError(r)
 
     def Replace(self, doc, data):
-        self.stream.Write("REPLACE %s %d" % (doc, len(data)))
+        wire_data = data if isinstance(data, bytes) else data.encode("utf-8")
+        self.stream.Write("REPLACE %s %d" % (doc, len(wire_data)))
 
         r = self.stream.GetResponse()
         if r.code != 2002:
             raise CommandError(r)
 
-        self.stream.WriteRaw(data)
+        self.stream.WriteRaw(wire_data)
 
         r = self.stream.GetResponse()
         (guid, junk) = r.message.split(" ", 1)
@@ -877,7 +874,7 @@ class StoreClient:
             command = command + " G%s" % guid
 
         if timeCreated is not None:
-            command += " T" + timeCreated
+            command += " T" + str(timeCreated)
 
         if flags is not None:
             command = command + " Z%d" % flags
@@ -904,8 +901,10 @@ class StoreClient:
     def Write(self, collection, type, data, filename=None, index=None, guid=None, timeCreated=None, flags=None, link=None, noProcess=None):
         if data is None:
             data = ""
+        wire_data = data if isinstance(data, bytes) else data.encode("utf-8")
 
-        command = "WRITE \"%s\" %d %d" % (collection, type, len(data))
+        command = "WRITE \"%s\" %d %d" % (
+            collection, type, len(wire_data))
 
         if index is not None:
             command = command + " I%s" % index
@@ -917,7 +916,7 @@ class StoreClient:
             command = command + " G%s" % guid
 
         if timeCreated is not None:
-            command += " T" + timeCreated
+            command += " T" + str(timeCreated)
 
         if flags is not None:
             command = command + " Z%d" % flags
@@ -935,7 +934,7 @@ class StoreClient:
         if r.code != 2002:
             raise CommandError(r)
 
-        self.stream.WriteRaw(data)
+        self.stream.WriteRaw(wire_data)
 
         r = self.stream.GetResponse()
         if r.code != 1000:
@@ -943,7 +942,3 @@ class StoreClient:
 
         (guid, junk) = r.message.split(" ", 1)
         return guid
-
-    def Reset(self) :
-        self.stream.Write("RESET\r\n")
-        self.stream.GetResponse()

src/libs/python/bongo/table.py

@@ -1,8 +1,6 @@
 # some simple code for formatting text tables
 
 def format_row(elts, widths, sep=" | "):
-    import string
-    
     row = []
     for i in range(len(elts)):
         if elts[i] is None:
@@ -11,7 +9,7 @@ def format_row(elts, widths, sep=" | "):
             elt = elts[i]
 
         row.append("%-*s" % (widths[i], elt))
-    return string.join(row, sep)
+    return sep.join(row)
 
 def format_separator(widths):
     elts = [ "-" * width for width in widths ]
@@ -41,5 +39,4 @@ def format_table(labels, rows):
         else:
             table.append(format_row(row, widths).rstrip())
 
-    import string
-    return string.join(table, "\n")
+    return "\n".join(table)