diff --git a/client/equo.py b/client/equo.py index 8bb36da2d..4be5d0e77 100644 --- a/client/equo.py +++ b/client/equo.py @@ -886,7 +886,7 @@ def handle_exception(exc_class, exc_instance, exc_tb): try_to_kill_cacher() raise SystemExit(1) - exception_data = entropy.tools.print_exception(returndata = True, + exception_data = entropy.tools.print_exception(silent = True, tb_data = exc_tb, all_frame_data = True) exception_stack = t_back ferror.write(const_convert_to_rawstring("\nRevision: " + \ diff --git a/libraries/entropy/cache.py b/libraries/entropy/cache.py index 2d5d83577..95bb0b503 100644 --- a/libraries/entropy/cache.py +++ b/libraries/entropy/cache.py @@ -18,12 +18,13 @@ import os import sys import tempfile -import time from entropy.const import etpConst, etpUi, const_debug_write, \ const_pid_exists, const_setup_perms from entropy.core import Singleton from entropy.misc import TimeScheduled, Lifo import time +import threading +import copy import entropy.dump import entropy.tools @@ -60,31 +61,31 @@ class EntropyCacher(Singleton): Sample code: - >>> # import module - >>> from entropy.cache import EntropyCacher - ... - >>> # first EntropyCacher load, start it - >>> cacher = EntropyCacher() - >>> cacher.start() - ... - >>> # now store something into its cache - >>> cacher.push('my_identifier1', [1, 2, 3]) - >>> # now store something synchronously - >>> cacher.push('my_identifier2', [1, 2, 3], async = False) - ... - >>> # now flush all the caches to disk, and make sure all - >>> # is written - >>> cacher.sync() - ... - >>> # now fetch something from the cache - >>> data = cacher.pop('my_identifier1') - [1, 2, 3] - ... - >>> # now discard all the cached (async) writes - >>> cacher.discard() - ... - >>> # and stop EntropyCacher - >>> cacher.stop() + >>> # import module + >>> from entropy.cache import EntropyCacher + ... + >>> # first EntropyCacher load, start it + >>> cacher = EntropyCacher() + >>> cacher.start() + ... + >>> # now store something into its cache + >>> cacher.push('my_identifier1', [1, 2, 3]) + >>> # now store something synchronously + >>> cacher.push('my_identifier2', [1, 2, 3], async = False) + ... + >>> # now flush all the caches to disk, and make sure all + >>> # is written + >>> cacher.sync() + ... + >>> # now fetch something from the cache + >>> data = cacher.pop('my_identifier1') + [1, 2, 3] + ... + >>> # now discard all the cached (async) writes + >>> cacher.discard() + ... + >>> # and stop EntropyCacher + >>> cacher.stop() """ @@ -94,8 +95,6 @@ class EntropyCacher(Singleton): This is the place where all the properties initialization takes place. """ - import threading - import copy self.__copy = copy self.__alive = False self.__cache_writer = None diff --git a/libraries/entropy/client/interfaces/cache.py b/libraries/entropy/client/interfaces/cache.py index e6c002563..2e79da0bd 100644 --- a/libraries/entropy/client/interfaces/cache.py +++ b/libraries/entropy/client/interfaces/cache.py @@ -13,9 +13,7 @@ import os import shutil from entropy.const import etpConst, const_setup_perms from entropy.exceptions import RepositoryError -from entropy.output import red, darkred, darkgreen from entropy.cache import EntropyCacher -from entropy.i18n import _ from entropy.db.exceptions import OperationalError class CacheMixin: diff --git a/libraries/entropy/client/interfaces/client.py b/libraries/entropy/client/interfaces/client.py index e038e0e3f..cf7786fb3 100644 --- a/libraries/entropy/client/interfaces/client.py +++ b/libraries/entropy/client/interfaces/client.py @@ -11,7 +11,6 @@ """ import os -import sys from entropy.core import Singleton from entropy.output import TextInterface, bold, red, darkred, blue from entropy.client.interfaces.loaders import LoadersMixin @@ -20,7 +19,7 @@ from entropy.client.interfaces.dep import CalculatorsMixin from entropy.client.interfaces.methods import RepositoryMixin, MiscMixin, \ MatchMixin from entropy.client.interfaces.noticeboard import NoticeBoardMixin -from entropy.const import etpConst, etpUi, const_debug_write, \ +from entropy.const import etpConst, const_debug_write, \ const_convert_to_unicode from entropy.core.settings.base import SystemSettings from entropy.core.settings.plugins.skel import SystemSettingsPlugin @@ -352,10 +351,8 @@ class ClientSystemSettingsPlugin(SystemSettingsPlugin): } # parse license whitelist - """ - Parser returning licenses considered accepted by default - (= GPL compatibles) read from package.lic_whitelist. - """ + # Parser returning licenses considered accepted by default + # (= GPL compatibles) read from package.lic_whitelist. for repoid in self.__repos_files['repos_license_whitelist']: sys_settings_instance.validate_entropy_cache( self.__repos_files['repos_license_whitelist'][repoid], @@ -367,10 +364,8 @@ class ClientSystemSettingsPlugin(SystemSettingsPlugin): self.__repos_files['repos_license_whitelist'][repoid]) # package masking - """ - Parser returning packages masked at repository level read from - packages.db.mask inside the repository database directory. - """ + # Parser returning packages masked at repository level read from + # packages.db.mask inside the repository database directory. for repoid in self.__repos_files['repos_mask']: sys_settings_instance.validate_entropy_cache( self.__repos_files['repos_mask'][repoid], @@ -381,10 +376,8 @@ class ClientSystemSettingsPlugin(SystemSettingsPlugin): self.__repos_files['repos_mask'][repoid]) # keywords masking - """ - Parser returning packages masked at repository level read from - packages.db.keywords inside the repository database directory. - """ + # Parser returning packages masked at repository level read from + # packages.db.keywords inside the repository database directory. for repoid in self.__repos_files['repos_keywords']: sys_settings_instance.validate_entropy_cache( self.__repos_files['repos_keywords'][repoid], @@ -399,12 +392,10 @@ class ClientSystemSettingsPlugin(SystemSettingsPlugin): sys_settings_instance) # critical updates - """ - Parser returning critical packages list metadata read from - packages.db.critical file inside the repository directory. - This file contains packages that should be always updated - before anything else. - """ + # Parser returning critical packages list metadata read from + # packages.db.critical file inside the repository directory. + # This file contains packages that should be always updated + # before anything else. for repoid in self.__repos_files['repos_critical_updates']: sys_settings_instance.validate_entropy_cache( self.__repos_files['repos_critical_updates'][repoid], @@ -416,11 +407,9 @@ class ClientSystemSettingsPlugin(SystemSettingsPlugin): # conflicts map - """ - Parser returning packages that could have been installed because - they aren't in the same scope, but ending up creating critical - issues. You can see it as a configurable conflict map. - """ + # Parser returning packages that could have been installed because + # they aren't in the same scope, but ending up creating critical + # issues. You can see it as a configurable conflict map. # keep priority order repoids = [x for x in sys_settings_instance['repositories']['order'] \ if x in self.__repos_files['conflicting_tagged_packages']] @@ -640,9 +629,7 @@ class Client(Singleton, TextInterface, LoadersMixin, CacheMixin, CalculatorsMixi self.openclientdb = True # setup package settings (masking and other stuff) - # TODO: kept for backward compatibility, will be removed - self.SystemSettings = SystemSettings() - self._settings = self.SystemSettings + self._settings = SystemSettings() const_debug_write(__name__, "SystemSettings loaded") # class init diff --git a/libraries/entropy/client/interfaces/db.py b/libraries/entropy/client/interfaces/db.py index 4599fd0e9..efa7d0802 100644 --- a/libraries/entropy/client/interfaces/db.py +++ b/libraries/entropy/client/interfaces/db.py @@ -213,7 +213,7 @@ class AvailablePackagesRepositoryUpdater(object): elif 'added' not in data or \ 'removed' not in data or \ 'secure_checksum' not in data: - return None, None, None + return None, None, None return data['added'], data['removed'], data['secure_checksum'] @@ -711,8 +711,8 @@ class AvailablePackagesRepositoryUpdater(object): "dbdumplight", "dbdumplightck", "compck", ) if (item in items_needing_cmethod) and (cmethod is None): - mytxt = "For %s, cmethod can't be None" % (item,) - raise AttributeError(mytxt) + mytxt = "For %s, cmethod can't be None" % (item,) + raise AttributeError(mytxt) avail_data = self._settings['repositories']['available'] repo_data = avail_data[self.__repository_id] @@ -1372,9 +1372,10 @@ class AvailablePackagesRepositoryUpdater(object): myidpackages, session) if (None in (added_ids, removed_ids, secure_checksum)) or \ (not added_ids and not removed_ids and self.__force): - mydbconn.closeDB() - self.__eapi3_close(eapi3_interface, session) - return False + + mydbconn.closeDB() + self.__eapi3_close(eapi3_interface, session) + return False elif not secure_checksum: # {added_ids, removed_ids, secure_checksum} == False @@ -1427,6 +1428,7 @@ class AvailablePackagesRepositoryUpdater(object): count = 0 maxcount = len(added_segments) product = self._settings['repositories']['product'] + segment = None for segment in added_segments: count += 1 diff --git a/libraries/entropy/client/interfaces/dep.py b/libraries/entropy/client/interfaces/dep.py index df53ff0b4..a3ce95676 100644 --- a/libraries/entropy/client/interfaces/dep.py +++ b/libraries/entropy/client/interfaces/dep.py @@ -9,14 +9,18 @@ B{Entropy Package Manager Client Dependency handling Interface}. """ -from entropy.const import * -from entropy.exceptions import * +import os + +from entropy.const import etpConst, const_debug_write, const_isstring, \ + const_isnumber +from entropy.exceptions import RepositoryError, SystemDatabaseError, \ + DependenciesNotFound, DependenciesNotRemovable from entropy.graph import Graph from entropy.misc import Lifo from entropy.cache import EntropyCacher -from entropy.output import bold, darkgreen, darkred, blue, red, purple +from entropy.output import bold, darkgreen, darkred, blue, purple from entropy.i18n import _ -from entropy.db.exceptions import IntegrityError, OperationalError, Error, \ +from entropy.db.exceptions import IntegrityError, OperationalError, \ DatabaseError, InterfaceError from entropy.db.skel import EntropyRepositoryBase @@ -165,7 +169,7 @@ class CalculatorsMixin: # set([((14789, '3.3.8b', '', 0), 'sabayonlinux.org')]) matches = [(x[0][0], x[1],) for x in data] for m_id, m_repo in matches: - # FIXME: there is a bug up the queue somewhere + # NOTE: there is a bug up the queue somewhere # but current error report tool didn't provide full # stack variables (only for the innermost frame) #if isinstance(m_id, tuple): @@ -189,21 +193,7 @@ class CalculatorsMixin: def atom_match(self, atom, match_slot = None, mask_filter = True, multi_match = False, multi_repo = False, match_repo = None, - extended_results = False, use_cache = True, **kwargs): - - # TODO: remove this on 20101010, backward compatiblity - if kwargs: - import warnings - warnings.warn( - "Client.atom_match() called with deprecated args %s" % ( - kwargs,)) - match_slot = kwargs.get('matchSlot', match_slot) - mask_filter = kwargs.get('packagesFilter', mask_filter) - multi_match = kwargs.get('multiMatch', multi_match) - multi_repo = kwargs.get('multiRepo', multi_repo) - match_repo = kwargs.get('matchRepo', match_repo) - extended_results = kwargs.get('extendedResults', extended_results) - use_cache = kwargs.get('useCache', use_cache) + extended_results = False, use_cache = True): # support match in repository from shell # atom@repo1,repo2,repo3 @@ -452,7 +442,7 @@ class CalculatorsMixin: av_tags = [x for x in \ _my_get_available_tags(dependency, None) if x] if av_tags: - # XXX: since tags replace slots, use them as slots + # NOTE: since tags replace slots, use them as slots i_key = entropy.tools.dep_getkey(dependency) matching_tags = set() for a_tag in av_tags: @@ -858,7 +848,7 @@ class CalculatorsMixin: try: pkg_key, pkg_slot = repo_db.retrieveKeySlot(pkg_id) except TypeError: - deps_not_found("unknown_%s_%s" % (pkg_id, repo_id,)) + deps_not_found.add("unknown_%s_%s" % (pkg_id, repo_id,)) continue cm_idpackage, cm_result = self._installed_repository.atomMatch( pkg_key, matchSlot = pkg_slot) @@ -1044,9 +1034,9 @@ class CalculatorsMixin: soname = ".so" repo_needed = match_db.retrieveNeeded(match_idpackage, - extended = True, format = True) + extended = True, formatted = True) client_needed = self._installed_repository.retrieveNeeded(client_idpackage, - extended = True, format = True) + extended = True, formatted = True) repo_split = [x.split(soname)[0] for x in repo_needed] client_split = [x.split(soname)[0] for x in client_needed] @@ -1330,9 +1320,6 @@ class CalculatorsMixin: ) if self.xcache: cached = self._cacher.pop(c_hash) - # XXX drop old cache object format - if not isinstance(cached, dict): - cached = None if cached is not None: return cached @@ -1957,9 +1944,9 @@ class CalculatorsMixin: if idpackage == -1: treelevel += 1 if atoms: - mydict = {myatom: idreason,} + mydict = {myatom: idreason} else: - mydict = {package_match: idreason,} + mydict = {package_match: idreason} if flat: maskedtree.update(mydict) else: diff --git a/libraries/entropy/client/interfaces/loaders.py b/libraries/entropy/client/interfaces/loaders.py index f17ef5029..54203b77f 100644 --- a/libraries/entropy/client/interfaces/loaders.py +++ b/libraries/entropy/client/interfaces/loaders.py @@ -21,7 +21,6 @@ class LoadersMixin: def __init__(self): self._spm_cache = {} - from entropy.client.interfaces.client import Client from entropy.client.interfaces.trigger import Trigger from entropy.client.interfaces.repository import Repository from entropy.client.interfaces.package import Package diff --git a/libraries/entropy/client/interfaces/methods.py b/libraries/entropy/client/interfaces/methods.py index c51738ab3..1226140a3 100644 --- a/libraries/entropy/client/interfaces/methods.py +++ b/libraries/entropy/client/interfaces/methods.py @@ -24,10 +24,10 @@ from datetime import datetime from entropy.i18n import _ from entropy.const import etpConst, const_debug_write, etpSys, \ const_setup_file, initconfig_entropy_constants, const_pid_exists, \ - const_set_nice_level, const_setup_perms, const_setup_entropy_pid, \ + const_setup_perms, const_setup_entropy_pid, \ const_isstring, const_convert_to_unicode, const_isnumber -from entropy.exceptions import RepositoryError, InvalidPackageSet,\ - SystemDatabaseError +from entropy.exceptions import RepositoryError, SystemDatabaseError, \ + RepositoryPluginError from entropy.db import EntropyRepository from entropy.cache import EntropyCacher from entropy.client.interfaces.db import ClientEntropyRepositoryPlugin, \ @@ -229,17 +229,18 @@ class RepositoryMixin: if (repoid not in self._treeupdates_repos) and \ (entropy.tools.is_root()) and \ (not repoid.endswith(etpConst['packagesext'])): - # only as root due to Portage - try: - updated = self.repository_packages_spm_sync(repoid, conn) - except (OperationalError, DatabaseError,): - updated = False - if updated: - self._cacher.discard() - EntropyCacher.clear_cache_item( - EntropyCacher.CACHE_IDS['world_update']) - EntropyCacher.clear_cache_item( - EntropyCacher.CACHE_IDS['critical_update']) + + # only as root due to Portage + try: + updated = self.repository_packages_spm_sync(repoid, conn) + except (OperationalError, DatabaseError,): + updated = False + if updated: + self._cacher.discard() + EntropyCacher.clear_cache_item( + EntropyCacher.CACHE_IDS['world_update']) + EntropyCacher.clear_cache_item( + EntropyCacher.CACHE_IDS['critical_update']) return conn def get_repository_revision(self, reponame): @@ -251,8 +252,6 @@ class RepositoryMixin: def add_repository(self, repodata): - product = self._settings['repositories']['product'] - branch = self._settings['repositories']['branch'] avail_data = self._settings['repositories']['available'] repoid = repodata['repoid'] @@ -629,7 +628,7 @@ class RepositoryMixin: except SystemDatabaseError: try: conn.closeDB() - except: + except (RepositoryPluginError, OSError, IOError): pass entropy.tools.print_traceback(f = self.clientLog) conn = load_db_from_ram() @@ -690,7 +689,8 @@ class RepositoryMixin: compress_level = 9 backup_dir = os.path.dirname(dbpath) - if not backup_dir: backup_dir = os.path.dirname(dbpath) + if not backup_dir: + backup_dir = os.path.dirname(dbpath) dbname = os.path.basename(dbpath) bytes_required = 1024000*300 if not (os.access(backup_dir, os.W_OK) and \ @@ -734,7 +734,7 @@ class RepositoryMixin: try: entropy.tools.compress_file(dbpath, comp_dbpath, bz2.BZ2File, compress_level) - except: + except (IOError, OSError): if not silent: entropy.tools.print_traceback() return False, _("Unable to compress") @@ -760,19 +760,19 @@ class RepositoryMixin: os.path.isfile(backup_path) and os.access(backup_path, os.R_OK) and \ entropy.tools.check_required_space(db_dir, bytes_required)): - if not silent: - mytxt = "%s: %s, %s" % ( - darkred(_("Cannot restore selected backup")), - blue(os.path.basename(backup_path)), - darkred(_("permission denied")), - ) - self.output( - mytxt, - importance = 1, - level = "error", - header = red(" @@ ") - ) - return False, mytxt + if not silent: + mytxt = "%s: %s, %s" % ( + darkred(_("Cannot restore selected backup")), + blue(os.path.basename(backup_path)), + darkred(_("permission denied")), + ) + self.output( + mytxt, + importance = 1, + level = "error", + header = red(" @@ ") + ) + return False, mytxt if not silent: mytxt = "%s: %s => %s ..." % ( @@ -791,7 +791,7 @@ class RepositoryMixin: try: entropy.tools.uncompress_file(backup_path, db_destination, bz2.BZ2File) - except: + except (IOError, OSError): if not silent: entropy.tools.print_traceback() return False, _("Unable to unpack") diff --git a/libraries/entropy/client/interfaces/package.py b/libraries/entropy/client/interfaces/package.py index 58f627972..1b2cf0201 100644 --- a/libraries/entropy/client/interfaces/package.py +++ b/libraries/entropy/client/interfaces/package.py @@ -17,19 +17,16 @@ import shutil import tempfile import time -from entropy.const import etpConst, etpUi, etpSys, const_setup_perms, \ +from entropy.const import etpConst, etpUi, const_setup_perms, \ const_isunicode, const_convert_to_unicode from entropy.exceptions import PermissionDenied, SPMError from entropy.i18n import _ -from entropy.output import TextInterface, brown, blue, bold, darkgreen, \ +from entropy.output import brown, blue, bold, darkgreen, \ darkblue, red, purple, darkred, teal -from entropy.misc import TimeScheduled -from entropy.db import EntropyRepository from entropy.client.interfaces.client import Client from entropy.client.mirrors import StatusInterface from entropy.core.settings.base import SystemSettings -from entropy.security import System as SystemSecurity, \ - Repository as RepositorySecurity +from entropy.security import Repository as RepositorySecurity import entropy.tools @@ -131,7 +128,6 @@ class Package: for url, dest_path, cksum in url_data_list: count += 1 - filename = os.path.basename(url) dest_dir = os.path.dirname(dest_path) if not os.path.isdir(dest_dir): os.makedirs(dest_dir, 0o775) @@ -205,7 +201,7 @@ class Package: return False # set to 30 for convenience mirror_status.set_failing_mirror_status(best_mirror, 30) - mirrorcount = repo_uris[repo].index(best_mirror)+1 + mirrorcount = repo_uris[repository].index(best_mirror)+1 mytxt = "( mirror #%s ) " % (mirrorcount,) mytxt += blue(" %s: ") % (_("Mirror"),) mytxt += red(entropy.tools.spliturl(best_mirror)[1]) @@ -381,7 +377,6 @@ class Package: pass fetch_abort_function = self.pkgmeta.get('fetch_abort_function') - filename = os.path.basename(url) filepath_dir = os.path.dirname(save_path) # symlink support if not os.path.isdir(os.path.realpath(filepath_dir)): @@ -689,7 +684,7 @@ class Package: if isinstance(signatures, dict): for hash_type in sorted(signatures): hash_val = signatures[hash_type] - # XXX workaround bug on unreleased + # NOTE: workaround bug on unreleased # entropy versions if hash_val in signatures: continue @@ -1815,7 +1810,7 @@ class Package: # symlink doesn't need permissions, also # until os.walk ends they might be broken - # XXX also, added os.access() check because + # NOTE: also, added os.access() check because # there might be directories/files unwritable # what to do otherwise? user = os.stat(imagepath_dir)[stat.ST_UID] @@ -2123,7 +2118,6 @@ class Package: """ protected = False - tofile_before_protect = tofile do_continue = False in_mask = False encoded_protect = [x.encode('raw_unicode_escape') for x in protect] diff --git a/libraries/entropy/client/interfaces/qa.py b/libraries/entropy/client/interfaces/qa.py index f2142cfa4..159a19c55 100644 --- a/libraries/entropy/client/interfaces/qa.py +++ b/libraries/entropy/client/interfaces/qa.py @@ -13,7 +13,7 @@ from entropy.qa import ErrorReportInterface from entropy.client.interfaces import Client from entropy.core.settings.base import SystemSettings from entropy.const import etpConst -from entropy.exceptions import OnlineMirrorError, PermissionDenied +from entropy.exceptions import PermissionDenied from entropy.i18n import _ class UGCErrorReportInterface(ErrorReportInterface): diff --git a/libraries/entropy/client/interfaces/repository.py b/libraries/entropy/client/interfaces/repository.py index a6b6e6be3..82a3e6947 100644 --- a/libraries/entropy/client/interfaces/repository.py +++ b/libraries/entropy/client/interfaces/repository.py @@ -11,11 +11,8 @@ """ import os -import sys -import subprocess from entropy.i18n import _ -from entropy.const import etpConst, const_debug_write from entropy.exceptions import RepositoryError, PermissionDenied from entropy.output import blue, darkred, red, darkgreen, bold, purple, teal, \ brown @@ -34,7 +31,6 @@ class Repository: if repo_identifiers is None: repo_identifiers = [] - self._entropy = entropy_client_instance self._settings = SystemSettings() self._pkg_size_warning_th = 512*1024000 # 500mb @@ -61,7 +57,6 @@ class Repository: def _run_sync(self): self.updated = False - repolength = len(self.repo_ids) for repo in self.repo_ids: # handle diff --git a/libraries/entropy/client/interfaces/sets.py b/libraries/entropy/client/interfaces/sets.py index f1964c7e2..51fd4c184 100644 --- a/libraries/entropy/client/interfaces/sets.py +++ b/libraries/entropy/client/interfaces/sets.py @@ -103,8 +103,6 @@ class Sets: while True: - # check inside SystemSettings - # XXX: remove this in future # ALLOW server-side caller to match sets in /etc/entropy/sets # if not server_repos: sys_pkgsets = self._settings['system_package_sets'] @@ -168,7 +166,8 @@ class Sets: raise InvalidPackageSet("InvalidPackageSet: %s %s '%s'" % ( set_name, _("cannot start with"), etpConst['packagesetprefix'],)) set_match, rc = self.match(set_name) - if rc: return -1, _("Name already taken") + if rc: + return -1, _("Name already taken") _ensure_package_sets_dir() set_file = os.path.join(etpConst['confsetsdir'], set_name) @@ -181,7 +180,8 @@ class Sets: return -3, _("Cannot create the element") f = open(set_file, "w") - for x in set_atoms: f.write("%s\n" % (x,)) + for x in set_atoms: + f.write("%s\n" % (x,)) f.flush() f.close() self._settings['system_package_sets'][set_name] = set(set_atoms) @@ -201,7 +201,8 @@ class Sets: etpConst['packagesetprefix'],)) set_match, rc = self.match(set_name) - if not rc: return -1, _("Already removed") + if not rc: + return -1, _("Already removed") set_id, set_x, set_y = set_match if set_id != etpConst['userpackagesetsid']: diff --git a/libraries/entropy/client/interfaces/trigger.py b/libraries/entropy/client/interfaces/trigger.py index 538e45656..1e2498b17 100644 --- a/libraries/entropy/client/interfaces/trigger.py +++ b/libraries/entropy/client/interfaces/trigger.py @@ -12,7 +12,7 @@ import sys import os import subprocess -import shutil + from entropy.client.interfaces.client import Client from entropy.const import etpConst, const_isunicode, etpSys, etpUi from entropy.output import brown, bold, darkred, red @@ -234,7 +234,8 @@ class Trigger: pr = entropy.tools.dep_get_spm_revision(pv) pvr = pv - if pr == "r0": pvr += "-%s" % (pr,) + if pr == "r0": + pvr += "-%s" % (pr,) pet = pkgdata.get('versiontag') if const_isunicode(pet): @@ -376,7 +377,8 @@ class Trigger: tg_pfx = "%s/trigger-" % (etpConst['entropyunpackdir'],) while True: triggerfile = "%s%s" % (tg_pfx, entropy.tools.get_random_number(),) - if not os.path.isfile(triggerfile): break + if not os.path.isfile(triggerfile): + break triggerdir = os.path.dirname(triggerfile) if not os.path.isdir(triggerdir): @@ -387,7 +389,8 @@ class Trigger: start = 0 while True: buf = self.pkgdata['trigger'][start:] - if not buf: break + if not buf: + break f.write(buf) start += chunk f.flush() diff --git a/libraries/entropy/client/misc.py b/libraries/entropy/client/misc.py index 74fa85b39..044e9157c 100644 --- a/libraries/entropy/client/misc.py +++ b/libraries/entropy/client/misc.py @@ -138,7 +138,7 @@ class FileUpdates: continue filepath = os.path.join(currentdir, item) - # FIXME: with Python 3.x we can remove const_convert... + # NOTE: with Python 3.x we can remove const_convert... # and not use path.encode('utf-8') if item.startswith(const_convert_to_rawstring("._cfg")): @@ -276,31 +276,34 @@ class FileUpdates: mydict['automerge'] = True if (not mydict['automerge']): # is it trivial? + if not os.path.lexists(filepath): # if file does not even exist + return mydict + if os.path.islink(filepath): + # if it's broken, skip diff and automerge + if not os.path.exists(filepath): + return mydict + result = 1 try: - if not os.path.lexists(filepath): # if file does not even exist + result = getstatusoutput('diff -Nua "%s" "%s" | grep "^[+-][^+-]" | grep -v \'# .Header:.*\'' % (filepath, tofilepath,))[1] + except (OSError, IOError): + pass + if not result: + mydict['automerge'] = True + # another test + if not mydict['automerge']: + # if file does not even exist + if not os.path.lexists(filepath): return mydict if os.path.islink(filepath): # if it's broken, skip diff and automerge if not os.path.exists(filepath): return mydict - result = getstatusoutput('diff -Nua "%s" "%s" | grep "^[+-][^+-]" | grep -v \'# .Header:.*\'' % (filepath, tofilepath,))[1] - if not result: - mydict['automerge'] = True - except: - pass - # another test - if not mydict['automerge']: + result = 0 try: - # if file does not even exist - if not os.path.lexists(filepath): - return mydict - if os.path.islink(filepath): - # if it's broken, skip diff and automerge - if not os.path.exists(filepath): - return mydict - result = subprocess.call('diff -Bbua "%s" "%s" | egrep \'^[+-]\' | egrep -v \'^[+-][\t ]*#|^--- |^\+\+\+ \' | egrep -qv \'^[-+][\t ]*$\'' % (filepath, tofilepath,), shell = True) - if result == 1: - mydict['automerge'] = True - except: + result = subprocess.call('diff -Bbua "%s" "%s" | egrep \'^[+-]\' | egrep -v \'^[+-][\t ]*#|^--- |^\+\+\+ \' | egrep -qv \'^[-+][\t ]*$\'' % (filepath, tofilepath,), + shell = True) + except (IOError, OSError,): pass + if result == 1: + mydict['automerge'] = True return mydict diff --git a/libraries/entropy/client/services/system/commands.py b/libraries/entropy/client/services/system/commands.py index 1067c5e40..9814c7fbd 100644 --- a/libraries/entropy/client/services/system/commands.py +++ b/libraries/entropy/client/services/system/commands.py @@ -194,13 +194,20 @@ class Repository(Client): s_fetchonly = "0" s_buildonly = "0" s_nodeps = "0" - if pretend: s_pretend = "1" - if oneshot: s_oneshot = "1" - if verbose: s_verbose = "1" - if nocolor: s_nocolor = "1" - if fetchonly: s_fetchonly = "1" - if buildonly: s_buildonly = "1" - if nodeps: s_nodeps = "1" + if pretend: + s_pretend = "1" + if oneshot: + s_oneshot = "1" + if verbose: + s_verbose = "1" + if nocolor: + s_nocolor = "1" + if fetchonly: + s_fetchonly = "1" + if buildonly: + s_buildonly = "1" + if nodeps: + s_nodeps = "1" mydict = { 'atoms': ' '.join(atoms), 'pretend': s_pretend, @@ -228,9 +235,12 @@ class Repository(Client): s_pretend = "0" s_verbose = "0" s_nocolor = "0" - if pretend: s_pretend = "1" - if verbose: s_verbose = "1" - if nocolor: s_nocolor = "1" + if pretend: + s_pretend = "1" + if verbose: + s_verbose = "1" + if nocolor: + s_nocolor = "1" mydict = { 'atoms': ' '.join(atoms), 'pretend': s_pretend, diff --git a/libraries/entropy/client/services/system/interfaces.py b/libraries/entropy/client/services/system/interfaces.py index de7bffede..28cac8d25 100644 --- a/libraries/entropy/client/services/system/interfaces.py +++ b/libraries/entropy/client/services/system/interfaces.py @@ -11,10 +11,9 @@ """ import time from entropy.const import const_isstring, etpUi -from entropy.exceptions import * +from entropy.exceptions import ConnectionError from entropy.i18n import _ from entropy.misc import TimeScheduled -from entropy.i18n import _ class Client: @@ -70,7 +69,7 @@ class Client: self.shutdown = False self.connection_killer = None - # XXX actually session cache doesn't work when the connection is closed and re-opened + # NOTE actually session cache doesn't work when the connection is closed and re-opened # when the server is spawning requests under a child process (fork_requests = True) # this should be fixed by pushing the cache to disk but triggers a possible security issue # since sessions and their password are stored in memory and kept alive there until those @@ -127,7 +126,7 @@ class Client: if self.do_cache_connection: key = self.get_connection_cache_key() srv = self.connection_cache.get(key) - # FIXME: if you enable cache connection, you should also + # NOTE: if you enable cache connection, you should also # consider to clear the socket buffer # srv.sock_conn # srv.real_sock_conn @@ -207,7 +206,7 @@ class Client: else: kwargs['socket_timeout'] = timeout from entropy.services.ugc.interfaces import Client - srv = Client(*args,**kwargs) + srv = Client(*args, **kwargs) srv.connect(self.hostname, self.hostport) return srv diff --git a/libraries/entropy/client/services/system/methods.py b/libraries/entropy/client/services/system/methods.py index f97e22840..32fde49b5 100644 --- a/libraries/entropy/client/services/system/methods.py +++ b/libraries/entropy/client/services/system/methods.py @@ -44,12 +44,6 @@ class BaseMixin: 'call': self.get_queue_id_stdout, 'private': True, }, - 'get_queue_id_stdout': { - 'desc': _("Get queued command result using its queue unique identifier"), - 'params': [('queue_id', int, _('Queue Identifier'), True,)], - 'call': self.get_queue_id_result, - 'private': True, - }, 'remove_queue_ids': { 'desc': _("Remove queued commands using their queue unique identifiers"), 'params': [('queue_ids', list, _('Queue Identifiers'), True,)], diff --git a/libraries/entropy/client/services/ugc/commands.py b/libraries/entropy/client/services/ugc/commands.py index 3b9e6b698..89747ddaa 100644 --- a/libraries/entropy/client/services/ugc/commands.py +++ b/libraries/entropy/client/services/ugc/commands.py @@ -11,7 +11,7 @@ """ import os -from entropy.exceptions import * +from entropy.exceptions import SSLError from entropy.const import etpConst, const_get_stringtype, const_debug_write, \ const_convert_to_rawstring from entropy.output import darkblue, bold, blue, darkgreen, darkred, brown @@ -148,7 +148,7 @@ class Base: try: data = self.Service.receive() return data - except: + except Exception: entropy.tools.print_traceback() return None diff --git a/libraries/entropy/client/services/ugc/interfaces.py b/libraries/entropy/client/services/ugc/interfaces.py index 538e8c138..b3b5dbb40 100644 --- a/libraries/entropy/client/services/ugc/interfaces.py +++ b/libraries/entropy/client/services/ugc/interfaces.py @@ -74,7 +74,7 @@ class Client: if timeout is not None: kwargs['socket_timeout'] = timeout - srv = Client(*args,**kwargs) + srv = Client(*args, **kwargs) srv.connect(url, port) return srv @@ -133,7 +133,7 @@ class Client: if not aware: return False, _('repository does not support EAPI3') - def fake_callback(*args,**kwargs): + def fake_callback(*args, **kwargs): return True attempts = 3 @@ -278,9 +278,12 @@ class Client: def add_vote(self, repository, pkgkey, vote): data = self.do_cmd(repository, True, "ugc_do_vote", [pkgkey, vote], {}) - if isinstance(data, tuple): voted, add_err_msg = data - else: return False, 'wrong server answer' - if voted: self.get_vote(repository, pkgkey) + if isinstance(data, tuple): + voted, add_err_msg = data + else: + return False, 'wrong server answer' + if voted: + self.get_vote(repository, pkgkey) return voted, add_err_msg def get_vote(self, repository, pkgkey): @@ -298,8 +301,10 @@ class Client: def get_downloads(self, repository, pkgkey): data = self.do_cmd(repository, False, "ugc_get_downloads", [pkgkey], {}) - if isinstance(data, tuple): downloads, err_msg = data - else: return False, 'wrong server answer' + if isinstance(data, tuple): + downloads, err_msg = data + else: + return False, 'wrong server answer' if downloads: mydict = {pkgkey: downloads} self.UGCCache.update_downloads_cache(repository, mydict) @@ -484,14 +489,22 @@ class AuthStore(Singleton): store.appendChild(repo) self.xmldoc.appendChild(store) - try: + f = None + try: f = open(self.access_file, "w") f.writelines(self.xmldoc.toprettyxml(indent=" ")) f.flush() - f.close() self.setup_permissions() except IOError: - pass # wtf! no permissions? + # no permissions? + pass + finally: + if f is not None: + try: + f.close() + except IOError: + pass + self.parse_document() def remove_login(self, repository, save = True): diff --git a/libraries/entropy/const.py b/libraries/entropy/const.py index 63eb015fc..b8a544d9d 100644 --- a/libraries/entropy/const.py +++ b/libraries/entropy/const.py @@ -73,8 +73,6 @@ etpUi = { 'ask': False, 'pretend': False, 'mute': False, - # TODO: remove this after 20101010 - 'nolog': False, 'clean': False, 'warn': True, } @@ -155,7 +153,6 @@ def const_default_settings(rootdir): default_etp_dir = os.getenv('DEV_ETP_VAR_DIR', rootdir+"/var/lib/entropy") default_etp_tmpdir = "/tmp" - default_etp_repodir = "/packages/"+ETP_ARCH_CONST default_etp_portdir = rootdir+"/usr/portage" default_etp_distfilesdir = "/distfiles" default_etp_dbdir = "/database/"+ETP_ARCH_CONST diff --git a/libraries/entropy/core/settings/base.py b/libraries/entropy/core/settings/base.py index 0997adb64..87a4414cf 100644 --- a/libraries/entropy/core/settings/base.py +++ b/libraries/entropy/core/settings/base.py @@ -62,7 +62,6 @@ class SystemSettings(Singleton, EntropyPluginStore): from threading import RLock self.__lock = RLock() - from entropy.cache import EntropyCacher self.__cacher = EntropyCacher() self.__data = {} self.__is_destroyed = False diff --git a/libraries/entropy/core/settings/plugins/skel.py b/libraries/entropy/core/settings/plugins/skel.py index 23ea24e28..09a092898 100644 --- a/libraries/entropy/core/settings/plugins/skel.py +++ b/libraries/entropy/core/settings/plugins/skel.py @@ -29,23 +29,23 @@ class SystemSettingsPlugin(object): Sample code: - >>> # load SystemSettings - >>> from entropy.core.settings.base import SystemSettings - >>> from entropy.core.settings.plugins.skel import SystemSettingsPlugin - >>> system_settings = SystemSettings() - >>> class MyPlugin(SystemSettingsPlugin): - >>> pass - >>> my_plugin = MyPlugin('mystuff', None) - >>> def myparsing_function(): - >>> return {'abc': 1 } - >>> my_plugin.add_parser('parser_no_1', myparsing_function) - >>> system_settings.add_plugin(my_plugin) - >>> print(system_settings['mystuff']['parser_no_1']) - {'abc': 1 } - >>> # let's remove it - >>> system_settings.remove_plugin('mystuff') # through its plugin_id - >>> print(system_settings.get('mystuff')) - None + >>> # load SystemSettings + >>> from entropy.core.settings.base import SystemSettings + >>> from entropy.core.settings.plugins.skel import SystemSettingsPlugin + >>> system_settings = SystemSettings() + >>> class MyPlugin(SystemSettingsPlugin): + >>> pass + >>> my_plugin = MyPlugin('mystuff', None) + >>> def myparsing_function(): + >>> return {'abc': 1 } + >>> my_plugin.add_parser('parser_no_1', myparsing_function) + >>> system_settings.add_plugin(my_plugin) + >>> print(system_settings['mystuff']['parser_no_1']) + {'abc': 1 } + >>> # let's remove it + >>> system_settings.remove_plugin('mystuff') # through its plugin_id + >>> print(system_settings.get('mystuff')) + None """ diff --git a/libraries/entropy/db/__init__.py b/libraries/entropy/db/__init__.py index ea8e00b59..a5433fb3d 100644 --- a/libraries/entropy/db/__init__.py +++ b/libraries/entropy/db/__init__.py @@ -418,9 +418,6 @@ class EntropyRepository(EntropyRepositoryBase): EntropyRepositoryBase.__init__(self, readOnly, xcache, temporary, dbname, indexing) - # FIXME: remove before 20101010, backward compatibility - self.dbFile = dbFile - self._db_path = dbFile if self._db_path is None: raise AttributeError("valid database path needed") @@ -591,6 +588,28 @@ class EntropyRepository(EntropyRepositoryBase): def __del__(self): self.closeDB() + @staticmethod + def update(entropy_client, repository_id, force, gpg): + """ + Reimplemented from EntropyRepositoryBase. + """ + return EntropyRepositoryBase.update(entropy_client, repository_id, + force, gpg) + + @staticmethod + def revision(repository_id): + """ + Reimplemented from EntropyRepositoryBase. + """ + return EntropyRepositoryBase.revision(repository_id) + + @staticmethod + def remote_revision(repository_id): + """ + Reimplemented from EntropyRepositoryBase. + """ + return EntropyRepositoryBase.remote_revision(repository_id) + def closeDB(self): """ Reimplemented from EntropyRepositoryBase. @@ -680,10 +699,8 @@ class EntropyRepository(EntropyRepositoryBase): pkgdata['dependencies'][manual_dep] = \ etpConst['dependency_type_ids']['mdepend_id'] - # FIXME: this is Entropy Client related but also part of the - # currently implemented metaphor, so let's wait to have a Rule - # Engine in place before removing the oddity of client_repo - # metadatum. + # Client repositories behave differently + # TODO: move to Client repository class? client_repo = self.get_plugins_metadata().get('client_repo') if client_repo: remove_conflicting_packages(pkg_data) @@ -920,7 +937,7 @@ class EntropyRepository(EntropyRepositoryBase): new_way = self.getSetting("on_delete_cascade") except KeyError: new_way = '' - # TODO: deprecate this someday + # TODO: remove this before 31-12-2011 (deprecate) if new_way: # this will work thanks to ON DELETE CASCADE ! self._cursor().execute( @@ -948,7 +965,7 @@ class EntropyRepository(EntropyRepositoryBase): DELETE FROM injected WHERE idpackage = %d; DELETE FROM installedtable WHERE idpackage = %d; """ % r_tup) - # FIXME: incorportate in query above after 2011 + # TODO: incorportate in query above after 2011 if self._doesTableExist("packagedesktopmime"): self._cursor().execute(""" DELETE FROM packagedesktopmime WHERE idpackage = (?)""", @@ -1352,10 +1369,6 @@ class EntropyRepository(EntropyRepositoryBase): tuples of length 3 containing library name, path and ELF class. @type libs_metadata: list """ - # TODO: remove this in future - if not self._doesTableExist('provided_libs'): - self._createProvidedLibs() - self._cursor().executemany(""" INSERT INTO provided_libs VALUES (?,?,?,?) """, [(package_id, x, y, z,) for x, y, z in libs_metadata]) @@ -1526,14 +1539,9 @@ class EntropyRepository(EntropyRepositoryBase): @keyword gpg: GPG signature file content @type gpg: string """ - try: - self._cursor().execute(""" - INSERT INTO packagesignatures VALUES (?,?,?,?,?) - """, (package_id, sha1, sha256, sha512, gpg)) - except OperationalError: # FIXME: remove this before 2010-12-31 - self._cursor().execute(""" - INSERT INTO packagesignatures VALUES (?,?,?,?) - """, (package_id, sha1, sha256, sha512)) + self._cursor().execute(""" + INSERT INTO packagesignatures VALUES (?,?,?,?,?) + """, (package_id, sha1, sha256, sha512, gpg)) def _insertDesktopMime(self, package_id, metadata): """ @@ -1544,9 +1552,6 @@ class EntropyRepository(EntropyRepositoryBase): @param metadata: list of dict() containing file association metadata @type metadata: list """ - # FIXME: remove this before 2010-12-31 - if not self._doesTableExist("packagedesktopmime"): - self._createPackageDesktopMimeTable() mime_data = [(package_id, x['name'], x['mimetype'], x['executable'], x['icon']) for x in metadata] self._cursor().executemany(""" @@ -1563,9 +1568,6 @@ class EntropyRepository(EntropyRepositoryBase): @param mimetypes: list of mimetypes supported by package @type mimetypes: list """ - # FIXME: remove this before 2010-12-31 - if not self._doesTableExist("provided_mime"): - self._createProvidedMimeTable() self._cursor().executemany(""" INSERT INTO provided_mime VALUES (?,?)""", [(x, package_id) for x in mimetypes]) @@ -1597,9 +1599,6 @@ class EntropyRepository(EntropyRepositoryBase): @param repository: Source Package Manager repository @type repository: string """ - # FIXME backward compatibility - if not self._doesTableExist('packagespmrepository'): - self._createPackagespmrepository() self._cursor().execute(""" INSERT INTO packagespmrepository VALUES (?,?) """, (package_id, repository,)) @@ -1658,7 +1657,7 @@ class EntropyRepository(EntropyRepositoryBase): @param provides: list of atom strings @type provides: list """ - # FIXME: backward compat, remove someday + # TODO: remove this before 31-12-2011 # this adds default provide information to set data if not available my_provides = set() for prov in provides: @@ -1864,7 +1863,7 @@ class EntropyRepository(EntropyRepositoryBase): if self is dbconn: raise AttributeError("cannot diff inside the same db") - self._connection().text_factory = lambda x: const_convert_to_unicode(x) + self._connection().text_factory = const_convert_to_unicode # setup random table name randomtable = "cdiff%s" % (entropy.tools.get_random_number(),) @@ -1877,8 +1876,7 @@ class EntropyRepository(EntropyRepositoryBase): ) try: - dbconn._connection().text_factory = lambda x: \ - const_convert_to_unicode(x) + dbconn._connection().text_factory = const_convert_to_unicode cur = dbconn._cursor().execute(""" SELECT file FROM content WHERE idpackage = (?) @@ -2330,7 +2328,7 @@ class EntropyRepository(EntropyRepositoryBase): """ Reimplemented from EntropyRepositoryBase. """ - # FIXME backward compatibility + # TODO: remove this before 31-12-2011 if not self._doesTableExist('packagesets'): return {} @@ -2481,7 +2479,7 @@ class EntropyRepository(EntropyRepositoryBase): """ Reimplemented from EntropyRepositoryBase. """ - # FIXME backward compatibility + # TODO: remove this before 31-12-2011 if not self._doesTableExist('packagesignatures'): return None, None, None, None @@ -2492,7 +2490,7 @@ class EntropyRepository(EntropyRepositoryBase): """, (package_id,)) data = cur.fetchone() except OperationalError: - # FIXME: backward compat + # TODO: remove this before 31-12-2011 cur = self._cursor().execute(""" SELECT sha1, sha256, sha512 FROM packagesignatures WHERE idpackage = (?) LIMIT 1 @@ -2633,7 +2631,7 @@ class EntropyRepository(EntropyRepositoryBase): """ Reimplemented from EntropyRepositoryBase. """ - # FIXME backward compatibility + # TODO: remove this before 31-12-2011 if not self._doesTableExist('packagespmphases'): return None @@ -2649,7 +2647,7 @@ class EntropyRepository(EntropyRepositoryBase): """ Reimplemented from EntropyRepositoryBase. """ - # FIXME backward compatibility + # TODO: remove this before 31-12-2011 if not self._doesTableExist('packagespmrepository'): return None cur = self._cursor().execute(""" @@ -2701,7 +2699,7 @@ class EntropyRepository(EntropyRepositoryBase): needed.idneeded = neededreference.idneeded""", (package_id,)) return self._cur2set(cur) - def retrieveNeeded(self, package_id, extended = False, format = False): + def retrieveNeeded(self, package_id, extended = False, formatted = False): """ Reimplemented from EntropyRepositoryBase. """ @@ -2723,7 +2721,7 @@ class EntropyRepository(EntropyRepositoryBase): """, (package_id,)) needed = self._cur2list(cur) - if extended and format: + if extended and formatted: return dict((lib, elfclass,) for lib, elfclass in needed) return needed @@ -2731,7 +2729,7 @@ class EntropyRepository(EntropyRepositoryBase): """ Reimplemented from EntropyRepositoryBase. """ - # TODO: remove this in future, backward compat. + # TODO: remove this before 31-12-2011 if not self._doesTableExist('provided_libs'): return set() @@ -2754,7 +2752,7 @@ class EntropyRepository(EntropyRepositoryBase): """ Reimplemented from EntropyRepositoryBase. """ - # FIXME: added for backward compatibility, remove someday + # TODO: remove this before 31-12-2011 is_default_str = ',0' if self._doesColumnInTableExist("provide", "is_default"): is_default_str = ',is_default ' @@ -2920,12 +2918,12 @@ class EntropyRepository(EntropyRepositoryBase): """ Reimplemented from EntropyRepositoryBase. """ - # FIXME backward compatibility + # TODO: remove this before 31-12-2011 if not self._doesTableExist('automergefiles'): self._createAutomergefilesTable() # like portage does - self._connection().text_factory = lambda x: const_convert_to_unicode(x) + self._connection().text_factory = const_convert_to_unicode cur = self._cursor().execute(""" SELECT configfile, md5 FROM automergefiles WHERE idpackage = (?) @@ -2993,10 +2991,10 @@ class EntropyRepository(EntropyRepositoryBase): raise did_try = True - # FIXME support for old entropy db entries, which were + # TODO: remove this before 31-12-2011 + # Support for old entropy db entries, which were # not inserted in utf-8 - self._connection().text_factory = lambda x: \ - const_convert_to_unicode(x) + self._connection().text_factory = const_convert_to_unicode continue return fl @@ -3005,7 +3003,7 @@ class EntropyRepository(EntropyRepositoryBase): """ Reimplemented from EntropyRepositoryBase. """ - # FIXME backward compatibility + # TODO: remove this before 31-12-2011 if not self._doesTableExist('packagechangelogs'): return None @@ -3030,11 +3028,11 @@ class EntropyRepository(EntropyRepositoryBase): """ Reimplemented from EntropyRepositoryBase. """ - # FIXME backward compatibility + # TODO: remove this before 31-12-2011 if not self._doesTableExist('packagechangelogs'): return None - self._connection().text_factory = lambda x: const_convert_to_unicode(x) + self._connection().text_factory = const_convert_to_unicode cur = self._cursor().execute(""" SELECT changelog FROM packagechangelogs WHERE category = (?) AND @@ -3162,7 +3160,7 @@ class EntropyRepository(EntropyRepositoryBase): """ Reimplemented from EntropyRepositoryBase. """ - self._connection().text_factory = lambda x: const_convert_to_unicode(x) + self._connection().text_factory = const_convert_to_unicode cur = self._cursor().execute(""" SELECT text FROM licensedata WHERE licensename = (?) LIMIT 1 @@ -3383,7 +3381,7 @@ class EntropyRepository(EntropyRepositoryBase): """ Reimplemented from EntropyRepositoryBase. """ - # TODO: remove this in future, backward compat. + # TODO: remove this before 31-12-2011 if not self._doesTableExist('provided_libs'): self._createProvidedLibs() @@ -3643,7 +3641,7 @@ class EntropyRepository(EntropyRepositoryBase): return result[0] return -1 - def searchBelongs(self, file, like = False): + def searchBelongs(self, bfile, like = False): """ Reimplemented from EntropyRepositoryBase. """ @@ -3651,11 +3649,11 @@ class EntropyRepository(EntropyRepositoryBase): cur = self._cursor().execute(""" SELECT content.idpackage FROM content,baseinfo WHERE file LIKE (?) AND - content.idpackage = baseinfo.idpackage""", (file,)) + content.idpackage = baseinfo.idpackage""", (bfile,)) else: cur = self._cursor().execute("""SELECT content.idpackage FROM content, baseinfo WHERE file = (?) - AND content.idpackage = baseinfo.idpackage""", (file,)) + AND content.idpackage = baseinfo.idpackage""", (bfile,)) return self._cur2set(cur) @@ -3917,7 +3915,7 @@ class EntropyRepository(EntropyRepositoryBase): @return: found PROVIDE metadata @rtype: list """ - # FIXME: this small snippet is here for backward compat + # TODO: remove this before 31-12-2011 if self._doesColumnInTableExist("provide", "is_default"): get_def_string = ",provide.is_default" else: @@ -4195,7 +4193,7 @@ class EntropyRepository(EntropyRepositoryBase): """ Reimplemented from EntropyRepositoryBase. """ - self._connection().text_factory = lambda x: const_convert_to_unicode(x) + self._connection().text_factory = const_convert_to_unicode if count: cur = self._cursor().execute('SELECT count(file) FROM content') @@ -4554,8 +4552,7 @@ class EntropyRepository(EntropyRepositoryBase): q += ", ".join(["'||quote(" + x + ")||'" for x in cols]) q += ")' FROM '%(tbl_name)s'" self._cursor().execute(q % {'tbl_name': name}) - self._connection().text_factory = lambda x: \ - const_convert_to_unicode(x) + self._connection().text_factory = const_convert_to_unicode for row in self._cursor(): dumpfile.write(toraw("%s;\n" % (row[0],))) @@ -4567,10 +4564,9 @@ class EntropyRepository(EntropyRepositoryBase): dumpfile.write(toraw("%s;\n" % sql)) dumpfile.write(toraw("COMMIT;\n")) - try: + if hasattr(dumpfile, 'flush'): dumpfile.flush() - except: - pass + self.output( red(_("Database Export complete.")), importance = 0, @@ -4672,7 +4668,6 @@ class EntropyRepository(EntropyRepositoryBase): if strings: m = hashlib.md5() - if not self._doesTableExist("baseinfo"): if strings: m.update(const_convert_to_rawstring("~empty~")) @@ -4702,7 +4697,7 @@ class EntropyRepository(EntropyRepositoryBase): b_hash = hash(tuple(cur.fetchall())) if include_signatures: - # TODO: backward compatibility, remove this in future + # TODO: remove this before 31-12-2011 gpg_str = ", gpg" if not self._doesColumnInTableExist("packagesignatures", "gpg"): gpg_str = "" @@ -5204,13 +5199,13 @@ class EntropyRepository(EntropyRepositoryBase): def _foreignKeySupport(self): - # TODO: remove this by 2011/12/31 + # TODO: remove this before 31-12-2011 # entropy.qa uses this dbname, must skip migration if self.reponame in ("qa_testing", "mem_repo"): return - tables = ("extrainfo", "dependencies" ,"provide", + tables = ("extrainfo", "dependencies" , "provide", "conflicts", "configprotect", "configprotectmask", "sources", "useflags", "keywords", "content", "counters", "sizes", "eclasses", "needed", "triggers", "systempackages", "injected", @@ -5328,7 +5323,6 @@ class EntropyRepository(EntropyRepositoryBase): """) self.__clearLiveCache("_doesTableExist") - # TODO: this is going to be removed soon client_repo = self.get_plugins_metadata().get('client_repo') if client_repo and (self.reponame != etpConst['clientdbid']): diff --git a/libraries/entropy/db/skel.py b/libraries/entropy/db/skel.py index 2775198ba..ce5695caa 100644 --- a/libraries/entropy/db/skel.py +++ b/libraries/entropy/db/skel.py @@ -16,7 +16,7 @@ from entropy.i18n import _ from entropy.exceptions import InvalidAtom from entropy.const import etpConst, const_cmp from entropy.output import TextInterface, brown, bold, red, blue, purple, \ - darkred, darkgreen + darkred from entropy.cache import EntropyCacher from entropy.core import EntropyPluginStore from entropy.core.settings.base import SystemSettings @@ -26,138 +26,6 @@ from entropy.db.exceptions import OperationalError import entropy.tools -class EntropyRepositoryPluginStore(EntropyPluginStore): - - """ - EntropyRepository plugin interface. This is the EntropyRepository part - aimed to handle connected plugins. - """ - - _PERMANENT_PLUGINS = {} - - def __init__(self): - EntropyPluginStore.__init__(self) - permanent_plugs = EntropyRepositoryPluginStore.get_permanent_plugins() - for plug in permanent_plugs.values(): - plug.add_plugin_hook(self) - - def add_plugin(self, entropy_repository_plugin): - """ - Overloaded from EntropyPluginStore, adds support for hooks execution. - """ - from entropy.db.skel import EntropyRepositoryPlugin - inst = entropy_repository_plugin - if not isinstance(inst, EntropyRepositoryPlugin): - raise AttributeError("EntropyRepositoryPluginStore: " + \ - "expected valid EntropyRepositoryPlugin instance") - EntropyPluginStore.add_plugin(self, inst.get_id(), inst) - inst.add_plugin_hook(self) - - def remove_plugin(self, plugin_id): - """ - Overloaded from EntropyPluginStore, adds support for hooks execution. - """ - plugins = self.get_plugins() - plug_inst = plugins.get(plugin_id) - if plug_inst is not None: - plug_inst.remove_plugin_hook(self) - return EntropyPluginStore.remove_plugin(self, plugin_id) - - @staticmethod - def add_permanent_plugin(entropy_repository_plugin): - """ - Add EntropyRepository permanent plugin. This plugin object will be - used across all the instantiated EntropyRepositoryPluginStore classes. - Each time a new instance is created, add_plugin_hook will be executed - for all the permanent plugins. - - @param entropy_repository_plugin: EntropyRepositoryPlugin instance - @type entropy_repository_plugin: EntropyRepositoryPlugin instance - """ - from entropy.db.skel import EntropyRepositoryPlugin - inst = entropy_repository_plugin - if not isinstance(inst, EntropyRepositoryPlugin): - raise AttributeError("EntropyRepositoryPluginStore: " + \ - "expected valid EntropyRepositoryPlugin instance") - EntropyRepositoryPluginStore._PERMANENT_PLUGINS[inst.get_id()] = inst - - @staticmethod - def remove_permanent_plugin(plugin_id): - """ - Remove EntropyRepository permanent plugin. This plugin object will be - removed across all the EntropyRepository instances around. - Please note: due to the fact that there are no destructors around, - the "remove_plugin_hook" callback won't be executed when calling this - static method. - - @param plugin_id: EntropyRepositoryPlugin identifier - @type plugin_id: string - @raise KeyError: in case of unavailable plugin identifier - """ - del EntropyRepositoryPluginStore._PERMANENT_PLUGINS[plugin_id] - - @staticmethod - def get_permanent_plugins(): - """ - Return EntropyRepositoryStore installed permanent plugins. - - @return: copy of internal permanent plugins dict - @rtype: dict - """ - return EntropyRepositoryPluginStore._PERMANENT_PLUGINS.copy() - - def get_plugins(self): - """ - Overloaded from EntropyPluginStore, adds support for permanent plugins. - """ - plugins = EntropyPluginStore.get_plugins(self) - plugins.update(EntropyRepositoryPluginStore.get_permanent_plugins()) - return plugins - - def get_plugins_metadata(self): - """ - Return EntropyRepositoryPluginStore registered plugins metadata. - - @return: plugins metadata - @rtype: dict - """ - plugins = self.get_plugins() - meta = {} - for plugin_id in plugins: - meta.update(plugins[plugin_id].get_metadata()) - return meta - - def get_plugin_metadata(self, plugin_id, key): - """ - Return EntropyRepositoryPlugin metadata value referenced by "key". - - @param plugin_id. EntropyRepositoryPlugin identifier - @type plugin_id: string - @param key: EntropyRepositoryPlugin metadatum identifier - @type key: string - @return: metadatum value - @rtype: any Python object - @raise KeyError: if provided key or plugin_id is not available - """ - plugins = self.get_plugins() - return plugins[plugin_id][key] - - def set_plugin_metadata(self, plugin_id, key, value): - """ - Set EntropyRepositoryPlugin stored metadata. - - @param plugin_id. EntropyRepositoryPlugin identifier - @type plugin_id: string - @param key: EntropyRepositoryPlugin metadatum identifier - @type key: string - @param value: value to set - @type value: any valid Python object - @raise KeyError: if plugin_id is not available - """ - plugins = self.get_plugins() - meta = plugins[plugin_id].get_metadata() - meta[key] = value - class EntropyRepositoryPlugin(object): """ This is the base class for implementing EntropyRepository plugin hooks. @@ -349,6 +217,136 @@ class EntropyRepositoryPlugin(object): """ return 0 +class EntropyRepositoryPluginStore(EntropyPluginStore): + + """ + EntropyRepository plugin interface. This is the EntropyRepository part + aimed to handle connected plugins. + """ + + _PERMANENT_PLUGINS = {} + + def __init__(self): + EntropyPluginStore.__init__(self) + permanent_plugs = EntropyRepositoryPluginStore.get_permanent_plugins() + for plug in permanent_plugs.values(): + plug.add_plugin_hook(self) + + def add_plugin(self, entropy_repository_plugin): + """ + Overloaded from EntropyPluginStore, adds support for hooks execution. + """ + inst = entropy_repository_plugin + if not isinstance(inst, EntropyRepositoryPlugin): + raise AttributeError("EntropyRepositoryPluginStore: " + \ + "expected valid EntropyRepositoryPlugin instance") + EntropyPluginStore.add_plugin(self, inst.get_id(), inst) + inst.add_plugin_hook(self) + + def remove_plugin(self, plugin_id): + """ + Overloaded from EntropyPluginStore, adds support for hooks execution. + """ + plugins = self.get_plugins() + plug_inst = plugins.get(plugin_id) + if plug_inst is not None: + plug_inst.remove_plugin_hook(self) + return EntropyPluginStore.remove_plugin(self, plugin_id) + + @staticmethod + def add_permanent_plugin(entropy_repository_plugin): + """ + Add EntropyRepository permanent plugin. This plugin object will be + used across all the instantiated EntropyRepositoryPluginStore classes. + Each time a new instance is created, add_plugin_hook will be executed + for all the permanent plugins. + + @param entropy_repository_plugin: EntropyRepositoryPlugin instance + @type entropy_repository_plugin: EntropyRepositoryPlugin instance + """ + inst = entropy_repository_plugin + if not isinstance(inst, EntropyRepositoryPlugin): + raise AttributeError("EntropyRepositoryPluginStore: " + \ + "expected valid EntropyRepositoryPlugin instance") + EntropyRepositoryPluginStore._PERMANENT_PLUGINS[inst.get_id()] = inst + + @staticmethod + def remove_permanent_plugin(plugin_id): + """ + Remove EntropyRepository permanent plugin. This plugin object will be + removed across all the EntropyRepository instances around. + Please note: due to the fact that there are no destructors around, + the "remove_plugin_hook" callback won't be executed when calling this + static method. + + @param plugin_id: EntropyRepositoryPlugin identifier + @type plugin_id: string + @raise KeyError: in case of unavailable plugin identifier + """ + del EntropyRepositoryPluginStore._PERMANENT_PLUGINS[plugin_id] + + @staticmethod + def get_permanent_plugins(): + """ + Return EntropyRepositoryStore installed permanent plugins. + + @return: copy of internal permanent plugins dict + @rtype: dict + """ + return EntropyRepositoryPluginStore._PERMANENT_PLUGINS.copy() + + def get_plugins(self): + """ + Overloaded from EntropyPluginStore, adds support for permanent plugins. + """ + plugins = EntropyPluginStore.get_plugins(self) + plugins.update(EntropyRepositoryPluginStore.get_permanent_plugins()) + return plugins + + def get_plugins_metadata(self): + """ + Return EntropyRepositoryPluginStore registered plugins metadata. + + @return: plugins metadata + @rtype: dict + """ + plugins = self.get_plugins() + meta = {} + for plugin_id in plugins: + meta.update(plugins[plugin_id].get_metadata()) + return meta + + def get_plugin_metadata(self, plugin_id, key): + """ + Return EntropyRepositoryPlugin metadata value referenced by "key". + + @param plugin_id. EntropyRepositoryPlugin identifier + @type plugin_id: string + @param key: EntropyRepositoryPlugin metadatum identifier + @type key: string + @return: metadatum value + @rtype: any Python object + @raise KeyError: if provided key or plugin_id is not available + """ + plugins = self.get_plugins() + return plugins[plugin_id][key] + + def set_plugin_metadata(self, plugin_id, key, value): + """ + Set EntropyRepositoryPlugin stored metadata. + + @param plugin_id. EntropyRepositoryPlugin identifier + @type plugin_id: string + @param key: EntropyRepositoryPlugin metadatum identifier + @type key: string + @param value: value to set + @type value: any valid Python object + @raise KeyError: if plugin_id is not available + """ + plugins = self.get_plugins() + meta = plugins[plugin_id].get_metadata() + meta[key] = value + class EntropyRepositoryBase(TextInterface, EntropyRepositoryPluginStore, object): """ @@ -379,8 +377,6 @@ class EntropyRepositoryBase(TextInterface, EntropyRepositoryPluginStore, object) self.xcache = xcache self.temporary = temporary self.indexing = indexing - # XXX: backward compatibility - self.readOnly = readonly self.dbname = reponame self.reponame = reponame self._settings = SystemSettings() @@ -1355,7 +1351,7 @@ class EntropyRepositoryBase(TextInterface, EntropyRepositoryPluginStore, object) } provide_extended = self.retrieveProvide(package_id) - # FIXME: backward compat, remove before 2010-12-31 + # TODO: remove this before 31-12-2011 old_provide = set() for x in provide_extended: if isinstance(x, tuple): @@ -1474,7 +1470,7 @@ class EntropyRepositoryBase(TextInterface, EntropyRepositoryPluginStore, object) try: spm = get_spm(self) spm.packages_repositories_metadata_update() - except: + except Exception: entropy.tools.print_traceback() spm_moves = set() @@ -1730,7 +1726,7 @@ class EntropyRepositoryBase(TextInterface, EntropyRepositoryPluginStore, object) try: spm = get_spm(self) - except: + except Exception: entropy.tools.print_traceback() continue @@ -2221,7 +2217,7 @@ class EntropyRepositoryBase(TextInterface, EntropyRepositoryPluginStore, object) """ raise NotImplementedError() - def retrieveNeeded(self, package_id, extended = False, format = False): + def retrieveNeeded(self, package_id, extended = False, formatted = False): """ Return "NEEDED" elf metadata for libraries contained in given package. @@ -2230,9 +2226,9 @@ class EntropyRepositoryBase(TextInterface, EntropyRepositoryPluginStore, object) @keyword extended: also return ELF class information for every library name @type extended: bool - @keyword format: properly format output, returning a dictionary with + @keyword formatted: properly format output, returning a dictionary with library name as key and ELF class as value - @type format: bool + @type formatted: bool @return: "NEEDED" metadata for libraries contained in given package. @rtype: list or set """ @@ -2799,12 +2795,12 @@ class EntropyRepositoryBase(TextInterface, EntropyRepositoryPluginStore, object) """ raise NotImplementedError() - def searchBelongs(self, file, like = False): + def searchBelongs(self, bfile, like = False): """ Search packages which given file path belongs to. - @param file: file path to search - @type file: string + @param bfile: file path to search + @type bfile: string @keyword like: do not match exact case @type like: bool @return: list (set) of package identifiers owning given file @@ -3479,9 +3475,7 @@ class EntropyRepositoryBase(TextInterface, EntropyRepositoryPluginStore, object) """ raise NotImplementedError() - """ - Update status flags, self explanatory. - """ + # Update status flags, self explanatory. REPOSITORY_ALREADY_UPTODATE = -1 REPOSITORY_NOT_AVAILABLE = -2 REPOSITORY_GENERIC_ERROR = -3 @@ -3704,7 +3698,7 @@ class EntropyRepositoryBase(TextInterface, EntropyRepositoryPluginStore, object) # WORKAROUND for buggy entries # ** is fine then - # FIXME: remove after 2011 + # TODO: remove this before 31-12-2011 mykeywords = self.retrieveKeywords(package_id) if mykeywords == set([""]): mykeywords = set(['**']) @@ -3938,10 +3932,8 @@ class EntropyRepositoryBase(TextInterface, EntropyRepositoryPluginStore, object) self.__atomMatchStoreCache(("idpackageValidator", package_id, live), result = result) - # FIXME: this is Entropy Client related but also part of the - # currently implemented metaphor, so let's wait to have a Rule - # Engine in place before removing the oddity of client_repo - # metadatum. + # non-client repos don't use validation here + # TODO: fix this client_repo = self.get_plugins_metadata().get('client_repo') if not client_repo: # server-side repositories don't make any use of package_id validator @@ -4204,14 +4196,12 @@ class EntropyRepositoryBase(TextInterface, EntropyRepositoryPluginStore, object) ) return x[0], 0 - """ # if a default_package_id is given by __generate_found_ids_match # we need to respect it # NOTE: this is only used by old-style virtual packages - if (len(found_ids) > 1) and (default_package_id is not None): - if default_package_id in found_ids: - found_ids = set([default_package_id]) - """ + # if (len(found_ids) > 1) and (default_package_id is not None): + # if default_package_id in found_ids: + # found_ids = set([default_package_id]) dbpkginfo = list(dbpkginfo) if default_package_ids is not None: # at this point, if default_package_ids is not None (== set()) diff --git a/libraries/entropy/dump.py b/libraries/entropy/dump.py index 5fefcdcb6..d015db018 100644 --- a/libraries/entropy/dump.py +++ b/libraries/entropy/dump.py @@ -25,7 +25,7 @@ import sys import os -from entropy.const import etpConst, const_setup_perms, const_setup_file +from entropy.const import etpConst, const_setup_file # Always use MAX pickle protocol to <=2, to allow Python 2 and 3 support COMPAT_PICKLE_PROTOCOL = 0 diff --git a/libraries/entropy/fetchers.py b/libraries/entropy/fetchers.py index 1879c3d85..ea0dbc2c5 100644 --- a/libraries/entropy/fetchers.py +++ b/libraries/entropy/fetchers.py @@ -23,7 +23,7 @@ else: import urllib2 as urlmod_error from entropy.exceptions import InterruptError -from entropy.tools import print_traceback, get_file_size, \ +from entropy.tools import print_traceback, \ convert_seconds_to_fancy_output, bytes_into_human, spliturl, \ add_proxy_opener, md5sum from entropy.const import etpConst, const_isfileobj @@ -79,8 +79,6 @@ class UrlFetcher(TextInterface): speed_limit = \ self.__system_settings['repositories']['transfer_limit'] - import socket - self.socket = socket self.__timeout = \ self.__system_settings['repositories']['timeout'] self.__th_id = 0 @@ -94,7 +92,7 @@ class UrlFetcher(TextInterface): self.__disallow_redirect = disallow_redirect self.__speedlimit = speed_limit # kbytes/sec self.__existed_before = False - self.localfile = None + self.__localfile = None # important to have this here too self.__datatransfer = 0 @@ -140,10 +138,11 @@ class UrlFetcher(TextInterface): # if client uses this instance more than # once, make sure we close previously opened # files. - if const_isfileobj(self.localfile): + if const_isfileobj(self.__localfile): try: - self.localfile.flush() - self.localfile.close() + if hasattr(self.__localfile, 'flush'): + self.__localfile.flush() + self.__localfile.close() except (IOError, OSError,): pass @@ -151,14 +150,14 @@ class UrlFetcher(TextInterface): if os.path.isfile(self.__path_to_save) and \ os.access(self.__path_to_save, os.W_OK) and self.__resume: - self.localfile = open(self.__path_to_save, "ab") - self.localfile.seek(0, os.SEEK_END) - self.__startingposition = int(self.localfile.tell()) + self.__localfile = open(self.__path_to_save, "ab") + self.__localfile.seek(0, os.SEEK_END) + self.__startingposition = int(self.__localfile.tell()) self.__last_downloadedsize = self.__startingposition self.__resumed = True return - self.localfile = open(self.__path_to_save, "wb") + self.__localfile = open(self.__path_to_save, "wb") def _setup_proxy(self): # setup proxy, doing here because config is dynamic @@ -214,13 +213,16 @@ class UrlFetcher(TextInterface): req = self.__url u_agent_error = False + do_return = False while True: + # get file size if available try: self.__remotefile = urlmod.urlopen(req, None, self.__timeout) except KeyboardInterrupt: self.__close(False) raise + except urlmod_error.HTTPError as e: if (e.code == 405) and not u_agent_error: # server doesn't like our user agent @@ -229,33 +231,38 @@ class UrlFetcher(TextInterface): continue self.__close(True) self.__status = "-3" - return self.__status + do_return = True except urlmod_error.URLError as err: # timeout error self.__close(True) self.__status = "-3" - return self.__status + do_return = True except httplib.BadStatusLine: # obviously, something to cope with self.__close(True) self.__status = "-3" - return self.__status + do_return = True + except socket.timeout: # arghv!! self.__close(True) self.__status = "-3" + do_return = True except ValueError: # malformed, unsupported URL? raised by urllib self.__close(True) self.__status = "-3" - return self.__status + do_return = True - except: + except Exception: print_traceback() raise break + if do_return: + return self.__status + try: self.__remotesize = int(self.__remotefile.headers.get( "content-length")) @@ -263,7 +270,7 @@ class UrlFetcher(TextInterface): except KeyboardInterrupt: self.__close(False) raise - except: + except Exception: pass # handle user stupidity @@ -322,7 +329,7 @@ class UrlFetcher(TextInterface): except KeyboardInterrupt: self.__close(False) raise - except self.socket.timeout: + except socket.timeout: self.__close(False) self.__status = "-4" return self.__status @@ -339,14 +346,14 @@ class UrlFetcher(TextInterface): self.__updatestep, self.__show_speed, self.__datatransfer, self.__time_remaining, self.__time_remaining_secs ) - self.output() + self.update() self.__oldaverage = self.__average if self.__speedlimit: while self.__datatransfer > self.__speedlimit*1024: time.sleep(0.1) self._update_speed() if self.__show_speed: - self.output() + self.update() self.__oldaverage = self.__average self._push_progress_to_output() @@ -364,9 +371,9 @@ class UrlFetcher(TextInterface): def _commit(self, mybuffer): # writing file buffer - self.localfile.write(mybuffer) + self.__localfile.write(mybuffer) # update progress info - self.__downloadedsize = self.localfile.tell() + self.__downloadedsize = self.__localfile.tell() kbytecount = float(self.__downloadedsize)/1024 average = int((kbytecount/self.__remotesize)*100) if average > 100: @@ -377,9 +384,10 @@ class UrlFetcher(TextInterface): def __close(self, errored): self._update_speed() try: - if const_isfileobj(self.localfile): - self.localfile.flush() - self.localfile.close() + if const_isfileobj(self.__localfile): + if hasattr(self.__localfile, 'flush'): + self.__localfile.flush() + self.__localfile.close() except IOError: pass if (not self.__existed_before) and errored: @@ -532,6 +540,10 @@ class UrlFetcher(TextInterface): TextInterface.output(self, current_txt, back = True) def output(self): + """ @deprecated, remove after 2010 """ + return self.update() + + def update(self): """ Main fetch progress callback. You can reimplement this to refresh your output devices. @@ -634,7 +646,7 @@ class MultipleUrlFetcher(TextInterface): klass.__init__(self, *args, **kwargs) self.__multiple_fetcher = multiple - def output(self): + def update(self): return self.__multiple_fetcher.output() def _push_progress_to_output(self): @@ -860,7 +872,7 @@ class MultipleUrlFetcher(TextInterface): self.__old_average = average - def output(self): + def update(self): """ Main fetch progress callback. You can reimplement this to refresh your output devices. diff --git a/libraries/entropy/i18n.py b/libraries/entropy/i18n.py index aa62e51ef..6dc3db48e 100644 --- a/libraries/entropy/i18n.py +++ b/libraries/entropy/i18n.py @@ -34,7 +34,7 @@ try: envdir = os.getenv('TEXTDOMAINDIR') if envdir is not None: localedir = envdir - kwargs = {"localedir": localedir,} + kwargs = {"localedir": localedir} if sys.hexversion < 0x3000000: kwargs['unicode'] = True gettext.install('entropy', **kwargs) @@ -65,16 +65,16 @@ def change_language(lang): """ global _ # change in environ - for var in ("LANGUAGE", "LC_ALL", "LANG",): + for var in ("LANGUAGE", "LC_ALL", "LANG"): os.environ[var] = lang # reinstall gettext # remove _ from global scope so that gettext will readd it old_ = _ del _ - kwargs = {"localedir": localedir,} + kw_args = {"localedir": localedir} if sys.hexversion < 0x3000000: - kwargs['unicode'] = True - gettext.install('entropy', **kwargs) + kw_args['unicode'] = True + gettext.install('entropy', **kw_args) _ = _ # redeclare "_" in all loaded modules for module in list(sys.modules.values()): diff --git a/libraries/entropy/misc.py b/libraries/entropy/misc.py index b49388ed8..05346157e 100644 --- a/libraries/entropy/misc.py +++ b/libraries/entropy/misc.py @@ -19,11 +19,12 @@ if sys.hexversion >= 0x3000000: import urllib.request, urllib.error, urllib.parse UrllibBaseHandler = urllib.request.BaseHandler else: + import urllib import urllib2 UrllibBaseHandler = urllib2.BaseHandler import logging import threading -from entropy.const import etpConst, etpUi, const_isunicode, const_isstring, \ +from entropy.const import etpConst, const_isunicode, \ const_isfileobj, const_convert_log_level import entropy.tools @@ -977,6 +978,8 @@ class RSS: class LogFile: + """ Entropy simple logging interface, works as file object """ + LEVELS = { "debug": logging.DEBUG, "info": logging.INFO, @@ -988,8 +991,6 @@ class LogFile: DATE_FORMAT = "[%H:%M:%S %d/%m/%Y %Z]" _HANDLER_CACHE = {} - """ Entropy simple logging interface, works as file object """ - def __init__(self, level = None, filename = None, header = "[LOG]"): """ LogFile constructor. @@ -1178,7 +1179,6 @@ class MultipartPostHandler(UrllibBaseHandler): if sys.hexversion >= 0x3000000: data = urllib.parse.urlencode(v_vars, doseq) else: - import urllib data = urllib.urlencode(v_vars, doseq) else: boundary, data = self.multipart_encode(v_vars, v_files) diff --git a/libraries/entropy/qa.py b/libraries/entropy/qa.py index 5db0419e2..7cf32e9a8 100644 --- a/libraries/entropy/qa.py +++ b/libraries/entropy/qa.py @@ -413,7 +413,7 @@ class QAInterface(TextInterface, EntropyPluginStore): rc_ask = self.ask_question(_("Want to add?")) if rc_ask == _("Yes"): newmissing.add(dependency) - ### XXX: disabled, devs are not able to use it properly + ### NOTE: disabled, devs are not able to use it properly ### needs usability fixes #else: #rc_ask = self.ask_question( diff --git a/libraries/entropy/security.py b/libraries/entropy/security.py index 6a33945e7..c31613f8f 100644 --- a/libraries/entropy/security.py +++ b/libraries/entropy/security.py @@ -20,7 +20,6 @@ import tempfile import time from entropy.exceptions import EntropyException -from entropy.misc import LogFile from entropy.const import etpConst, etpUi, const_setup_perms, \ const_debug_write, const_setup_file from entropy.i18n import _ @@ -34,9 +33,7 @@ class System: """ ~~ GIVES YOU WINGS ~~ - """ - """ @note: thanks to Gentoo "gentoolkit" package, License below: @note: This program is licensed under the GPL, version 2 @@ -58,17 +55,17 @@ class System: "securityurl" parameter inside repositories.conf. To sum up, you as distributor should: - 1. implement your security advisories xml files by looking at - docs/metadata/glsa.dtd specifications. - 2. setup a cronjob that compresses your unpacked list of advisories - to a file inside a publicly available URL as well as a valid .md5 - file. - 3. provide a default repositories.conf file with securityurl| pointing - to that file (HTTP, FTP and FILE protocols supported). - 4. Optionally, in the same dir you could make available a GPG public - key and a GPG signature of your security advisories .tar.* file. - The former MUST be named signature.asc while the latter must match - securityurl value plus ".asc" + 1. implement your security advisories xml files by looking at + docs/metadata/glsa.dtd specifications. + 2. setup a cronjob that compresses your unpacked list of advisories + to a file inside a publicly available URL as well as a valid .md5 + file. + 3. provide a default repositories.conf file with securityurl| pointing + to that file (HTTP, FTP and FILE protocols supported). + 4. Optionally, in the same dir you could make available a GPG public + key and a GPG signature of your security advisories .tar.* file. + The former MUST be named signature.asc while the latter must match + securityurl value plus ".asc" """ _CACHE_ID = 'advisories_cache_' @@ -290,7 +287,10 @@ class System: header = red(" # ") ) - easy_url = entropy.tools.spliturl(self.security_url).netloc + easy_url = "N/A" + splitres = entropy.tools.spliturl(self.security_url) + if hasattr(splitres, 'netloc'): + easy_url = splitres.netloc if pk_avail: @@ -304,8 +304,8 @@ class System: except repo_sec.GPGError: downloaded_key_fp = None - fingerprint = \ - repo_sec.get_key_metadata(self.security_url)['fingerprint'] + fingerprint = repo_sec.get_key_metadata( + self.security_url)['fingerprint'] shutil.rmtree(tmp_dir, True) if downloaded_key_fp != fingerprint and \ @@ -1256,8 +1256,6 @@ class Repository: a nice testing codebase. """ - from entropy.exceptions import EntropyException - class GPGError(EntropyException): """Errors during GPG commands execution""" @@ -1292,6 +1290,7 @@ class Repository: rvk = revocation key ''' def __init__(self): + list.__init__(self) self.curkey = None self.fingerprints = [] @@ -1495,7 +1494,7 @@ class Repository: key = key.replace('_','-').title() parms[key] = val parms.setdefault('Key-Type','RSA') - parms.setdefault('Key-Length',1024) + parms.setdefault('Key-Length', 1024) parms.setdefault('Name-Real', "Autogenerated Key") parms.setdefault('Name-Comment', "Generated by gnupg.py") try: @@ -1746,7 +1745,7 @@ class Repository: @raise Repository.KeyExpired: if key is expired """ try: - key = self.get_pubkey(repository_identifier) + self.get_pubkey(repository_identifier) except KeyError: return False diff --git a/libraries/entropy/server/interfaces/main.py b/libraries/entropy/server/interfaces/main.py index 0c2704a1d..f8df8e3ec 100644 --- a/libraries/entropy/server/interfaces/main.py +++ b/libraries/entropy/server/interfaces/main.py @@ -17,23 +17,21 @@ import tempfile import time import re -from entropy.core import Singleton from entropy.exceptions import OnlineMirrorError, PermissionDenied, \ SystemDatabaseError from entropy.const import etpConst, etpSys, const_setup_perms, \ const_create_working_dirs, etpUi, \ const_setup_file, const_get_stringtype, const_debug_write -from entropy.output import TextInterface, purple, red, darkgreen, \ +from entropy.output import purple, red, darkgreen, \ bold, brown, blue, darkred, teal from entropy.cache import EntropyCacher from entropy.server.interfaces.mirrors import Server as MirrorsServer -from entropy.server.interfaces.rss import ServerRssMetadata from entropy.i18n import _ from entropy.core.settings.base import SystemSettings from entropy.core.settings.plugins.skel import SystemSettingsPlugin from entropy.transceivers import EntropyTransceiver from entropy.db import EntropyRepository -from entropy.db.skel import EntropyRepositoryPlugin, EntropyRepositoryBase +from entropy.db.skel import EntropyRepositoryPlugin from entropy.server.interfaces.db import ServerRepositoryStatus from entropy.spm.plugins.factory import get_default_instance as get_spm, \ get_default_class as get_spm_class @@ -63,6 +61,9 @@ class ServerPackagesRepository(EntropyRepository): srv = Server() return srv.get_remote_repository_revision(repo = repository_id) + @staticmethod + def update(entropy_client, repository_id, force, gpg): + raise NotImplementedError() class ServerEntropyRepositoryPlugin(EntropyRepositoryPlugin): @@ -130,8 +131,6 @@ class ServerEntropyRepositoryPlugin(EntropyRepositoryPlugin): # better than having a completely broken db self._metadata['read_only'] = False entropy_repository_instance.readonly = False - # XXX remove this in future - entropy_repository_instance.readOnly = False entropy_repository_instance.initializeRepository() entropy_repository_instance.commitChanges() @@ -570,11 +569,11 @@ class ServerSystemSettingsPlugin(SystemSettingsPlugin): split_line = line.split("|") split_line_len = len(split_line) - # TODO: remove this in future, supported for backward compat. + # NOTE: remove this in future, supported for backward compat. if (line.find("officialserverrepositoryid|") != -1) and \ (not line.startswith("#")) and (split_line_len == 2): - # TODO: added for backward and mixed compat. + # NOTE: added for backward and mixed compat. if default_repo_changed: continue @@ -2102,7 +2101,6 @@ class ServerPackagesHandlingMixin: # and grab the new "download" metadatum value using our # license check callback. It has to be done here because # we need the new path. - srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server'] def _package_injector_check_license(pkg_data): licenses = pkg_data['license'].split() @@ -2117,11 +2115,11 @@ class ServerPackagesHandlingMixin: # check if pkg is restricted # and check if pkg is free, we must do this step in any case - # XXX: it sucks! + # NOTE: it sucks! tmp_data = self.Spm().extract_package_metadata(from_file, license_callback = _package_injector_check_license, restricted_callback = _package_injector_check_restricted) - # XXX: since ~0.tbz2 << revision is lost, we need to trick + # NOTE: since ~0.tbz2 << revision is lost, we need to trick # the logic. updated_package_rel_path = os.path.join( os.path.dirname(tmp_data['download']), @@ -4084,8 +4082,8 @@ class ServerRepositoryMixin: if not wl_licenses: return True # no whitelist ! - for license in pkg_licenses: - if license not in wl_licenses: + for lic in pkg_licenses: + if lic not in wl_licenses: return False return True @@ -4594,7 +4592,6 @@ class ServerMiscMixin: to_be_injected = set() my_settings = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server'] exp_based_scope = my_settings['exp_based_scope'] - excluded_dep_types = [etpConst['dependency_type_ids']['bdepend_id']] server_repos = list(my_settings['repositories'].keys()) diff --git a/libraries/entropy/server/interfaces/mirrors.py b/libraries/entropy/server/interfaces/mirrors.py index 2e73fdbd8..3b0fca4a0 100644 --- a/libraries/entropy/server/interfaces/mirrors.py +++ b/libraries/entropy/server/interfaces/mirrors.py @@ -14,11 +14,10 @@ import tempfile import shutil import time -from entropy.exceptions import OnlineMirrorError, ConnectionError, \ - EntropyPackageException, TransceiverError +from entropy.exceptions import OnlineMirrorError, EntropyPackageException from entropy.output import red, darkgreen, bold, brown, blue, darkred, \ darkblue, purple, teal -from entropy.const import etpConst, const_setup_perms, const_setup_file +from entropy.const import etpConst, const_setup_file from entropy.cache import EntropyCacher from entropy.i18n import _ from entropy.misc import RSS @@ -646,7 +645,6 @@ class Server(ServerNoticeBoardMixin): if repo is None: repo = self._entropy.default_repository - pkgfile = os.path.basename(pkg_relative_path) crippled_uri = EntropyTransceiver.get_uri_name(uri) tries = 0 @@ -1265,7 +1263,7 @@ class Server(ServerNoticeBoardMixin): data[myname] = myfile extra_text_files.append(myfile) - # XXX/FIXME: for symlinks, we read their link and send a file with that + # NOTE: for symlinks, we read their link and send a file with that # content. This is the default behaviour for now and allows to send # /etc/make.profile link pointer correctly. for symname, symfile in spm_syms.items(): @@ -2038,7 +2036,7 @@ class Server(ServerNoticeBoardMixin): if local_revision < highest_remote_revision: for remote_item in remote_status: if remote_item[1] == highest_remote_revision: - download_latest = x + download_latest = remote_item break if download_latest: @@ -2122,8 +2120,8 @@ class Server(ServerNoticeBoardMixin): if upload_queue and not no_upload: - # XXX QA checks, - # please group them into entropy.qa + # Some internal QA checks, make sure everything is fine + # on the repo srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server'] base_repo = srv_set['base_repository_id'] @@ -3167,7 +3165,7 @@ class Server(ServerNoticeBoardMixin): ) exc_txt = entropy.tools.print_exception( - returndata = True) + silent = True) for line in exc_txt: self._entropy.output( repr(line), diff --git a/libraries/entropy/server/transceivers.py b/libraries/entropy/server/transceivers.py index 7778e9210..61058b5dc 100644 --- a/libraries/entropy/server/transceivers.py +++ b/libraries/entropy/server/transceivers.py @@ -10,11 +10,10 @@ """ import os -import sys -import time + from entropy.const import const_isstring, const_isnumber, etpConst from entropy.output import darkred, blue, brown, darkgreen, red, bold -from entropy.exceptions import * +from entropy.exceptions import ConnectionError from entropy.i18n import _ from entropy.core.settings.base import SystemSettings from entropy.transceivers import EntropyTransceiver @@ -187,7 +186,6 @@ class TransceiverServerHandler: base_dir = self.txc_basedir - mycwd = None if isinstance(mypath, tuple): if len(mypath) < 2: continue diff --git a/libraries/entropy/services/auth_interfaces.py b/libraries/entropy/services/auth_interfaces.py index 329ab798f..2e87a33b1 100644 --- a/libraries/entropy/services/auth_interfaces.py +++ b/libraries/entropy/services/auth_interfaces.py @@ -14,7 +14,7 @@ import os import time import random from entropy.services.skel import Authenticator, RemoteDatabase -from entropy.exceptions import * +from entropy.exceptions import PermissionDenied from entropy.const import etpConst, const_isstring, const_convert_to_unicode from entropy.i18n import _ @@ -77,27 +77,36 @@ class phpBB3Auth(Authenticator, RemoteDatabase): self.check_connection() self.cursor.execute('SELECT user_id FROM '+self.TABLE_PREFIX+'users WHERE `username_clean` = %s OR LOWER(`username`) = %s', (username_clean, username.lower(),)) data = self.cursor.fetchone() - if not data: return False - if not isinstance(data, dict): return False - if 'user_id' not in data: return False + if not data: + return False + if not isinstance(data, dict): + return False + if 'user_id' not in data: + return False return True def does_email_exist(self, email): self.check_connection() self.cursor.execute('SELECT user_id FROM '+self.TABLE_PREFIX+'users WHERE `user_email` = %s', (email,)) data = self.cursor.fetchone() - if not data: return False - if not isinstance(data, dict): return False - if 'user_id' not in data: return False + if not data: + return False + if not isinstance(data, dict): + return False + if 'user_id' not in data: + return False return True def is_username_allowed(self, username): self.check_connection() self.cursor.execute('SELECT disallow_id FROM '+self.TABLE_PREFIX+'disallow WHERE `disallow_username` = %s', (username,)) data = self.cursor.fetchone() - if not data: return True - if not isinstance(data, dict): return True - if 'disallow_id' not in data: return True + if not data: + return True + if not isinstance(data, dict): + return True + if 'disallow_id' not in data: + return True return False def validate_username_string(self, username, username_clean): @@ -118,10 +127,12 @@ class phpBB3Auth(Authenticator, RemoteDatabase): return False, 'Invalid username' exists = self.does_username_exist(username, username_clean) - if exists: return False, 'Username already taken' + if exists: + return False, 'Username already taken' allowed = self.is_username_allowed(username) - if not allowed: return False, 'Username not allowed' + if not allowed: + return False, 'Username not allowed' return True, 'All fine' @@ -157,11 +168,13 @@ class phpBB3Auth(Authenticator, RemoteDatabase): # check username validity status, err_msg = self.validate_username_string(username, username_clean) - if not status: return False, err_msg + if not status: + return False, err_msg # check email exists = self.does_email_exist(email) - if exists: return False, 'Email already in use' + if exists: + return False, 'Email already in use' # now cross fingers status, user_id = self.__register(username, username_clean, password, email, activate) @@ -178,7 +191,8 @@ class phpBB3Auth(Authenticator, RemoteDatabase): time_now = int(time.time()) user_type = self.USER_INACTIVE - if activate: user_type = self.USER_NORMAL + if activate: + user_type = self.USER_NORMAL registration_data = { 'username': username, @@ -260,7 +274,8 @@ class phpBB3Auth(Authenticator, RemoteDatabase): if isinstance(data, dict): if 'group_colour' in data: gcolor = data['group_colour'] - if gcolor: self._set_config_value('newest_user_colour', gcolor) + if gcolor: + self._set_config_value('newest_user_colour', gcolor) return True, user_id @@ -548,7 +563,8 @@ class phpBB3Auth(Authenticator, RemoteDatabase): my_params = {} for param in valid_params: d = profile_data.get(param) - if d == None: continue + if d is None: + continue my_params[param] = d if not my_params: @@ -700,7 +716,8 @@ class phpBB3Auth(Authenticator, RemoteDatabase): return myoutput count_log2 = self.itoa64.find(setting[3]) - if count_log2 == -1: count_log2 = 0 + if count_log2 == -1: + count_log2 = 0 if (count_log2 < 7) or (count_log2 > 30): return myoutput diff --git a/libraries/entropy/services/authenticators.py b/libraries/entropy/services/authenticators.py index 1f6cc0524..d813696e6 100644 --- a/libraries/entropy/services/authenticators.py +++ b/libraries/entropy/services/authenticators.py @@ -11,10 +11,9 @@ """ from entropy.const import const_get_stringtype -from entropy.services.skel import Authenticator from entropy.services.auth_interfaces import phpBB3Auth from entropy.services.skel import SocketAuthenticator -from entropy.exceptions import * +from entropy.exceptions import PermissionDenied # Authenticator that can be used by SocketHostInterface based instances class phpBB3(phpBB3Auth, SocketAuthenticator): diff --git a/libraries/entropy/services/interfaces.py b/libraries/entropy/services/interfaces.py index edfe8f860..acf1d0a22 100644 --- a/libraries/entropy/services/interfaces.py +++ b/libraries/entropy/services/interfaces.py @@ -204,12 +204,14 @@ class SocketHost: self.ssl_authorized_clients_only = authorized_clients_only if self.SSL: - socketserver.BaseServer.__init__(self, server_address, RequestHandlerClass) + socketserver.BaseServer.__init__(self, server_address, + RequestHandlerClass) self.load_ssl_context() self.make_ssl_connection_alive() else: try: - socketserver.TCPServer.__init__(self, server_address, RequestHandlerClass) + socketserver.TCPServer.__init__(self, server_address, + RequestHandlerClass) except self.socket_mod.error as e: if e[0] == 13: raise ConnectionError('ConnectionError: %s' % (_("Cannot bind the service"),)) @@ -247,8 +249,10 @@ class SocketHost: def verify_request(self, request, client_address): self.do_ssl = self.HostInterface.SSL - if self.do_ssl: self.do_ssl = True - else: self.do_ssl = False + if self.do_ssl: + self.do_ssl = True + else: + self.do_ssl = False allowed = self.ip_blacklist_check(client_address[0]) if allowed: allowed = self.ip_max_connections_check(client_address[0]) @@ -570,7 +574,8 @@ class SocketHost: if hasattr(self.server.processor.HostInterface, 'ForkLock'): x = getattr(self.server.processor.HostInterface, 'ForkLock') if hasattr(x, 'acquire') and hasattr(x, 'release') and hasattr(x, 'locked'): - if x.locked(): x.release() + if x.locked(): + x.release() def handle(self): # not using spawnFunction because it causes some mess @@ -693,8 +698,8 @@ class SocketHost: if (session in self.HostInterface.initialization_commands) or \ (session in self.HostInterface.no_session_commands) or \ len(args) < 2: - cmd = args[0] - session = None + cmd = args[0] + session = None else: cmd = args[1] args = args[1:] # remove session @@ -705,7 +710,8 @@ class SocketHost: if stream_enabled and (cmd not in self.HostInterface.config_commands): session_len = 0 - if session: session_len = len(session)+1 + if session: + session_len = len(session)+1 return cmd, [string[session_len+len(cmd)+1:]], session else: myargs = [] @@ -753,7 +759,7 @@ class SocketHost: def load_authenticator(self): f, args, kwargs = self.HostInterface.AuthenticatorInst - myinst = f(*args,**kwargs) + myinst = f(*args, **kwargs) return myinst def load_service_interface(self, session): @@ -1060,7 +1066,7 @@ class SocketHost: def _do_fork(self, f, authenticator, uid, gid, *args, **kwargs): authenticator.set_exc_permissions(uid, gid) - rc = f(*args,**kwargs) + rc = f(*args, **kwargs) return rc class BuiltInCommands(SocketCommands): @@ -1403,7 +1409,6 @@ class SocketHost: def docmd_hello(self, transmitter): from entropy.tools import getstatusoutput - from entropy.core.settings.base import SystemSettings sys_settings = SystemSettings() uname = os.uname() kern_string = uname[2] @@ -1633,29 +1638,30 @@ class SocketHost: os.path.isfile(self.SSL['ca_pkey']) and \ os.path.isfile(self.SSL['key']) and \ os.path.isfile(self.SSL['cert'])): - self.create_ca_server_certs( - self.SSL['serial'], - self.SSL['digest'], - self.SSL['not_before'], - self.SSL['not_after'], - self.SSL['ca_pkey'], - self.SSL['ca_cert'], - self.SSL['key'], - self.SSL['cert'] - ) - os.chmod(self.SSL['ca_cert'], 0o644) - try: - os.chown(self.SSL['ca_cert'], -1, 0) - except OSError: - pass - try: - os.chmod(self.SSL['ca_pkey'], 0o600) - except OSError: - pass - try: - os.chown(self.SSL['ca_pkey'], -1, 0) - except OSError: - pass + + self.create_ca_server_certs( + self.SSL['serial'], + self.SSL['digest'], + self.SSL['not_before'], + self.SSL['not_after'], + self.SSL['ca_pkey'], + self.SSL['ca_cert'], + self.SSL['key'], + self.SSL['cert'] + ) + os.chmod(self.SSL['ca_cert'], 0o644) + try: + os.chown(self.SSL['ca_cert'], -1, 0) + except OSError: + pass + try: + os.chmod(self.SSL['ca_pkey'], 0o600) + except OSError: + pass + try: + os.chown(self.SSL['ca_pkey'], -1, 0) + except OSError: + pass try: os.chmod(self.SSL['key'], 0o600) @@ -2026,5 +2032,5 @@ class SocketHost: etpConst['logging']['normal_loglevel_id'], str(args[0])) if self.__output is not None and self.stdout_logging: - self.__output.output(*args,**kwargs) + self.__output.output(*args, **kwargs) self.last_print = message diff --git a/libraries/entropy/services/repository/commands.py b/libraries/entropy/services/repository/commands.py index b1274dd2a..432bceb27 100644 --- a/libraries/entropy/services/repository/commands.py +++ b/libraries/entropy/services/repository/commands.py @@ -158,7 +158,7 @@ class Repository(SocketCommands): metadata['sets'] = dbconn.retrievePackageSets() metadata['treeupdates_actions'] = dbconn.listAllTreeUpdatesActions() metadata['treeupdates_digest'] = dbconn.retrieveRepositoryUpdatesDigest(repository) - # FIXME: kept for backward compatibility (<=0.99.0.x) remove in future + # NOTE: kept for backward compatibility (<=0.99.0.x) remove in future metadata['library_idpackages'] = [] metadata['revision'] = self.get_database_revision(repository, arch, product, branch) @@ -299,7 +299,7 @@ class Repository(SocketCommands): content_insert_formatted = format_content_for_insert, get_content = False, get_changelog = False ) - except: + except Exception: tb = entropy.tools.get_traceback() print(tb) self.HostInterface.socketLog.write(tb) diff --git a/libraries/entropy/services/repository/interfaces.py b/libraries/entropy/services/repository/interfaces.py index e8963aebc..9d8adc0da 100644 --- a/libraries/entropy/services/repository/interfaces.py +++ b/libraries/entropy/services/repository/interfaces.py @@ -13,12 +13,10 @@ import os import shutil from entropy.core.settings.base import SystemSettings -from entropy.output import TextInterface, blue, brown, darkred, darkgreen, teal +from entropy.output import TextInterface, blue, brown, darkred, teal from entropy.const import etpConst from entropy.misc import TimeScheduled from entropy.cache import EntropyCacher -from entropy.i18n import _ -from entropy.db.exceptions import ProgrammingError from entropy.services.interfaces import SocketHost from entropy.services.repository.commands import Repository from entropy.client.interfaces import Client @@ -300,7 +298,7 @@ class Server(SocketHost): raise AttributeError("cmethod not specified for: %s" % (x,)) if self.repositories[x]['cmethod'] not in \ etpConst['etpdatabasesupportedcformats']: - raise AttributeError("wrong cmethod for: %s" % (x,)) + raise AttributeError("wrong cmethod for: %s" % (x,)) # repository is locked by default, its db needs to be unpacked self.repositories[x]['locked'] = True diff --git a/libraries/entropy/services/skel.py b/libraries/entropy/services/skel.py index c6c9eca52..ce84fee3c 100644 --- a/libraries/entropy/services/skel.py +++ b/libraries/entropy/services/skel.py @@ -11,7 +11,8 @@ """ import os from entropy.const import const_convert_to_unicode -from entropy.exceptions import * +from entropy.exceptions import PermissionDenied, ConnectionError, \ + LibraryNotFound from entropy.i18n import _ class SocketCommands: @@ -148,8 +149,8 @@ class RemoteDatabase: if e[0] != 2006: raise else: - self.connect() - return True + self.connect() + return True return False def _raise_not_implemented_error(self): @@ -227,7 +228,7 @@ class RemoteDatabase: return self.cursor.fetchall() def fetchmany(self, *args, **kwargs): - return self.cursor.fetchmany(*args,**kwargs) + return self.cursor.fetchmany(*args, **kwargs) def lastrowid(self): return self.cursor.lastrowid diff --git a/libraries/entropy/services/system/commands.py b/libraries/entropy/services/system/commands.py index a06c2979f..f8dee4eae 100644 --- a/libraries/entropy/services/system/commands.py +++ b/libraries/entropy/services/system/commands.py @@ -754,7 +754,8 @@ class Repository(SocketCommands): 'buildonly, nodeps, custom_use, ldflags, cflags' atoms = mydict.get('atoms') - if atoms: atoms = atoms.split() + if atoms: + atoms = atoms.split() pretend = mydict.get('pretend') oneshot = mydict.get('oneshot') verbose = mydict.get('verbose') @@ -766,20 +767,34 @@ class Repository(SocketCommands): ldflags = mydict.get('ldflags') cflags = mydict.get('cflags') - if pretend == "1": pretend = True - else: pretend = False - if oneshot == "1": oneshot = True - else: oneshot = False - if verbose == "1": verbose = True - else: verbose = False - if nocolor == "1": nocolor = True - else: nocolor = False - if fetchonly == "1": fetchonly = True - else: fetchonly = False - if buildonly == "1": buildonly = True - else: buildonly = False - if nodeps == "1": nodeps = True - else: nodeps = False + if pretend == "1": + pretend = True + else: + pretend = False + if oneshot == "1": + oneshot = True + else: + oneshot = False + if verbose == "1": + verbose = True + else: + verbose = False + if nocolor == "1": + nocolor = True + else: + nocolor = False + if fetchonly == "1": + fetchonly = True + else: + fetchonly = False + if buildonly == "1": + buildonly = True + else: + buildonly = False + if nodeps == "1": + nodeps = True + else: + nodeps = False status, userdata, err_str = authenticator.docmd_userdata() uid = userdata.get('uid') @@ -819,17 +834,24 @@ class Repository(SocketCommands): return None, 'wrong dict arguments, xml must have 4 items with attr value -> atoms, pretend, verbose, nocolor' atoms = mydict.get('atoms') - if atoms: atoms = atoms.split() + if atoms: + atoms = atoms.split() pretend = mydict.get('pretend') verbose = mydict.get('verbose') nocolor = mydict.get('nocolor') - if pretend == "1": pretend = True - else: pretend = False - if verbose == "1": verbose = True - else: verbose = False - if nocolor == "1": nocolor = True - else: nocolor = False + if pretend == "1": + pretend = True + else: + pretend = False + if verbose == "1": + verbose = True + else: + verbose = False + if nocolor == "1": + nocolor = True + else: + nocolor = False status, userdata, err_str = authenticator.docmd_userdata() uid = userdata.get('uid') @@ -842,7 +864,8 @@ class Repository(SocketCommands): } queue_id = self.HostInterface.add_to_queue(cmd, ' '.join(myargs), uid, gid, 'spm_remove_atoms', [atoms[:]], add_dict.copy(), False, False, interactive = True) - if queue_id < 0: return False, queue_id + if queue_id < 0: + return False, queue_id return True, queue_id def docmd_get_spm_categories_updates(self, cmd, myargs, authenticator): @@ -854,7 +877,8 @@ class Repository(SocketCommands): gid = userdata.get('gid') queue_id = self.HostInterface.add_to_queue(cmd, ' '.join(myargs), uid, gid, 'get_spm_categories_updates', [myargs], {}, True, True, interactive = True) - if queue_id < 0: return False, queue_id + if queue_id < 0: + return False, queue_id return True, queue_id def docmd_get_spm_categories_installed(self, cmd, myargs, authenticator): @@ -864,7 +888,8 @@ class Repository(SocketCommands): gid = userdata.get('gid') queue_id = self.HostInterface.add_to_queue(cmd, ' '.join(myargs), uid, gid, 'get_spm_categories_installed', [myargs], {}, True, True, interactive = True) - if queue_id < 0: return False, queue_id + if queue_id < 0: + return False, queue_id return True, queue_id def docmd_enable_uses_for_atoms(self, cmd, myargs, authenticator): @@ -881,15 +906,18 @@ class Repository(SocketCommands): atoms = mydict.get('atoms') useflags = mydict.get('useflags') - if atoms: atoms = atoms.split() - if useflags: useflags = useflags.split() + if atoms: + atoms = atoms.split() + if useflags: + useflags = useflags.split() status, userdata, err_str = authenticator.docmd_userdata() uid = userdata.get('uid') gid = userdata.get('gid') queue_id = self.HostInterface.add_to_queue(cmd, ' '.join(myargs), uid, gid, 'enable_uses_for_atoms', [atoms, useflags], {}, True, True) - if queue_id < 0: return False, queue_id + if queue_id < 0: + return False, queue_id return True, queue_id def docmd_disable_uses_for_atoms(self, cmd, myargs, authenticator): @@ -906,15 +934,18 @@ class Repository(SocketCommands): atoms = mydict.get('atoms') useflags = mydict.get('useflags') - if atoms: atoms = atoms.split() - if useflags: useflags = useflags.split() + if atoms: + atoms = atoms.split() + if useflags: + useflags = useflags.split() status, userdata, err_str = authenticator.docmd_userdata() uid = userdata.get('uid') gid = userdata.get('gid') queue_id = self.HostInterface.add_to_queue(cmd, ' '.join(myargs), uid, gid, 'disable_uses_for_atoms', [atoms, useflags], {}, True, True) - if queue_id < 0: return False, queue_id + if queue_id < 0: + return False, queue_id return True, queue_id def docmd_get_spm_atoms_info(self, cmd, myargs, authenticator): @@ -926,7 +957,8 @@ class Repository(SocketCommands): gid = userdata.get('gid') queue_id = self.HostInterface.add_to_queue(cmd, ' '.join(myargs), uid, gid, 'get_spm_atoms_info', [myargs], {}, True, True) - if queue_id < 0: return False, queue_id + if queue_id < 0: + return False, queue_id return True, queue_id def docmd_run_spm_info(self, cmd, authenticator): @@ -936,7 +968,8 @@ class Repository(SocketCommands): gid = userdata.get('gid') queue_id = self.HostInterface.add_to_queue(cmd, '', uid, gid, 'run_spm_info', [], {}, True, False, interactive = True) - if queue_id < 0: return False, queue_id + if queue_id < 0: + return False, queue_id return True, queue_id def docmd_run_custom_shell_command(self, cmd, myargs, authenticator): @@ -949,7 +982,8 @@ class Repository(SocketCommands): command = ' '.join(myargs) queue_id = self.HostInterface.add_to_queue(cmd, command, uid, gid, 'run_custom_shell_command', [command], {}, True, False, interactive = True) - if queue_id < 0: return False, queue_id + if queue_id < 0: + return False, queue_id return True, queue_id def docmd_get_spm_glsa_data(self, cmd, myargs, authenticator): @@ -961,7 +995,8 @@ class Repository(SocketCommands): gid = userdata.get('gid') queue_id = self.HostInterface.add_to_queue(cmd, ' '.join(myargs), uid, gid, 'get_spm_glsa_data', [myargs[0]], {}, True, True) - if queue_id < 0: return False, queue_id + if queue_id < 0: + return False, queue_id return True, queue_id def docmd_get_available_repositories(self): @@ -1010,7 +1045,8 @@ class Repository(SocketCommands): for idpackage in idpackages: data = self._get_entropy_pkginfo(dbconn, idpackage, repoid) - if not data: continue + if not data: + continue package_data['data'][idpackage] = data.copy() dbconn.closeDB() return True, package_data @@ -1076,7 +1112,8 @@ class Repository(SocketCommands): [from_repo, to_repo, idpackages, do_copy], {}, False, True, interactive = True ) - if queue_id < 0: return False, queue_id + if queue_id < 0: + return False, queue_id return True, queue_id def docmd_scan_entropy_packages_database_changes(self, cmd, authenticator): @@ -1086,7 +1123,8 @@ class Repository(SocketCommands): gid = userdata.get('gid') queue_id = self.HostInterface.add_to_queue(cmd, '', uid, gid, 'scan_entropy_packages_database_changes', [], {}, True, True, interactive = True) - if queue_id < 0: return False, queue_id + if queue_id < 0: + return False, queue_id return True, queue_id def docmd_run_entropy_dependency_test(self, cmd, authenticator): @@ -1096,7 +1134,8 @@ class Repository(SocketCommands): gid = userdata.get('gid') queue_id = self.HostInterface.add_to_queue(cmd, '', uid, gid, 'run_entropy_dependency_test', [], {}, True, True, interactive = True) - if queue_id < 0: return False, queue_id + if queue_id < 0: + return False, queue_id return True, queue_id def docmd_run_entropy_library_test(self, cmd, authenticator): @@ -1106,7 +1145,8 @@ class Repository(SocketCommands): gid = userdata.get('gid') queue_id = self.HostInterface.add_to_queue(cmd, '', uid, gid, 'run_entropy_library_test', [], {}, True, True, interactive = True) - if queue_id < 0: return False, queue_id + if queue_id < 0: + return False, queue_id return True, queue_id def docmd_run_entropy_checksum_test(self, cmd, myargs, authenticator): @@ -1120,7 +1160,8 @@ class Repository(SocketCommands): gid = userdata.get('gid') queue_id = self.HostInterface.add_to_queue(cmd, ' '.join(myargs), uid, gid, 'run_entropy_checksum_test', [repoid, mode], {}, True, False, interactive = True) - if queue_id < 0: return False, queue_id + if queue_id < 0: + return False, queue_id return True, queue_id def docmd_run_entropy_treeupdates(self, cmd, myargs, authenticator): @@ -1133,7 +1174,8 @@ class Repository(SocketCommands): gid = userdata.get('gid') queue_id = self.HostInterface.add_to_queue(cmd, ' '.join(myargs), uid, gid, 'run_entropy_treeupdates', [myargs[0]], {}, False, False, interactive = True) - if queue_id < 0: return False, queue_id + if queue_id < 0: + return False, queue_id return True, queue_id def docmd_scan_entropy_mirror_updates(self, cmd, myargs, authenticator): @@ -1146,7 +1188,8 @@ class Repository(SocketCommands): gid = userdata.get('gid') queue_id = self.HostInterface.add_to_queue(cmd, ' '.join(myargs), uid, gid, 'scan_entropy_mirror_updates', [myargs], {}, True, True, interactive = True) - if queue_id < 0: return False, queue_id + if queue_id < 0: + return False, queue_id return True, queue_id def docmd_run_entropy_mirror_updates(self, cmd, myargs, authenticator): @@ -1165,7 +1208,8 @@ class Repository(SocketCommands): gid = userdata.get('gid') queue_id = self.HostInterface.add_to_queue(cmd, '', uid, gid, 'run_entropy_mirror_updates', [mydict], {}, False, False, interactive = True) - if queue_id < 0: return False, queue_id + if queue_id < 0: + return False, queue_id return True, queue_id def docmd_run_entropy_database_updates(self, cmd, myargs, authenticator): @@ -1210,7 +1254,8 @@ class Repository(SocketCommands): 'run_entropy_database_updates', [to_add, to_remove, to_inject], {}, False, True, interactive = True ) - if queue_id < 0: return False, queue_id + if queue_id < 0: + return False, queue_id return True, queue_id @@ -1240,7 +1285,8 @@ class Repository(SocketCommands): results = dbconn.searchPackages(mystring, order_by = "atom") for atom, idpackage, branch in results: data = self._get_entropy_pkginfo(dbconn, idpackage, repoid) - if not data: continue + if not data: + continue search_results['ordered_idpackages'].add(idpackage) search_results['data'][idpackage] = data.copy() @@ -1318,7 +1364,8 @@ class Repository(SocketCommands): gid = userdata.get('gid') queue_id = self.HostInterface.add_to_queue(cmd, ' '.join(myargs), uid, gid, 'get_notice_board', [repoid], {}, True, True, interactive = True) - if queue_id < 0: return False, queue_id + if queue_id < 0: + return False, queue_id return True, queue_id def docmd_remove_notice_board_entries(self, cmd, myargs, authenticator): @@ -1336,7 +1383,8 @@ class Repository(SocketCommands): cmd, ' '.join([str(x) for x in myargs]), uid, gid, 'remove_notice_board_entries', [repoid, entry_ids], {}, True, False, interactive = True ) - if queue_id < 0: return False, queue_id + if queue_id < 0: + return False, queue_id return True, queue_id def docmd_add_notice_board_entry(self, cmd, myargs, authenticator): @@ -1366,7 +1414,8 @@ class Repository(SocketCommands): cmd, ' '.join(myargs), uid, gid, 'add_notice_board_entry', [repoid, title, notice_text, link], {}, True, False, interactive = True ) - if queue_id < 0: return False, queue_id + if queue_id < 0: + return False, queue_id return True, queue_id def _get_entropy_pkginfo(self, dbconn, idpackage, repoid): diff --git a/libraries/entropy/services/system/executors.py b/libraries/entropy/services/system/executors.py index dac7cca41..a1a3edc82 100644 --- a/libraries/entropy/services/system/executors.py +++ b/libraries/entropy/services/system/executors.py @@ -156,7 +156,7 @@ class Base: def sync_portage(self, queue_id): queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True) - if queue_data == None: + if queue_data is None: return False, 'no item in queue' stdout_err = open(queue_data['stdout'], "a+") @@ -182,7 +182,7 @@ class Base: sys_intf = self.SystemManagerExecutor.SystemInterface queue_data, key = sys_intf.get_item_by_queue_id(queue_id, copy = True) - if queue_data == None: + if queue_data is None: return False, 'no item in queue' def set_proc_pid(pid): @@ -222,7 +222,7 @@ class Base: sys_intf = self.SystemManagerExecutor.SystemInterface queue_data, key = sys_intf.get_item_by_queue_id(queue_id, copy = True) - if queue_data == None: + if queue_data is None: return False, 'no item in queue' def set_proc_pid(pid): @@ -253,7 +253,7 @@ class Base: def enable_uses_for_atoms(self, queue_id, atoms, useflags): queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True) - if queue_data == None: + if queue_data is None: return False, 'no item in queue' use_data = {} @@ -273,7 +273,7 @@ class Base: def disable_uses_for_atoms(self, queue_id, atoms, useflags): queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True) - if queue_data == None: + if queue_data is None: return False, 'no item in queue' use_data = {} @@ -293,7 +293,7 @@ class Base: def get_spm_atoms_info(self, queue_id, atoms): queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True) - if queue_data == None: + if queue_data is None: return False, 'no item in queue' atoms_data = {} @@ -322,7 +322,7 @@ class Base: def get_spm_categories_updates(self, queue_id, categories): queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True) - if queue_data == None: + if queue_data is None: return False, 'no item in queue' spm = self.SystemManagerExecutor.SystemInterface.Entropy.Spm() @@ -343,7 +343,7 @@ class Base: def get_spm_categories_installed(self, queue_id, categories): queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True) - if queue_data == None: + if queue_data is None: return False, 'no item in queue' spm = self.SystemManagerExecutor.SystemInterface.Entropy.Spm() @@ -365,7 +365,7 @@ class Base: sys_intf = self.SystemManagerExecutor.SystemInterface queue_data, key = sys_intf.get_item_by_queue_id(queue_id, copy = True) - if queue_data == None: + if queue_data is None: return False, 'no item in queue' def set_proc_pid(pid): @@ -394,7 +394,7 @@ class Base: def run_custom_shell_command(self, queue_id, command): queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True) - if queue_data == None: + if queue_data is None: return False, 'no item in queue' stdout_err = open(queue_data['stdout'], "a+") @@ -418,7 +418,7 @@ class Base: def move_entropy_packages_to_repository(self, queue_id, from_repo, to_repo, idpackages, do_copy): queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True) - if queue_data == None: + if queue_data is None: return False, 'no item in queue' # run @@ -463,7 +463,7 @@ class Base: def scan_entropy_packages_database_changes(self, queue_id): queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True) - if queue_data == None: + if queue_data is None: return False, 'no item in queue' stdout_err = open(queue_data['stdout'], "a+") @@ -522,7 +522,7 @@ class Base: def run_entropy_database_updates(self, queue_id, to_add, to_remove, to_inject): queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True) - if queue_data == None: + if queue_data is None: return False, 'no item in queue' stdout_err = open(queue_data['stdout'], "a+") @@ -533,13 +533,15 @@ class Base: sys.stdout = stdout_err sys.stderr = stdout_err mystdin = self._get_stdin(queue_id) - if mystdin: sys.stdin = os.fdopen(mystdin, 'rb') + if mystdin: + sys.stdin = os.fdopen(mystdin, 'rb') try: atoms_removed = [] matches_injected = set() - if to_inject: Entropy.output(_("Running package injection")) + if to_inject: + Entropy.output(_("Running package injection")) # run inject for idpackage, repoid in to_inject: @@ -547,7 +549,8 @@ class Base: Entropy._transform_package_into_injected(idpackage, repo = repoid) - if to_remove: Entropy.output(_("Running package removal")) + if to_remove: + Entropy.output(_("Running package removal")) # run remove remdata = {} @@ -634,7 +637,7 @@ class Base: def run_entropy_dependency_test(self, queue_id): queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True) - if queue_data == None: + if queue_data is None: return False, 'no item in queue' stdout_err = open(queue_data['stdout'], "a+") @@ -643,7 +646,8 @@ class Base: sys.stdout = stdout_err sys.stderr = stdout_err mystdin = self._get_stdin(queue_id) - if mystdin: sys.stdin = os.fdopen(mystdin, 'rb') + if mystdin: + sys.stdin = os.fdopen(mystdin, 'rb') try: deps_not_matched = self.SystemManagerExecutor.SystemInterface.Entropy.dependencies_test() return True, deps_not_matched @@ -668,7 +672,7 @@ class Base: def run_entropy_library_test(self, queue_id): queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True) - if queue_data == None: + if queue_data is None: return False, 'no item in queue' stdout_err = open(queue_data['stdout'], "a+") @@ -677,7 +681,8 @@ class Base: sys.stdout = stdout_err sys.stderr = stdout_err mystdin = self._get_stdin(queue_id) - if mystdin: sys.stdin = os.fdopen(mystdin, 'rb') + if mystdin: + sys.stdin = os.fdopen(mystdin, 'rb') try: return self.SystemManagerExecutor.SystemInterface.Entropy.test_shared_objects() except Exception as e: @@ -698,14 +703,16 @@ class Base: stdout_err.close() mystatus = False - if status == 0: mystatus = True - if not result: result = set() + if status == 0: + mystatus = True + if not result: + result = set() return mystatus, result def run_entropy_checksum_test(self, queue_id, repoid, mode): queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True) - if queue_data == None: + if queue_data is None: return False, 'no item in queue' stdout_err = open(queue_data['stdout'], "a+") @@ -714,7 +721,8 @@ class Base: sys.stdout = stdout_err sys.stderr = stdout_err mystdin = self._get_stdin(queue_id) - if mystdin: sys.stdin = os.fdopen(mystdin, 'rb') + if mystdin: + sys.stdin = os.fdopen(mystdin, 'rb') try: if mode == "local": data = self.SystemManagerExecutor.SystemInterface.Entropy.verify_local_packages([], ask = False, repo = repoid) @@ -742,7 +750,7 @@ class Base: def run_entropy_treeupdates(self, queue_id, repoid): queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True) - if queue_data == None: + if queue_data is None: return False, 'no item in queue' stdout_err = open(queue_data['stdout'], "a+") @@ -751,7 +759,8 @@ class Base: sys.stdout = stdout_err sys.stderr = stdout_err mystdin = self._get_stdin(queue_id) - if mystdin: sys.stdin = os.fdopen(mystdin, 'rb') + if mystdin: + sys.stdin = os.fdopen(mystdin, 'rb') try: sys.stdout.write(_("Opening database to let it run treeupdates. If you won't see anything below, it's just fine.").encode('utf-8')+"\n") dbconn = self.SystemManagerExecutor.SystemInterface.Entropy.open_server_repository( @@ -780,7 +789,7 @@ class Base: def scan_entropy_mirror_updates(self, queue_id, repositories): queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True) - if queue_data == None: + if queue_data is None: return False, 'no item in queue' stdout_err = open(queue_data['stdout'], "a+") @@ -792,7 +801,8 @@ class Base: sys.stdout = stdout_err sys.stderr = stdout_err mystdin = self._get_stdin(queue_id) - if mystdin: sys.stdin = os.fdopen(mystdin, 'rb') + if mystdin: + sys.stdin = os.fdopen(mystdin, 'rb') try: sys.stdout.write(_("Scanning").encode('utf-8')+"\n") @@ -885,7 +895,7 @@ class Base: def run_entropy_mirror_updates(self, queue_id, repository_data): queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True) - if queue_data == None: + if queue_data is None: return False, 'no item in queue' stdout_err = open(queue_data['stdout'], "a+") @@ -922,7 +932,8 @@ class Base: sys.stdout = stdout_err sys.stderr = stdout_err mystdin = self._get_stdin(queue_id) - if mystdin: sys.stdin = os.fdopen(mystdin, 'rb') + if mystdin: + sys.stdin = os.fdopen(mystdin, 'rb') try: repo_data = {} @@ -972,7 +983,8 @@ class Base: if mirrors_tainted and sys_settings[sys_settings_srv_plugin_id]['server']['rss']['enabled']: commit_msg = repository_data[repoid]['commit_msg'] - if not commit_msg: commit_msg = "Autodriven update" + if not commit_msg: + commit_msg = "Autodriven update" ServerRssMetadata()['commitmessage'] = commit_msg errors, fine, broken = sync_remote_databases(repoid, repository_data[repoid]['pretend']) @@ -1010,7 +1022,7 @@ class Base: def get_spm_glsa_data(self, queue_id, list_type): queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True) - if queue_data == None: + if queue_data is None: return False, 'no item in queue' spm = self.SystemManagerExecutor.SystemInterface.Entropy.Spm() @@ -1026,7 +1038,7 @@ class Base: def get_notice_board(self, queue_id, repoid): queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True) - if queue_data == None: + if queue_data is None: return False, 'no item in queue' stdout_err = open(queue_data['stdout'], "a+") @@ -1035,10 +1047,11 @@ class Base: sys.stdout = stdout_err sys.stderr = stdout_err mystdin = self._get_stdin(queue_id) - if mystdin: sys.stdin = os.fdopen(mystdin, 'rb') + if mystdin: + sys.stdin = os.fdopen(mystdin, 'rb') try: data = self.SystemManagerExecutor.SystemInterface.Entropy.Mirrors.read_notice_board(repo = repoid) - if data == None: + if data is None: return False, None return True, data except Exception as e: @@ -1062,7 +1075,7 @@ class Base: def remove_notice_board_entries(self, queue_id, repoid, entry_ids): queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True) - if queue_data == None: + if queue_data is None: return False, 'no item in queue' stdout_err = open(queue_data['stdout'], "a+") @@ -1071,7 +1084,8 @@ class Base: sys.stdout = stdout_err sys.stderr = stdout_err mystdin = self._get_stdin(queue_id) - if mystdin: sys.stdin = os.fdopen(mystdin, 'rb') + if mystdin: + sys.stdin = os.fdopen(mystdin, 'rb') try: for entry_id in entry_ids: data = self.SystemManagerExecutor.SystemInterface.Entropy.Mirrors.remove_from_notice_board(entry_id, repo = repoid) @@ -1098,7 +1112,7 @@ class Base: def add_notice_board_entry(self, queue_id, repoid, title, notice_text, link): queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True) - if queue_data == None: + if queue_data is None: return False, 'no item in queue' stdout_err = open(queue_data['stdout'], "a+") @@ -1107,7 +1121,8 @@ class Base: sys.stdout = stdout_err sys.stderr = stdout_err mystdin = self._get_stdin(queue_id) - if mystdin: sys.stdin = os.fdopen(mystdin, 'rb') + if mystdin: + sys.stdin = os.fdopen(mystdin, 'rb') try: data = self.SystemManagerExecutor.SystemInterface.Entropy.Mirrors.update_notice_board(title, notice_text, link = link, repo = repoid) return True, data @@ -1132,7 +1147,8 @@ class Base: def _get_stdin(self, queue_id): mystdin = None std_data = self.SystemManagerExecutor.SystemInterface.ManagerQueueStdInOut.get(queue_id) - if std_data != None: mystdin = std_data[0] + if std_data is not None: + mystdin = std_data[0] return mystdin def _file_output(self, f, *myargs, **mykwargs): @@ -1143,7 +1159,8 @@ class Base: header = mykwargs.get("header") percent = mykwargs.get("percent") text = myargs[0].encode('utf-8') - if not header: header = '' + if not header: + header = '' count_str = "" if count: @@ -1167,7 +1184,8 @@ class Base: seek_till_newline(f) txt = header+count_str+text else: - if not is_last_newline(f): f.write("\n") + if not is_last_newline(f): + f.write("\n") txt = header+count_str+text+"\n" f.write(txt) diff --git a/libraries/entropy/services/system/interfaces.py b/libraries/entropy/services/system/interfaces.py index 17fd741f5..5c55fde86 100644 --- a/libraries/entropy/services/system/interfaces.py +++ b/libraries/entropy/services/system/interfaces.py @@ -174,7 +174,7 @@ class Server(SocketHost): def handle_executor_command_classes_initialization(self): for myclass, args, kwargs in self.ExecutorCommandClasses: - myintf = myclass(self.SystemExecutor, *args,**kwargs) + myintf = myclass(self.SystemExecutor, *args, **kwargs) if hasattr(myintf, 'available_commands'): self.SystemExecutor.register(myintf.available_commands) self.ExecutorCommandInstances.append(myintf) @@ -342,7 +342,8 @@ class Server(SocketHost): self.ManagerQueue[key+"_order"].remove(queue_id) removed = True self.remove_queue_ext_rc(queue_id) - if removed: self.store_queue() + if removed: + self.store_queue() return removed def kill_processing_queue_id(self, queue_id): @@ -443,7 +444,8 @@ class Server(SocketHost): def _queue_processor(self, fork_data): # queue processing is stopped until there's a process running - if self.ForkLock.locked(): return + if self.ForkLock.locked(): + return with self.ForkLock: with self.QueueLock: @@ -452,10 +454,13 @@ class Server(SocketHost): command_data, queue_id = self._queue_copy_obj(fork_data) else: self.load_queue() - if self.ManagerQueue['pause']: return - if not self.ManagerQueue['queue_order']: return + if self.ManagerQueue['pause']: + return + if not self.ManagerQueue['queue_order']: + return command_data, queue_id = self._pop_item_from_queue() - if not command_data: return + if not command_data: + return command_data = self._queue_copy_obj(command_data) command_data['processing_ts'] = "%s" % (self.get_ts(),) self.ManagerQueue['processing'][queue_id] = command_data @@ -470,7 +475,8 @@ class Server(SocketHost): return done, result = self.SystemExecutor.execute_task(command_data) except Exception as e: - if self.QueueLock.locked(): self.QueueLock.release() + if self.QueueLock.locked(): + self.QueueLock.release() entropy.tools.print_traceback() done = False result = (False, str(e),) diff --git a/libraries/entropy/services/test/interfaces.py b/libraries/entropy/services/test/interfaces.py index 0413e7dc4..1aecea4c8 100644 --- a/libraries/entropy/services/test/interfaces.py +++ b/libraries/entropy/services/test/interfaces.py @@ -10,7 +10,6 @@ """ from entropy.services.interfaces import SocketHost -from entropy.const import etpConst from entropy.output import TextInterface class Server(SocketHost): @@ -21,7 +20,9 @@ class Server(SocketHost): self.Text.output(":: FakeServiceInterface loaded ::") def __init__(self, do_ssl = False, stdout_logging = True, - entropy_interface_kwargs = {}, **kwargs): + entropy_interface_kwargs = None, **kwargs): + if entropy_interface_kwargs is None: + entropy_interface_kwargs = {} from entropy.services.system.commands import Base if 'external_cmd_classes' not in kwargs: diff --git a/libraries/entropy/services/ugc/commands.py b/libraries/entropy/services/ugc/commands.py index 25cefcd43..2eb2cc684 100644 --- a/libraries/entropy/services/ugc/commands.py +++ b/libraries/entropy/services/ugc/commands.py @@ -317,9 +317,12 @@ class UGC(SocketCommands): if doc_type not in self.SUPPORTED_DOCFILE_TYPES: return None, 'unsupported doc type (SUPPORTED_DOCFILE_TYPES)' - if not title: title = 'No title' - if not description: description = 'No description' - if not keywords: keywords = '' + if not title: + title = 'No title' + if not description: + description = 'No description' + if not keywords: + keywords = '' userid = self._get_userid(authenticator) if userid == None: diff --git a/libraries/entropy/services/ugc/interfaces.py b/libraries/entropy/services/ugc/interfaces.py index ef6092822..f7039f8c1 100644 --- a/libraries/entropy/services/ugc/interfaces.py +++ b/libraries/entropy/services/ugc/interfaces.py @@ -20,7 +20,8 @@ import shutil import entropy.dump import entropy.tools from entropy.services.skel import RemoteDatabase -from entropy.exceptions import * +from entropy.exceptions import DumbException, ConnectionError, \ + TimeoutError, SSLError, PermissionDenied from entropy.const import etpConst, etpUi, const_setup_perms, \ const_set_chmod, const_setup_file, const_get_stringtype, \ const_convert_to_rawstring, const_convert_to_unicode, const_debug_write @@ -225,17 +226,19 @@ class Server(RemoteDatabase): def cache_results(self): for cache_item in self.cached_results: fdata = self.cached_results.get(cache_item) - if fdata == None: return + if fdata is None: + return func, args, kwargs, exp_time = fdata key = self.get_cache_item_key(cache_item) - r = func(*args,**kwargs) + r = func(*args, **kwargs) entropy.dump.dumpobj(key, r) def get_cache_item_key(self, cache_item): return os.path.join(Server.CACHE_ID, cache_item) def cache_result(self, cache_item, r): - if not self.cached_results.get(cache_item): return None + if not self.cached_results.get(cache_item): + return None key = self.get_cache_item_key(cache_item) entropy.dump.dumpobj(key, r) @@ -251,7 +254,8 @@ class Server(RemoteDatabase): # expired get_ugc_alldownloads 0 86400 1228577077 def get_cached_result(self, cache_item): fdata = self.cached_results.get(cache_item) - if fdata == None: return None + if fdata is None: + return None func, args, kwargs, exp_time = fdata key = self.get_cache_item_key(cache_item) @@ -316,13 +320,15 @@ class Server(RemoteDatabase): def insert_iddoctype(self, iddoctype, description, do_commit = False): self.check_connection() self.execute_query('INSERT INTO entropy_doctypes VALUES (%s,%s)', (iddoctype, description,)) - if do_commit: self.commit() + if do_commit: + self.commit() def insert_pkgkey(self, key, do_commit = False): self.check_connection() self.execute_query('INSERT INTO entropy_base VALUES (%s,%s)', (None, key,)) myid = self.lastrowid() - if do_commit: self.commit() + if do_commit: + self.commit() return myid def insert_download(self, key, ddate, count = 0, do_commit = False): @@ -330,28 +336,32 @@ class Server(RemoteDatabase): idkey = self.handle_pkgkey(key) self.execute_query('INSERT INTO entropy_downloads VALUES (%s,%s,%s,%s)', (None, idkey, ddate, count)) myid = self.lastrowid() - if do_commit: self.commit() + if do_commit: + self.commit() return myid def insert_entropy_branch(self, branch, do_commit = False): self.check_connection() self.execute_query('INSERT INTO entropy_branches VALUES (%s,%s)', (None, branch,)) myid = self.lastrowid() - if do_commit: self.commit() + if do_commit: + self.commit() return myid def insert_entropy_release_string(self, release_string, do_commit = False): self.check_connection() self.execute_query('INSERT INTO entropy_release_strings VALUES (%s,%s)', (None, release_string,)) myid = self.lastrowid() - if do_commit: self.commit() + if do_commit: + self.commit() return myid def insert_entropy_ip_locations_id(self, ip_latitude, ip_longitude, do_commit = False): self.check_connection() self.execute_query('INSERT INTO entropy_ip_locations VALUES (%s,%s,%s)', (None, ip_latitude, ip_longitude,)) myid = self.lastrowid() - if do_commit: self.commit() + if do_commit: + self.commit() return myid def handle_entropy_ip_locations_id(self, ip_addr): @@ -371,14 +381,16 @@ class Server(RemoteDatabase): def update_download(self, iddownload, do_commit = False): self.check_connection() self.execute_query('UPDATE entropy_downloads SET `count` = `count`+1 WHERE `iddownload` = %s', (iddownload,)) - if do_commit: self.commit() + if do_commit: + self.commit() return iddownload def store_download_data(self, iddownloads, ip_addr, do_commit = False): entropy_ip_locations_id = self.handle_entropy_ip_locations_id(ip_addr) mydata = [(x, ip_addr, entropy_ip_locations_id,) for x in iddownloads] self.execute_many('INSERT INTO entropy_downloads_data VALUES (%s,%s,%s)', mydata) - if do_commit: self.commit() + if do_commit: + self.commit() def get_date(self): mytime = time.time() @@ -448,7 +460,8 @@ class Server(RemoteDatabase): self.check_connection() self.execute_query('SELECT `key` FROM entropy_base WHERE `idkey` = %s', (idkey,)) data = self.fetchone() - if data: return data['key'] + if data: + return data['key'] def get_ugc_metadata(self, pkgkey): self.check_connection() @@ -575,7 +588,8 @@ class Server(RemoteDatabase): # cached? cache_item = 'get_ugc_alldownloads' cached = self.get_cached_result(cache_item) - if cached != None: return cached + if cached != None: + return cached self.check_connection() self.execute_query(""" @@ -627,7 +641,8 @@ class Server(RemoteDatabase): # cached? cache_item = 'get_total_downloads_count' cached = self.get_cached_result(cache_item) - if cached != None: return cached + if cached != None: + return cached self.check_connection() self.execute_query('SELECT SQL_CACHE sum(entropy_downloads.`count`) as downloads FROM entropy_downloads') @@ -648,7 +663,7 @@ class Server(RemoteDatabase): SELECT Row, col_a FROM (SELECT @row := @row + 1 AS Row, userid AS col_a FROM entropy_user_scores ORDER BY score DESC) As derived1 WHERE col_a = %s""", (userid,)) data = self.fetchone() or {} - ranking = data.get('Row', 0) # key can be avail but == None + ranking = data.get('Row', 0) # key can be avail but is None if not ranking: return 0 return ranking @@ -702,7 +717,7 @@ class Server(RemoteDatabase): self.execute_query('SELECT score FROM entropy_user_scores WHERE userid = %s', (userid,)) data = self.fetchone() or {} myscore = data.get('score') - if myscore == None: + if myscore is None: myscore = self.update_user_score(userid) return myscore @@ -1152,7 +1167,7 @@ class Server(RemoteDatabase): return False elif 'ts' not in data: return False - elif data['ts'] == None: + elif data['ts'] is None: return False delta = self.datetime.fromtimestamp(time.time()) - data['ts'] if (delta.days == 0) and (delta.seconds <= self.FLOOD_INTERVAL): @@ -1197,10 +1212,13 @@ class Server(RemoteDatabase): key = key.strip().split()[0] except IndexError: continue - if not key: continue - if not key.isalnum(): continue + if not key: + continue + if not key.isalnum(): + continue key = key[:self.entropy_docs_keyword_len] - if key in clean_keys: continue + if key in clean_keys: + continue clean_keys.append(key) if not clean_keys: @@ -1364,7 +1382,8 @@ class Server(RemoteDatabase): self.do_entropy_hardware_usage_stats(entropy_distribution_usage_id, hw_hash) - if do_commit: self.commit() + if do_commit: + self.commit() return True def do_entropy_hardware_usage_stats(self, entropy_distribution_usage_id, hw_hash): @@ -1397,7 +1416,8 @@ class Server(RemoteDatabase): description, keywords, doc_type = None, do_commit = False): self.check_connection() idkey = self.handle_pkgkey(pkgkey) - if doc_type == None: doc_type = self.DOC_TYPES['bbcode_doc'] + if doc_type is None: + doc_type = self.DOC_TYPES['bbcode_doc'] iddoc = self.insert_generic_doc(idkey, userid, username, doc_type, text, title, description, keywords) if isinstance(iddoc, const_get_stringtype()): @@ -1468,7 +1488,8 @@ class Server(RemoteDatabase): # flood control flood_risk = self.insert_flood_control_check(userid) - if flood_risk: return False, 'flooding detected' + if flood_risk: + return False, 'flooding detected' # validity check if doc_type == self.DOC_TYPES['image']: @@ -1611,7 +1632,7 @@ class Server(RemoteDatabase): shutil.move(video_path, new_video_path) yt_service = self.get_youtube_service() - if yt_service == None: + if yt_service is None: return False, None mykeywords = ', '.join([x.strip().strip(',') for x in \ @@ -1658,7 +1679,7 @@ class Server(RemoteDatabase): userid = self.get_iddoc_userid(iddoc) yt_service = self.get_youtube_service() - if yt_service == None: + if yt_service is None: return False, None def do_remove(): @@ -1691,7 +1712,8 @@ class Server(RemoteDatabase): if deleted: do_remove() - if userid: self.update_user_score(userid) + if userid: + self.update_user_score(userid) return deleted, (iddoc, video_id,) def get_youtube_service(self): @@ -1774,17 +1796,15 @@ class Client: self.pyopenssl = True self.context = None - ''' - self.server_cert = server_cert - self.server_ca_cert = server_ca_cert - self.ssl_pkey = None - self.ssl_cert = None - self.ssl_CN = 'Entropy Repository Service Client' - self.ssl_digest = 'md5' - self.ssl_serial = 1 - self.ssl_not_before = 0 - self.ssl_not_after = 60*60*24*1 # 1 day - ''' + # self.server_cert = server_cert + # self.server_ca_cert = server_ca_cert + # self.ssl_pkey = None + # self.ssl_cert = None + # self.ssl_CN = 'Entropy Repository Service Client' + # self.ssl_digest = 'md5' + # self.ssl_serial = 1 + # self.ssl_not_before = 0 + # self.ssl_not_after = 60*60*24*1 # 1 day if self.ssl: @@ -2041,7 +2061,7 @@ class Client: cl_answer = self.answers['cl'] data = do_receive() - if self.buffer_length == None: + if self.buffer_length is None: self.buffered_data = const_convert_to_rawstring('') if (not data) or (data == cl_answer): # nein! no support, KAPUTT! diff --git a/libraries/entropy/spm/plugins/factory.py b/libraries/entropy/spm/plugins/factory.py index ee75348b9..ba6041dd9 100644 --- a/libraries/entropy/spm/plugins/factory.py +++ b/libraries/entropy/spm/plugins/factory.py @@ -9,10 +9,8 @@ B{Entropy Source Package Manager Plugins factory module}. """ -from entropy.const import etpConst from entropy.core import EntropyPluginFactory from entropy.core.settings.base import SystemSettings -from entropy.i18n import _ from entropy.spm.plugins.skel import SpmPlugin import entropy.spm.plugins.interfaces as plugs diff --git a/libraries/entropy/spm/plugins/interfaces/portage_plugin/__init__.py b/libraries/entropy/spm/plugins/interfaces/portage_plugin/__init__.py index 68b039e89..1e9551dfc 100644 --- a/libraries/entropy/spm/plugins/interfaces/portage_plugin/__init__.py +++ b/libraries/entropy/spm/plugins/interfaces/portage_plugin/__init__.py @@ -667,8 +667,6 @@ class PortagePlugin(SpmPlugin): file_save_path += "/" + pkgname + etpConst['packagesext'] dbdir = os.path.join(self._get_vdb_path(), pkgcat, pkgname) - import tarfile - import stat trees = self._portage.db["/"] vartree = trees["vartree"] dblnk = self._portage.dblink(pkgcat, pkgname, "/", vartree.settings, @@ -784,6 +782,7 @@ class PortagePlugin(SpmPlugin): possible_kernel_owned_path = os.path.join(kmod_pfx, kern_vermagic) owners = self.search_paths_owners([possible_kernel_owned_path]) owner_data = None + k_atom = None for k_atom, k_slot in owners: k_cat, k_name, k_ver, k_rev = entropy.tools.catpkgsplit(k_atom) if k_cat == PortagePlugin.KERNEL_CATEGORY: @@ -798,7 +797,8 @@ class PortagePlugin(SpmPlugin): k_cat, k_name, k_ver, k_rev = owner_data if k_rev != "r0": k_ver += "-%s" % (k_rev,) - kern_dep_key = "=%s~-1" % (k_atom,) + if k_atom is not None: + kern_dep_key = "=%s~-1" % (k_atom,) return kern_dep_key @@ -808,7 +808,6 @@ class PortagePlugin(SpmPlugin): return defaults[0] def __source_env_get_var(self, env_file, env_var): - bash_exec = "" cmd = "/bin/bash -c \"source " + env_file + \ " && echo ${" + env_var + "}\"" tmp_fd, tmp_file = tempfile.mkstemp(prefix = "etp_portage") @@ -1347,7 +1346,6 @@ class PortagePlugin(SpmPlugin): """ if root is None: root = etpConst['systemroot'] + os.path.sep - mytree = self._get_portage_vartree(root) cat, pkgv = package.split("/") return sorted(self._portage.dblink(cat, pkgv, root, @@ -2210,13 +2208,13 @@ class PortagePlugin(SpmPlugin): mydest = entropy_server._get_local_store_directory(repo = repo) try: mypath = self.generate_package(myatom, mydest) - except: + except Exception: + entropy.tools.print_traceback() # remove broken bin before raising mypath = os.path.join(mydest, os.path.basename(myatom) + etpConst['packagesext']) if os.path.isfile(mypath): os.remove(mypath) - entropy.tools.print_traceback() mytxt = "%s: %s: %s, %s." % ( bold(_("WARNING")), red(_("Cannot complete quickpkg for atom")), @@ -2295,8 +2293,6 @@ class PortagePlugin(SpmPlugin): # force parameters entropy_repository.readonly = False - # XXX: remove this in future - entropy_repository.readOnly = False # disable upload trigger from entropy.server.interfaces.main import \ ServerEntropyRepositoryPlugin @@ -2678,7 +2674,7 @@ class PortagePlugin(SpmPlugin): myslot = package_metadata['slot'][:] # old slot protocol for kernel packages - # FIXME: remove before 2010-12-31 + # XXX: remove before 2011-12-31 if (package_metadata['versiontag'] == package_metadata['slot']) \ and package_metadata['versiontag']: # usually kernel packages @@ -2710,18 +2706,18 @@ class PortagePlugin(SpmPlugin): os.access(world_dir, os.W_OK) and \ entropy.tools.istextfile(world_file): - world_atoms.discard(key) - world_atoms.add(keyslot) - world_file_tmp = world_file+".entropy_inst" + world_atoms.discard(key) + world_atoms.add(keyslot) + world_file_tmp = world_file+".entropy_inst" - newline = const_convert_to_rawstring("\n") - with open(world_file_tmp, "wb") as world_f: - for item in sorted(world_atoms): - world_f.write( - const_convert_to_rawstring(item + newline)) - world_f.flush() + newline = const_convert_to_rawstring("\n") + with open(world_file_tmp, "wb") as world_f: + for item in sorted(world_atoms): + world_f.write( + const_convert_to_rawstring(item + newline)) + world_f.flush() - os.rename(world_file_tmp, world_file) + os.rename(world_file_tmp, world_file) except (UnicodeDecodeError, UnicodeEncodeError,) as e: @@ -3032,8 +3028,10 @@ class PortagePlugin(SpmPlugin): try: myval = '='.join(differences[var].strip().split("=")[1:]) if myval: - if myval[0] in ("'", '"',): myval = myval[1:] - if myval[-1] in ("'", '"',): myval = myval[:-1] + if myval[0] in ("'", '"',): + myval = myval[1:] + if myval[-1] in ("'", '"',): + myval = myval[:-1] except IndexError: myval = '' os.environ[var] = myval @@ -3385,7 +3383,8 @@ class PortagePlugin(SpmPlugin): for line in content: changed, elaborated_line = handle_line(line, useflags) - if changed: atom_found = True + if changed: + atom_found = True new_content.append(elaborated_line) if not atom_found: @@ -3736,10 +3735,10 @@ class PortagePlugin(SpmPlugin): def _strip_empty(self, myarr): """ - # deps.py -- Portage dependency resolution functions - # Copyright 2003-2004 Gentoo Foundation - # Distributed under the terms of the GNU General Public License v2 - # $Id: portage_dep.py 9174 2008-01-11 05:49:02Z zmedico $ + # deps.py -- Portage dependency resolution functions + # Copyright 2003-2004 Gentoo Foundation + # Distributed under the terms of the GNU General Public License v2 + # $Id: portage_dep.py 9174 2008-01-11 05:49:02Z zmedico $ Strip all empty elements from an array @@ -3749,8 +3748,8 @@ class PortagePlugin(SpmPlugin): @return: The array with empty elements removed """ for x in range(len(myarr)-1, -1, -1): - if not myarr[x]: - del myarr[x] + if not myarr[x]: + del myarr[x] return myarr def _use_reduce(self, deparray, uselist = None, masklist = None, @@ -3810,8 +3809,8 @@ class PortagePlugin(SpmPlugin): if additions: rlist.append(additions) elif rlist and rlist[-1] == "||": - #XXX: Currently some DEPEND strings have || lists without default atoms. - # raise portage_exception.InvalidDependString("No default atom(s) in \""+paren_enclose(deparray)+"\"") + # Currently some DEPEND strings have || lists without default atoms. + # raise portage_exception.InvalidDependString("No default atom(s) in \""+paren_enclose(deparray)+"\"") rlist.append([]) else: if head[-1] == "?": # Use reduce next group on fail. @@ -3870,8 +3869,8 @@ class PortagePlugin(SpmPlugin): break elif head not in masklist: if not matchall and head not in uselist: - ismatch = False - break + ismatch = False + break else: ismatch = False if missing_flag: @@ -3886,7 +3885,7 @@ class PortagePlugin(SpmPlugin): if isinstance(target, list): additions = self._use_reduce(target, uselist, masklist, matchall, excludeall) if additions: - rlist.append(additions) + rlist.append(additions) elif not _dep_check_strict: # The old deprecated behavior. rlist.append(target) diff --git a/libraries/entropy/spm/plugins/interfaces/portage_plugin/xpak.py b/libraries/entropy/spm/plugins/interfaces/portage_plugin/xpak.py index e7255a5fa..5a1c1426f 100644 --- a/libraries/entropy/spm/plugins/interfaces/portage_plugin/xpak.py +++ b/libraries/entropy/spm/plugins/interfaces/portage_plugin/xpak.py @@ -19,7 +19,7 @@ import sys import os, shutil, errno -from stat import * +from stat import ST_SIZE, ST_MTIME, ST_CTIME from entropy.const import const_convert_to_rawstring if sys.hexversion >= 0x3000000: @@ -32,16 +32,17 @@ else: STOP = "STOP" def _addtolist(mylist, curdir, _nested = False): - if not _nested: - curdir = os.path.normpath(curdir) """(list, dir) --- Takes an array(list) and appends all files from dir down the directory tree. Returns nothing. list is modified.""" + + if not _nested: + curdir = os.path.normpath(curdir) for x in os.listdir(curdir): x_path = os.path.join(curdir, x) if os.path.isdir(x_path): _addtolist(mylist, x_path, _nested = True) elif x_path not in mylist: - mylist.append(x_path) + mylist.append(x_path) if not _nested: for idx in xrange(len(mylist)): @@ -50,29 +51,29 @@ def _addtolist(mylist, curdir, _nested = False): def encodeint(myint): """Takes a 4 byte integer and converts it into a string of 4 characters. Returns the characters in a string.""" - part1=chr((myint >> 24 ) & 0x000000ff) - part2=chr((myint >> 16 ) & 0x000000ff) - part3=chr((myint >> 8 ) & 0x000000ff) - part4=chr(myint & 0x000000ff) + part1 = chr((myint >> 24 ) & 0x000000ff) + part2 = chr((myint >> 16 ) & 0x000000ff) + part3 = chr((myint >> 8 ) & 0x000000ff) + part4 = chr(myint & 0x000000ff) if sys.hexversion >= 0x3000000: - return bytes(part1+part2+part3+part4, 'raw_unicode_escape') + return bytes(part1 + part2 + part3 + part4, 'raw_unicode_escape') else: - return part1+part2+part3+part4 + return part1 + part2 + part3 + part4 def decodeint(mystring): """Takes a 4 byte string and converts it into a 4 byte integer. Returns an integer.""" - myint=0 + myint = 0 if sys.hexversion >= 0x3000000: - myint=myint+mystring[3] - myint=myint+(mystring[2] << 8) - myint=myint+(mystring[1] << 16) - myint=myint+(mystring[0] << 24) + myint = myint+mystring[3] + myint = myint+(mystring[2] << 8) + myint = myint+(mystring[1] << 16) + myint = myint+(mystring[0] << 24) else: - myint=myint+ord(mystring[3]) - myint=myint+(ord(mystring[2]) << 8) - myint=myint+(ord(mystring[1]) << 16) - myint=myint+(ord(mystring[0]) << 24) + myint = myint+ord(mystring[3]) + myint = myint+(ord(mystring[2]) << 8) + myint = myint+(ord(mystring[1]) << 16) + myint = myint+(ord(mystring[0]) << 24) return myint def xpak(rootdir, outfile=None): @@ -127,18 +128,18 @@ def xsplit(infile): """(infile) -- Splits the infile into two files. 'infile.index' contains the index segment. 'infile.dat' contails the data segment.""" - myfile=open(infile, "rb") - mydat=myfile.read() + myfile = open(infile, "rb") + mydat = myfile.read() myfile.close() splits = xsplit_mem(mydat) if not splits: return False - myfile=open(infile+".index", "wb") + myfile = open(infile+".index", "wb") myfile.write(splits[0]) myfile.close() - myfile=open(infile+".dat", "wb") + myfile = open(infile+".dat", "wb") myfile.write(splits[1]) myfile.close() return True @@ -148,34 +149,34 @@ def xsplit_mem(mydat): return None if mydat[-8:] != XPAKSTOP: return None - indexsize=decodeint(mydat[8:12]) - #datasize=decodeint(mydat[12:16]) not used + indexsize = decodeint(mydat[8:12]) + #datasize = decodeint(mydat[12:16]) not used return (mydat[16:indexsize+16], mydat[indexsize+16:-8]) def getindex(infile): """(infile) -- grabs the index segment from the infile and returns it.""" - myfile=open(infile, "rb") - myheader=myfile.read(16) + myfile = open(infile, "rb") + myheader = myfile.read(16) if myheader[0:8] != XPAKPACK: myfile.close() return - indexsize=decodeint(myheader[8:12]) - myindex=myfile.read(indexsize) + indexsize = decodeint(myheader[8:12]) + myindex = myfile.read(indexsize) myfile.close() return myindex def getboth(infile): """(infile) -- grabs the index and data segments from the infile. Returns an array [indexSegment,dataSegment]""" - myfile=open(infile, "rb") - myheader=myfile.read(16) + myfile = open(infile, "rb") + myheader = myfile.read(16) if myheader[0:8] != XPAKPACK: myfile.close() return - indexsize=decodeint(myheader[8:12]) - datasize=decodeint(myheader[12:16]) - myindex=myfile.read(indexsize) - mydata=myfile.read(datasize) + indexsize = decodeint(myheader[8:12]) + datasize = decodeint(myheader[12:16]) + myindex = myfile.read(indexsize) + mydata = myfile.read(datasize) myfile.close() return myindex, mydata @@ -186,35 +187,35 @@ def listindex(myindex): def getindex_mem(myindex): """Returns the filenames listed in the indexglob passed in.""" - myindexlen=len(myindex) - startpos=0 - myret=[] - while ((startpos+8)= 0x3000000: xpak_end = b"XPAKSTOP" xpak_start = b"XPAKPACK" @@ -114,8 +113,8 @@ def suck_xpak(tbz2file, outputpath): while counter >= (0 - chunk_len): - old.seek(counter - bytes, os.SEEK_END) - if (bytes - (abs(counter - bytes))) < chunk_len: + old.seek(counter - n_bytes, os.SEEK_END) + if (n_bytes - (abs(counter - n_bytes))) < chunk_len: chunk_len = 1 read_bytes = old.read(chunk_len) read_len = len(read_bytes) diff --git a/libraries/entropy/tools.py b/libraries/entropy/tools.py index 64c1a0116..739a499c6 100644 --- a/libraries/entropy/tools.py +++ b/libraries/entropy/tools.py @@ -27,8 +27,7 @@ import pwd import hashlib import random import traceback -from entropy.output import TextInterface, print_generic, red, \ - darkgreen, green, blue, purple, teal, brown +from entropy.output import print_generic from entropy.const import etpConst, const_kill_threads, const_islive, \ const_isunicode, const_convert_to_unicode, const_convert_to_rawstring, \ const_cmp, const_israwstring, const_cmp @@ -73,7 +72,7 @@ def is_user_in_entropy_group(uid = None): if not etp_group_users or \ username not in etp_group_users: - return False + return False return True @@ -175,21 +174,21 @@ def get_traceback(tb_obj = None): #traceback.print_last(file = buf) return buf.getvalue() -def print_exception(returndata = False, tb_data = None, all_frame_data = False): +def print_exception(silent = False, tb_data = None, all_frame_data = False): """ Print last Python exception and frame variables values (if available) to stdout. - @keyword returndata: do not print data but return - @type returndata: bool + @keyword silent: do not print to stdout + @type silent: bool @keyword tb_data: Python traceback object @type tb_data: Python traceback instance @keyword all_frame_data: print all variables in every frame @type all_frame_data: bool @return: exception data - @rtype: string + @rtype: list of strings """ - if not returndata: + if not silent: traceback.print_last() data = [] if tb_data is not None: @@ -212,30 +211,26 @@ def print_exception(returndata = False, tb_data = None, all_frame_data = False): #if not returndata: print for frame in stack: - if not returndata: - sys.stdout.write("\n") + if not silent: + print_generic("") print_generic("Frame %s in %s at line %s" % (frame.f_code.co_name, - frame.f_code.co_filename, - frame.f_lineno)) - else: - data.append("Frame %s in %s at line %s\n" % (frame.f_code.co_name, - frame.f_code.co_filename, - frame.f_lineno)) + frame.f_code.co_filename, frame.f_lineno)) + data.append("Frame %s in %s at line %s\n" % (frame.f_code.co_name, + frame.f_code.co_filename, frame.f_lineno)) + for key, value in list(frame.f_locals.items()): cur_str = '' cur_str = "\t%20s = " % key try: cur_str += repr(value) + "\n" - except: + except (AttributeError, NameError, TypeError): cur_str += "\n" - if not returndata: + if not silent: sys.stdout.write(cur_str) - else: - data.append(cur_str) + data.append(cur_str) - if returndata: - return data + return data # Get the content of an online page # @returns content: if the file exists @@ -276,14 +271,21 @@ def get_remote_data(url, timeout = 5): # unset urlmod._opener = None + item = None try: item = urlmod.urlopen(url, timeout = timeout) - result = item.readlines() - item.close() - except: + except Exception: + # urllib2.HTTPError + # urllib2.URLError + # httplib.BadStatusLine + # httplib.InvalidURL + # ValueError + # IOError return False finally: + if item is not None: + item.close() socket.setdefaulttimeout(2) if not result: @@ -542,7 +544,8 @@ def getstatusoutput(cmd): pipe = os.popen('{ ' + cmd + '; } 2>&1', 'r') text = pipe.read() sts = pipe.close() - if sts is None: sts = 0 + if sts is None: + sts = 0 if text[-1:] == '\n': text = text[:-1] return sts, text @@ -566,7 +569,6 @@ def movefile(src, dest, src_basedir = None): @return: True, if file was moved successfully @rtype: bool """ - sstat = os.lstat(src) destexists = 1 try: @@ -607,7 +609,7 @@ def movefile(src, dest, src_basedir = None): try: os.rename(src, dest) renamefailed = False - except Exception as e: + except OSError as e: if e[0] != errno.EXDEV: # Some random error. print_generic("!!! Failed to move", src, "to", dest) @@ -628,7 +630,7 @@ def movefile(src, dest, src_basedir = None): didcopy = True except SystemExit as e: raise - except Exception as e: + except (OSError, IOError, shutil.Error) as e: print_generic('!!! copy', src, '->', dest, 'failed.') print_generic("!!!", repr(e)) return False @@ -2238,10 +2240,12 @@ def spawn_function(f, *args, **kwds): """ uid = kwds.get('spf_uid') - if uid is not None: kwds.pop('spf_uid') + if uid is not None: + kwds.pop('spf_uid') gid = kwds.get('spf_gid') - if gid is not None: kwds.pop('spf_gid') + if gid is not None: + kwds.pop('spf_gid') write_pid_func = kwds.get('write_pid_func') if write_pid_func is not None: @@ -2315,7 +2319,7 @@ def uncompress_tarball(filepath, extract_path = None, catch_empty = False): except ValueError: ugdata_valid = True try: - if ugdata_valid: # FIXME: will be removed in 2011 + if ugdata_valid: # NOTE: backward compat. remove after 2012 # get uid/gid # if not found, returns -1 that won't change anything uid, gid = get_uid_from_user(uname), \ @@ -2560,7 +2564,8 @@ def is_valid_string(string): @rtype: bool """ invalid = [ord(x) for x in string if ord(x) not in list(range(32, 127))] - if invalid: return False + if invalid: + return False return True valid_path_regexp = re.compile("^([A-Za-z0-9/\.:\-_]+)$") @@ -2783,7 +2788,8 @@ def xml_from_dict_extended(dictionary): elif value is None: mytype = "None" value = "None" - else: TypeError + else: + raise TypeError() item.setAttribute('type', mytype) item_value = doc.createTextNode("%s" % (value,)) item.appendChild(item_value) @@ -3050,7 +3056,7 @@ def recursive_directory_relative_listing(empty_list, base_directory, recursive_directory_relative_listing(empty_list, x_path, _nested = True) elif x_path not in empty_list: - empty_list.append(x_path) + empty_list.append(x_path) if not _nested: for idx in xrange(len(empty_list)): diff --git a/libraries/entropy/transceivers/__init__.py b/libraries/entropy/transceivers/__init__.py index 1cd3e058c..bf6798e3a 100644 --- a/libraries/entropy/transceivers/__init__.py +++ b/libraries/entropy/transceivers/__init__.py @@ -26,26 +26,23 @@ from entropy.transceivers.uri_handlers.skel import EntropyUriHandler class EntropyTransceiver(TextInterface): - _URI_HANDLERS = [] - """ Base class for Entropy transceivers. This provides a common API across all the available URI handlers. - # FIXME: allow to provide other OutputInterfaces - How to use this class: - Let's consider that we have a valid EntropyUriHandler for ftp:// protocol - already installed via "add_uri_handler". - - >> txc = EntropyTransceiver("ftp://myuser:mypwd@myhost") - >> txc.set_speed_limit(150) # set speed limit to 150kb/sec - >> handler = txc.swallow() - >> handler.download("ftp://myuser:mypwd@myhost/myfile.txt", "/tmp") - # download + Let's consider that we have a valid EntropyUriHandler for ftp:// protocol + already installed via "add_uri_handler". + >> txc = EntropyTransceiver("ftp://myuser:mypwd@myhost") + >> txc.set_speed_limit(150) # set speed limit to 150kb/sec + >> handler = txc.swallow() + >> handler.download("ftp://myuser:mypwd@myhost/myfile.txt", "/tmp") + # download """ + _URI_HANDLERS = [] + @staticmethod def add_uri_handler(entropy_uri_handler_class): """ diff --git a/libraries/entropy/transceivers/uri_handlers/plugins/interfaces/file_plugin.py b/libraries/entropy/transceivers/uri_handlers/plugins/interfaces/file_plugin.py index 330047574..783e74680 100644 --- a/libraries/entropy/transceivers/uri_handlers/plugins/interfaces/file_plugin.py +++ b/libraries/entropy/transceivers/uri_handlers/plugins/interfaces/file_plugin.py @@ -11,15 +11,11 @@ """ import os -import tempfile import pwd import grp import shutil -from entropy.const import const_isnumber, const_setup_perms, etpConst -from entropy.output import brown, darkgreen, teal -from entropy.i18n import _ -from entropy.exceptions import ConnectionError +from entropy.const import const_setup_perms, etpConst from entropy.transceivers.uri_handlers.skel import EntropyUriHandler from entropy.tools import md5sum @@ -91,12 +87,12 @@ def filemode(mode): class EntropyFileUriHandler(EntropyUriHandler): - PLUGIN_API_VERSION = 1 - """ EntropyUriHandler based FILE (local) transceiver plugin. """ + PLUGIN_API_VERSION = 1 + @staticmethod def approve_uri(uri): if uri.startswith("file://"): diff --git a/libraries/entropy/transceivers/uri_handlers/plugins/interfaces/ftp_plugin.py b/libraries/entropy/transceivers/uri_handlers/plugins/interfaces/ftp_plugin.py index 0983299bf..19ea1b7f4 100644 --- a/libraries/entropy/transceivers/uri_handlers/plugins/interfaces/ftp_plugin.py +++ b/libraries/entropy/transceivers/uri_handlers/plugins/interfaces/ftp_plugin.py @@ -11,6 +11,7 @@ """ import os import time +import socket from entropy.tools import print_traceback, get_file_size, \ convert_seconds_to_fancy_output, bytes_into_human, spliturl @@ -21,12 +22,12 @@ from entropy.transceivers.uri_handlers.skel import EntropyUriHandler class EntropyFtpUriHandler(EntropyUriHandler): - PLUGIN_API_VERSION = 1 - """ EntropyUriHandler based FTP transceiver plugin. """ + PLUGIN_API_VERSION = 1 + _DEFAULT_TIMEOUT = 60 @staticmethod @@ -57,8 +58,8 @@ class EntropyFtpUriHandler(EntropyUriHandler): def __init__(self, uri): EntropyUriHandler.__init__(self, uri) - import socket, ftplib - self.socket, self.ftplib = socket, ftplib + import ftplib + self.ftplib = ftplib self.__connected = False self.__ftpconn = None self.__currentdir = '.' @@ -142,9 +143,9 @@ class EntropyFtpUriHandler(EntropyUriHandler): self.__ftpconn = self.ftplib.FTP() self.__ftpconn.connect(self.__ftphost, self.__ftpport, timeout) break - except (self.socket.gaierror,) as e: + except (socket.gaierror,) as e: raise ConnectionError('ConnectionError: %s' % (e,)) - except (self.socket.error,) as e: + except (socket.error,) as e: if not count: raise ConnectionError('ConnectionError: %s' % (e,)) except: @@ -185,7 +186,7 @@ class EntropyFtpUriHandler(EntropyUriHandler): # try to disconnect try: self.__ftpconn.quit() - except (EOFError, self.socket, self.socket.timeout, + except (EOFError, socket.error, socket.timeout, self.ftplib.error_reply,): # AttributeError is raised when socket gets trashed # EOFError is raised when the connection breaks @@ -343,12 +344,14 @@ class EntropyFtpUriHandler(EntropyUriHandler): tries = 10 while tries: - tries -= 1 + tries -= 1 self._init_vars() + self.__filekbcount = 0 + rc = '' + try: - self.__filekbcount = 0 # get the file size self.__filesize = self._get_file_size_compat(path) if (self.__filesize): @@ -363,16 +366,9 @@ class EntropyFtpUriHandler(EntropyUriHandler): with open(tmp_save_path, "wb") as f: rc = self.__ftpconn.retrbinary('RETR ' + path, writer, 8192) f.flush() - self._update_progress(force = True) - done = rc.find("226") != -1 - if done: - # download complete, atomic mv - os.rename(tmp_save_path, save_path) - - return done - - except Exception as e: # connection reset by peer + except (IOError, self.ftplib.error_reply, socket.error) as e: + # connection reset by peer print_traceback() mytxt = red("%s: %s, %s... #%s") % ( @@ -388,6 +384,22 @@ class EntropyFtpUriHandler(EntropyUriHandler): header = " " ) self._reconnect() # reconnect + continue + + finally: + try: + os.remove(tmp_save_path) + except OSError: + pass + + self._update_progress(force = True) + done = rc.find("226") != -1 + if done: + # download complete, atomic mv + os.rename(tmp_save_path, save_path) + + return done + def download_many(self, remote_paths, save_dir): for remote_path in remote_paths: diff --git a/libraries/entropy/transceivers/uri_handlers/plugins/interfaces/ssh_plugin.py b/libraries/entropy/transceivers/uri_handlers/plugins/interfaces/ssh_plugin.py index 3392542fe..4acf53396 100644 --- a/libraries/entropy/transceivers/uri_handlers/plugins/interfaces/ssh_plugin.py +++ b/libraries/entropy/transceivers/uri_handlers/plugins/interfaces/ssh_plugin.py @@ -22,12 +22,12 @@ from entropy.transceivers.uri_handlers.skel import EntropyUriHandler class EntropySshUriHandler(EntropyUriHandler): - PLUGIN_API_VERSION = 1 - """ EntropyUriHandler based SSH (with pubkey) transceiver plugin. """ + PLUGIN_API_VERSION = 1 + _DEFAULT_TIMEOUT = 60 _DEFAULT_PORT = 22 _TXC_CMD = "/usr/bin/scp" diff --git a/libraries/entropy/transceivers/uri_handlers/skel.py b/libraries/entropy/transceivers/uri_handlers/skel.py index 14441453a..f6b6481f7 100644 --- a/libraries/entropy/transceivers/uri_handlers/skel.py +++ b/libraries/entropy/transceivers/uri_handlers/skel.py @@ -14,10 +14,6 @@ from entropy.output import TextInterface class EntropyUriHandler(object, TextInterface): - BASE_PLUGIN_API_VERSION = 1 - - TMP_TXC_FILE_EXT = ".tmp-entropy-txc" - """ Base class for EntropyTransceiver URI handler interfaces. This provides a common API for implementing custom URI handlers. @@ -26,6 +22,11 @@ class EntropyUriHandler(object, TextInterface): >>> EntropyTransceiver.add_uri_handler(my_entropy_transceiver_based_instance) "add_uri_handler" is a EntropyTransceiver static method. """ + + BASE_PLUGIN_API_VERSION = 1 + + TMP_TXC_FILE_EXT = ".tmp-entropy-txc" + def __init__(self, uri): """ EntropyUriHandler constructor. diff --git a/services/client-updates-daemon b/services/client-updates-daemon index 12457f256..6a27d0899 100644 --- a/services/client-updates-daemon +++ b/services/client-updates-daemon @@ -113,7 +113,7 @@ class DaemonUrlFetcher(UrlFetcher): self.__remotesize = total_size self.__datatransfer = data_transfer - def output(self): + def update(self): myavg = abs(int(round(float(self.__average), 1))) if abs((myavg - self.daemon_last_avg)) < 1: return diff --git a/sulfur/src/sulfur/entropyapi.py b/sulfur/src/sulfur/entropyapi.py index 5ea6474af..ff3419bc8 100644 --- a/sulfur/src/sulfur/entropyapi.py +++ b/sulfur/src/sulfur/entropyapi.py @@ -454,7 +454,7 @@ class GuiUrlFetcher(UrlFetcher): self.__remotesize = total_size self.__datatransfer = data_transfer - def output(self): + def update(self): if self.progress == None: return diff --git a/sulfur/src/sulfur_client.py b/sulfur/src/sulfur_client.py index 9aa538dca..0cce701df 100644 --- a/sulfur/src/sulfur_client.py +++ b/sulfur/src/sulfur_client.py @@ -64,14 +64,15 @@ def handle_exception(exc_class, exc_instance, exc_tb): t_back = entropy.tools.get_traceback(tb_obj = exc_tb) t_back += "\n" - t_back += ''.join(entropy.tools.print_exception(True, tb_data = exc_tb)) + t_back += ''.join(entropy.tools.print_exception(silent = True, + tb_data = exc_tb)) if "--debug-catch" in sys.argv: print(t_back) import pdb pdb.set_trace() - exc_data = entropy.tools.print_exception(returndata = True, + exc_data = entropy.tools.print_exception(silent = True, tb_data = exc_tb, all_frame_data = True) my = ExceptionDialog()