diff --git a/client/text_configuration.py b/client/text_configuration.py index 58f2240af..a86c3e196 100644 --- a/client/text_configuration.py +++ b/client/text_configuration.py @@ -315,11 +315,10 @@ def showdiff(fromfile, tofile): coloured.append(line + "\n") fd, tmp_path = tempfile.mkstemp() - f = open(tmp_path, "w") + f = os.fdopen(fd, "w") f.writelines(coloured) f.flush() f.close() - os.close(fd) print("") args = ["less", "--no-init", "--QUIT-AT-EOF", tmp_path] @@ -330,7 +329,7 @@ def showdiff(fromfile, tofile): raise args = ["cat", tmp_path] subprocess.call(args) - + os.remove(tmp_path) if output == ['']: diff --git a/client/text_ui.py b/client/text_ui.py index ba448f3e7..c715c5494 100644 --- a/client/text_ui.py +++ b/client/text_ui.py @@ -16,6 +16,7 @@ # import os import shutil +import tempfile from entropy.exceptions import SystemDatabaseError, DependenciesNotRemovable from entropy.db.exceptions import OperationalError @@ -1490,12 +1491,12 @@ def install_packages(entropy_client, def get_text_license(license_name, repoid): dbconn = entropy_client.open_repository(repoid) text = dbconn.retrieveLicenseText(license_name) - tempfile = entropy.tools.get_random_temp_file() - f = open(tempfile, "w") - f.write(text) - f.flush() - f.close() - return tempfile + tmp_fd, tmp_path = tempfile.mkstemp() + tmp_f = os.fdopen(tmp_fd, "w") + tmp_f.write(text) + tmp_f.flush() + tmp_f.close() + return tmp_path if licenses: print_info(red(" @@ ")+blue("%s:" % (_("You need to accept the licenses below"),) )) diff --git a/libraries/entropy/client/interfaces/dep.py b/libraries/entropy/client/interfaces/dep.py index 509945fbb..377346205 100644 --- a/libraries/entropy/client/interfaces/dep.py +++ b/libraries/entropy/client/interfaces/dep.py @@ -402,6 +402,9 @@ class CalculatorsMixin: unsatisfied = set() for dependency in dependencies: + if dependency == "~x11-drivers/xf86-video-virtualbox-3.2.8": + import pdb; pdb.set_trace() + if dependency in depcache: # already analized ? is_unsat = depcache[dependency] diff --git a/libraries/entropy/client/interfaces/methods.py b/libraries/entropy/client/interfaces/methods.py index 36ef88ec6..189c76af0 100644 --- a/libraries/entropy/client/interfaces/methods.py +++ b/libraries/entropy/client/interfaces/methods.py @@ -667,7 +667,8 @@ class RepositoryMixin: if dbname is None: dbname = etpConst['genericdbid'] if temp_file is None: - temp_file = entropy.tools.get_random_temp_file() + tmp_fd, temp_file = tempfile.mkstemp() + os.close(tmp_fd) dbc = GenericRepository( readOnly = False, @@ -1443,48 +1444,50 @@ class MiscMixin: pkg_mirrors = repo_data['plain_packages'] mirror_stats = {} - tmp_fd, tmp_path = tempfile.mkstemp() - os.close(tmp_fd) retries = 3 for mirror in pkg_mirrors: - url_data = entropy.tools.spliturl(mirror) - mytxt = "%s: %s" % ( - blue(_("Checking response time of")), - purple(url_data.hostname), - ) - self.output( - mytxt, - importance = 1, - level = "info", - header = purple(" @@ "), - back = True - ) + tmp_fd, tmp_path = tempfile.mkstemp() + try: - start_time = time.time() - for idx in range(retries): - fetcher = self.urlFetcher(mirror, tmp_path, resume = False, - show_speed = False) - fetcher.download() - end_time = time.time() + url_data = entropy.tools.spliturl(mirror) + mytxt = "%s: %s" % ( + blue(_("Checking response time of")), + purple(url_data.hostname), + ) + self.output( + mytxt, + importance = 1, + level = "info", + header = purple(" @@ "), + back = True + ) - result_time = (end_time - start_time)/retries - mirror_stats[mirror] = result_time + start_time = time.time() + for idx in range(retries): + fetcher = self.urlFetcher(mirror, tmp_path, resume = False, + show_speed = False) + fetcher.download() + end_time = time.time() - mytxt = "%s: %s, %s" % ( - blue(_("Mirror response time")), - purple(url_data.hostname), - teal(str(result_time)), - ) - self.output( - mytxt, - importance = 1, - level = "info", - header = brown(" @@ ") - ) + result_time = (end_time - start_time)/retries + mirror_stats[mirror] = result_time - os.remove(tmp_path) + mytxt = "%s: %s, %s" % ( + blue(_("Mirror response time")), + purple(url_data.hostname), + teal(str(result_time)), + ) + self.output( + mytxt, + importance = 1, + level = "info", + header = brown(" @@ ") + ) + finally: + os.close(tmp_fd) + os.remove(tmp_path) # calculate new order new_pkg_mirrors = sorted(mirror_stats.keys(), @@ -1597,16 +1600,18 @@ class MiscMixin: def _inject_entropy_database_into_package(self, package_filename, data, treeupdates_actions = None): tmp_fd, tmp_path = tempfile.mkstemp() - os.close(tmp_fd) - dbconn = self.open_generic_repository(tmp_path) - dbconn.initializeRepository() - dbconn.addPackage(data, revision = data['revision']) - if treeupdates_actions != None: - dbconn.bumpTreeUpdatesActions(treeupdates_actions) - dbconn.commitChanges() - dbconn.closeDB() - entropy.tools.aggregate_entropy_metadata(package_filename, tmp_path) - os.remove(tmp_path) + try: + dbconn = self.open_generic_repository(tmp_path) + dbconn.initializeRepository() + dbconn.addPackage(data, revision = data['revision']) + if treeupdates_actions != None: + dbconn.bumpTreeUpdatesActions(treeupdates_actions) + dbconn.commitChanges() + dbconn.closeDB() + entropy.tools.aggregate_entropy_metadata(package_filename, tmp_path) + finally: + os.close(tmp_fd) + os.remove(tmp_path) def quickpkg(self, pkgdata, dirpath, edb = True, fake = False, compression = "bz2", shiftpath = ""): @@ -1909,10 +1914,9 @@ class MatchMixin: for mask_file in new_mask_list: tmp_fd, tmp_path = tempfile.mkstemp() - os.close(tmp_fd) with open(mask_file, "r") as mask_f: - with open(tmp_path, "w") as tmp_f: + with os.fdopen(tmp_fd, "w") as tmp_f: for line in mask_f.readlines(): strip_line = line.strip() diff --git a/libraries/entropy/client/interfaces/package.py b/libraries/entropy/client/interfaces/package.py index 1b2cf0201..d804971b5 100644 --- a/libraries/entropy/client/interfaces/package.py +++ b/libraries/entropy/client/interfaces/package.py @@ -644,8 +644,7 @@ class Package: # write gpg signature to disk for verification tmp_fd, tmp_path = tempfile.mkstemp() - os.close(tmp_fd) - with open(tmp_path, "wb") as tmp_f: + with os.fdopen(tmp_fd, "wb") as tmp_f: tmp_f.write(hash_val) tmp_f.flush() diff --git a/libraries/entropy/qa.py b/libraries/entropy/qa.py index 22974c20f..7f7100c85 100644 --- a/libraries/entropy/qa.py +++ b/libraries/entropy/qa.py @@ -1077,49 +1077,51 @@ class QAInterface(TextInterface, EntropyPluginStore): from entropy.db import EntropyRepository from entropy.db.exceptions import Error fd, tmp_path = tempfile.mkstemp() - dump_rc = entropy.tools.dump_entropy_metadata(pkg_path, tmp_path) - if not dump_rc: - os.remove(tmp_path) - os.close(fd) - return False # error! + dbc = None try: - dbc = EntropyRepository( - readOnly = False, - dbFile = tmp_path, - dbname = 'qa_testing', - xcache = False, - indexing = False, - skipChecks = False - ) - etp_repo_meta = { - 'output_interface': self, - } - repo_plug = QAEntropyRepositoryPlugin(self, - metadata = etp_repo_meta) - dbc.add_plugin(repo_plug) + dump_rc = entropy.tools.dump_entropy_metadata(pkg_path, tmp_path) + if not dump_rc: + return False # error! - except Error: - os.remove(tmp_path) - os.close(fd) - return False - - valid = True - try: - dbc.validate() - except SystemDatabaseError: - valid = False - - if valid: + valid = True try: - for idpackage in dbc.listAllPackageIds(): - dbc.retrieveContent(idpackage, extended = True, - formatted = True, insert_formatted = True) + dbc = EntropyRepository( + readOnly = False, + dbFile = tmp_path, + dbname = 'qa_testing', + xcache = False, + indexing = False, + skipChecks = False + ) + etp_repo_meta = { + 'output_interface': self, + } + repo_plug = QAEntropyRepositoryPlugin(self, + metadata = etp_repo_meta) + dbc.add_plugin(repo_plug) + except Error: valid = False - dbc.closeDB() - os.remove(tmp_path) - os.close(fd) + if valid: + try: + dbc.validate() + except SystemDatabaseError: + valid = False + + if valid: + try: + for idpackage in dbc.listAllPackageIds(): + dbc.retrieveContent(idpackage, extended = True, + formatted = True, insert_formatted = True) + except Error: + valid = False + + finally: + if dbc is not None: + dbc.closeDB() + os.remove(tmp_path) + os.close(fd) return valid diff --git a/libraries/entropy/server/interfaces/main.py b/libraries/entropy/server/interfaces/main.py index 8cd89a705..4b7fee515 100644 --- a/libraries/entropy/server/interfaces/main.py +++ b/libraries/entropy/server/interfaces/main.py @@ -872,9 +872,7 @@ class ServerQAInterfacePlugin(QAInterfacePlugin): return False def __extract_edb_analyze_metadata(self, package_path): - tmp_fd, tmp_f = tempfile.mkstemp() - os.close(tmp_fd) - + tmp_fd, tmp_f = tempfile.mkstemp(prefix = 'entropy.server') dbc = None try: found_edb = entropy.tools.dump_entropy_metadata(package_path, tmp_f) @@ -896,10 +894,8 @@ class ServerQAInterfacePlugin(QAInterfacePlugin): finally: if dbc is not None: dbc.closeDB() - try: - os.remove(tmp_f) - except OSError: - pass + os.close(tmp_fd) + os.remove(tmp_f) return True @@ -1620,7 +1616,7 @@ class ServerPackagesHandlingMixin: all_fine = True - tmp_down_dir = tempfile.mkdtemp() + tmp_down_dir = tempfile.mkdtemp(prefix = "entropy.server") download_queue = {} dbconn = self.open_server_repository(read_only = False, @@ -3382,7 +3378,7 @@ class ServerQAMixin: pkg_list_path = None if dump_results_to_file: - tmp_dir = tempfile.mkdtemp() + tmp_dir = tempfile.mkdtemp(prefix = "entropy.server") pkg_list_path = os.path.join(tmp_dir, "libtest_broken.txt") dmp_data = [ (_("Broken and matched packages list"), pkg_list_path,), @@ -3808,7 +3804,8 @@ class ServerRepositoryMixin: @type output_interface: entropy.output.TextInterface based instance """ if temp_file is None: - temp_file = entropy.tools.get_random_temp_file() + tmp_fd, temp_file = tempfile.mkstemp(prefix = 'entropy.server') + os.close(tmp_fd) conn = ServerPackagesRepository( readOnly = False, diff --git a/libraries/entropy/server/interfaces/mirrors.py b/libraries/entropy/server/interfaces/mirrors.py index 3b0fca4a0..db07eeac5 100644 --- a/libraries/entropy/server/interfaces/mirrors.py +++ b/libraries/entropy/server/interfaces/mirrors.py @@ -39,7 +39,7 @@ class ServerNoticeBoardMixin: repo = self._entropy.default_repository mirrors = self._entropy.get_remote_repository_mirrors(repo) rss_path = self._entropy._get_local_database_notice_board_file(repo) - mytmpdir = tempfile.mkdtemp() + mytmpdir = tempfile.mkdtemp(prefix = "entropy.server") self._entropy.output( "[repo:%s] %s %s" % ( @@ -340,7 +340,7 @@ class Server(ServerNoticeBoardMixin): # nothing to do, not a file continue - tmp_dir = tempfile.mkdtemp() + tmp_dir = tempfile.mkdtemp(prefix = "entropy.server") down_path = os.path.join(tmp_dir, os.path.basename(filename)) tries = 4 @@ -819,33 +819,48 @@ class Server(ServerNoticeBoardMixin): if not (rc1 and rc2): return [uri, revision] - tmp_fd, rev_tmp_path = tempfile.mkstemp() + tmp_fd, rev_tmp_path = tempfile.mkstemp(prefix = "entropy.server") + try: - dlcount = 5 - dled = False - while dlcount: - remote_rev_path = os.path.join(remote_dir, revfilename) - dled = handler.download(remote_rev_path, rev_tmp_path) - if dled: - break - dlcount -= 1 - os.close(tmp_fd) + dlcount = 5 + dled = False + while dlcount: + remote_rev_path = os.path.join(remote_dir, revfilename) + dled = handler.download(remote_rev_path, rev_tmp_path) + if dled: + break + dlcount -= 1 - crippled_uri = EntropyTransceiver.get_uri_name(uri) + crippled_uri = EntropyTransceiver.get_uri_name(uri) - if os.access(rev_tmp_path, os.R_OK) and \ - os.path.isfile(rev_tmp_path): + if os.access(rev_tmp_path, os.R_OK) and \ + os.path.isfile(rev_tmp_path): - f_rev = open(rev_tmp_path, "r") - try: - revision = int(f_rev.readline().strip()) - except ValueError: - mytxt = _("mirror hasn't valid database revision file") + f_rev = open(rev_tmp_path, "r") + try: + revision = int(f_rev.readline().strip()) + except ValueError: + mytxt = _("mirror hasn't valid database revision file") + self._entropy.output( + "[repo:%s|%s] %s: %s" % ( + brown(repo), + darkgreen(crippled_uri), + blue(mytxt), + bold(revision), + ), + importance = 1, + level = "error", + header = darkred(" !!! ") + ) + revision = 0 + f_rev.close() + + elif dlcount == 0: self._entropy.output( "[repo:%s|%s] %s: %s" % ( brown(repo), darkgreen(crippled_uri), - blue(mytxt), + blue(_("unable to download repository revision")), bold(revision), ), importance = 1, @@ -853,37 +868,25 @@ class Server(ServerNoticeBoardMixin): header = darkred(" !!! ") ) revision = 0 - f_rev.close() - elif dlcount == 0: - self._entropy.output( - "[repo:%s|%s] %s: %s" % ( - brown(repo), - darkgreen(crippled_uri), - blue(_("unable to download repository revision")), - bold(revision), - ), - importance = 1, - level = "error", - header = darkred(" !!! ") - ) - revision = 0 + else: + self._entropy.output( + "[repo:%s|%s] %s: %s" % ( + brown(repo), + darkgreen(crippled_uri), + blue(_("mirror doesn't have valid revision file")), + bold(revision), + ), + importance = 1, + level = "error", + header = darkred(" !!! ") + ) + revision = 0 - else: - self._entropy.output( - "[repo:%s|%s] %s: %s" % ( - brown(repo), - darkgreen(crippled_uri), - blue(_("mirror doesn't have valid revision file")), - bold(revision), - ), - importance = 1, - level = "error", - header = darkred(" !!! ") - ) - revision = 0 + finally: + os.close(tmp_fd) + os.remove(rev_tmp_path) - os.remove(rev_tmp_path) return [uri, revision] def get_remote_repositories_status(self, repo = None, mirrors = None): @@ -1266,10 +1269,11 @@ class Server(ServerNoticeBoardMixin): # NOTE: for symlinks, we read their link and send a file with that # content. This is the default behaviour for now and allows to send # /etc/make.profile link pointer correctly. + tmp_dirs = [] for symname, symfile in spm_syms.items(): - tmp_file = entropy.tools.get_random_temp_file() - mytmpdir = os.path.dirname(tmp_file) + mytmpdir = tempfile.mkdtemp(dir = etpConst['entropyunpackdir']) + tmp_dirs.append(mytmpdir) mytmpfile = os.path.join(mytmpdir, os.path.basename(symfile)) mylink = os.readlink(symfile) f_mkp = open(mytmpfile, "w") @@ -1281,7 +1285,7 @@ class Server(ServerNoticeBoardMixin): data[symname] = mytmpfile extra_text_files.append(mytmpfile) - return data, critical, extra_text_files, gpg_signed_files + return data, critical, extra_text_files, tmp_dirs, gpg_signed_files def _show_package_sets_messages(self, repo): @@ -1710,7 +1714,7 @@ class Server(ServerNoticeBoardMixin): # create pkglist service file self._create_repository_pkglist(repo) - upload_data, critical, text_files, gpg_to_sign_files = \ + upload_data, critical, text_files, tmp_dirs, gpg_to_sign_files = \ self._get_files_to_sync(cmethod, repo = repo, disabled_eapis = disabled_eapis) @@ -1883,6 +1887,14 @@ class Server(ServerNoticeBoardMixin): if not fine_uris: upload_errors = True + + # remove temporary directories + for tmp_dir in tmp_dirs: + try: + shutil.rmtree(tmp_dir, True) + except shutil.Error: + continue + return upload_errors, broken_uris, fine_uris @@ -1909,111 +1921,122 @@ class Server(ServerNoticeBoardMixin): database_path = self._entropy._get_local_database_file(repo) database_dir_path = os.path.dirname( self._entropy._get_local_database_file(repo)) - download_data, critical, text_files, gpg_to_verify_files = \ - self._get_files_to_sync(cmethod, download = True, - repo = repo, disabled_eapis = disabled_eapis) - mytmpdir = tempfile.mkdtemp() - self._entropy.output( - "[repo:%s|%s|%s] %s" % ( - brown(repo), - darkgreen(crippled_uri), - red(_("download")), - blue(_("preparing to download database from mirror")), - ), - importance = 1, - level = "info", - header = darkgreen(" * ") - ) - files_to_sync = sorted(download_data.keys()) - for myfile in files_to_sync: + download_data, critical, text_files, tmp_dirs, \ + gpg_to_verify_files = self._get_files_to_sync(cmethod, + download = True, repo = repo, + disabled_eapis = disabled_eapis) + try: + + mytmpdir = tempfile.mkdtemp(prefix = "entropy.server") + self._entropy.output( - "%s: %s" % ( - blue(_("download path")), - brown(download_data[myfile]), + "[repo:%s|%s|%s] %s" % ( + brown(repo), + darkgreen(crippled_uri), + red(_("download")), + blue(_("preparing to download database from mirror")), ), - importance = 0, + importance = 1, level = "info", - header = brown(" # ") + header = darkgreen(" * ") ) - - if lock_check: - given_up = self._mirror_lock_check(uri, repo = repo) - if given_up: - download_errors = True - broken_uris.add(uri) - continue - - # avoid having others messing while we're downloading - self.lock_mirrors(True, [uri], repo = repo) - - if not pretend: - # download - downloader = self.TransceiverServerHandler( - self._entropy, [uri], - [download_data[x] for x in download_data], download = True, - local_basedir = mytmpdir, critical_files = critical, - repo = repo - ) - errors, m_fine_uris, m_broken_uris = downloader.go() - if errors: - my_broken_uris = sorted([ - (EntropyTransceiver.get_uri_name(x[0]), - x[1]) for x in m_broken_uris]) + files_to_sync = sorted(download_data.keys()) + for myfile in files_to_sync: self._entropy.output( - "[repo:%s|%s|%s] %s" % ( - brown(repo), - darkgreen(crippled_uri), - red(_("errors")), - blue(_("failed to download from mirror")), + "%s: %s" % ( + blue(_("download path")), + brown(download_data[myfile]), ), importance = 0, - level = "error", - header = darkred(" !!! ") + level = "info", + header = brown(" # ") ) - # get reason - reason = my_broken_uris[0][1] - self._entropy.output( - blue("%s: %s" % (_("reason"), reason,)), - importance = 0, - level = "error", - header = blue(" # ") + + if lock_check: + given_up = self._mirror_lock_check(uri, repo = repo) + if given_up: + download_errors = True + broken_uris.add(uri) + continue + + # avoid having others messing while we're downloading + self.lock_mirrors(True, [uri], repo = repo) + + if not pretend: + # download + downloader = self.TransceiverServerHandler( + self._entropy, [uri], + [download_data[x] for x in download_data], download = True, + local_basedir = mytmpdir, critical_files = critical, + repo = repo ) - download_errors = True - broken_uris |= m_broken_uris - self.lock_mirrors(False, [uri], repo = repo) - continue + errors, m_fine_uris, m_broken_uris = downloader.go() + if errors: + my_broken_uris = sorted([ + (EntropyTransceiver.get_uri_name(x[0]), + x[1]) for x in m_broken_uris]) + self._entropy.output( + "[repo:%s|%s|%s] %s" % ( + brown(repo), + darkgreen(crippled_uri), + red(_("errors")), + blue(_("failed to download from mirror")), + ), + importance = 0, + level = "error", + header = darkred(" !!! ") + ) + # get reason + reason = my_broken_uris[0][1] + self._entropy.output( + blue("%s: %s" % (_("reason"), reason,)), + importance = 0, + level = "error", + header = blue(" # ") + ) + download_errors = True + broken_uris |= m_broken_uris + self.lock_mirrors(False, [uri], repo = repo) + continue - # all fine then, we need to move data from mytmpdir - # to database_dir_path + # all fine then, we need to move data from mytmpdir + # to database_dir_path - # EAPI 1 -- unpack database - if 1 not in disabled_eapis: - compressed_db_filename = os.path.basename( - download_data['compressed_database_path']) - uncompressed_db_filename = os.path.basename(database_path) - compressed_file = os.path.join(mytmpdir, - compressed_db_filename) - uncompressed_file = os.path.join(mytmpdir, - uncompressed_db_filename) - entropy.tools.uncompress_file(compressed_file, - uncompressed_file, cmethod[0]) + # EAPI 1 -- unpack database + if 1 not in disabled_eapis: + compressed_db_filename = os.path.basename( + download_data['compressed_database_path']) + uncompressed_db_filename = os.path.basename(database_path) + compressed_file = os.path.join(mytmpdir, + compressed_db_filename) + uncompressed_file = os.path.join(mytmpdir, + uncompressed_db_filename) + entropy.tools.uncompress_file(compressed_file, + uncompressed_file, cmethod[0]) - # now move - for myfile in os.listdir(mytmpdir): - fromfile = os.path.join(mytmpdir, myfile) - tofile = os.path.join(database_dir_path, myfile) - shutil.move(fromfile, tofile) - const_setup_file(tofile, etpConst['entropygid'], 0o664) + # now move + for myfile in os.listdir(mytmpdir): + fromfile = os.path.join(mytmpdir, myfile) + tofile = os.path.join(database_dir_path, myfile) + shutil.move(fromfile, tofile) + const_setup_file(tofile, etpConst['entropygid'], 0o664) - if os.path.isdir(mytmpdir): - shutil.rmtree(mytmpdir) - if os.path.isdir(mytmpdir): - os.rmdir(mytmpdir) + if os.path.isdir(mytmpdir): + shutil.rmtree(mytmpdir) + if os.path.isdir(mytmpdir): + os.rmdir(mytmpdir) + fine_uris.add(uri) + self.lock_mirrors(False, [uri], repo = repo) - fine_uris.add(uri) - self.lock_mirrors(False, [uri], repo = repo) + finally: + # remove temporary directories + for tmp_dir in tmp_dirs: + try: + shutil.rmtree(tmp_dir, True) + except shutil.Error: + continue return download_errors, fine_uris, broken_uris diff --git a/libraries/entropy/services/interfaces.py b/libraries/entropy/services/interfaces.py index acf1d0a22..53df9082b 100644 --- a/libraries/entropy/services/interfaces.py +++ b/libraries/entropy/services/interfaces.py @@ -11,6 +11,7 @@ """ import sys import os +import tempfile import select import shutil import time @@ -1909,9 +1910,12 @@ class SocketHost: self.sessions[rng]['compression'] = None self.sessions[rng]['stream_mode'] = False try: - self.sessions[rng]['stream_path'] = entropy.tools.get_random_temp_file() + tmp_fd, self.sessions[rng]['stream_path'] = tempfile.mkstemp() + self.sessions[rng]['steam_stat'] = os.fstat(tmp_fd) + os.close(tmp_fd) except (IOError, OSError,): self.sessions[rng]['stream_path'] = '' + self.sessions[rng]['steam_stat'] = None self.sessions[rng]['t'] = time.time() self.sessions[rng]['ip_address'] = ip_address return rng @@ -1939,12 +1943,25 @@ class SocketHost: def _destroy_session(self, session): if session in self.sessions: stream_path = self.sessions[session]['stream_path'] + orig_st_data = self.sessions[session]['stream_stat'] del self.sessions[session] - if os.path.isfile(stream_path) and os.access(stream_path, os.W_OK) and not os.path.islink(stream_path): + if os.path.isfile(stream_path) and os.access(stream_path, os.W_OK) \ + and not os.path.islink(stream_path): + ack_remove = False try: - os.remove(stream_path) - except OSError: - pass + st_data = os.lstat(stream_path) + st_data_t = (st_data.st_ino, st_data.st_dev) + orig_st_data_t = (orig_st_data.st_ino, orig_st_data.st_dev) + if st_data_t == orig_st_data_t: + ack_remove = True + except (OSError, IOError): + # cannot get enough info to safely remove file + ack_remove = False + if ack_remove: + try: + os.remove(stream_path) + except OSError: + pass return True return False diff --git a/libraries/entropy/services/ugc/commands.py b/libraries/entropy/services/ugc/commands.py index 2eb2cc684..160c39275 100644 --- a/libraries/entropy/services/ugc/commands.py +++ b/libraries/entropy/services/ugc/commands.py @@ -801,17 +801,18 @@ class UGC(SocketCommands): for key in keys_to_file: if key not in mydict: continue + fd, path = tempfile.mkstemp(suffix = "__%s.txt" % (key,)) try: - f_path = open(path, "w") - f_path.write(mydict.get(key, '')) - f_path.flush() - f_path.close() + with os.fdopen(fd, "w") as f_path: + f_path.write(mydict.get(key, '')) + f_path.flush() except IOError: continue - os.close(fd) + finally: + rm_paths.append(path) + files.append(path) - rm_paths.append(path) sender = EmailSender() sender.send_mime_email(sender_email, [UGC.ERROR_REPORT_MAIL], subject, diff --git a/libraries/entropy/spm/plugins/interfaces/portage_plugin/__init__.py b/libraries/entropy/spm/plugins/interfaces/portage_plugin/__init__.py index 1e9551dfc..bf547be32 100644 --- a/libraries/entropy/spm/plugins/interfaces/portage_plugin/__init__.py +++ b/libraries/entropy/spm/plugins/interfaces/portage_plugin/__init__.py @@ -740,17 +740,21 @@ class PortagePlugin(SpmPlugin): def read_kern_vermagic(ko_path): tmp_fd, tmp_file = tempfile.mkstemp() - os.close(tmp_fd) + try: + with os.fdopen(tmp_fd, "w") as tmp_fw: + rc = subprocess.call((modinfo_path, "-F", "vermagic", + ko_path), stdout = tmp_fw, stderr = tmp_fw) + tmp_fw.flush() - tmp_fw = open(tmp_file, "w") - rc = subprocess.call((modinfo_path, "-F", "vermagic", - ko_path), stdout = tmp_fw, stderr = tmp_fw) - tmp_fw.flush() - tmp_fw.close() - tmp_r = open(tmp_file, "r") - modinfo_output = tmp_r.read().strip() - tmp_r.close() - os.remove(tmp_file) + tmp_r = open(tmp_file, "r") + modinfo_output = tmp_r.read().strip() + tmp_r.close() + finally: + try: + os.close(tmp_fd) + except OSError: + pass + os.remove(tmp_file) if rc != 0: import warnings @@ -811,19 +815,23 @@ class PortagePlugin(SpmPlugin): cmd = "/bin/bash -c \"source " + env_file + \ " && echo ${" + env_var + "}\"" tmp_fd, tmp_file = tempfile.mkstemp(prefix = "etp_portage") - os.close(tmp_fd) - std_f = open(tmp_file, "w") + try: + with os.fdopen(tmp_fd, "w") as std_f: + cmd = const_convert_to_rawstring(cmd) + proc = subprocess.Popen(shlex.split(cmd), stdout = std_f, + stderr = std_f) + sts = proc.wait() + std_f.flush() - cmd = const_convert_to_rawstring(cmd) - proc = subprocess.Popen(shlex.split(cmd), stdout = std_f, stderr = std_f) - sts = proc.wait() - std_f.flush() - std_f.close() + with open(tmp_file, "r") as std_f: + output = std_f.read() + finally: + try: + os.close(tmp_fd) + except OSError: + pass + os.remove(tmp_file) - std_f = open(tmp_file, "r") - output = std_f.read() - std_f.close() - os.remove(tmp_file) if sts != 0: raise IOError("cannot source %s and get %s => %s" % (env_file, env_var, repr(output))) @@ -1692,18 +1700,6 @@ class PortagePlugin(SpmPlugin): if licenses is None: licenses = [] - ### SETUP ENVIRONMENT - # if mute, supress portage output - if etpUi['mute']: - tmp_fd, tmp_file = tempfile.mkstemp() - os.close(tmp_fd) - tmp_fw = open(tmp_file, "w") - - oldsysstdout = sys.stdout - oldsysstderr = sys.stderr - sys.stdout = tmp_fw - sys.stderr = tmp_fw - root = etpConst['systemroot'] + os.path.sep # old way to avoid loop of deaths for entropy portage hooks @@ -1817,6 +1813,15 @@ class PortagePlugin(SpmPlugin): header = "" ) + # if mute, supress portage output + if etpUi['mute']: + tmp_fd, tmp_file = tempfile.mkstemp() + tmp_fw = os.fdopen(tmp_fd, "w") + oldsysstdout = sys.stdout + oldsysstderr = sys.stderr + sys.stdout = tmp_fw + sys.stderr = tmp_fw + try: rc = self._portage.doebuild( myebuild = str(myebuild), diff --git a/libraries/entropy/tools.py b/libraries/entropy/tools.py index 739a499c6..0e6da1351 100644 --- a/libraries/entropy/tools.py +++ b/libraries/entropy/tools.py @@ -1020,9 +1020,8 @@ def unpack_gzip(gzipfilepath): import gzip filepath = gzipfilepath[:-3] # remove .gz fd, tmp_path = tempfile.mkstemp(dir=os.path.dirname(filepath)) - os.close(fd) - item = open(tmp_path, "wb") + item = os.fdopen(fd, "wb") filegz = gzip.GzipFile(gzipfilepath, "rb") chunk = filegz.read(8192) while chunk: @@ -1046,8 +1045,7 @@ def unpack_bzip2(bzip2filepath): import bz2 filepath = bzip2filepath[:-4] # remove .bz2 fd, tmp_path = tempfile.mkstemp(dir=os.path.dirname(filepath)) - os.close(fd) - item = open(tmp_path, "wb") + item = os.fdopen(fd, "wb") filebz2 = bz2.BZ2File(bzip2filepath, "rb") chunk = filebz2.read(16384) while chunk: @@ -2409,6 +2407,7 @@ def bytes_into_human(xbytes): def get_random_temp_file(): """ Return random temporary file path. + @deprecated @return: temporary, random file path @rtype: string diff --git a/libraries/entropy/transceivers/uri_handlers/plugins/interfaces/ssh_plugin.py b/libraries/entropy/transceivers/uri_handlers/plugins/interfaces/ssh_plugin.py index 4acf53396..a6e4137c9 100644 --- a/libraries/entropy/transceivers/uri_handlers/plugins/interfaces/ssh_plugin.py +++ b/libraries/entropy/transceivers/uri_handlers/plugins/interfaces/ssh_plugin.py @@ -175,24 +175,20 @@ class EntropySshUriHandler(EntropyUriHandler): fd, tmp_path = tempfile.mkstemp() fd_err, tmp_path_err = tempfile.mkstemp() - os.close(fd) - os.close(fd_err) + try: + with os.fdopen(fd, "wb") as std_f: + with os.fdopen(fd_err, "wb") as std_f_err: + proc = self._subprocess.Popen(args, stdout = std_f, + stderr = std_f_err) + exec_rc = proc.wait() + with open(tmp_path, "rb") as std_f: + output = std_f.read() + with open(tmp_path_err, "rb") as std_f: + error = std_f.read() + finally: + os.remove(tmp_path) + os.remove(tmp_path_err) - with open(tmp_path, "wb") as std_f: - with open(tmp_path_err, "wb") as std_f_err: - proc = self._subprocess.Popen(args, stdout = std_f, - stderr = std_f_err) - exec_rc = proc.wait() - - std_f = open(tmp_path, "rb") - output = std_f.read() - std_f.close() - std_f = open(tmp_path_err, "rb") - error = std_f.read() - std_f.close() - - os.remove(tmp_path) - os.remove(tmp_path_err) return exec_rc, output, error def _setup_common_args(self, remote_path): @@ -293,48 +289,49 @@ class EntropySshUriHandler(EntropyUriHandler): # first of all, copy files renaming them tmp_file_map = {} - for load_path in load_path_list: - tmp_fd, tmp_path = tempfile.mkstemp( - suffix = EntropyUriHandler.TMP_TXC_FILE_EXT, - prefix = "._%s" % (os.path.basename(load_path),)) - os.close(tmp_fd) - shutil.copy2(load_path, tmp_path) - tmp_file_map[tmp_path] = load_path + try: + for load_path in load_path_list: + tmp_fd, tmp_path = tempfile.mkstemp( + suffix = EntropyUriHandler.TMP_TXC_FILE_EXT, + prefix = "._%s" % (os.path.basename(load_path),)) + os.close(tmp_fd) + shutil.copy2(load_path, tmp_path) + tmp_file_map[tmp_path] = load_path - args = [EntropySshUriHandler._TXC_CMD] - c_args, remote_str = self._setup_common_args(remote_dir) + args = [EntropySshUriHandler._TXC_CMD] + c_args, remote_str = self._setup_common_args(remote_dir) - args += c_args - args += ["-B", "-P", str(self.__port)] - args += sorted(tmp_file_map.keys()) - args += [remote_str] + args += c_args + args += ["-B", "-P", str(self.__port)] + args += sorted(tmp_file_map.keys()) + args += [remote_str] - upload_sts = self._fork_cmd(args) == 0 - if not upload_sts: + upload_sts = self._fork_cmd(args) == 0 + if not upload_sts: + return False + + # atomic rename + rename_fine = True + for tmp_path, orig_path in tmp_file_map.items(): + tmp_file = os.path.basename(tmp_path) + orig_file = os.path.basename(orig_path) + tmp_remote_path = os.path.join(remote_dir, tmp_file) + remote_path = os.path.join(remote_dir, orig_file) + self.output( + "<-> %s %s %s" % ( + brown(tmp_file), + teal("=>"), + darkgreen(orig_file), + ), + header = " ", + back = True + ) + rc = self.rename(tmp_remote_path, remote_path) + if not rc: + rename_fine = False + finally: for path in tmp_file_map.keys(): do_rm(path) - return False - - # atomic rename - rename_fine = True - for tmp_path, orig_path in tmp_file_map.items(): - tmp_file = os.path.basename(tmp_path) - orig_file = os.path.basename(orig_path) - tmp_remote_path = os.path.join(remote_dir, tmp_file) - remote_path = os.path.join(remote_dir, orig_file) - self.output( - "<-> %s %s %s" % ( - brown(tmp_file), - teal("=>"), - darkgreen(orig_file), - ), - header = " ", - back = True - ) - rc = self.rename(tmp_remote_path, remote_path) - if not rc: - rename_fine = False - do_rm(tmp_path) return rename_fine diff --git a/server/server_activator.py b/server/server_activator.py index 08c8c9eb5..01f01aa26 100644 --- a/server/server_activator.py +++ b/server/server_activator.py @@ -87,8 +87,7 @@ def sync(options, just_tidy = False): if (not do_noask) and rss_enabled: tmp_fd, tmp_commit_path = tempfile.mkstemp() - os.close(tmp_fd) - with open(tmp_commit_path, "w") as tmp_f: + with os.fdopen(tmp_fd, "w") as tmp_f: tmp_f.write(DEFAULT_REPO_COMMIT_MSG) if successfull_mirrors: tmp_f.write("# Changes to be committed:\n")