From f6c7cb49e091acdb608071c8a506d5f78c673984 Mon Sep 17 00:00:00 2001 From: lxnay Date: Sun, 9 Nov 2008 17:38:21 +0000 Subject: [PATCH] Entropy/EquoInterface: - split extract_pkg_metadata(): cross fingers, needs testing Entropy/entropyTools: - add sum_file_sizes function git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@2660 cd1c1023-2f26-0410-ae45-c471fc1f0318 --- libraries/entropy.py | 605 ++++++++++++++++++-------------------- libraries/entropyTools.py | 9 + 2 files changed, 292 insertions(+), 322 deletions(-) diff --git a/libraries/entropy.py b/libraries/entropy.py index 62ea5340e..b7ef9395b 100644 --- a/libraries/entropy.py +++ b/libraries/entropy.py @@ -3601,6 +3601,244 @@ class EquoInterface(TextInterface): self.spmCache[myroot] = conn.intf return conn.intf + def _extract_pkg_metadata_generate_extraction_dict(self): + data = { + 'chost': { + 'path': etpConst['spm']['xpak_entries']['chost'], + 'critical': True, + }, + 'description': { + 'path': etpConst['spm']['xpak_entries']['description'], + 'critical': False, + }, + 'homepage': { + 'path': etpConst['spm']['xpak_entries']['homepage'], + 'critical': False, + }, + 'slot': { + 'path': etpConst['spm']['xpak_entries']['slot'], + 'critical': False, + }, + 'cflags': { + 'path': etpConst['spm']['xpak_entries']['cflags'], + 'critical': False, + }, + 'cxxflags': { + 'path': etpConst['spm']['xpak_entries']['cxxflags'], + 'critical': False, + }, + 'category': { + 'path': etpConst['spm']['xpak_entries']['category'], + 'critical': True, + }, + 'rdepend': { + 'path': etpConst['spm']['xpak_entries']['rdepend'], + 'critical': False, + }, + 'pdepend': { + 'path': etpConst['spm']['xpak_entries']['pdepend'], + 'critical': False, + }, + 'depend': { + 'path': etpConst['spm']['xpak_entries']['depend'], + 'critical': False, + }, + 'use': { + 'path': etpConst['spm']['xpak_entries']['use'], + 'critical': False, + }, + 'iuse': { + 'path': etpConst['spm']['xpak_entries']['iuse'], + 'critical': False, + }, + 'license': { + 'path': etpConst['spm']['xpak_entries']['license'], + 'critical': False, + }, + 'provide': { + 'path': etpConst['spm']['xpak_entries']['provide'], + 'critical': False, + }, + 'sources': { + 'path': etpConst['spm']['xpak_entries']['src_uri'], + 'critical': False, + }, + 'eclasses': { + 'path': etpConst['spm']['xpak_entries']['inherited'], + 'critical': False, + }, + 'counter': { + 'path': etpConst['spm']['xpak_entries']['counter'], + 'critical': False, + }, + 'keywords': { + 'path': etpConst['spm']['xpak_entries']['keywords'], + 'critical': False, + }, + } + return data + + def _extract_pkg_metadata_content(self, content_file, package_path): + + pkg_content = {} + + if os.path.isfile(content_file): + f = open(content_file,"r") + content = f.readlines() + f.close() + outcontent = set() + for line in content: + line = line.strip().split() + try: + datatype = line[0] + datafile = line[1:] + if datatype == 'obj': + datafile = datafile[:-2] + datafile = ' '.join(datafile) + elif datatype == 'dir': + datafile = ' '.join(datafile) + elif datatype == 'sym': + datafile = datafile[:-3] + datafile = ' '.join(datafile) + else: + myexc = "InvalidData: %s %s. %s." % ( + datafile, + _("not supported"), + _("Probably Portage API has changed"), + ) + raise exceptionTools.InvalidData(myexc) + outcontent.add((datafile,datatype)) + except: + pass + + _outcontent = set() + for i in outcontent: + i = list(i) + datatype = i[1] + _outcontent.add((i[0],i[1])) + outcontent = list(_outcontent) + outcontent.sort() + for i in outcontent: + pkg_content[i[0]] = i[1] + + else: + + # CONTENTS is not generated when a package is emerged with portage and the option -B + # we have to unpack the tbz2 and generate content dict + mytempdir = etpConst['packagestmpdir']+"/"+os.path.basename(package_path)+".inject" + if os.path.isdir(mytempdir): + shutil.rmtree(mytempdir) + if not os.path.isdir(mytempdir): + os.makedirs(mytempdir) + + self.entropyTools.uncompressTarBz2(package_path, extractPath = mytempdir, catchEmpty = True) + for currentdir, subdirs, files in os.walk(mytempdir): + pkg_content[currentdir[len(mytempdir):]] = "dir" + for item in files: + item = currentdir+"/"+item + if os.path.islink(item): + pkg_content[item[len(mytempdir):]] = "sym" + else: + pkg_content[item[len(mytempdir):]] = "obj" + + # now remove + shutil.rmtree(mytempdir,True) + try: + os.rmdir(mytempdir) + except: + pass + + return pkg_content + + def _extract_pkg_metadata_needed(self, needed_file): + + pkg_needed = set() + lines = [] + + try: + f = open(needed_file,"r") + lines = [x.strip() for x in f.readlines() if x.strip()] + f.close() + except IOError: + return lines + + for line in lines: + needed = line.split() + if len(needed) == 2: + ownlib = needed[0] + ownelf = -1 + if os.access(ownlib,os.R_OK): + ownelf = self.entropyTools.read_elf_class(ownlib) + for lib in needed[1].split(","): + #if lib.find(".so") != -1: + pkg_needed.add((lib,ownelf)) + + return list(pkg_needed) + + def _extract_pkg_metadata_messages(self, log_dir, category, name, version, silent = False): + + pkg_messages = [] + + if os.path.isdir(log_dir): + + elogfiles = os.listdir(log_dir) + myelogfile = "%s:%s-%s" % (category, name, version,) + foundfiles = [x for x in elogfiles if x.startswith(myelogfile)] + if foundfiles: + elogfile = foundfiles[0] + if len(foundfiles) > 1: + # get the latest + mtimes = [] + for item in foundfiles: mtimes.append((self.entropyTools.getFileUnixMtime(os.path.join(log_dir,item)),item)) + mtimes = sorted(mtimes) + elogfile = mtimes[-1][1] + messages = self.entropyTools.extractElog(os.path.join(log_dir,elogfile)) + for message in messages: + message = message.replace("emerge","install") + pkg_messages.append(message) + + elif not silent: + + mytxt = " %s, %s" % (_("not set"),_("have you configured make.conf properly?"),) + self.updateProgress( + red(log_dir)+mytxt, + importance = 1, + type = "warning", + header = brown(" * ") + ) + + return pkg_messages + + def _extract_pkg_metadata_license_data(self, licenses_dir, license_string): + + pkg_licensedata = {} + if licenses_dir and os.path.isdir(licenses_dir): + licdata = [x.strip() for x in license_string.split() if x.strip() and self.entropyTools.is_valid_string(x.strip())] + for mylicense in licdata: + licfile = os.path.join(licenses_dir,mylicense) + if os.access(licfile,os.R_OK): + if self.entropyTools.istextfile(licfile): + f = open(licfile) + pkg_licensedata[mylicense] = f.read() + f.close() + + return pkg_licensedata + + def _extract_pkg_metadata_mirror_links(self, Spm, sources_list): + + # =mirror://openoffice|link1|link2|link3 + pkg_links = [] + for i in sources_list: + if i.startswith("mirror://"): + # parse what mirror I need + mirrorURI = i.split("/")[2] + mirrorlist = Spm.get_third_party_mirrors(mirrorURI) + pkg_links.append([mirrorURI,mirrorlist]) + # mirrorURI = openoffice and mirrorlist = [link1, link2, link3] + + return pkg_links + + # This function extracts all the info from a .tbz2 file and returns them def extract_pkg_metadata(self, package, etpBranch = etpConst['branch'], silent = False, inject = False): @@ -3644,14 +3882,10 @@ class EquoInterface(TextInterface): # Fill Package name and version data['name'] = pkgname data['version'] = pkgver - - # .tbz2 md5 data['digest'] = self.entropyTools.md5sum(tbz2File) data['datecreation'] = str(self.entropyTools.getFileUnixMtime(tbz2File)) - # .tbz2 byte size data['size'] = str(self.entropyTools.get_file_size(tbz2File)) - # unpack file tbz2TmpDir = etpConst['packagestmpdir']+"/"+data['name']+"-"+data['version']+"/" if not os.path.isdir(tbz2TmpDir): if os.path.lexists(tbz2TmpDir): @@ -3659,159 +3893,39 @@ class EquoInterface(TextInterface): os.makedirs(tbz2TmpDir) self.entropyTools.extractXpak(tbz2File,tbz2TmpDir) - # Fill chost - f = open(tbz2TmpDir+etpConst['spm']['xpak_entries']['chost'],"r") - data['chost'] = f.readline().strip() - f.close() - - # Fill branch + data['injected'] = False + if inject: data['injected'] = True data['branch'] = etpBranch - # Fill description - data['description'] = "" - try: - f = open(tbz2TmpDir+etpConst['spm']['xpak_entries']['description'],"r") - data['description'] = f.readline().strip() - f.close() - except IOError: - pass - - # Fill homepage - data['homepage'] = "" - try: - f = open(tbz2TmpDir+etpConst['spm']['xpak_entries']['homepage'],"r") - data['homepage'] = f.readline().strip() - f.close() - except IOError: - pass - - # fill slot, if it is - data['slot'] = "" - try: - f = open(tbz2TmpDir+etpConst['spm']['xpak_entries']['slot'],"r") - data['slot'] = f.readline().strip() - f.close() - except IOError: - pass - - # fill slot, if it is - if inject: - data['injected'] = True - else: - data['injected'] = False - - # fill eclasses list - data['eclasses'] = [] - try: - f = open(tbz2TmpDir+etpConst['spm']['xpak_entries']['inherited'],"r") - data['eclasses'] = f.readline().strip().split() - f.close() - except IOError: - pass - - # fill needed list - data['needed'] = set() - try: - f = open(tbz2TmpDir+etpConst['spm']['xpak_entries']['needed'],"r") - lines = f.readlines() - f.close() - for line in lines: - line = line.strip() - if line: - needed = line.split() - if len(needed) == 2: - ownlib = needed[0] - ownelf = -1 - if os.access(ownlib,os.R_OK): - ownelf = self.entropyTools.read_elf_class(ownlib) - libs = needed[1].split(",") - for lib in libs: - if (lib.find(".so") != -1): - data['needed'].add((lib,ownelf)) - except IOError: - pass - data['needed'] = list(data['needed']) - - data['content'] = {} - if os.path.isfile(tbz2TmpDir+etpConst['spm']['xpak_entries']['contents']): - f = open(tbz2TmpDir+etpConst['spm']['xpak_entries']['contents'],"r") - content = f.readlines() - f.close() - outcontent = set() - for line in content: - line = line.strip().split() - try: - datatype = line[0] - datafile = line[1:] - if datatype == 'obj': - datafile = datafile[:-2] - datafile = ' '.join(datafile) - elif datatype == 'dir': - datafile = ' '.join(datafile) - elif datatype == 'sym': - datafile = datafile[:-3] - datafile = ' '.join(datafile) - else: - myexc = "InvalidData: %s %s. %s." % ( - datafile, - _("not supported"), - _("Probably Portage API has changed"), - ) - raise exceptionTools.InvalidData(myexc) - outcontent.add((datafile,datatype)) - except: - pass - - _outcontent = set() - for i in outcontent: - i = list(i) - datatype = i[1] - _outcontent.add((i[0],i[1])) - outcontent = list(_outcontent) - outcontent.sort() - for i in outcontent: - data['content'][i[0]] = i[1] - - else: - # CONTENTS is not generated when a package is emerged with portage and the option -B - # we have to unpack the tbz2 and generate content dict - mytempdir = etpConst['packagestmpdir']+"/"+os.path.basename(filepath)+".inject" - if os.path.isdir(mytempdir): - shutil.rmtree(mytempdir) - if not os.path.isdir(mytempdir): - os.makedirs(mytempdir) - self.entropyTools.uncompressTarBz2(filepath, extractPath = mytempdir, catchEmpty = True) - - for currentdir, subdirs, files in os.walk(mytempdir): - data['content'][currentdir[len(mytempdir):]] = "dir" - for item in files: - item = currentdir+"/"+item - if os.path.islink(item): - data['content'][item[len(mytempdir):]] = "sym" - else: - data['content'][item[len(mytempdir):]] = "obj" - - # now remove - shutil.rmtree(mytempdir,True) + portage_entries = self._extract_pkg_metadata_generate_extraction_dict() + for item in portage_entries: + value = '' try: - os.rmdir(mytempdir) - except: - pass + f = open(os.path.join(tbz2TmpDir,portage_entries[item]['path']),"r") + value = f.readline().strip() + f.close() + except IOError: + if portage_entries[item]['critical']: + raise + data[item] = value - # files size on disk - if (data['content']): - data['disksize'] = 0 - for item in data['content']: - try: - size = self.entropyTools.get_file_size(item) - data['disksize'] += size - except: - pass - else: - data['disksize'] = 0 + # setup vars + data['eclasses'] = data['eclasses'].split() + try: + data['counter'] = int(data['counter']) + except ValueError: + data['counter'] = -2 # -2 values will be insterted as incremental negative values into the database + data['keywords'] = [x.strip() for x in data['keywords'].split() if x.strip()] + if not data['keywords']: data['keywords'].insert(0,"") # support for packages with no keywords + needed_file = os.path.join(tbz2TmpDir,etpConst['spm']['xpak_entries']['needed']) + data['needed'] = self._extract_pkg_metadata_needed(needed_file) + content_file = os.path.join(tbz2TmpDir,etpConst['spm']['xpak_entries']['contents']) + data['content'] = self._extract_pkg_metadata_content(content_file, filepath) + data['disksize'] = self.entropyTools.sum_file_sizes(data['content']) # [][][] Kernel dependent packages hook [][][] data['versiontag'] = '' + versiontag = "" kernelstuff = False kernelstuff_kernel = False for item in data['content']: @@ -3830,129 +3944,39 @@ class EquoInterface(TextInterface): kver = kmodver.split("-")[0] break # validate the results above - if (kernelstuff): + if kernelstuff: matchatom = "linux-%s-%s" % (kname,kver,) if (matchatom == data['name']+"-"+data['version']): kernelstuff_kernel = True - # Fill category - f = open(tbz2TmpDir+etpConst['spm']['xpak_entries']['category'],"r") - data['category'] = f.readline().strip() - f.close() - - # Fill download relative URI - if (kernelstuff): data['versiontag'] = kmodver if not kernelstuff_kernel: data['slot'] = kmodver # if you change this behaviour, # you must change "reagent update" # and "equo database gentoosync" consequentially versiontag = "#"+data['versiontag'] - else: - versiontag = "" + data['download'] = etpConst['packagesrelativepath'] + data['branch'] + "/" data['download'] += data['category']+":"+data['name']+"-"+data['version'] data['download'] += versiontag+etpConst['packagesext'] - # Fill counter - try: - f = open(tbz2TmpDir+etpConst['spm']['xpak_entries']['counter'],"r") - data['counter'] = int(f.readline().strip()) - f.close() - except IOError: - data['counter'] = -2 # -2 values will be insterted as incremental negative values into the database - data['trigger'] = "" if os.path.isfile(etpConst['triggersdir']+"/"+data['category']+"/"+data['name']+"/"+etpConst['triggername']): f = open(etpConst['triggersdir']+"/"+data['category']+"/"+data['name']+"/"+etpConst['triggername'],"rb") data['trigger'] = f.read() f.close() - # Fill CFLAGS - data['cflags'] = "" - try: - f = open(tbz2TmpDir+etpConst['spm']['xpak_entries']['cflags'],"r") - data['cflags'] = f.readline().strip() - f.close() - except IOError: - pass - - # Fill CXXFLAGS - data['cxxflags'] = "" - try: - f = open(tbz2TmpDir+etpConst['spm']['xpak_entries']['cxxflags'],"r") - data['cxxflags'] = f.readline().strip() - f.close() - except IOError: - pass - - # fill KEYWORDS - data['keywords'] = [] - try: - f = open(tbz2TmpDir+etpConst['spm']['xpak_entries']['keywords'],"r") - cnt = f.readline().strip().split() - if not cnt: - data['keywords'].append("") # support for packages with no keywords - else: - for i in cnt: - if i: - data['keywords'].append(i) - f.close() - except IOError: - pass - - - f = open(tbz2TmpDir+etpConst['spm']['xpak_entries']['rdepend'],"r") - rdepend = f.readline().strip() - f.close() - - f = open(tbz2TmpDir+etpConst['spm']['xpak_entries']['pdepend'],"r") - pdepend = f.readline().strip() - f.close() - - f = open(tbz2TmpDir+etpConst['spm']['xpak_entries']['depend'],"r") - depend = f.readline().strip() - f.close() - - f = open(tbz2TmpDir+etpConst['spm']['xpak_entries']['use'],"r") - use = f.readline().strip() - f.close() - - try: - f = open(tbz2TmpDir+etpConst['spm']['xpak_entries']['iuse'],"r") - iuse = f.readline().strip() - f.close() - except IOError: - iuse = "" - - try: - f = open(tbz2TmpDir+etpConst['spm']['xpak_entries']['license'],"r") - lics = f.readline().strip() - f.close() - except IOError: - lics = "" - - try: - f = open(tbz2TmpDir+etpConst['spm']['xpak_entries']['provide'],"r") - provide = f.readline().strip() - except IOError: - provide = "" - - try: - f = open(tbz2TmpDir+etpConst['spm']['xpak_entries']['src_uri'],"r") - sources = f.readline().strip() - f.close() - except IOError: - sources = "" - Spm = self.Spm() - portage_metadata = Spm.calculate_dependencies(iuse, use, lics, depend, rdepend, pdepend, provide, sources) + portage_metadata = Spm.calculate_dependencies( + data['iuse'], data['use'], data['license'], data['depend'], + data['rdepend'], data['pdepend'], data['provide'], data['sources'] + ) data['provide'] = portage_metadata['PROVIDE'].split() data['license'] = portage_metadata['LICENSE'] data['useflags'] = [] - for x in use.split(): + for x in data['use'].split(): if x.startswith("+"): x = x[1:] elif x.startswith("-"): @@ -3984,98 +4008,35 @@ class EquoInterface(TextInterface): # Get License text if possible licenses_dir = os.path.join(Spm.get_spm_setting('PORTDIR'),'licenses') - data['licensedata'] = {} - if licenses_dir: - licdata = [str(x.strip()) for x in data['license'].split() if str(x.strip()) and self.entropyTools.is_valid_string(x.strip())] - for mylicense in licdata: - - licfile = os.path.join(licenses_dir,mylicense) - if os.access(licfile,os.R_OK): - if self.entropyTools.istextfile(licfile): - f = open(licfile) - data['licensedata'][mylicense] = f.read() - f.close() - - # manage data['sources'] to create data['mirrorlinks'] - # =mirror://openoffice|link1|link2|link3 - data['mirrorlinks'] = [] - for i in data['sources']: - if i.startswith("mirror://"): - # parse what mirror I need - mirrorURI = i.split("/")[2] - mirrorlist = Spm.get_third_party_mirrors(mirrorURI) - data['mirrorlinks'].append([mirrorURI,mirrorlist]) - # mirrorURI = openoffice and mirrorlist = [link1, link2, link3] + data['licensedata'] = self._extract_pkg_metadata_license_data(licenses_dir, data['license']) + data['mirrorlinks'] = self._extract_pkg_metadata_mirror_links(Spm, data['sources']) # write only if it's a systempackage data['systempackage'] = False - systemPackages = Spm.get_atoms_in_system() - for x in systemPackages: - x = self.entropyTools.dep_getkey(x) - y = data['category']+"/"+data['name'] - if x == y: - # found - data['systempackage'] = True - break + system_packages = [self.entropyTools.dep_getkey(x) for x in Spm.get_atoms_in_system()] + if data['category']+"/"+data['name'] in system_packages: + data['systempackage'] = True # write only if it's a systempackage protect, mask = Spm.get_config_protect_and_mask() data['config_protect'] = protect data['config_protect_mask'] = mask - # fill data['messages'] - # etpConst['logdir']+"/elog" - if not os.path.isdir(etpConst['logdir']+"/elog"): - os.makedirs(etpConst['logdir']+"/elog") - data['messages'] = [] - if os.path.isdir(etpConst['logdir']+"/elog"): - elogfiles = os.listdir(etpConst['logdir']+"/elog") - myelogfile = data['category']+":"+data['name']+"-"+data['version'] - foundfiles = [] - for item in elogfiles: - if item.startswith(myelogfile): - foundfiles.append(item) - if foundfiles: - elogfile = foundfiles[0] - if len(foundfiles) > 1: - # get the latest - mtimes = [] - for item in foundfiles: - mtimes.append((self.entropyTools.getFileUnixMtime(etpConst['logdir']+"/elog/"+item),item)) - mtimes.sort() - elogfile = mtimes[len(mtimes)-1][1] - messages = self.entropyTools.extractElog(etpConst['logdir']+"/elog/"+elogfile) - for message in messages: - message = message.replace("emerge","install") - data['messages'].append(message) - else: - if not silent: - mytxt = "%s, %s" % (_("not set"),_("have you configured make.conf properly?"),) - self.updateProgress( - red(etpConst['logdir']+"/elog ")+mytxt, - importance = 1, - type = "warning", - header = brown(" * ") - ) - - # write API info + log_dir = etpConst['logdir']+"/elog" + if not os.path.isdir(log_dir): os.makedirs(log_dir) + data['messages'] = self._extract_pkg_metadata_messages(log_dir, data['category'], data['name'], data['version'], silent = silent) data['etpapi'] = etpConst['etpapi'] # removing temporary directory shutil.rmtree(tbz2TmpDir,True) if os.path.isdir(tbz2TmpDir): - try: - os.remove(tbz2TmpDir) - except OSError: - pass + try: os.remove(tbz2TmpDir) + except OSError: pass if not silent: self.updateProgress( - red(info_package+_("Package extraction complete")), - importance = 0, - type = "info", - header = brown(" * "), - back = True + red(info_package+_("Package extraction complete")), importance = 0, + type = "info", header = brown(" * "), back = True ) return data diff --git a/libraries/entropyTools.py b/libraries/entropyTools.py index 7f24861c3..9fcff5c23 100644 --- a/libraries/entropyTools.py +++ b/libraries/entropyTools.py @@ -306,6 +306,15 @@ def get_file_size(file_path): mystat = os.lstat(file_path) return int(mystat.st_size) +def sum_file_sizes(file_list): + size = 0 + for myfile in file_list: + try: + size += get_file_size(myfile) + except (OSError,IOError,): + continue + return size + def check_required_space(mountpoint, bytes_required): import statvfs st = os.statvfs(mountpoint)