1
2 '''
3 # DESCRIPTION:
4 # Entropy Object Oriented Interface
5
6 Copyright (C) 2007-2009 Fabio Erculiani
7
8 This program is free software; you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation; either version 2 of the License, or
11 (at your option) any later version.
12
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 '''
22
23 from __future__ import with_statement
24 import os
25 import shutil
26 from entropy.core import Singleton
27 from entropy.exceptions import OnlineMirrorError, PermissionDenied, \
28 SystemDatabaseError
29 from entropy.const import etpConst, etpSys, const_setup_perms, \
30 const_create_working_dirs, const_extract_srv_repo_params
31 from entropy.output import TextInterface, purple, red, darkgreen, \
32 bold, brown, blue, darkred
33 from entropy.server.interfaces.mirrors import Server as MirrorsServer
34 from entropy.i18n import _
35 from entropy.core import SystemSettings, SystemSettingsPlugin
36 from entropy.transceivers import FtpInterface
37 from entropy.db import LocalRepository
38
40
42
43 """
44 Parses Entropy server system configuration file.
45
46 @return dict data
47 """
48
49 data = {
50 'repositories': etpConst['server_repositories'].copy(),
51 'branches': etpConst['branches'][:],
52 'default_repository_id': etpConst['officialserverrepositoryid'],
53 'packages_expiration_days': etpConst['packagesexpirationdays'],
54 'database_file_format': etpConst['etpdatabasefileformat'],
55 'disabled_eapis': set(),
56 'exp_based_scope': etpConst['expiration_based_scope'],
57 'rss': {
58 'enabled': etpConst['rss-feed'],
59 'name': etpConst['rss-name'],
60 'base_url': etpConst['rss-base-url'],
61 'website_url': etpConst['rss-website-url'],
62 'editor': etpConst['rss-managing-editor'],
63 'max_entries': etpConst['rss-max-entries'],
64 'light_max_entries': etpConst['rss-light-max-entries'],
65 },
66 }
67
68 fake_instance = self._helper.fake_default_repo
69
70 if not os.access(etpConst['serverconf'], os.R_OK):
71 return data
72
73 with open(etpConst['serverconf'],"r") as server_f:
74 serverconf = [x.strip() for x in server_f.readlines() if x.strip()]
75
76 for line in serverconf:
77
78 split_line = line.split("|")
79 split_line_len = len(split_line)
80
81 if line.startswith("branches|") and (split_line_len == 2):
82
83 branches = split_line[1]
84 data['branches'] = []
85 for branch in branches.split():
86 data['branches'].append(branch)
87 if sys_set['repositories']['branch'] not in data['branches']:
88 data['branches'].append(sys_set['repositories']['branch'])
89 data['branches'] = sorted(data['branches'])
90
91 elif (line.find("officialserverrepositoryid|") != -1) and \
92 (not line.startswith("#")) and (split_line_len == 2):
93
94 if not fake_instance:
95 data['default_repository_id'] = split_line[1].strip()
96
97 elif line.startswith("expiration-days|") and (split_line_len == 2):
98
99 mydays = split_line[1].strip()
100 try:
101 mydays = int(mydays)
102 data['packages_expiration_days'] = mydays
103 except ValueError:
104 continue
105
106 elif line.startswith("expiration-based-scope|") and \
107 (split_line_len == 2):
108
109 exp_opt = split_line[1].strip().lower()
110 if exp_opt in ("enable", "enabled", "true", "1", "yes"):
111 data['exp_based_scope'] = True
112 else:
113 data['exp_based_scope'] = False
114
115 elif line.startswith("disabled-eapis|") and (split_line_len == 2):
116
117 mydis = split_line[1].strip().split(",")
118 try:
119 mydis = [int(x) for x in mydis]
120 mydis = set([x for x in mydis if x in (1, 2, 3,)])
121 except ValueError:
122 continue
123 if (len(mydis) < 3) and mydis:
124 data['disabled_eapis'] = mydis
125
126
127 elif line.startswith("repository|") and (split_line_len in [5, 6]) \
128 and (not fake_instance):
129
130 repoid, repodata = const_extract_srv_repo_params(line,
131 product = sys_set['repositories']['product'])
132 if repoid in data['repositories']:
133
134 data['repositories'][repoid]['mirrors'].extend(
135 repodata['mirrors'])
136 else:
137 data['repositories'][repoid] = repodata.copy()
138
139 elif line.startswith("database-format|") and (split_line_len == 2):
140
141 fmt = split_line[1]
142 if fmt in etpConst['etpdatabasesupportedcformats']:
143 data['database_file_format'] = fmt
144
145 elif line.startswith("rss-feed|") and (split_line_len == 2):
146
147 feed = split_line[1]
148 if feed in ("enable", "enabled", "true", "1"):
149 data['rss']['enabled'] = True
150 elif feed in ("disable", "disabled", "false", "0", "no",):
151 data['rss']['enabled'] = False
152
153 elif line.startswith("rss-name|") and (split_line_len == 2):
154
155 feedname = line.split("rss-name|")[1].strip()
156 data['rss']['name'] = feedname
157
158 elif line.startswith("rss-base-url|") and (split_line_len == 2):
159
160 data['rss']['base_url'] = line.split("rss-base-url|")[1].strip()
161 if not data['rss']['base_url'][-1] == "/":
162 data['rss']['base_url'] += "/"
163
164 elif line.startswith("rss-website-url|") and (split_line_len == 2):
165
166 data['rss']['website_url'] = split_line[1].strip()
167
168 elif line.startswith("managing-editor|") and (split_line_len == 2):
169
170 data['rss']['editor'] = split_line[1].strip()
171
172 elif line.startswith("max-rss-entries|") and (split_line_len == 2):
173
174 try:
175 entries = int(split_line[1].strip())
176 data['rss']['max_entries'] = entries
177 except (ValueError, IndexError,):
178 continue
179
180 elif line.startswith("max-rss-light-entries|") and \
181 (split_line_len == 2):
182
183 try:
184 entries = int(split_line[1].strip())
185 data['rss']['light_max_entries'] = entries
186 except (ValueError, IndexError,):
187 continue
188
189
190 if self._helper.community_repo:
191 data['repositories'][etpConst['clientserverrepoid']] = {}
192 mydata = {}
193 mydata['description'] = "Community Repositories System Database"
194 mydata['mirrors'] = []
195 mydata['community'] = False
196 data['repositories'][etpConst['clientserverrepoid']].update(mydata)
197
198
199 for repoid in data['repositories']:
200 data['repositories'][repoid]['packages_dir'] = \
201 os.path.join( etpConst['entropyworkdir'],
202 "server",
203 repoid,
204 "packages",
205 etpSys['arch']
206 )
207 data['repositories'][repoid]['store_dir'] = \
208 os.path.join( etpConst['entropyworkdir'],
209 "server",
210 repoid,
211 "store",
212 etpSys['arch']
213 )
214 data['repositories'][repoid]['upload_dir'] = \
215 os.path.join( etpConst['entropyworkdir'],
216 "server",
217 repoid,
218 "upload",
219 etpSys['arch']
220 )
221 data['repositories'][repoid]['database_dir'] = \
222 os.path.join( etpConst['entropyworkdir'],
223 "server",
224 repoid,
225 "database",
226 etpSys['arch']
227 )
228 data['repositories'][repoid]['packages_relative_path'] = \
229 os.path.join( sys_set['repositories']['product'],
230 repoid,
231 "packages",
232 etpSys['arch']
233 )+"/"
234 data['repositories'][repoid]['database_relative_path'] = \
235 os.path.join( sys_set['repositories']['product'],
236 repoid,
237 "database",
238 etpSys['arch']
239 )+"/"
240
241
242 shell_repoid = os.getenv('ETP_REPO')
243 if shell_repoid:
244 data['default_repository_id'] = shell_repoid
245
246 expiration_days = os.getenv('ETP_EXPIRATION_DAYS')
247 if expiration_days:
248 try:
249 expiration_days = int(expiration_days)
250 data['packages_expiration_days'] = expiration_days
251 except ValueError:
252 pass
253
254 return data
255
257
258 import entropy.tools as entropyTools
259
261
262 data = {}
263 srv_plug_id = etpConst['system_settings_plugins_ids']['server_plugin']
264
265 srv_parser_data = sys_set[srv_plug_id]['server']
266 if not srv_parser_data['exp_based_scope']:
267 return data
268
269
270 for repoid in srv_parser_data['repositories']:
271
272
273
274 if repoid == etpConst['clientserverrepoid']:
275 continue
276
277 idpackages = set()
278 exp_fp = self._helper.get_local_exp_based_pkgs_rm_whitelist_file(
279 repo = repoid)
280 dbconn = self._helper.open_server_repository(
281 just_reading = True, repo = repoid)
282
283 if os.access(exp_fp, os.R_OK | os.F_OK):
284 pkgs = self.entropyTools.generic_file_content_parser(exp_fp)
285 if '*' in pkgs:
286 idpackages.add(-1)
287 else:
288 for pkg in pkgs:
289 idpackage, rc_match = dbconn.atomMatch(pkg)
290 if rc_match:
291 continue
292 idpackages.add(idpackage)
293
294 data[repoid] = idpackages
295
296 return data
297
298
299 -class Server(Singleton, TextInterface):
300
301 - def init_singleton(self, default_repository = None, save_repository = False,
302 community_repo = False, fake_default_repo = False,
303 fake_default_repo_id = '::fake::',
304 fake_default_repo_desc = 'this is a fake repository'):
305
306 self.__instance_destroyed = False
307 if etpConst['uid'] != 0:
308 mytxt = _("Entropy Server interface must be run as root")
309 raise PermissionDenied("PermissionDenied: %s" % (mytxt,))
310
311
312 from entropy.misc import LogFile
313 self.SystemSettings = SystemSettings()
314 self.community_repo = community_repo
315 from entropy.db import dbapi2
316 self.dbapi2 = dbapi2
317 etpSys['serverside'] = True
318 self._memory_db_instances = {}
319 self.fake_default_repo = fake_default_repo
320 self.indexing = False
321 self.xcache = False
322 self.MirrorsService = None
323 self.__server_dbcache = {}
324 self.repository_treeupdate_digests = {}
325 self.__settings_to_backup = []
326 self.__do_save_repository = save_repository
327 self.__sync_lock_cache = set()
328 self.rssMessages = {
329 'added': {},
330 'removed': {},
331 'commitmessage': "",
332 'light': {},
333 }
334
335 if fake_default_repo:
336 default_repository = fake_default_repo_id
337 etpConst['officialserverrepositoryid'] = fake_default_repo_id
338 self.init_generic_memory_server_repository(fake_default_repo_id,
339 fake_default_repo_desc)
340
341
342 self.sys_settings_plugin_id = \
343 etpConst['system_settings_plugins_ids']['server_plugin']
344 self.sys_settings_plugin = ServerSystemSettingsPlugin(
345 self.sys_settings_plugin_id, self)
346 self.SystemSettings.add_plugin(self.sys_settings_plugin)
347
348
349 self.sys_settings_fatscope_plugin_id = \
350 etpConst['system_settings_plugins_ids']['server_plugin_fatscope']
351 self.sys_settings_fatscope_plugin = ServerFatscopeSystemSettingsPlugin(
352 self.sys_settings_fatscope_plugin_id, self)
353 self.SystemSettings.add_plugin(self.sys_settings_fatscope_plugin)
354
355 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
356 self.default_repository = default_repository
357 if self.default_repository == None:
358 self.default_repository = srv_set['default_repository_id']
359
360 if self.default_repository in srv_set['repositories']:
361 self.ensure_paths(self.default_repository)
362 self.migrate_repository_databases_to_new_branched_path()
363
364 if self.default_repository not in srv_set['repositories']:
365 raise PermissionDenied("PermissionDenied: %s %s" % (
366 self.default_repository,
367 _("repository not configured"),
368 )
369 )
370 if etpConst['clientserverrepoid'] == self.default_repository:
371 raise PermissionDenied("PermissionDenied: %s %s" % (
372 etpConst['clientserverrepoid'],
373 _("protected repository id, can't use this, sorry dude..."),
374 )
375 )
376
377 self.switch_default_repository(self.default_repository)
378
380 self.__instance_destroyed = True
381 if hasattr(self,'ClientService'):
382 self.ClientService.destroy()
383 if hasattr(self,'sys_settings_server_plugin'):
384 try:
385 self.SystemSettings.remove_plugin(self.sys_settings_plugin_id)
386 except KeyError:
387 pass
388 if hasattr(self,'sys_settings_fatscope_plugin'):
389 try:
390 self.SystemSettings.remove_plugin(
391 self.sys_settings_fatscope_plugin)
392 except KeyError:
393 pass
394 self.close_server_databases()
395
397 return self.__instance_destroyed
398
401
410
411
413 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
414 migrated_filename = '.branch_migrated'
415 for repoid in srv_set['repositories'].keys():
416
417 if repoid == etpConst['clientserverrepoid']: continue
418 mydir = srv_set['repositories'][repoid]['database_dir']
419 if not os.path.isdir(mydir):
420 continue
421
422 migrated_filepath = os.path.join(mydir, migrated_filename)
423 if os.path.isfile(migrated_filepath):
424 continue
425
426 my_branched_dir = self.get_local_database_dir(repoid)
427 if os.path.isdir(my_branched_dir):
428 continue
429
430 self.updateProgress(
431 "[%s:%s] %s: %s, %s: %s" % (
432 brown("repo"),
433 purple(repoid),
434 _("migrating database path from"),
435 brown(mydir),
436 _('to'),
437 brown(my_branched_dir),
438 ),
439 importance = 1,
440 type = "info",
441 header = bold(" @@ ")
442 )
443
444 repo_files = [os.path.join(mydir, x) for x in os.listdir(mydir) if \
445 (os.path.isfile(os.path.join(mydir, x)) and \
446 os.access(os.path.join(mydir, x), os.W_OK))
447 ]
448 os.makedirs(my_branched_dir)
449 const_setup_perms(my_branched_dir, etpConst['entropygid'])
450
451 for repo_file in repo_files:
452 repo_filename = os.path.basename(repo_file)
453 shutil.move(repo_file,
454 os.path.join(my_branched_dir, repo_filename))
455
456 f_migrated = open(migrated_filepath,"w")
457 f_migrated.write("done\n")
458 f_migrated.flush()
459 f_migrated.close()
460
462 self.setup_entropy_settings()
463 cs_name = 'ClientService'
464 if hasattr(self, cs_name):
465 obj = getattr(self, cs_name)
466 obj.destroy()
467 from entropy.client.interfaces import Client
468 self.ClientService = Client(
469 indexing = self.indexing,
470 xcache = self.xcache,
471 repo_validation = False,
472 noclientdb = 1
473 )
474 from entropy.cache import EntropyCacher
475 self.Cacher = EntropyCacher()
476 self.ClientService.updateProgress = self.updateProgress
477 self.validRepositories = self.ClientService.validRepositories
478 self.entropyTools = self.ClientService.entropyTools
479 self.dumpTools = self.ClientService.dumpTools
480 self.QA = self.ClientService.QA
481 self.backup_entropy_settings()
482 self.SpmService = self.ClientService.Spm()
483 self.MirrorsService = MirrorsServer(self)
484
486 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
487 curr_repoid = srv_set['default_repository_id']
488 backup_list = [
489 'etpdatabaseclientfilepath',
490 'clientdbid',
491 {'server': srv_set.copy()},
492 ]
493 for setting in backup_list:
494 if setting not in self.__settings_to_backup:
495 self.__settings_to_backup.append(setting)
496
497 if not self.community_repo:
498 etpConst['etpdatabaseclientfilepath'] = \
499 self.get_local_database_file(repo)
500 etpConst['clientdbid'] = etpConst['serverdbid']
501 const_create_working_dirs()
502
504 if hasattr(self,'serverDbCache'):
505 for item in self.__server_dbcache:
506 try:
507 self.__server_dbcache[item].closeDB()
508 except self.dbapi2.ProgrammingError:
509 pass
510 self.__server_dbcache.clear()
511
513 found = None
514 for item in self.__server_dbcache:
515 if dbinstance == self.__server_dbcache[item]:
516 found = item
517 break
518 if found:
519 instance = self.__server_dbcache.pop(found)
520 instance.closeDB()
521
523 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
524 return srv_set['repositories'].copy()
525
553
555 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
556 if self.community_repo:
557 for repoid in srv_set['repositories']:
558 srv_set['repositories'][repoid]['community'] = True
559
560
562
563 if not self.is_repository_initialized(repoid):
564 mytxt = blue("%s.") % (
565 _("Your default repository is not initialized"),)
566 self.updateProgress(
567 "[%s:%s] %s" % (
568 brown("repo"),
569 purple(repoid),
570 mytxt,
571 ),
572 importance = 1,
573 type = "warning",
574 header = darkred(" !!! ")
575 )
576 answer = self.askQuestion(
577 _("Do you want to initialize your default repository ?"))
578 if answer == "No":
579 mytxt = red("%s.") % (
580 _("Continuing with an uninitialized repository"),)
581 self.updateProgress(
582 "[%s:%s] %s" % (
583 brown("repo"),
584 purple(repoid),
585 mytxt,
586 ),
587 importance = 1,
588 type = "warning",
589 header = darkred(" !!! ")
590 )
591 else:
592
593 dbfile = self.get_local_database_file(repoid)
594 if os.path.isfile(dbfile):
595 shutil.move(dbfile, dbfile+".backup")
596 self.initialize_server_database(empty = True,
597 repo = repoid, warnings = False)
598
599
601 type_txt = _("server-side repository")
602 if self.community_repo:
603 type_txt = _("community repository")
604
605 mytxt = _("Entropy Server Interface Instance on repository")
606 self.updateProgress(
607 blue("%s: %s, %s: %s (%s: %s)" % (
608 mytxt,
609 red(self.default_repository),
610 _("current branch"),
611 darkgreen(self.SystemSettings['repositories']['branch']),
612 purple(_("type")),
613 bold(type_txt),
614 )
615 ),
616 importance = 2,
617 type = "info",
618 header = red(" @@ ")
619 )
620 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
621 repos = srv_set['repositories'].keys()
622 mytxt = blue("%s:") % (_("Currently configured repositories"),)
623 self.updateProgress(
624 mytxt,
625 importance = 1,
626 type = "info",
627 header = red(" @@ ")
628 )
629 for repo in repos:
630 self.updateProgress(
631 darkgreen(repo),
632 importance = 0,
633 type = "info",
634 header = brown(" # ")
635 )
636
637
639
640
641 if repoid == etpConst['clientserverrepoid']:
642 return
643
644 if os.path.isfile(etpConst['serverconf']):
645 f_srv = open(etpConst['serverconf'],"r")
646 content = f_srv.readlines()
647 f_srv.close()
648 content = [x.strip() for x in content]
649 found = False
650 new_content = []
651 for line in content:
652 if line.strip().startswith("officialserverrepositoryid|"):
653 line = "officialserverrepositoryid|%s" % (repoid,)
654 found = True
655 new_content.append(line)
656 if not found:
657 new_content.append("officialserverrepositoryid|%s" % (repoid,))
658 f_srv_t = open(etpConst['serverconf']+".save_default_repo_tmp","w")
659 for line in new_content:
660 f_srv_t.write(line+"\n")
661 f_srv_t.flush()
662 f_srv_t.close()
663 os.rename(etpConst['serverconf']+".save_default_repo_tmp",
664 etpConst['serverconf'])
665 else:
666 f_srv = open(etpConst['serverconf'],"w")
667 f_srv.write("officialserverrepositoryid|%s\n" % (repoid,))
668 f_srv.flush()
669 f_srv.close()
670
672
673
674 if repoid == etpConst['clientserverrepoid']:
675 return False
676
677 if not os.path.isfile(etpConst['serverconf']):
678 return None
679 f_srv = open(etpConst['serverconf'])
680 tmpfile = etpConst['serverconf']+".switch"
681 mycontent = [x.strip() for x in f_srv.readlines()]
682 f_srv.close()
683 f_tmp = open(tmpfile,"w")
684 st = "repository|%s" % (repoid,)
685 status = False
686 for line in mycontent:
687 if enable:
688 if (line.find(st) != -1) and line.startswith("#") and \
689 (len(line.split("|")) == 5):
690 line = line[1:]
691 status = True
692 else:
693 if (line.find(st) != -1) and not line.startswith("#") and \
694 (len(line.split("|")) == 5):
695 line = "#"+line
696 status = True
697 f_tmp.write(line+"\n")
698 f_tmp.flush()
699 f_tmp.close()
700 shutil.move(tmpfile, etpConst['serverconf'])
701 if status:
702 self.close_server_databases()
703 self.SystemSettings.clear()
704 self.setup_services()
705 self.show_interface_status()
706 return status
707
714
723
724 dbc = self.open_server_repository(just_reading = True, repo = repo)
725 valid = do_validate(dbc)
726 self.close_server_database(dbc)
727 if not valid:
728 dbc = self.open_server_repository(read_only = False,
729 no_upload = True, repo = repo, is_new = True)
730 valid = do_validate(dbc)
731 self.close_server_database(dbc)
732
733 return valid
734
736
737 if repo == None:
738 repo = self.default_repository
739
740
741 lock_file = self.MirrorsService.get_database_lockfile(repo)
742 if os.path.isfile(lock_file):
743 self.updateProgress(
744 red(_("Entropy database is already locked by you :-)")),
745 importance = 1,
746 type = "info",
747 header = red(" * ")
748 )
749 else:
750
751 mytxt = "%s ..." % (_("Locking and Syncing Entropy database"),)
752 self.updateProgress(
753 red(mytxt),
754 importance = 1,
755 type = "info",
756 header = red(" * "),
757 back = True
758 )
759 for uri in self.get_remote_mirrors(repo):
760 given_up = self.MirrorsService.mirror_lock_check(uri,
761 repo = repo)
762 if given_up:
763 crippled_uri = \
764 self.entropyTools.extract_ftp_host_from_uri(uri)
765 mytxt = "%s:" % (_("Mirrors status table"),)
766 self.updateProgress(
767 darkgreen(mytxt),
768 importance = 1,
769 type = "info",
770 header = brown(" * ")
771 )
772 dbstatus = self.MirrorsService.get_mirrors_lock(repo = repo)
773 for db_uri, db_st1, db_st2 in dbstatus:
774 db_st1_info = darkgreen(_("Unlocked"))
775 if db_st1:
776 db_st1_info = red(_("Locked"))
777 db_st2_info = darkgreen(_("Unlocked"))
778 if db_st2:
779 db_st2_info = red(_("Locked"))
780
781 crippled_uri = \
782 self.entropyTools.extract_ftp_host_from_uri(db_uri)
783 self.updateProgress(
784 "%s: [%s: %s] [%s: %s]" % (
785 bold(crippled_uri),
786 brown(_("database")),
787 db_st1_info,
788 brown(_("download")),
789 db_st2_info,
790 ),
791 importance = 1,
792 type = "info",
793 header = "\t"
794 )
795
796 raise OnlineMirrorError("OnlineMirrorError: %s %s" % (
797 _("cannot lock mirror"),
798 crippled_uri,
799 )
800 )
801
802
803 self.MirrorsService.lock_mirrors(True, repo = repo)
804 self.MirrorsService.sync_databases(no_upload, repo = repo)
805
806
809
810 if mirrors is None:
811 mirrors = []
812 product = self.SystemSettings['repositories']['product']
813 dbc = self.open_memory_database(dbname = etpConst['serverdbid']+repoid)
814 self._memory_db_instances[repoid] = dbc
815
816 eapi3_port = int(etpConst['socket_service']['port'])
817 eapi3_ssl_port = int(etpConst['socket_service']['ssl_port'])
818
819 repodata = {
820 'repoid': repoid,
821 'description': description,
822 'mirrors': mirrors,
823 'community': community_repo,
824 'service_port': eapi3_port,
825 'ssl_service_port': eapi3_ssl_port,
826 'service_url': service_url,
827 'handler': '',
828 'in_memory': True,
829 }
830
831 etpConst['server_repositories'][repoid] = repodata
832 self.SystemSettings.clear()
833
834 return dbc
835
837 if dbname == None:
838 dbname = etpConst['genericdbid']
839 dbc = LocalRepository(
840 readOnly = False,
841 dbFile = ':memory:',
842 clientDatabase = True,
843 dbname = dbname,
844 xcache = False,
845 indexing = False,
846 OutputInterface = self,
847 skipChecks = True,
848 ServiceInterface = self
849 )
850 dbc.initializeDatabase()
851 return dbc
852
853 - def open_server_repository(
854 self,
855 read_only = True,
856 no_upload = True,
857 just_reading = False,
858 repo = None,
859 indexing = True,
860 warnings = True,
861 do_cache = True,
862 use_branch = None,
863 lock_remote = True,
864 is_new = False
865 ):
866
867 if repo == None:
868 repo = self.default_repository
869
870 if repo == etpConst['clientserverrepoid'] and self.community_repo:
871 return self.ClientService.clientDbconn
872
873
874 if repo in self._memory_db_instances:
875 return self._memory_db_instances.get(repo)
876
877 if just_reading:
878 read_only = True
879 no_upload = True
880
881 local_dbfile = self.get_local_database_file(repo, use_branch)
882 if do_cache:
883 cached = self.__server_dbcache.get(
884 (repo, etpConst['systemroot'], local_dbfile, read_only,
885 no_upload, just_reading, use_branch, lock_remote,)
886 )
887 if cached != None:
888 return cached
889
890 local_dbfile_dir = os.path.dirname(local_dbfile)
891 if not os.path.isdir(local_dbfile_dir):
892 os.makedirs(local_dbfile_dir)
893
894 if (not read_only) and (lock_remote) and \
895 (repo not in self.__sync_lock_cache):
896 self.do_server_repository_sync_lock(repo, no_upload)
897 self.__sync_lock_cache.add(repo)
898
899 local_dbfile_exists = os.path.lexists(local_dbfile)
900 conn = LocalRepository(
901 readOnly = read_only,
902 dbFile = local_dbfile,
903 noUpload = no_upload,
904 OutputInterface = self,
905 ServiceInterface = self,
906 dbname = etpConst['serverdbid']+repo,
907 useBranch = use_branch,
908 lockRemote = lock_remote
909 )
910 if not local_dbfile_exists:
911
912 conn.readOnly = False
913 conn.initializeDatabase()
914 conn.commitChanges()
915
916 valid = True
917 try:
918 conn.validateDatabase()
919 except SystemDatabaseError:
920 valid = False
921
922
923
924 if (repo not in etpConst['server_treeupdatescalled']) and \
925 (not just_reading):
926
927
928 if valid:
929 conn.serverUpdatePackagesData()
930 elif warnings and not is_new:
931 mytxt = _("Entropy database is corrupted!")
932 self.updateProgress(
933 darkred(mytxt),
934 importance = 1,
935 type = "warning",
936 header = bold(" !!! ")
937 )
938
939 if not read_only and valid and indexing:
940
941 self.updateProgress(
942 "[repo:%s|%s] %s" % (
943 blue(repo),
944 red(_("database")),
945 blue(_("indexing database")),
946 ),
947 importance = 1,
948 type = "info",
949 header = brown(" @@ "),
950 back = True
951 )
952 conn.createAllIndexes()
953
954 if do_cache:
955
956
957 self.__server_dbcache[
958 (repo, etpConst['systemroot'], local_dbfile, read_only,
959 no_upload, just_reading, use_branch, lock_remote,)] = conn
960
961
962 if (not read_only) and (not is_new):
963 cur_sets = conn.retrievePackageSets()
964 sys_sets = self.get_configured_package_sets(repo)
965 if cur_sets != sys_sets:
966 self.update_database_package_sets(repo, dbconn = conn)
967 conn.commitChanges()
968
969 return conn
970
972
973 sys_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
974 server_repos = sys_set['repositories'].keys()
975 installed_packages = set()
976
977 if default_repo:
978 server_repos = [default_repo]
979
980 for repo in server_repos:
981 dbconn = self.open_server_repository(read_only = True,
982 no_upload = True, repo = repo)
983 installed_packages |= set([(x, repo) for x in \
984 dbconn.listAllIdpackages()])
985
986
987 deps_not_satisfied = set()
988 length = str((len(installed_packages)))
989 count = 0
990 mytxt = _("Checking")
991
992 for idpackage, repo in installed_packages:
993 count += 1
994 dbconn = self.open_server_repository(read_only = True,
995 no_upload = True, repo = repo)
996
997 if (count%150 == 0) or (count == length) or (count == 1):
998 atom = dbconn.retrieveAtom(idpackage)
999 self.updateProgress(
1000 darkgreen(mytxt)+" "+bold(atom),
1001 importance = 0,
1002 type = "info",
1003 back = True,
1004 count = (count, length),
1005 header = darkred(" @@ ")
1006 )
1007
1008 xdeps = dbconn.retrieveDependencies(idpackage)
1009 for xdep in xdeps:
1010 xid, xuseless = self.atom_match(xdep)
1011 if xid == -1:
1012 deps_not_satisfied.add(xdep)
1013
1014 return deps_not_satisfied
1015
1017
1018 mytxt = "%s %s" % (blue(_("Running dependencies test")), red("..."))
1019 self.updateProgress(
1020 mytxt,
1021 importance = 2,
1022 type = "info",
1023 header = red(" @@ ")
1024 )
1025
1026 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
1027 server_repos = srv_set['repositories'].keys()
1028 deps_not_matched = self.deps_tester(repo)
1029
1030 if deps_not_matched:
1031
1032 crying_atoms = {}
1033 for atom in deps_not_matched:
1034 for repo in server_repos:
1035 dbconn = self.open_server_repository(just_reading = True,
1036 repo = repo)
1037 riddep = dbconn.searchDependency(atom)
1038 if riddep == -1:
1039 continue
1040 ridpackages = dbconn.searchIdpackageFromIddependency(riddep)
1041 for i in ridpackages:
1042 iatom = dbconn.retrieveAtom(i)
1043 if not crying_atoms.has_key(atom):
1044 crying_atoms[atom] = set()
1045 crying_atoms[atom].add((iatom, repo))
1046
1047 mytxt = blue("%s:") % (_("These are the dependencies not found"),)
1048 self.updateProgress(
1049 mytxt,
1050 importance = 1,
1051 type = "info",
1052 header = red(" @@ ")
1053 )
1054 mytxt = "%s:" % (_("Needed by"),)
1055 for atom in deps_not_matched:
1056 self.updateProgress(
1057 red(atom),
1058 importance = 1,
1059 type = "info",
1060 header = blue(" # ")
1061 )
1062 if crying_atoms.has_key(atom):
1063 self.updateProgress(
1064 red(mytxt),
1065 importance = 0,
1066 type = "info",
1067 header = blue(" # ")
1068 )
1069 for my_dep, myrepo in crying_atoms[atom]:
1070 self.updateProgress(
1071 "[%s:%s] %s" % (
1072 blue(_("by repo")),
1073 darkred(myrepo),
1074 darkgreen(my_dep),
1075 ),
1076 importance = 0,
1077 type = "info",
1078 header = blue(" # ")
1079 )
1080 else:
1081
1082 mytxt = blue(_("Every dependency is satisfied. It's all fine."))
1083 self.updateProgress(
1084 mytxt,
1085 importance = 2,
1086 type = "info",
1087 header = red(" @@ ")
1088 )
1089
1090 return deps_not_matched
1091
1093
1094
1095 dbconn = self.open_server_repository(read_only = True,
1096 no_upload = True, repo = repo)
1097 packages_matched, brokenexecs, status = \
1098 self.ClientService.libraries_test(dbconn = dbconn,
1099 broken_symbols = False)
1100 if status != 0:
1101 return 1, None
1102
1103 if get_files:
1104 return 0, brokenexecs
1105
1106 if (not brokenexecs) and (not packages_matched):
1107 mytxt = "%s." % (_("System is healthy"),)
1108 self.updateProgress(
1109 blue(mytxt),
1110 importance = 2,
1111 type = "info",
1112 header = red(" @@ ")
1113 )
1114 return 0, None
1115
1116 mytxt = "%s..." % (_("Matching libraries with Spm, please wait"),)
1117 self.updateProgress(
1118 blue(mytxt),
1119 importance = 1,
1120 type = "info",
1121 header = red(" @@ ")
1122 )
1123
1124 packages = self.SpmService.query_belongs_multiple(brokenexecs)
1125
1126 if packages:
1127 mytxt = "%s:" % (_("These are the matched packages"),)
1128 self.updateProgress(
1129 red(mytxt),
1130 importance = 1,
1131 type = "info",
1132 header = red(" @@ ")
1133 )
1134 for package_slot in packages:
1135 self.updateProgress(
1136 blue(unicode(package_slot)),
1137 importance = 0,
1138 type = "info",
1139 header = red(" # ")
1140 )
1141 for filename in sorted(list(packages[package_slot])):
1142 self.updateProgress(
1143 blue(filename),
1144 importance = 0,
1145 type = "info",
1146 header = brown(" => ")
1147 )
1148
1149 pkgstring = ' '.join(["%s:%s" % (
1150 self.entropyTools.dep_getkey(x[0]), x[1],) for x \
1151 in sorted(packages)])
1152 mytxt = "%s: %s" % (darkgreen(_("Packages string")), pkgstring,)
1153 self.updateProgress(
1154 mytxt,
1155 importance = 1,
1156 type = "info",
1157 header = red(" @@ ")
1158 )
1159 else:
1160 self.updateProgress(
1161 red(_("No matched packages")),
1162 importance = 1,
1163 type = "info",
1164 header = red(" @@ ")
1165 )
1166
1167 return 0, packages
1168
1170
1171 mytxt = "%s %s" % (
1172 blue(_("Running orphaned SPM packages test")), red("..."),)
1173 self.updateProgress(
1174 mytxt,
1175 importance = 2,
1176 type = "info",
1177 header = red(" @@ ")
1178 )
1179 installed_packages, length = self.SpmService.get_installed_packages()
1180 not_found = {}
1181 count = 0
1182 for installed_package in installed_packages:
1183 count += 1
1184 self.updateProgress(
1185 "%s: %s" % (
1186 darkgreen(_("Scanning package")),
1187 brown(installed_package),),
1188 importance = 0,
1189 type = "info",
1190 back = True,
1191 count = (count, length),
1192 header = darkred(" @@ ")
1193 )
1194 key, slot = (self.entropyTools.dep_getkey(installed_package),
1195 self.SpmService.get_installed_package_slot(installed_package),)
1196 pkg_atom = "%s:%s" % (key, slot,)
1197 tree_atom = self.SpmService.get_best_atom(pkg_atom)
1198 if not tree_atom:
1199 not_found[installed_package] = pkg_atom
1200 self.updateProgress(
1201 "%s: %s" % (
1202 blue(pkg_atom),
1203 darkred(_("not found anymore")),
1204 ),
1205 importance = 0,
1206 type = "warning",
1207 count = (count, length),
1208 header = darkred(" @@ ")
1209 )
1210
1211 if not_found:
1212 not_found_list = ' '.join([not_found[x] for x in sorted(not_found)])
1213 self.updateProgress(
1214 "%s: %s" % (
1215 blue(_("Packages string")),
1216 not_found_list,
1217 ),
1218 importance = 0,
1219 type = "warning",
1220 count = (count, length),
1221 header = darkred(" @@ ")
1222 )
1223
1224 return not_found
1225
1232
1239
1241 if dbpath == None:
1242 dbpath = self.get_local_database_file(repo)
1243
1244 dbdir = os.path.dirname(dbpath)
1245 if not os.path.isdir(dbdir):
1246 os.makedirs(dbdir)
1247
1248 mytxt = red("%s ...") % (_("Initializing an empty database"),)
1249 self.updateProgress(
1250 mytxt,
1251 importance = 1,
1252 type = "info",
1253 header = darkgreen(" * "),
1254 back = True
1255 )
1256 dbconn = self.ClientService.open_generic_database(dbpath)
1257 dbconn.initializeDatabase()
1258 dbconn.commitChanges()
1259 dbconn.closeDB()
1260 mytxt = "%s %s %s." % (
1261 red(_("Entropy database file")),
1262 bold(dbpath),
1263 red(_("successfully initialized")),
1264 )
1265 self.updateProgress(
1266 mytxt,
1267 importance = 1,
1268 type = "info",
1269 header = darkgreen(" * ")
1270 )
1271
1272 - def tag_packages(self, package_tag, idpackages, repo = None, ask = True):
1273
1274
1275
1276 try:
1277 package_tag = str(package_tag)
1278 if " " in package_tag:
1279 raise ValueError
1280 except (UnicodeDecodeError, UnicodeEncodeError, ValueError,):
1281 self.updateProgress(
1282 "%s: %s" % (
1283 blue(_("Invalid tag specified")),
1284 package_tag,
1285 ),
1286 importance = 1, type = "error", header = darkred(" !! ")
1287 )
1288 return 1, package_tag
1289
1290 if repo == None:
1291 repo = self.default_repository
1292
1293
1294 invalid_atoms = []
1295 dbconn = self.open_server_repository(read_only = True,
1296 no_upload = True, repo = repo)
1297 for idpackage in idpackages:
1298 ver_tag = dbconn.retrieveVersionTag(idpackage)
1299 if ver_tag:
1300 invalid_atoms.append(dbconn.retrieveAtom(idpackage))
1301
1302 if invalid_atoms:
1303 self.updateProgress(
1304 "%s: %s" % (
1305 blue(_("Packages already tagged, action aborted")),
1306 ', '.join([darkred(unicode(x)) for x in invalid_atoms]),
1307 ),
1308 importance = 1, type = "error", header = darkred(" !! ")
1309 )
1310 return 2, invalid_atoms
1311
1312 matches = [(x, repo) for x in idpackages]
1313 status = 0
1314 data = self.move_packages(
1315 matches, to_repo = repo, from_repo = repo, ask = ask,
1316 do_copy = True, new_tag = package_tag
1317 )
1318 return status, data
1319
1321 """
1322 When creating a new branch, for space reasons, packages are not
1323 moved to a new location. This works fine until old branch is removed.
1324 To avoid inconsistences, before deciding to do that, all the packages
1325 in the old branch should be flushed back to the the currently configured
1326 branch.
1327
1328 @param from_branches -- list of branches to move packages from
1329 @type from_branches -- list
1330 @param repo -- repository to work on
1331 @type repo -- str
1332 @param ask -- user interactivity
1333 @type ask -- bool
1334
1335 @return status
1336 """
1337
1338 status = True
1339 if repo == None:
1340 repo = self.default_repository
1341 branch = self.SystemSettings['repositories']['branch']
1342
1343 if branch in from_branches:
1344 from_branches = [x for x in from_branches if x != branch]
1345
1346 self.updateProgress(
1347 "[%s=>%s|%s] %s" % (
1348 darkgreen(', '.join(from_branches)),
1349 darkred(branch),
1350 brown(repo),
1351 blue(_("flushing back selected packages from branches")),
1352 ),
1353 importance = 2,
1354 type = "info",
1355 header = red(" @@ ")
1356 )
1357
1358 dbconn = self.open_server_repository(read_only = True,
1359 no_upload = True, repo = repo)
1360
1361 idpackage_map = dict(((x, [],) for x in from_branches))
1362 idpackages = dbconn.listAllIdpackages(order_by = 'atom')
1363 for idpackage in idpackages:
1364 download_url = dbconn.retrieveDownloadURL(idpackage)
1365 url_br = self.ClientService.get_branch_from_download_relative_uri(
1366 download_url)
1367 if url_br in from_branches:
1368 idpackage_map[url_br].append(idpackage)
1369
1370 mapped_branches = [x for x in idpackage_map if idpackage_map[x]]
1371 if not mapped_branches:
1372 self.updateProgress(
1373 "[%s=>%s|%s] %s !" % (
1374 darkgreen(', '.join(from_branches)),
1375 darkred(branch),
1376 brown(repo),
1377 blue(_("nothing to do")),
1378 ),
1379 importance = 0,
1380 type = "warning",
1381 header = blue(" @@ ")
1382 )
1383 return status
1384
1385
1386 all_fine = True
1387 tmp_down_dir = self.entropyTools.get_random_temp_file()
1388 os.makedirs(tmp_down_dir)
1389
1390 download_queue = {}
1391 local_up_dir = self.get_local_upload_directory(repo)
1392 local_basedir = os.path.join(local_up_dir, branch)
1393 dbconn = self.open_server_repository(read_only = False,
1394 no_upload = True, repo = repo)
1395
1396 def generate_queue(branch, repo, from_branch, down_q, idpackage_map):
1397
1398 self.updateProgress(
1399 "[%s=>%s|%s] %s" % (
1400 darkgreen(from_branch),
1401 darkred(branch),
1402 brown(repo),
1403 brown(_("these are the packages that will be flushed")),
1404 ),
1405 importance = 1,
1406 type = "info",
1407 header = brown(" @@ ")
1408 )
1409
1410
1411 for idpackage in idpackage_map[from_branch]:
1412 atom = dbconn.retrieveAtom(idpackage)
1413 self.updateProgress(
1414 "[%s=>%s|%s] %s" % (
1415 darkgreen(from_branch),
1416 darkred(branch),
1417 brown(repo),
1418 purple(atom),
1419 ),
1420 importance = 0,
1421 type = "info",
1422 header = blue(" # ")
1423 )
1424 pkg_fp = os.path.basename(dbconn.retrieveDownloadURL(idpackage))
1425 pkg_fp = os.path.join(tmp_down_dir, pkg_fp)
1426 down_q.append((pkg_fp, idpackage,))
1427
1428
1429 for from_branch in sorted(mapped_branches):
1430
1431 download_queue[from_branch] = []
1432 all_fine = False
1433 generate_queue(branch, repo, from_branch,
1434 download_queue[from_branch], idpackage_map)
1435
1436 if ask:
1437 rc_question = self.askQuestion(
1438 _("Would you like to continue ?"))
1439 if rc_question == "No":
1440 continue
1441
1442 remote_relative_path = self.get_remote_packages_relative_path(repo)
1443
1444 for uri in self.get_remote_mirrors(repo):
1445
1446 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri)
1447 ftp_basedir = os.path.join(remote_relative_path, from_branch)
1448
1449 downloader_queue = [x[0] for x in download_queue[from_branch]]
1450 downloader = self.MirrorsService.FtpServerHandler(
1451 FtpInterface,
1452 self,
1453 [uri],
1454 downloader_queue,
1455 critical_files = downloader_queue,
1456 use_handlers = True,
1457 ftp_basedir = ftp_basedir,
1458 local_basedir = tmp_down_dir,
1459 download = True,
1460 repo = repo
1461 )
1462
1463 errors, m_fine_uris, m_broken_uris = downloader.go()
1464
1465 if errors:
1466 my_broken_uris = [
1467 (self.entropyTools.extract_ftp_host_from_uri(x), y,) \
1468 for x, y in m_broken_uris]
1469
1470 reason = my_broken_uris[0][1]
1471
1472 self.updateProgress(
1473 "[%s=>%s|%s] %s, %s: %s" % (
1474 darkgreen(from_branch),
1475 darkred(branch),
1476 brown(repo),
1477 blue(_("download errors")),
1478 blue(_("reason")),
1479 reason,
1480 ),
1481 importance = 1,
1482 type = "error",
1483 header = darkred(" !!! ")
1484 )
1485
1486 continue
1487
1488 all_fine = True
1489
1490 self.updateProgress(
1491 "[%s=>%s|%s] %s: %s" % (
1492 darkgreen(from_branch),
1493 darkred(branch),
1494 brown(repo),
1495 blue(_("download completed successfully")),
1496 darkgreen(crippled_uri),
1497 ),
1498 importance = 1,
1499 type = "info",
1500 header = darkgreen(" * ")
1501 )
1502
1503 if not all_fine:
1504 self.updateProgress(
1505 "[%s=>%s|%s] %s" % (
1506 darkgreen(', '.join(from_branches)),
1507 darkred(branch),
1508 brown(repo),
1509 blue(_("error downloading packages from mirrors")),
1510 ),
1511 importance = 2,
1512 type = "error",
1513 header = darkred(" !!! ")
1514 )
1515 return False
1516
1517 tmp_db_path = self.entropyTools.get_random_temp_file()
1518 for from_branch in sorted(mapped_branches):
1519
1520 self.updateProgress(
1521 "[%s=>%s|%s] %s: %s" % (
1522 darkgreen(from_branch),
1523 darkred(branch),
1524 brown(repo),
1525 blue(_("working on branch")),
1526 darkgreen(from_branch),
1527 ),
1528 importance = 1,
1529 type = "info",
1530 header = brown(" @@ ")
1531 )
1532
1533 down_queue = download_queue[from_branch]
1534 for package_path, idpackage in down_queue:
1535
1536 self.updateProgress(
1537 "[%s=>%s|%s|%s] %s: %s" % (
1538 darkgreen(from_branch),
1539 darkred(branch),
1540 brown(repo),
1541 dbconn.retrieveAtom(idpackage),
1542 blue(_("updating package")),
1543 darkgreen(os.path.basename(package_path)),
1544 ),
1545 importance = 1,
1546 type = "info",
1547 header = brown(" "),
1548 back = True
1549 )
1550
1551
1552 package_name = os.path.basename(package_path)
1553 new_package_path = os.path.join(local_basedir, package_name)
1554 shutil.move(package_path, new_package_path)
1555
1556
1557 download_url = dbconn.retrieveDownloadURL(idpackage)
1558 download_url = \
1559 self.ClientService.swap_branch_in_download_relative_uri(
1560 branch, download_url)
1561 dbconn.setDownloadURL(idpackage, download_url)
1562 dbconn.switchBranch(idpackage, branch)
1563 dbconn.commitChanges()
1564
1565 self.updateProgress(
1566 "[%s=>%s|%s|%s] %s: %s" % (
1567 darkgreen(from_branch),
1568 darkred(branch),
1569 brown(repo),
1570 dbconn.retrieveAtom(idpackage),
1571 blue(_("package flushed")),
1572 darkgreen(os.path.basename(package_path)),
1573 ),
1574 importance = 1,
1575 type = "info",
1576 header = brown(" ")
1577 )
1578
1579 try:
1580 os.rmdir(tmp_down_dir)
1581 except OSError:
1582 pass
1583
1584 return True
1585
1586 - def move_packages(self, matches, to_repo, from_repo = None, ask = True,
1587 do_copy = False, new_tag = None, pull_deps = False):
1588
1589 if from_repo == None:
1590 from_repo = self.default_repository
1591 switched = set()
1592
1593 my_matches = list(matches)
1594
1595
1596 if etpConst['clientserverrepoid'] in (to_repo, from_repo):
1597 self.updateProgress(
1598 "%s: %s" % (
1599 blue(_("Cannot touch system database")),
1600 red(etpConst['clientserverrepoid']),
1601 ),
1602 importance = 2, type = "warning", header = darkred(" @@ ")
1603 )
1604 return switched
1605
1606 if not my_matches and from_repo:
1607 dbconn = self.open_server_repository(read_only = True,
1608 no_upload = True, repo = from_repo)
1609 my_matches = set( \
1610 [(x, from_repo) for x in \
1611 dbconn.listAllIdpackages()]
1612 )
1613
1614 mytxt = _("Preparing to move selected packages to")
1615 if do_copy:
1616 mytxt = _("Preparing to copy selected packages to")
1617 self.updateProgress(
1618 "%s %s:" % (
1619 blue(mytxt),
1620 red(to_repo),
1621 ),
1622 importance = 2,
1623 type = "info",
1624 header = red(" @@ ")
1625 )
1626 self.updateProgress(
1627 "%s: %s" % (
1628 bold(_("Note")),
1629 red(_("all old packages with conflicting scope will be " \
1630 "removed from destination repo unless injected")),
1631 ),
1632 importance = 1,
1633 type = "info",
1634 header = red(" @@ ")
1635 )
1636
1637 new_tag_string = ''
1638 if new_tag != None:
1639 new_tag_string = "[%s: %s]" % (darkgreen(_("new tag")),
1640 brown(new_tag),)
1641
1642 my_qa = self.QA()
1643 branch = self.SystemSettings['repositories']['branch']
1644 pull_deps_matches = []
1645 for idpackage, repo in my_matches:
1646 dbconn = self.open_server_repository(read_only = True,
1647 no_upload = True, repo = repo)
1648 self.updateProgress(
1649 "[%s=>%s|%s] %s " % (
1650 darkgreen(repo),
1651 darkred(to_repo),
1652 brown(branch),
1653 blue(dbconn.retrieveAtom(idpackage)),
1654 ) + new_tag_string,
1655 importance = 0,
1656 type = "info",
1657 header = brown(" # ")
1658 )
1659
1660 if pull_deps:
1661 dep_idpackages = my_qa.get_deep_dependency_list(dbconn,
1662 idpackage)
1663 for dep_idpackage in dep_idpackages:
1664
1665 my_dep_match = (dep_idpackage, repo,)
1666 if my_dep_match in pull_deps_matches:
1667 continue
1668 if my_dep_match in my_matches:
1669 continue
1670
1671 pull_deps_matches.append(my_dep_match)
1672 self.updateProgress(
1673 "[%s|%s] %s" % (
1674 brown(branch),
1675 blue(_("dependency")),
1676 purple(dbconn.retrieveAtom(dep_idpackage)),
1677 ),
1678 importance = 0,
1679 type = "info",
1680 header = purple(" >> ")
1681 )
1682
1683 if pull_deps:
1684
1685 my_matches = pull_deps_matches + [x for x in my_matches if x not \
1686 in pull_deps_matches]
1687
1688 if ask:
1689 rc_question = self.askQuestion(_("Would you like to continue ?"))
1690 if rc_question == "No":
1691 return switched
1692
1693 for idpackage, repo in my_matches:
1694 dbconn = self.open_server_repository(read_only = False,
1695 no_upload = True, repo = repo)
1696 match_branch = dbconn.retrieveBranch(idpackage)
1697 match_atom = dbconn.retrieveAtom(idpackage)
1698 package_filename = os.path.basename(
1699 dbconn.retrieveDownloadURL(idpackage))
1700 self.updateProgress(
1701 "[%s=>%s|%s] %s: %s" % (
1702 darkgreen(repo),
1703 darkred(to_repo),
1704 brown(branch),
1705 blue(_("switching")),
1706 darkgreen(match_atom),
1707 ),
1708 importance = 0,
1709 type = "info",
1710 header = red(" @@ "),
1711 back = True
1712 )
1713
1714 from_file = os.path.join(self.get_local_packages_directory(repo),
1715 match_branch, package_filename)
1716 if not os.path.isfile(from_file):
1717 from_file = os.path.join(self.get_local_upload_directory(repo),
1718 match_branch, package_filename)
1719 if not os.path.isfile(from_file):
1720 self.updateProgress(
1721 "[%s=>%s|%s] %s: %s -> %s" % (
1722 darkgreen(repo),
1723 darkred(to_repo),
1724 brown(branch),
1725 bold(_("cannot switch, package not found, skipping")),
1726 darkgreen(),
1727 red(from_file),
1728 ),
1729 importance = 1,
1730 type = "warning",
1731 header = darkred(" !!! ")
1732 )
1733 continue
1734
1735 if new_tag != None:
1736 match_category = dbconn.retrieveCategory(idpackage)
1737 match_name = dbconn.retrieveName(idpackage)
1738 match_version = dbconn.retrieveVersion(idpackage)
1739 tagged_package_filename = \
1740 self.entropyTools.create_package_filename(
1741 match_category, match_name, match_version, new_tag)
1742 to_file = os.path.join(self.get_local_upload_directory(to_repo),
1743 match_branch, tagged_package_filename)
1744 else:
1745 to_file = os.path.join(self.get_local_upload_directory(to_repo),
1746 match_branch, package_filename)
1747 if not os.path.isdir(os.path.dirname(to_file)):
1748 os.makedirs(os.path.dirname(to_file))
1749
1750 copy_data = [
1751 (from_file, to_file,),
1752 (from_file + etpConst['packagesmd5fileext'],
1753 to_file + etpConst['packagesmd5fileext'],),
1754 (from_file + etpConst['packagesexpirationfileext'],
1755 to_file + etpConst['packagesexpirationfileext'],)
1756 ]
1757
1758 for from_item, to_item in copy_data:
1759 self.updateProgress(
1760 "[%s=>%s|%s] %s: %s" % (
1761 darkgreen(repo),
1762 darkred(to_repo),
1763 brown(branch),
1764 blue(_("moving file")),
1765 darkgreen(os.path.basename(from_item)),
1766 ),
1767 importance = 0,
1768 type = "info",
1769 header = red(" @@ "),
1770 back = True
1771 )
1772 if os.path.isfile(from_item):
1773 shutil.copy2(from_item, to_item)
1774
1775 self.updateProgress(
1776 "[%s=>%s|%s] %s: %s" % (
1777 darkgreen(repo),
1778 darkred(to_repo),
1779 brown(branch),
1780 blue(_("loading data from source database")),
1781 darkgreen(repo),
1782 ),
1783 importance = 0,
1784 type = "info",
1785 header = red(" @@ "),
1786 back = True
1787 )
1788
1789 data = dbconn.getPackageData(idpackage)
1790 if new_tag != None:
1791 data['versiontag'] = new_tag
1792
1793 todbconn = self.open_server_repository(read_only = False,
1794 no_upload = True, repo = to_repo)
1795
1796 self.updateProgress(
1797 "[%s=>%s|%s] %s: %s" % (
1798 darkgreen(repo),
1799 darkred(to_repo),
1800 brown(branch),
1801 blue(_("injecting data to destination database")),
1802 darkgreen(to_repo),
1803 ),
1804 importance = 0,
1805 type = "info",
1806 header = red(" @@ "),
1807 back = True
1808 )
1809 new_idpackage, new_revision, new_data = todbconn.handlePackage(data)
1810 del data
1811 todbconn.commitChanges()
1812
1813 if not do_copy:
1814 self.updateProgress(
1815 "[%s=>%s|%s] %s: %s" % (
1816 darkgreen(repo),
1817 darkred(to_repo),
1818 brown(branch),
1819 blue(_("removing entry from source database")),
1820 darkgreen(repo),
1821 ),
1822 importance = 0,
1823 type = "info",
1824 header = red(" @@ "),
1825 back = True
1826 )
1827
1828
1829 dbconn.removePackage(idpackage)
1830 dbconn.commitChanges()
1831
1832 self.updateProgress(
1833 "[%s=>%s|%s] %s: %s" % (
1834 darkgreen(repo),
1835 darkred(to_repo),
1836 brown(branch),
1837 blue(_("successfully handled atom")),
1838 darkgreen(match_atom),
1839 ),
1840 importance = 0,
1841 type = "info",
1842 header = blue(" @@ ")
1843 )
1844 switched.add((idpackage, repo,))
1845
1846
1847 self.dependencies_test(to_repo)
1848
1849 return switched
1850
1851
1853
1854 if repo == None:
1855 repo = self.default_repository
1856
1857 upload_dir = os.path.join(self.get_local_upload_directory(repo),
1858 self.SystemSettings['repositories']['branch'])
1859 if not os.path.isdir(upload_dir):
1860 os.makedirs(upload_dir)
1861
1862 dbconn = self.open_server_repository(read_only = False,
1863 no_upload = True, repo = repo)
1864 self.updateProgress(
1865 red("[repo: %s] %s: %s" % (
1866 darkgreen(repo),
1867 _("adding package"),
1868 bold(os.path.basename(package_file)),
1869 )
1870 ),
1871 importance = 1,
1872 type = "info",
1873 header = brown(" * "),
1874 back = True
1875 )
1876 mydata = self.SpmService.extract_pkg_metadata(package_file,
1877 inject = inject)
1878 idpackage, revision, mydata = dbconn.handlePackage(mydata)
1879
1880
1881 trashing_counters = set()
1882 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
1883 myserver_repos = srv_set['repositories'].keys()
1884 for myrepo in myserver_repos:
1885 mydbconn = self.open_server_repository(read_only = True,
1886 no_upload = True, repo = myrepo)
1887 mylist = mydbconn.retrieve_packages_to_remove(
1888 mydata['name'],
1889 mydata['category'],
1890 mydata['slot'],
1891 mydata['injected']
1892 )
1893 for myitem in mylist:
1894 trashing_counters.add(mydbconn.retrieveCounter(myitem))
1895
1896 for mycounter in trashing_counters:
1897 dbconn.setTrashedCounter(mycounter)
1898
1899
1900 dbconn.removePackageFromInstalledTable(idpackage)
1901 dbconn.addPackageToInstalledTable(idpackage, repo)
1902 atom = dbconn.retrieveAtom(idpackage)
1903
1904 self.updateProgress(
1905 "[repo:%s] %s: %s %s: %s" % (
1906 darkgreen(repo),
1907 blue(_("added package")),
1908 darkgreen(atom),
1909 blue(_("rev")),
1910 bold(str(revision)),
1911 ),
1912 importance = 1,
1913 type = "info",
1914 header = red(" @@ ")
1915 )
1916
1917 manual_deps = sorted(dbconn.retrieveManualDependencies(idpackage))
1918 if manual_deps:
1919 self.updateProgress(
1920 "[repo:%s] %s: %s" % (
1921 darkgreen(repo),
1922 blue(_("manual dependencies for")),
1923 darkgreen(atom),
1924 ),
1925 importance = 1,
1926 type = "warning",
1927 header = darkgreen(" ## ")
1928 )
1929 for m_dep in manual_deps:
1930 self.updateProgress(
1931 brown(m_dep),
1932 importance = 1,
1933 type = "warning",
1934 header = darkred(" # ")
1935 )
1936
1937 download_url = self._setup_repository_package_filename(idpackage,
1938 repo = repo)
1939 downloadfile = os.path.basename(download_url)
1940 destination_path = os.path.join(upload_dir, downloadfile)
1941 shutil.move(package_file, destination_path)
1942
1943 dbconn.commitChanges()
1944 return idpackage, destination_path
1945
1946
1948
1949 dbconn = self.open_server_repository(read_only = False,
1950 no_upload = True, repo = repo)
1951
1952 downloadurl = dbconn.retrieveDownloadURL(idpackage)
1953 packagerev = dbconn.retrieveRevision(idpackage)
1954 downloaddir = os.path.dirname(downloadurl)
1955 downloadfile = os.path.basename(downloadurl)
1956
1957 downloadfile = downloadfile[:-5]+"~%s%s" % (packagerev,
1958 etpConst['packagesext'],)
1959 downloadurl = os.path.join(downloaddir, downloadfile)
1960
1961
1962 dbconn.setDownloadURL(idpackage, downloadurl)
1963
1964 return downloadurl
1965
1968
1969 if repo == None:
1970 repo = self.default_repository
1971
1972 mycount = 0
1973 maxcount = len(packages_data)
1974 idpackages_added = set()
1975 to_be_injected = set()
1976 my_qa = self.QA()
1977 missing_deps_taint = False
1978 for package_filepath, inject in packages_data:
1979
1980 mycount += 1
1981 self.updateProgress(
1982 "[repo:%s] %s: %s" % (
1983 darkgreen(repo),
1984 blue(_("adding package")),
1985 darkgreen(os.path.basename(package_filepath)),
1986 ),
1987 importance = 1,
1988 type = "info",
1989 header = blue(" @@ "),
1990 count = (mycount, maxcount,)
1991 )
1992
1993 try:
1994
1995 idpackage, destination_path = self.package_injector(
1996 package_filepath,
1997 inject = inject,
1998 repo = repo
1999 )
2000 idpackages_added.add(idpackage)
2001 to_be_injected.add((idpackage, destination_path))
2002 except Exception, err:
2003 self.entropyTools.print_traceback()
2004 self.updateProgress(
2005 "[repo:%s] %s: %s" % (
2006 darkgreen(repo),
2007 darkred(_("Exception caught, closing tasks")),
2008 darkgreen(unicode(err)),
2009 ),
2010 importance = 1,
2011 type = "error",
2012 header = bold(" !!! "),
2013 count = (mycount, maxcount,)
2014 )
2015
2016 self.depends_table_initialize(repo)
2017
2018 self.library_paths_table_initialize(repo)
2019 if idpackages_added:
2020 dbconn = self.open_server_repository(read_only = False,
2021 no_upload = True, repo = repo)
2022 missing_deps_taint = my_qa.scan_missing_dependencies(
2023 idpackages_added,
2024 dbconn,
2025 ask = ask,
2026 repo = repo,
2027 self_check = True,
2028 black_list = \
2029 self.get_missing_dependencies_blacklist(
2030 repo = repo),
2031 black_list_adder = \
2032 self.add_missing_dependencies_blacklist_items
2033 )
2034 my_qa.test_depends_linking(idpackages_added, dbconn,
2035 repo = repo)
2036 if to_be_injected:
2037 self.inject_database_into_packages(to_be_injected,
2038 repo = repo)
2039
2040 if missing_deps_taint:
2041 self.depends_table_initialize(repo)
2042 self.close_server_databases()
2043 raise
2044
2045
2046 self.depends_table_initialize(repo)
2047
2048 self.library_paths_table_initialize(repo)
2049
2050 if idpackages_added:
2051 dbconn = self.open_server_repository(read_only = False,
2052 no_upload = True, repo = repo)
2053 missing_deps_taint = my_qa.scan_missing_dependencies(
2054 idpackages_added,
2055 dbconn,
2056 ask = ask,
2057 repo = repo,
2058 self_check = True,
2059 black_list = \
2060 self.get_missing_dependencies_blacklist(repo = repo),
2061 black_list_adder = \
2062 self.add_missing_dependencies_blacklist_items
2063 )
2064 my_qa.test_depends_linking(idpackages_added, dbconn, repo = repo)
2065
2066
2067 if missing_deps_taint:
2068 self.depends_table_initialize(repo)
2069
2070
2071 self.inject_database_into_packages(to_be_injected, repo = repo)
2072
2073 return idpackages_added
2074
2075
2077
2078 if repo == None:
2079 repo = self.default_repository
2080
2081
2082 self.updateProgress(
2083 "[repo:%s] %s:" % (
2084 darkgreen(repo),
2085 blue(_("Injecting entropy metadata into built packages")),
2086 ),
2087 importance = 1,
2088 type = "info",
2089 header = red(" @@ ")
2090 )
2091
2092 dbconn = self.open_server_repository(read_only = False,
2093 no_upload = True, repo = repo)
2094 for idpackage, package_path in injection_data:
2095 self.updateProgress(
2096 "[repo:%s|%s] %s: %s" % (
2097 darkgreen(repo),
2098 brown(str(idpackage)),
2099 blue(_("injecting entropy metadata")),
2100 darkgreen(os.path.basename(package_path)),
2101 ),
2102 importance = 1,
2103 type = "info",
2104 header = blue(" @@ "),
2105 back = True
2106 )
2107 data = dbconn.getPackageData(idpackage)
2108 treeupdates_actions = dbconn.listAllTreeUpdatesActions()
2109 dbpath = self.ClientService.inject_entropy_database_into_package(
2110 package_path, data, treeupdates_actions)
2111 digest = self.entropyTools.md5sum(package_path)
2112
2113 dbconn.setDigest(idpackage, digest)
2114
2115 signatures = data['signatures'].copy()
2116 for hash_key in sorted(signatures):
2117 hash_func = getattr(self.entropyTools, hash_key)
2118 signatures[hash_key] = hash_func(package_path)
2119 dbconn.setSignatures(idpackage, signatures)
2120 self.entropyTools.create_md5_file(package_path)
2121
2122 os.remove(dbpath)
2123 self.updateProgress(
2124 "[repo:%s|%s] %s: %s" % (
2125 darkgreen(repo),
2126 brown(str(idpackage)),
2127 blue(_("injection complete")),
2128 darkgreen(os.path.basename(package_path)),
2129 ),
2130 importance = 1,
2131 type = "info",
2132 header = red(" @@ ")
2133 )
2134 dbconn.commitChanges()
2135
2137 self.updateProgress(
2138 "[%s] %s" % (
2139 red(_("config files")),
2140 blue(_("checking system")),
2141 ),
2142 importance = 1,
2143 type = "info",
2144 header = blue(" @@ "),
2145 back = True
2146 )
2147
2148 scandata = self.ClientService.FileUpdates.scanfs(dcache = False)
2149 if scandata:
2150 self.updateProgress(
2151 "[%s] %s" % (
2152 red(_("config files")),
2153 blue(_("there are configuration files not updated yet")),
2154 ),
2155 importance = 1,
2156 type = "error",
2157 header = darkred(" @@ ")
2158 )
2159 for key in scandata:
2160 self.updateProgress(
2161 "%s" % (brown(etpConst['systemroot'] + \
2162 scandata[key]['destination'])),
2163 importance = 1,
2164 type = "info",
2165 header = "\t"
2166 )
2167 return True
2168 return False
2169
2171 return self.SpmService.quickpkg(atom, storedir)
2172
2173
2175
2176 if repo == None:
2177 repo = self.default_repository
2178
2179 dbconn = self.open_server_repository(read_only = False,
2180 no_upload = True, repo = repo)
2181 for idpackage in idpackages:
2182 atom = dbconn.retrieveAtom(idpackage)
2183 self.updateProgress(
2184 "[repo:%s] %s: %s" % (
2185 darkgreen(repo),
2186 blue(_("removing package")),
2187 darkgreen(atom),
2188 ),
2189 importance = 1,
2190 type = "info",
2191 header = brown(" @@ ")
2192 )
2193 dbconn.removePackage(idpackage)
2194 self.close_server_database(dbconn)
2195 self.updateProgress(
2196 "[repo:%s] %s" % (
2197 darkgreen(repo),
2198 blue(_("removal complete")),
2199 ),
2200 importance = 1,
2201 type = "info",
2202 header = brown(" @@ ")
2203 )
2204
2205
2211
2213 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2214 if repo == None:
2215 repo = self.default_repository
2216 return srv_set['repositories'][repo]['mirrors'][:]
2217
2219 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2220 if repo == None:
2221 repo = self.default_repository
2222 return srv_set['repositories'][repo]['packages_relative_path']
2223
2225 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2226 if repo == None:
2227 repo = self.default_repository
2228 return srv_set['repositories'][repo]['database_relative_path']
2229
2231 if repo == None:
2232 repo = self.default_repository
2233 return os.path.join(self.get_local_database_dir(repo, branch),
2234 etpConst['etpdatabasefile'])
2235
2237 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2238 if repo == None:
2239 repo = self.default_repository
2240 return srv_set['repositories'][repo]['store_dir']
2241
2243 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2244 if repo == None:
2245 repo = self.default_repository
2246 return srv_set['repositories'][repo]['upload_dir']
2247
2249 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2250 if repo == None:
2251 repo = self.default_repository
2252 return srv_set['repositories'][repo]['packages_dir']
2253
2255 if repo == None:
2256 repo = self.default_repository
2257 return os.path.join(self.get_local_database_dir(repo, branch),
2258 etpConst['etpdatabasetaintfile'])
2259
2261 if repo == None:
2262 repo = self.default_repository
2263 return os.path.join(self.get_local_database_dir(repo, branch),
2264 etpConst['etpdatabaserevisionfile'])
2265
2267 if repo == None:
2268 repo = self.default_repository
2269 return os.path.join(self.get_local_database_dir(repo, branch),
2270 etpConst['etpdatabasetimestampfile'])
2271
2273 if repo == None:
2274 repo = self.default_repository
2275 return os.path.join(self.get_local_database_dir(repo, branch),
2276 etpConst['etpdatabasecacertfile'])
2277
2279 if repo == None:
2280 repo = self.default_repository
2281 return os.path.join(self.get_local_database_dir(repo, branch),
2282 etpConst['etpdatabaseservercertfile'])
2283
2285 if repo == None:
2286 repo = self.default_repository
2287 return os.path.join(self.get_local_database_dir(repo, branch),
2288 etpConst['etpdatabasemaskfile'])
2289
2291 if repo == None:
2292 repo = self.default_repository
2293 return os.path.join(self.get_local_database_dir(repo, branch),
2294 etpConst['etpdatabasesytemmaskfile'])
2295
2297 if repo == None:
2298 repo = self.default_repository
2299 return os.path.join(self.get_local_database_dir(repo, branch),
2300 etpConst['etpdatabaseconflictingtaggedfile'])
2301
2304
2305 if repo == None:
2306 repo = self.default_repository
2307 return os.path.join(self.get_local_database_dir(repo, branch),
2308 etpConst['etpdatabaselicwhitelistfile'])
2309
2311 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2312 if repo == None:
2313 repo = self.default_repository
2314 return os.path.join(self.get_local_database_dir(repo, branch),
2315 srv_set['rss']['name'])
2316
2318 if repo == None:
2319 repo = self.default_repository
2320 return os.path.join(self.get_local_database_dir(repo, branch),
2321 etpConst['rss-light-name'])
2322
2324 if repo == None:
2325 repo = self.default_repository
2326 return os.path.join(self.get_local_database_dir(repo, branch),
2327 etpConst['rss-notice-board'])
2328
2330 if repo == None:
2331 repo = self.default_repository
2332 return os.path.join(self.get_local_database_dir(repo, branch),
2333 etpConst['etpdatabaseupdatefile'])
2334
2342
2350
2353 if repo == None:
2354 repo = self.default_repository
2355 return os.path.join(self.get_local_database_dir(repo, branch),
2356 etpConst['etpdatabaseexpbasedpkgsrm'])
2357
2359 if repo == None:
2360 repo = self.default_repository
2361 return os.path.join(self.get_local_database_dir(repo, branch),
2362 etpConst['confsetsdirname'])
2363
2365 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2366 if repo == None:
2367 repo = self.default_repository
2368 if branch == None:
2369 branch = self.SystemSettings['repositories']['branch']
2370 return os.path.join(srv_set['repositories'][repo]['database_dir'],
2371 branch)
2372
2375 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2376 if repo == None:
2377 repo = self.default_repository
2378 if branch == None:
2379 branch = self.SystemSettings['repositories']['branch']
2380 return os.path.join(srv_set['repositories'][repo]['database_dir'],
2381 branch, etpConst['etpdatabasemissingdepsblfile'])
2382
2384 if repo == None:
2385 repo = self.default_repository
2386 if branch == None:
2387 branch = self.SystemSettings['repositories']['branch']
2388 wl_file = self.get_missing_dependencies_blacklist_file(repo, branch)
2389 wl_data = []
2390 if os.path.isfile(wl_file) and os.access(wl_file, os.R_OK):
2391 f_wl = open(wl_file,"r")
2392 wl_data = [x.strip() for x in f_wl.readlines() if x.strip() and \
2393 not x.strip().startswith("#")]
2394 f_wl.close()
2395 return set(wl_data)
2396
2399
2400 if repo == None:
2401 repo = self.default_repository
2402 if branch == None:
2403 branch = self.SystemSettings['repositories']['branch']
2404 wl_file = self.get_missing_dependencies_blacklist_file(repo, branch)
2405 wl_dir = os.path.dirname(wl_file)
2406 if not (os.path.isdir(wl_dir) and os.access(wl_dir, os.W_OK)):
2407 return
2408 if os.path.isfile(wl_file) and not os.access(wl_file, os.W_OK):
2409 return
2410 f_wl = open(wl_file,"a+")
2411 f_wl.write('\n'.join(items)+'\n')
2412 f_wl.flush()
2413 f_wl.close()
2414
2416
2417 if repo == None:
2418 repo = self.default_repository
2419
2420 dbrev_file = self.get_local_database_revision_file(repo)
2421 if os.path.isfile(dbrev_file):
2422 f_rev = open(dbrev_file)
2423 rev = f_rev.readline().strip()
2424 f_rev.close()
2425 try:
2426 rev = int(rev)
2427 except ValueError:
2428 self.updateProgress(
2429 "[repo:%s] %s: %s - %s" % (
2430 darkgreen(repo),
2431 blue(_("invalid database revision")),
2432 bold(rev),
2433 blue(_("defaulting to 0")),
2434 ),
2435 importance = 2,
2436 type = "error",
2437 header = darkred(" !!! ")
2438 )
2439 rev = 0
2440 return rev
2441 else:
2442 return 0
2443
2445
2446 if repo == None:
2447 repo = self.default_repository
2448
2449 remote_status = self.MirrorsService.get_remote_databases_status(repo)
2450 if not [x for x in remote_status if x[1]]:
2451 remote_revision = 0
2452 else:
2453 remote_revision = max([x[1] for x in remote_status])
2454
2455 return remote_revision
2456
2459
2461 from datetime import datetime
2462 import time
2463 return "%s" % (datetime.fromtimestamp(time.time()),)
2464
2466 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2467 repos = srv_set['repositories'].keys()
2468 kwargs['server_repos'] = repos
2469 kwargs['serverInstance'] = self
2470 return self.ClientService.package_set_list(*args, **kwargs)
2471
2473 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2474 repos = srv_set['repositories'].keys()
2475 kwargs['server_repos'] = repos
2476 kwargs['serverInstance'] = self
2477 return self.ClientService.package_set_search(*args, **kwargs)
2478
2480 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2481 repos = srv_set['repositories'].keys()
2482 kwargs['server_repos'] = repos
2483 kwargs['serverInstance'] = self
2484 return self.ClientService.package_set_match(*args, **kwargs)
2485
2487 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2488 repos = srv_set['repositories'].keys()
2489 kwargs['server_repos'] = repos
2490 kwargs['serverInstance'] = self
2491 return self.ClientService.atom_match(*args, **kwargs)
2492
2494
2495 installed_packages = self.SpmService.get_installed_packages_counter()
2496 installed_counters = set()
2497 to_be_added = set()
2498 to_be_removed = set()
2499 to_be_injected = set()
2500 my_settings = self.SystemSettings[self.sys_settings_plugin_id]['server']
2501 exp_based_scope = my_settings['exp_based_scope']
2502
2503 server_repos = my_settings['repositories'].keys()
2504 exp_pkgs_cache = {}
2505
2506
2507 for spm_atom, spm_counter in installed_packages:
2508 found = False
2509 for server_repo in server_repos:
2510 installed_counters.add(spm_counter)
2511 server_dbconn = self.open_server_repository(read_only = True,
2512 no_upload = True, repo = server_repo)
2513 counter = server_dbconn.isCounterAvailable(spm_counter)
2514 if counter:
2515 found = True
2516 break
2517 if not found:
2518 to_be_added.add((spm_atom, spm_counter,))
2519
2520
2521 database_counters = {}
2522 for server_repo in server_repos:
2523 server_dbconn = self.open_server_repository(read_only = True,
2524 no_upload = True, repo = server_repo)
2525 database_counters[server_repo] = server_dbconn.listAllCounters(
2526 branch = self.SystemSettings['repositories']['branch'])
2527
2528 ordered_counters = set()
2529 for server_repo in database_counters:
2530 for data in database_counters[server_repo]:
2531 ordered_counters.add((data, server_repo))
2532 database_counters = ordered_counters
2533
2534 for (counter, idpackage,), xrepo in database_counters:
2535
2536 if counter < 0:
2537 continue
2538
2539 if counter in installed_counters:
2540 continue
2541
2542 dbconn = self.open_server_repository(read_only = True,
2543 no_upload = True, repo = xrepo)
2544
2545 dorm = True
2546
2547 if to_be_added:
2548
2549 dorm = False
2550 atom = dbconn.retrieveAtom(idpackage)
2551 atomkey = self.entropyTools.dep_getkey(atom)
2552 atomtag = self.entropyTools.dep_gettag(atom)
2553 atomslot = dbconn.retrieveSlot(idpackage)
2554
2555 add = True
2556 for spm_atom, spm_counter in to_be_added:
2557 addslot = self.SpmService.get_installed_package_slot(
2558 spm_atom)
2559 addkey = self.entropyTools.dep_getkey(spm_atom)
2560
2561 if addslot == None:
2562 addslot = '0'
2563
2564 if (atomkey == addkey) and \
2565 ((str(atomslot) == str(addslot)) or (atomtag != None)):
2566
2567 add = False
2568 break
2569
2570 if not add:
2571 continue
2572 dorm = True
2573
2574
2575
2576
2577 if dorm and (xrepo == self.default_repository):
2578 trashed = self.is_counter_trashed(counter)
2579 if trashed:
2580
2581 try:
2582 key, slot = dbconn.retrieveKeySlot(idpackage)
2583 trashed = self.SpmService.get_installed_atom(
2584 key+":"+slot)
2585 except TypeError:
2586 trashed = True
2587 if not trashed:
2588
2589 dbtag = dbconn.retrieveVersionTag(idpackage)
2590 if dbtag:
2591 is_injected = dbconn.isInjected(idpackage)
2592 if not is_injected:
2593 to_be_injected.add((idpackage, xrepo))
2594
2595 elif exp_based_scope:
2596
2597
2598 plg_id = self.sys_settings_fatscope_plugin_id
2599 exp_data = self.SystemSettings[plg_id]['repos'].get(
2600 xrepo, set())
2601
2602
2603
2604 if (idpackage not in exp_data) and (-1 not in exp_data):
2605 to_be_removed.add((idpackage, xrepo))
2606 continue
2607
2608 idpackage_expired = self.is_match_expired((idpackage,
2609 xrepo,))
2610
2611 if idpackage_expired:
2612
2613
2614 to_be_removed.add((idpackage, xrepo))
2615 for my_id in dbconn.retrieveDepends(idpackage):
2616 to_be_removed.add((my_id, xrepo))
2617
2618 else:
2619 to_be_removed.add((idpackage, xrepo))
2620
2621 return to_be_added, to_be_removed, to_be_injected
2622
2624
2625 idpackage, repoid = match
2626 dbconn = self.open_server_repository(repo = repoid, just_reading = True)
2627
2628 my_settings = self.SystemSettings[self.sys_settings_plugin_id]['server']
2629 pkg_exp_secs = my_settings['packages_expiration_days'] * 86400
2630 cur_unix_time = self.entropyTools.get_current_unix_time()
2631
2632
2633
2634 mydate = dbconn.retrieveDateCreation(idpackage)
2635
2636 mydelta = cur_unix_time - float(mydate)
2637 if mydelta > pkg_exp_secs:
2638 return True
2639 return False
2640
2642 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2643 server_repos = srv_set['repositories'].keys()
2644 for repo in server_repos:
2645 dbconn = self.open_server_repository(read_only = True,
2646 no_upload = True, repo = repo)
2647 if dbconn.isCounterTrashed(counter):
2648 return True
2649 return False
2650
2657
2660
2661 if repo == None:
2662 repo = self.default_repository
2663
2664 self.close_server_databases()
2665 revisions_match = {}
2666 treeupdates_actions = []
2667 injected_packages = set()
2668 idpackages = set()
2669 idpackages_added = set()
2670
2671 mytxt = red("%s ...") % (_("Initializing Entropy database"),)
2672 self.updateProgress(
2673 mytxt, importance = 1,
2674 type = "info", header = darkgreen(" * "),
2675 back = True
2676 )
2677
2678 if os.path.isfile(self.get_local_database_file(repo)):
2679
2680 dbconn = self.open_server_repository(read_only = True,
2681 no_upload = True, repo = repo, warnings = warnings)
2682
2683 if dbconn.doesTableExist("baseinfo") and \
2684 dbconn.doesTableExist("extrainfo"):
2685 idpackages = dbconn.listAllIdpackages()
2686
2687 if dbconn.doesTableExist("treeupdatesactions"):
2688 treeupdates_actions = dbconn.listAllTreeUpdatesActions()
2689
2690
2691 if dbconn.doesTableExist("injected") and \
2692 dbconn.doesTableExist("extrainfo"):
2693 injected_packages = dbconn.listAllInjectedPackages(
2694 justFiles = True)
2695 injected_packages = set([os.path.basename(x) for x \
2696 in injected_packages])
2697
2698 for idpackage in idpackages:
2699 url = dbconn.retrieveDownloadURL(idpackage)
2700 package = os.path.basename(url)
2701 branch = dbconn.retrieveBranch(idpackage)
2702 revision = dbconn.retrieveRevision(idpackage)
2703 revisions_match[package] = (branch, revision,)
2704
2705 self.close_server_database(dbconn)
2706
2707 mytxt = "%s: %s: %s" % (
2708 bold(_("WARNING")),
2709 red(_("database already exists")),
2710 self.get_local_database_file(repo),
2711 )
2712 self.updateProgress(
2713 mytxt,
2714 importance = 1,
2715 type = "warning",
2716 header = darkred(" !!! ")
2717 )
2718
2719 rc_question = self.askQuestion(_("Do you want to continue ?"))
2720 if rc_question == "No":
2721 return
2722 try:
2723 os.remove(self.get_local_database_file(repo))
2724 except OSError:
2725 pass
2726
2727
2728
2729 dbconn = self.open_server_repository(read_only = False,
2730 no_upload = True, repo = repo, is_new = True)
2731 dbconn.initializeDatabase()
2732
2733 if not empty:
2734
2735 revisions_file = "/entropy-revisions-dump.txt"
2736
2737 if revisions_match:
2738 self.updateProgress(
2739 "%s: %s" % (
2740 red(_("Dumping current revisions to file")),
2741 darkgreen(revisions_file),
2742 ),
2743 importance = 1,
2744 type = "info",
2745 header = darkgreen(" * ")
2746 )
2747 f_rev = open(revisions_file,"w")
2748 f_rev.write(str(revisions_match))
2749 f_rev.flush()
2750 f_rev.close()
2751
2752
2753 treeupdates_file = "/entropy-treeupdates-dump.txt"
2754 if treeupdates_actions:
2755 self.updateProgress(
2756 "%s: %s" % (
2757
2758 red(_("Dumping current 'treeupdates' actions to file")),
2759 bold(treeupdates_file),
2760 ),
2761 importance = 1,
2762 type = "info",
2763 header = darkgreen(" * ")
2764 )
2765 f_tree = open(treeupdates_file,"w")
2766 f_tree.write(str(treeupdates_actions))
2767 f_tree.flush()
2768 f_tree.close()
2769
2770 rc_question = self.askQuestion(
2771 _("Would you like to sync packages first (important!) ?"))
2772 if rc_question == "Yes":
2773 self.MirrorsService.sync_packages(repo = repo)
2774
2775
2776 if treeupdates_actions:
2777 dbconn.bumpTreeUpdatesActions(treeupdates_actions)
2778
2779
2780 pkg_branch_dir = os.path.join(
2781 self.get_local_packages_directory(repo),
2782 self.SystemSettings['repositories']['branch'])
2783 pkglist = os.listdir(pkg_branch_dir)
2784
2785 pkg_ext_len = len(etpConst['packagesext'])
2786 pkglist = [x for x in pkglist if (x[(pkg_ext_len*-1):] == \
2787 etpConst['packagesext']) and not \
2788 os.path.isfile(os.path.join(pkg_branch_dir,
2789 x + etpConst['packagesexpirationfileext']))]
2790
2791 if pkglist:
2792 self.updateProgress(
2793 "%s '%s' %s %s" % (
2794 red(_("Reinitializing Entropy database for branch")),
2795 bold(self.SystemSettings['repositories']['branch']),
2796 red(_("using Packages in the repository")),
2797 red("..."),
2798 ),
2799 importance = 1,
2800 type = "info",
2801 header = darkgreen(" * ")
2802 )
2803
2804 counter = 0
2805 maxcount = len(pkglist)
2806 branch = self.SystemSettings['repositories']['branch']
2807 for pkg in pkglist:
2808 counter += 1
2809
2810 self.updateProgress(
2811 "[repo:%s|%s] %s: %s" % (
2812 darkgreen(repo),
2813 brown(branch),
2814 blue(_("analyzing")),
2815 bold(pkg),
2816 ),
2817 importance = 1,
2818 type = "info",
2819 header = " ",
2820 back = True,
2821 count = (counter, maxcount,)
2822 )
2823
2824 doinject = False
2825 if pkg in injected_packages:
2826 doinject = True
2827
2828 pkg_path = os.path.join(self.get_local_packages_directory(repo),
2829 branch, pkg)
2830 mydata = self.SpmService.extract_pkg_metadata(pkg_path,
2831 inject = doinject)
2832
2833
2834 revision_avail = revisions_match.get(pkg)
2835 add_revision = 0
2836 if (revision_avail != None):
2837 if branch == revision_avail[0]:
2838 add_revision = revision_avail[1]
2839
2840 idpackage, revision, mydata_upd = dbconn.addPackage(mydata,
2841 revision = add_revision)
2842 idpackages_added.add(idpackage)
2843
2844 self.updateProgress(
2845 "[repo:%s] [%s:%s/%s] %s: %s, %s: %s" % (
2846 repo,
2847 brown(branch),
2848 darkgreen(str(counter)),
2849 blue(str(maxcount)),
2850 red(_("added package")),
2851 darkgreen(pkg),
2852 red(_("revision")),
2853 brown(str(revision)),
2854 ),
2855 importance = 1,
2856 type = "info",
2857 header = " ",
2858 back = True
2859 )
2860
2861 self.depends_table_initialize(repo)
2862 self.library_paths_table_initialize(repo)
2863
2864 my_qa = self.QA()
2865
2866 if idpackages_added:
2867 dbconn = self.open_server_repository(read_only = False,
2868 no_upload = True, repo = repo)
2869 my_qa.scan_missing_dependencies(
2870 idpackages_added, dbconn, ask = True,
2871 repo = repo, self_check = True,
2872 black_list = \
2873 self.get_missing_dependencies_blacklist(repo = repo),
2874 black_list_adder = \
2875 self.add_missing_dependencies_blacklist_items
2876 )
2877
2878 dbconn.commitChanges()
2879 self.close_server_databases()
2880
2881 return 0
2882
2884
2885 dbconn = self.open_server_repository(read_only = True,
2886 no_upload = True, repo = repo)
2887 if ("world" in packages) or not packages:
2888 return dbconn.listAllIdpackages(), True
2889 else:
2890 idpackages = set()
2891 for package in packages:
2892 matches = dbconn.atomMatch(package, multiMatch = True)
2893 if matches[1] == 0:
2894 idpackages |= matches[0]
2895 else:
2896 mytxt = "%s: %s: %s" % (
2897 red(_("Attention")),
2898 blue(_("cannot match")),
2899 bold(package),
2900 )
2901 self.updateProgress(
2902 mytxt,
2903 importance = 1,
2904 type = "warning",
2905 header = darkred(" !!! ")
2906 )
2907 return idpackages, False
2908
2910
2911 import urllib2
2912 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2913 if not srv_set['repositories'][repo].has_key('handler'):
2914 return None
2915 url = srv_set['repositories'][repo]['handler']
2916
2917
2918 filename = filename.replace("#","%23")
2919
2920 filename = filename.replace("+","%2b")
2921 request = os.path.join(url, etpConst['handlers']['md5sum'])
2922 request += filename+"&branch="+branch
2923
2924 proxy_settings = self.SystemSettings['system']['proxy']
2925 try:
2926 mydict = {}
2927 if proxy_settings['ftp']:
2928 mydict['ftp'] = proxy_settings['ftp']
2929 if proxy_settings['http']:
2930 mydict['http'] = proxy_settings['http']
2931 if mydict:
2932 mydict['username'] = proxy_settings['username']
2933 mydict['password'] = proxy_settings['password']
2934 self.entropyTools.add_proxy_opener(urllib2, mydict)
2935 else:
2936
2937 urllib2._opener = None
2938 item = urllib2.urlopen(request)
2939 result = item.readline().strip()
2940 item.close()
2941 del item
2942 return result
2943 except (urllib2.URLError, urllib2.HTTPError,):
2944 return None
2945
2947
2948 if repo == None:
2949 repo = self.default_repository
2950
2951 self.updateProgress(
2952 "[%s] %s:" % (
2953 red("remote"),
2954 blue(_("Integrity verification of the selected packages")),
2955 ),
2956 importance = 1,
2957 type = "info",
2958 header = blue(" @@ ")
2959 )
2960
2961 idpackages, world = self.match_packages(packages)
2962 dbconn = self.open_server_repository(read_only = True, no_upload = True,
2963 repo = repo)
2964 branch = self.SystemSettings['repositories']['branch']
2965
2966 if world:
2967 self.updateProgress(
2968 blue(
2969 _("All the packages in repository will be checked.")),
2970 importance = 1,
2971 type = "info",
2972 header = " "
2973 )
2974 else:
2975 mytxt = red("%s:") % (
2976 _("This is the list of the packages that would be checked"),)
2977 self.updateProgress(
2978 mytxt,
2979 importance = 1,
2980 type = "info",
2981 header = " "
2982 )
2983 for idpackage in idpackages:
2984 pkgatom = dbconn.retrieveAtom(idpackage)
2985 down_url = dbconn.retrieveDownloadURL(idpackage)
2986 pkgfile = os.path.basename(down_url)
2987 self.updateProgress(
2988 red(pkgatom) + " -> " + bold(os.path.join(branch, pkgfile)),
2989 importance = 1,
2990 type = "info",
2991 header = darkgreen(" - ")
2992 )
2993
2994 if ask:
2995 rc_question = self.askQuestion(
2996 _("Would you like to continue ?"))
2997 if rc_question == "No":
2998 return set(), set(), {}
2999
3000 match = set()
3001 not_match = set()
3002 broken_packages = {}
3003
3004 for uri in self.get_remote_mirrors(repo):
3005
3006 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri)
3007 self.updateProgress(
3008 "[repo:%s] %s: %s" % (
3009 darkgreen(repo),
3010 blue(_("Working on mirror")),
3011 brown(crippled_uri),
3012 ),
3013 importance = 1,
3014 type = "info",
3015 header = red(" @@ ")
3016 )
3017
3018
3019 totalcounter = len(idpackages)
3020 currentcounter = 0
3021 for idpackage in idpackages:
3022
3023 currentcounter += 1
3024 pkgfile = dbconn.retrieveDownloadURL(idpackage)
3025 orig_branch = self.get_branch_from_download_relative_uri(
3026 pkgfile)
3027
3028 self.updateProgress(
3029 "[%s] %s: %s" % (
3030 brown(crippled_uri),
3031 blue(_("checking hash")),
3032 darkgreen(pkgfile),
3033 ),
3034 importance = 1,
3035 type = "info",
3036 header = blue(" @@ "),
3037 back = True,
3038 count = (currentcounter, totalcounter,)
3039 )
3040
3041 checksum_ok = False
3042 ck_remote = self.get_remote_package_checksum(repo,
3043 os.path.basename(pkgfile), orig_branch)
3044 if ck_remote == None:
3045 self.updateProgress(
3046 "[%s] %s: %s %s" % (
3047 brown(crippled_uri),
3048 blue(_("digest verification of")),
3049 bold(pkgfile),
3050 blue(_("not supported")),
3051 ),
3052 importance = 1,
3053 type = "info",
3054 header = blue(" @@ "),
3055 count = (currentcounter, totalcounter,)
3056 )
3057 elif len(ck_remote) == 32:
3058 pkghash = dbconn.retrieveDigest(idpackage)
3059 if ck_remote == pkghash:
3060 checksum_ok = True
3061 else:
3062 self.updateProgress(
3063 "[%s] %s: %s %s" % (
3064 brown(crippled_uri),
3065 blue(_("digest verification of")),
3066 bold(pkgfile),
3067 blue(_("failed for unknown reasons")),
3068 ),
3069 importance = 1,
3070 type = "info",
3071 header = blue(" @@ "),
3072 count = (currentcounter, totalcounter,)
3073 )
3074
3075 if checksum_ok:
3076 match.add(idpackage)
3077 else:
3078 not_match.add(idpackage)
3079 self.updateProgress(
3080 "[%s] %s: %s %s" % (
3081 brown(crippled_uri),
3082 blue(_("package")),
3083 bold(pkgfile),
3084 red(_("NOT healthy")),
3085 ),
3086 importance = 1,
3087 type = "warning",
3088 header = darkred(" !!! "),
3089 count = (currentcounter, totalcounter,)
3090 )
3091 if not broken_packages.has_key(crippled_uri):
3092 broken_packages[crippled_uri] = []
3093 broken_packages[crippled_uri].append(pkgfile)
3094
3095 if broken_packages:
3096 mytxt = blue("%s:") % (
3097 _("This is the list of broken packages"),)
3098 self.updateProgress(
3099 mytxt,
3100 importance = 1,
3101 type = "info",
3102 header = red(" * ")
3103 )
3104 for mirror in broken_packages.keys():
3105 mytxt = "%s: %s" % (
3106 brown(_("Mirror")),
3107 bold(mirror),
3108 )
3109 self.updateProgress(
3110 mytxt,
3111 importance = 1,
3112 type = "info",
3113 header = red(" <> ")
3114 )
3115 for broken_package in broken_packages[mirror]:
3116 self.updateProgress(
3117 blue(broken_package),
3118 importance = 1,
3119 type = "info",
3120 header = red(" - ")
3121 )
3122
3123 self.updateProgress(
3124 "%s:" % (
3125 blue(_("Statistics")),
3126 ),
3127 importance = 1,
3128 type = "info",
3129 header = red(" @@ ")
3130 )
3131 self.updateProgress(
3132 "[%s] %s:\t%s" % (
3133 red(crippled_uri),
3134 brown(_("Number of checked packages")),
3135 brown(str(len(match) + len(not_match))),
3136 ),
3137 importance = 1,
3138 type = "info",
3139 header = brown(" # ")
3140 )
3141 self.updateProgress(
3142 "[%s] %s:\t%s" % (
3143 red(crippled_uri),
3144 darkgreen(_("Number of healthy packages")),
3145 darkgreen(str(len(match))),
3146 ),
3147 importance = 1,
3148 type = "info",
3149 header = brown(" # ")
3150 )
3151 self.updateProgress(
3152 "[%s] %s:\t%s" % (
3153 red(crippled_uri),
3154 darkred(_("Number of broken packages")),
3155 darkred(str(len(not_match))),
3156 ),
3157 importance = 1,
3158 type = "info",
3159 header = brown(" # ")
3160 )
3161
3162 return match, not_match, broken_packages
3163
3164
3166
3167 if repo == None:
3168 repo = self.default_repository
3169
3170 self.updateProgress(
3171 "[%s] %s:" % (
3172 red(_("local")),
3173 blue(_("Integrity verification of the selected packages")),
3174 ),
3175 importance = 1,
3176 type = "info",
3177 header = darkgreen(" * ")
3178 )
3179
3180 idpackages, world = self.match_packages(packages)
3181 dbconn = self.open_server_repository(read_only = True,
3182 no_upload = True, repo = repo)
3183
3184 if world:
3185 self.updateProgress(
3186 blue(_("All the packages in repository will be checked.")),
3187 importance = 1,
3188 type = "info",
3189 header = " "
3190 )
3191
3192 to_download = set()
3193 available = set()
3194 for idpackage in idpackages:
3195
3196 pkgatom = dbconn.retrieveAtom(idpackage)
3197 pkg_path = dbconn.retrieveDownloadURL(idpackage)
3198 pkg_rel_path = '/'.join(pkg_path.split("/")[2:])
3199
3200 bindir_path = os.path.join(self.get_local_packages_directory(repo),
3201 pkg_rel_path)
3202 uploaddir_path = os.path.join(self.get_local_upload_directory(repo),
3203 pkg_rel_path)
3204
3205 if os.path.isfile(bindir_path):
3206 if not world:
3207 self.updateProgress(
3208 "[%s] %s :: %s" % (
3209 darkgreen(_("available")),
3210 blue(pkgatom),
3211 darkgreen(pkg_rel_path),
3212 ),
3213 importance = 0,
3214 type = "info",
3215 header = darkgreen(" # ")
3216 )
3217 available.add(idpackage)
3218 elif os.path.isfile(uploaddir_path):
3219 if not world:
3220 self.updateProgress(
3221 "[%s] %s :: %s" % (
3222 darkred(_("upload/ignored")),
3223 blue(pkgatom),
3224 darkgreen(pkg_rel_path),
3225 ),
3226 importance = 0,
3227 type = "info",
3228 header = darkgreen(" # ")
3229 )
3230 else:
3231 self.updateProgress(
3232 "[%s] %s :: %s" % (
3233 brown(_("download")),
3234 blue(pkgatom),
3235 darkgreen(pkg_rel_path),
3236 ),
3237 importance = 0,
3238 type = "info",
3239 header = darkgreen(" # ")
3240 )
3241 to_download.add((idpackage, pkg_path,))
3242
3243 if ask:
3244 rc_question = self.askQuestion(_("Would you like to continue ?"))
3245 if rc_question == "No":
3246 return set(), set(), set(), set()
3247
3248
3249 fine = set()
3250 failed = set()
3251 downloaded_fine = set()
3252 downloaded_errors = set()
3253
3254 if to_download:
3255
3256 not_downloaded = set()
3257 mytxt = blue("%s ...") % (_("Starting to download missing files"),)
3258 self.updateProgress(
3259 mytxt,
3260 importance = 1,
3261 type = "info",
3262 header = " "
3263 )
3264 for uri in self.get_remote_mirrors(repo):
3265
3266 if not_downloaded:
3267 mytxt = blue("%s ...") % (
3268 _("Searching missing/broken files on another mirror"),)
3269 self.updateProgress(
3270 mytxt,
3271 importance = 1,
3272 type = "info",
3273 header = " "
3274 )
3275 to_download = not_downloaded.copy()
3276 not_downloaded = set()
3277
3278 for idpackage, pkg_path in to_download:
3279 rc_down = self.MirrorsService.download_package(uri,
3280 pkg_path, repo = repo)
3281 if rc_down:
3282 downloaded_fine.add(idpackage)
3283 available.add(idpackage)
3284 else:
3285 not_downloaded.add(pkg_path)
3286
3287 if not not_downloaded:
3288 self.updateProgress(
3289 red(_("Binary packages downloaded successfully.")),
3290 importance = 1,
3291 type = "info",
3292 header = " "
3293 )
3294 break
3295
3296 if not_downloaded:
3297 mytxt = blue("%s:") % (
3298 _("These are the packages that cannot be found online"),)
3299 self.updateProgress(
3300 mytxt,
3301 importance = 1,
3302 type = "info",
3303 header = " "
3304 )
3305 for pkg_path in not_downloaded:
3306 downloaded_errors.add(pkg_path)
3307 self.updateProgress(
3308 brown(pkg_path),
3309 importance = 1,
3310 type = "warning",
3311 header = red(" * ")
3312 )
3313 downloaded_errors |= not_downloaded
3314 mytxt = "%s." % (_("They won't be checked"),)
3315 self.updateProgress(
3316 mytxt,
3317 importance = 1,
3318 type = "warning",
3319 header = " "
3320 )
3321
3322 totalcounter = str(len(available))
3323 currentcounter = 0
3324 for idpackage in available:
3325 currentcounter += 1
3326 pkg_path = dbconn.retrieveDownloadURL(idpackage)
3327 orig_branch = self.get_branch_from_download_relative_uri(pkg_path)
3328 pkgfile = os.path.basename(pkg_path)
3329
3330 self.updateProgress(
3331 "[branch:%s] %s %s" % (
3332 brown(orig_branch),
3333 blue(_("checking hash of")),
3334 darkgreen(pkgfile),
3335 ),
3336 importance = 1,
3337 type = "info",
3338 header = " ",
3339 back = True,
3340 count = (currentcounter, totalcounter,)
3341 )
3342
3343 storedmd5 = dbconn.retrieveDigest(idpackage)
3344 pkgpath = os.path.join(self.get_local_packages_directory(repo),
3345 orig_branch, pkgfile)
3346 result = self.entropyTools.compare_md5(pkgpath, storedmd5)
3347 if result:
3348 fine.add(idpackage)
3349 else:
3350 failed.add(idpackage)
3351 self.updateProgress(
3352 "[branch:%s] %s %s %s: %s" % (
3353 brown(orig_branch),
3354 blue(_("package")),
3355 darkgreen(pkg_path),
3356 blue(_("is corrupted, stored checksum")),
3357 brown(storedmd5),
3358 ),
3359 importance = 1,
3360 type = "info",
3361 header = " ",
3362 count = (currentcounter, totalcounter,)
3363 )
3364
3365 if failed:
3366 mytxt = blue("%s:") % (_("This is the list of broken packages"),)
3367 self.updateProgress(
3368 mytxt,
3369 importance = 1,
3370 type = "warning",
3371 header = darkred(" # ")
3372 )
3373 for idpackage in failed:
3374 atom = dbconn.retrieveAtom(idpackage)
3375 down_p = dbconn.retrieveDownloadURL(idpackage)
3376 self.updateProgress(
3377 blue("[atom:%s] %s" % (atom, down_p,)),
3378 importance = 0,
3379 type = "warning",
3380 header = brown(" # ")
3381 )
3382
3383
3384 self.updateProgress(
3385 red("Statistics:"),
3386 importance = 1,
3387 type = "info",
3388 header = blue(" * ")
3389 )
3390 self.updateProgress(
3391 brown("%s => %s" % (
3392 len(fine) + len(failed),
3393 _("checked packages"),
3394 )
3395 ),
3396 importance = 0,
3397 type = "info",
3398 header = brown(" # ")
3399 )
3400 self.updateProgress(
3401 darkgreen("%s => %s" % (
3402 len(fine),
3403 _("healthy packages"),
3404 )
3405 ),
3406 importance = 0,
3407 type = "info",
3408 header = brown(" # ")
3409 )
3410 self.updateProgress(
3411 darkred("%s => %s" % (
3412 len(failed),
3413 _("broken packages"),
3414 )
3415 ),
3416 importance = 0,
3417 type = "info",
3418 header = brown(" # ")
3419 )
3420 self.updateProgress(
3421 blue("%s => %s" % (
3422 len(downloaded_fine),
3423 _("downloaded packages"),
3424 )
3425 ),
3426 importance = 0,
3427 type = "info",
3428 header = brown(" # ")
3429 )
3430 self.updateProgress(
3431 bold("%s => %s" % (
3432 len(downloaded_errors),
3433 _("failed downloads"),
3434 )
3435 ),
3436 importance = 0,
3437 type = "info",
3438 header = brown(" # ")
3439 )
3440
3441 self.close_server_database(dbconn)
3442 return fine, failed, downloaded_fine, downloaded_errors
3443
3444
3447
3448 if repo == None:
3449 repo = self.default_repository
3450
3451 if to_branch != self.SystemSettings['repositories']['branch']:
3452 mytxt = "%s: %s %s" % (
3453 blue(_("Please setup your branch to")),
3454 bold(to_branch),
3455 blue(_("and retry")),
3456 )
3457 self.updateProgress(
3458 mytxt,
3459 importance = 1,
3460 type = "error",
3461 header = darkred(" !! ")
3462 )
3463 return None
3464
3465 mytxt = red("%s ...") % (_("Moving database (if not exists)"),)
3466 self.updateProgress(
3467 mytxt,
3468 importance = 1,
3469 type = "info",
3470 header = darkgreen(" @@ ")
3471 )
3472 branch_dbdir = self.get_local_database_dir(repo)
3473 old_branch_dbdir = self.get_local_database_dir(repo, from_branch)
3474 if (not os.path.isdir(branch_dbdir)) and \
3475 os.path.isdir(old_branch_dbdir):
3476 shutil.copytree(old_branch_dbdir, branch_dbdir)
3477
3478 mytxt = red("%s ...") % (_("Switching packages"),)
3479 self.updateProgress(
3480 mytxt,
3481 importance = 1,
3482 type = "info",
3483 header = darkgreen(" @@ ")
3484 )
3485 dbconn = self.open_server_repository(read_only = False,
3486 no_upload = True, repo = repo, lock_remote = False)
3487
3488 already_switched = set()
3489 not_found = set()
3490 switched = set()
3491 ignored = set()
3492 no_checksum = set()
3493
3494 maxcount = len(idpackages)
3495 count = 0
3496 for idpackage in idpackages:
3497 count += 1
3498
3499 cur_branch = dbconn.retrieveBranch(idpackage)
3500 atom = dbconn.retrieveAtom(idpackage)
3501 if cur_branch == to_branch:
3502 already_switched.add(idpackage)
3503 self.updateProgress(
3504 red("%s %s, %s %s" % (
3505 _("Ignoring"),
3506 bold(atom),
3507 _("already in branch"),
3508 cur_branch,
3509 )
3510 ),
3511 importance = 0,
3512 type = "info",
3513 header = darkgreen(" @@ "),
3514 count = (count, maxcount,)
3515 )
3516 ignored.add(idpackage)
3517 continue
3518
3519 mytxt = blue("%s ...") % (_("configuring package information"),)
3520 self.updateProgress(
3521 "[%s=>%s] %s, %s" % (
3522 brown(cur_branch),
3523 bold(to_branch),
3524 darkgreen(atom),
3525 mytxt,
3526 ),
3527 importance = 0,
3528 type = "info",
3529 header = darkgreen(" @@ "),
3530 back = True,
3531 count = (count, maxcount,)
3532 )
3533 switch_status = dbconn.switchBranch(idpackage, to_branch)
3534 if not switch_status:
3535
3536 dbconn.removePackage(idpackage)
3537 dbconn.commitChanges()
3538 switched.add(idpackage)
3539
3540 dbconn.commitChanges()
3541
3542
3543 dbconn.moveCountersToBranch(to_branch)
3544
3545 self.close_server_database(dbconn)
3546 mytxt = blue("%s.") % (_("migration loop completed"),)
3547 self.updateProgress(
3548 "[%s=>%s] %s" % (
3549 brown(from_branch),
3550 bold(to_branch),
3551 mytxt,
3552 ),
3553 importance = 1,
3554 type = "info",
3555 header = darkgreen(" * ")
3556 )
3557
3558 return switched, already_switched, ignored, not_found, no_checksum
3559
3561
3562 if branch == None:
3563 branch = self.SystemSettings['repositories']['branch']
3564 if repo == None:
3565 repo = self.default_repository
3566
3567 sets_dir = self.get_local_database_sets_dir(repo, branch)
3568 if not (os.path.isdir(sets_dir) and os.access(sets_dir, os.R_OK)):
3569 return {}
3570
3571 mydata = {}
3572 items = os.listdir(sets_dir)
3573 for item in items:
3574
3575 try:
3576 item_clean = str(item)
3577 except (UnicodeEncodeError, UnicodeDecodeError,):
3578 continue
3579 item_path = os.path.join(sets_dir, item)
3580 if not (os.path.isfile(item_path) and \
3581 os.access(item_path, os.R_OK)):
3582 continue
3583 item_elements = self.entropyTools.extract_packages_from_set_file(
3584 item_path)
3585 if item_elements:
3586 mydata[item_clean] = item_elements.copy()
3587
3588 return mydata
3589
3620
3633