1
2 '''
3 # DESCRIPTION:
4 # Entropy Object Oriented Interface
5
6 Copyright (C) 2007-2009 Fabio Erculiani
7
8 This program is free software; you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation; either version 2 of the License, or
11 (at your option) any later version.
12
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 '''
22
23 from __future__ import with_statement
24 import os
25 import shutil
26 from entropy.core import Singleton
27 from entropy.exceptions import OnlineMirrorError, PermissionDenied, \
28 SystemDatabaseError
29 from entropy.const import etpConst, etpSys, const_setup_perms, \
30 const_create_working_dirs, const_extract_srv_repo_params, etpUi
31 from entropy.output import TextInterface, purple, red, darkgreen, \
32 bold, brown, blue, darkred
33 from entropy.server.interfaces.mirrors import Server as MirrorsServer
34 from entropy.i18n import _
35 from entropy.core import SystemSettings, SystemSettingsPlugin
36 from entropy.transceivers import FtpInterface
37 from entropy.db import EntropyRepository
38
40
42
43 """
44 Parses Entropy server system configuration file.
45
46 @return dict data
47 """
48
49 data = {
50 'repositories': etpConst['server_repositories'].copy(),
51 'default_repository_id': etpConst['officialserverrepositoryid'],
52 'packages_expiration_days': etpConst['packagesexpirationdays'],
53 'database_file_format': etpConst['etpdatabasefileformat'],
54 'disabled_eapis': set(),
55 'exp_based_scope': etpConst['expiration_based_scope'],
56 'rss': {
57 'enabled': etpConst['rss-feed'],
58 'name': etpConst['rss-name'],
59 'base_url': etpConst['rss-base-url'],
60 'website_url': etpConst['rss-website-url'],
61 'editor': etpConst['rss-managing-editor'],
62 'max_entries': etpConst['rss-max-entries'],
63 'light_max_entries': etpConst['rss-light-max-entries'],
64 },
65 }
66
67 fake_instance = self._helper.fake_default_repo
68
69 if not os.access(etpConst['serverconf'], os.R_OK):
70 return data
71
72 with open(etpConst['serverconf'],"r") as server_f:
73 serverconf = [x.strip() for x in server_f.readlines() if x.strip()]
74
75 for line in serverconf:
76
77 split_line = line.split("|")
78 split_line_len = len(split_line)
79
80 if (line.find("officialserverrepositoryid|") != -1) and \
81 (not line.startswith("#")) and (split_line_len == 2):
82
83 if not fake_instance:
84 data['default_repository_id'] = split_line[1].strip()
85
86 elif line.startswith("expiration-days|") and (split_line_len == 2):
87
88 mydays = split_line[1].strip()
89 try:
90 mydays = int(mydays)
91 data['packages_expiration_days'] = mydays
92 except ValueError:
93 continue
94
95 elif line.startswith("expiration-based-scope|") and \
96 (split_line_len == 2):
97
98 exp_opt = split_line[1].strip().lower()
99 if exp_opt in ("enable", "enabled", "true", "1", "yes"):
100 data['exp_based_scope'] = True
101 else:
102 data['exp_based_scope'] = False
103
104 elif line.startswith("disabled-eapis|") and (split_line_len == 2):
105
106 mydis = split_line[1].strip().split(",")
107 try:
108 mydis = [int(x) for x in mydis]
109 mydis = set([x for x in mydis if x in (1, 2, 3,)])
110 except ValueError:
111 continue
112 if (len(mydis) < 3) and mydis:
113 data['disabled_eapis'] = mydis
114
115
116 elif line.startswith("repository|") and (split_line_len in [5, 6]) \
117 and (not fake_instance):
118
119 repoid, repodata = const_extract_srv_repo_params(line,
120 product = sys_set['repositories']['product'])
121 if repoid in data['repositories']:
122
123 data['repositories'][repoid]['mirrors'].extend(
124 repodata['mirrors'])
125 else:
126 data['repositories'][repoid] = repodata.copy()
127
128 elif line.startswith("database-format|") and (split_line_len == 2):
129
130 fmt = split_line[1]
131 if fmt in etpConst['etpdatabasesupportedcformats']:
132 data['database_file_format'] = fmt
133
134 elif line.startswith("rss-feed|") and (split_line_len == 2):
135
136 feed = split_line[1]
137 if feed in ("enable", "enabled", "true", "1"):
138 data['rss']['enabled'] = True
139 elif feed in ("disable", "disabled", "false", "0", "no",):
140 data['rss']['enabled'] = False
141
142 elif line.startswith("rss-name|") and (split_line_len == 2):
143
144 feedname = line.split("rss-name|")[1].strip()
145 data['rss']['name'] = feedname
146
147 elif line.startswith("rss-base-url|") and (split_line_len == 2):
148
149 data['rss']['base_url'] = line.split("rss-base-url|")[1].strip()
150 if not data['rss']['base_url'][-1] == "/":
151 data['rss']['base_url'] += "/"
152
153 elif line.startswith("rss-website-url|") and (split_line_len == 2):
154
155 data['rss']['website_url'] = split_line[1].strip()
156
157 elif line.startswith("managing-editor|") and (split_line_len == 2):
158
159 data['rss']['editor'] = split_line[1].strip()
160
161 elif line.startswith("max-rss-entries|") and (split_line_len == 2):
162
163 try:
164 entries = int(split_line[1].strip())
165 data['rss']['max_entries'] = entries
166 except (ValueError, IndexError,):
167 continue
168
169 elif line.startswith("max-rss-light-entries|") and \
170 (split_line_len == 2):
171
172 try:
173 entries = int(split_line[1].strip())
174 data['rss']['light_max_entries'] = entries
175 except (ValueError, IndexError,):
176 continue
177
178
179 if self._helper.community_repo:
180 data['repositories'][etpConst['clientserverrepoid']] = {}
181 mydata = {}
182 mydata['description'] = "Community Repositories System Database"
183 mydata['mirrors'] = []
184 mydata['community'] = False
185 data['repositories'][etpConst['clientserverrepoid']].update(mydata)
186
187
188 for repoid in data['repositories']:
189 data['repositories'][repoid]['packages_dir'] = \
190 os.path.join( etpConst['entropyworkdir'],
191 "server",
192 repoid,
193 "packages",
194 etpSys['arch']
195 )
196 data['repositories'][repoid]['store_dir'] = \
197 os.path.join( etpConst['entropyworkdir'],
198 "server",
199 repoid,
200 "store",
201 etpSys['arch']
202 )
203 data['repositories'][repoid]['upload_dir'] = \
204 os.path.join( etpConst['entropyworkdir'],
205 "server",
206 repoid,
207 "upload",
208 etpSys['arch']
209 )
210 data['repositories'][repoid]['database_dir'] = \
211 os.path.join( etpConst['entropyworkdir'],
212 "server",
213 repoid,
214 "database",
215 etpSys['arch']
216 )
217 data['repositories'][repoid]['packages_relative_path'] = \
218 os.path.join( sys_set['repositories']['product'],
219 repoid,
220 "packages",
221 etpSys['arch']
222 )+"/"
223 data['repositories'][repoid]['database_relative_path'] = \
224 os.path.join( sys_set['repositories']['product'],
225 repoid,
226 "database",
227 etpSys['arch']
228 )+"/"
229
230
231 shell_repoid = os.getenv('ETP_REPO')
232 if shell_repoid:
233 data['default_repository_id'] = shell_repoid
234
235 expiration_days = os.getenv('ETP_EXPIRATION_DAYS')
236 if expiration_days:
237 try:
238 expiration_days = int(expiration_days)
239 data['packages_expiration_days'] = expiration_days
240 except ValueError:
241 pass
242
243 return data
244
246
247 import entropy.tools as entropyTools
248
250
251 data = {}
252 srv_plug_id = etpConst['system_settings_plugins_ids']['server_plugin']
253
254 srv_parser_data = sys_set[srv_plug_id]['server']
255 if not srv_parser_data['exp_based_scope']:
256 return data
257
258
259 for repoid in srv_parser_data['repositories']:
260
261
262
263 if repoid == etpConst['clientserverrepoid']:
264 continue
265
266 idpackages = set()
267 exp_fp = self._helper.get_local_exp_based_pkgs_rm_whitelist_file(
268 repo = repoid)
269 dbconn = self._helper.open_server_repository(
270 just_reading = True, repo = repoid)
271
272 if os.access(exp_fp, os.R_OK | os.F_OK):
273 pkgs = self.entropyTools.generic_file_content_parser(exp_fp)
274 if '*' in pkgs:
275 idpackages.add(-1)
276 else:
277 for pkg in pkgs:
278 idpackage, rc_match = dbconn.atomMatch(pkg)
279 if rc_match:
280 continue
281 idpackages.add(idpackage)
282
283 data[repoid] = idpackages
284
285 return data
286
287
288 -class Server(Singleton, TextInterface):
289
290 - def init_singleton(self, default_repository = None, save_repository = False,
291 community_repo = False, fake_default_repo = False,
292 fake_default_repo_id = '::fake::',
293 fake_default_repo_desc = 'this is a fake repository'):
294
295 self.__instance_destroyed = False
296 if etpConst['uid'] != 0:
297 mytxt = _("Entropy Server interface must be run as root")
298 raise PermissionDenied("PermissionDenied: %s" % (mytxt,))
299
300
301 from entropy.misc import LogFile
302 self.SystemSettings = SystemSettings()
303 self.community_repo = community_repo
304 from entropy.db import dbapi2
305 self.dbapi2 = dbapi2
306 etpSys['serverside'] = True
307 self._memory_db_instances = {}
308 self.fake_default_repo = fake_default_repo
309 self.indexing = False
310 self.xcache = False
311 self.MirrorsService = None
312 self.__server_dbcache = {}
313 self.repository_treeupdate_digests = {}
314 self.__settings_to_backup = []
315 self.__do_save_repository = save_repository
316 self.__sync_lock_cache = set()
317 self.rssMessages = {
318 'added': {},
319 'removed': {},
320 'commitmessage': "",
321 'light': {},
322 }
323
324 if fake_default_repo:
325 default_repository = fake_default_repo_id
326 etpConst['officialserverrepositoryid'] = fake_default_repo_id
327 self.init_generic_memory_server_repository(fake_default_repo_id,
328 fake_default_repo_desc)
329
330
331 self.sys_settings_plugin_id = \
332 etpConst['system_settings_plugins_ids']['server_plugin']
333 self.sys_settings_plugin = ServerSystemSettingsPlugin(
334 self.sys_settings_plugin_id, self)
335 self.SystemSettings.add_plugin(self.sys_settings_plugin)
336
337
338 self.sys_settings_fatscope_plugin_id = \
339 etpConst['system_settings_plugins_ids']['server_plugin_fatscope']
340 self.sys_settings_fatscope_plugin = ServerFatscopeSystemSettingsPlugin(
341 self.sys_settings_fatscope_plugin_id, self)
342 self.SystemSettings.add_plugin(self.sys_settings_fatscope_plugin)
343
344 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
345 self.default_repository = default_repository
346 if self.default_repository == None:
347 self.default_repository = srv_set['default_repository_id']
348
349 if self.default_repository in srv_set['repositories']:
350 self.ensure_paths(self.default_repository)
351
352
353
354
355 if self.default_repository not in srv_set['repositories']:
356 raise PermissionDenied("PermissionDenied: %s %s" % (
357 self.default_repository,
358 _("repository not configured"),
359 )
360 )
361 if etpConst['clientserverrepoid'] == self.default_repository:
362 raise PermissionDenied("PermissionDenied: %s %s" % (
363 etpConst['clientserverrepoid'],
364 _("protected repository id, can't use this, sorry dude..."),
365 )
366 )
367
368 self.switch_default_repository(self.default_repository)
369
371 self.__instance_destroyed = True
372 if hasattr(self,'ClientService'):
373 self.ClientService.destroy()
374 if hasattr(self,'sys_settings_server_plugin'):
375 try:
376 self.SystemSettings.remove_plugin(self.sys_settings_plugin_id)
377 except KeyError:
378 pass
379 if hasattr(self,'sys_settings_fatscope_plugin'):
380 try:
381 self.SystemSettings.remove_plugin(
382 self.sys_settings_fatscope_plugin)
383 except KeyError:
384 pass
385 self.close_server_databases()
386
388 return self.__instance_destroyed
389
392
401
402
404 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
405 migrated_filename = '.branch_migrated'
406 for repoid in srv_set['repositories'].keys():
407
408 if repoid == etpConst['clientserverrepoid']: continue
409 mydir = srv_set['repositories'][repoid]['database_dir']
410 if not os.path.isdir(mydir):
411 continue
412
413 migrated_filepath = os.path.join(mydir, migrated_filename)
414 if os.path.isfile(migrated_filepath):
415 continue
416
417 my_branched_dir = self.get_local_database_dir(repoid)
418 if os.path.isdir(my_branched_dir):
419 continue
420
421 self.updateProgress(
422 "[%s:%s] %s: %s, %s: %s" % (
423 brown("repo"),
424 purple(repoid),
425 _("migrating database path from"),
426 brown(mydir),
427 _('to'),
428 brown(my_branched_dir),
429 ),
430 importance = 1,
431 type = "info",
432 header = bold(" @@ ")
433 )
434
435 repo_files = [os.path.join(mydir, x) for x in os.listdir(mydir) if \
436 (os.path.isfile(os.path.join(mydir, x)) and \
437 os.access(os.path.join(mydir, x), os.W_OK))
438 ]
439 os.makedirs(my_branched_dir)
440 const_setup_perms(my_branched_dir, etpConst['entropygid'])
441
442 for repo_file in repo_files:
443 repo_filename = os.path.basename(repo_file)
444 shutil.move(repo_file,
445 os.path.join(my_branched_dir, repo_filename))
446
447 f_migrated = open(migrated_filepath,"w")
448 f_migrated.write("done\n")
449 f_migrated.flush()
450 f_migrated.close()
451
453 self.setup_entropy_settings()
454 cs_name = 'ClientService'
455 if hasattr(self, cs_name):
456 obj = getattr(self, cs_name)
457 obj.destroy()
458 from entropy.client.interfaces import Client
459 self.ClientService = Client(
460 indexing = self.indexing,
461 xcache = self.xcache,
462 repo_validation = False,
463 noclientdb = 1
464 )
465 from entropy.cache import EntropyCacher
466 self.Cacher = EntropyCacher()
467 self.ClientService.updateProgress = self.updateProgress
468 self.validRepositories = self.ClientService.validRepositories
469 self.entropyTools = self.ClientService.entropyTools
470 self.dumpTools = self.ClientService.dumpTools
471 self.QA = self.ClientService.QA
472 self.backup_entropy_settings()
473 self.SpmService = self.ClientService.Spm()
474 self.MirrorsService = MirrorsServer(self)
475
477 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
478 curr_repoid = srv_set['default_repository_id']
479 backup_list = [
480 'etpdatabaseclientfilepath',
481 'clientdbid',
482 {'server': srv_set.copy()},
483 ]
484 for setting in backup_list:
485 if setting not in self.__settings_to_backup:
486 self.__settings_to_backup.append(setting)
487
488 if not self.community_repo:
489 etpConst['etpdatabaseclientfilepath'] = \
490 self.get_local_database_file(repo)
491 etpConst['clientdbid'] = etpConst['serverdbid']
492 const_create_working_dirs()
493
495 if hasattr(self,'serverDbCache'):
496 for item in self.__server_dbcache:
497 try:
498 self.__server_dbcache[item].closeDB()
499 except self.dbapi2.ProgrammingError:
500 pass
501 self.__server_dbcache.clear()
502
504 found = None
505 for item in self.__server_dbcache:
506 if dbinstance == self.__server_dbcache[item]:
507 found = item
508 break
509 if found:
510 instance = self.__server_dbcache.pop(found)
511 instance.closeDB()
512
514 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
515 return srv_set['repositories'].copy()
516
544
546 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
547 if self.community_repo:
548 for repoid in srv_set['repositories']:
549 srv_set['repositories'][repoid]['community'] = True
550
551
553
554 if not self.is_repository_initialized(repoid):
555 mytxt = blue("%s.") % (
556 _("Your default repository is not initialized"),)
557 self.updateProgress(
558 "[%s:%s] %s" % (
559 brown("repo"),
560 purple(repoid),
561 mytxt,
562 ),
563 importance = 1,
564 type = "warning",
565 header = darkred(" !!! ")
566 )
567 answer = self.askQuestion(
568 _("Do you want to initialize your default repository ?"))
569 if answer == "No":
570 mytxt = red("%s.") % (
571 _("Continuing with an uninitialized repository"),)
572 self.updateProgress(
573 "[%s:%s] %s" % (
574 brown("repo"),
575 purple(repoid),
576 mytxt,
577 ),
578 importance = 1,
579 type = "warning",
580 header = darkred(" !!! ")
581 )
582 else:
583
584 dbfile = self.get_local_database_file(repoid)
585 if os.path.isfile(dbfile):
586 shutil.move(dbfile, dbfile+".backup")
587 self.initialize_server_database(empty = True,
588 repo = repoid, warnings = False)
589
590
592 type_txt = _("server-side repository")
593 if self.community_repo:
594 type_txt = _("community repository")
595
596 mytxt = _("Entropy Server Interface Instance on repository")
597 self.updateProgress(
598 blue("%s: %s, %s: %s (%s: %s)" % (
599 mytxt,
600 red(self.default_repository),
601 _("current branch"),
602 darkgreen(self.SystemSettings['repositories']['branch']),
603 purple(_("type")),
604 bold(type_txt),
605 )
606 ),
607 importance = 2,
608 type = "info",
609 header = red(" @@ ")
610 )
611 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
612 repos = srv_set['repositories'].keys()
613 mytxt = blue("%s:") % (_("Currently configured repositories"),)
614 self.updateProgress(
615 mytxt,
616 importance = 1,
617 type = "info",
618 header = red(" @@ ")
619 )
620 for repo in repos:
621 self.updateProgress(
622 darkgreen(repo),
623 importance = 0,
624 type = "info",
625 header = brown(" # ")
626 )
627
628
630
631
632 if repoid == etpConst['clientserverrepoid']:
633 return
634
635 if os.path.isfile(etpConst['serverconf']):
636 f_srv = open(etpConst['serverconf'],"r")
637 content = f_srv.readlines()
638 f_srv.close()
639 content = [x.strip() for x in content]
640 found = False
641 new_content = []
642 for line in content:
643 if line.strip().startswith("officialserverrepositoryid|"):
644 line = "officialserverrepositoryid|%s" % (repoid,)
645 found = True
646 new_content.append(line)
647 if not found:
648 new_content.append("officialserverrepositoryid|%s" % (repoid,))
649 f_srv_t = open(etpConst['serverconf']+".save_default_repo_tmp","w")
650 for line in new_content:
651 f_srv_t.write(line+"\n")
652 f_srv_t.flush()
653 f_srv_t.close()
654 os.rename(etpConst['serverconf']+".save_default_repo_tmp",
655 etpConst['serverconf'])
656 else:
657 f_srv = open(etpConst['serverconf'],"w")
658 f_srv.write("officialserverrepositoryid|%s\n" % (repoid,))
659 f_srv.flush()
660 f_srv.close()
661
663
664
665 if repoid == etpConst['clientserverrepoid']:
666 return False
667
668 if not os.path.isfile(etpConst['serverconf']):
669 return None
670 f_srv = open(etpConst['serverconf'])
671 tmpfile = etpConst['serverconf']+".switch"
672 mycontent = [x.strip() for x in f_srv.readlines()]
673 f_srv.close()
674 f_tmp = open(tmpfile,"w")
675 st = "repository|%s" % (repoid,)
676 status = False
677 for line in mycontent:
678 if enable:
679 if (line.find(st) != -1) and line.startswith("#") and \
680 (len(line.split("|")) == 5):
681 line = line[1:]
682 status = True
683 else:
684 if (line.find(st) != -1) and not line.startswith("#") and \
685 (len(line.split("|")) == 5):
686 line = "#"+line
687 status = True
688 f_tmp.write(line+"\n")
689 f_tmp.flush()
690 f_tmp.close()
691 shutil.move(tmpfile, etpConst['serverconf'])
692 if status:
693 self.close_server_databases()
694 self.SystemSettings.clear()
695 self.setup_services()
696 self.show_interface_status()
697 return status
698
705
714
715 dbc = self.open_server_repository(just_reading = True, repo = repo)
716 valid = do_validate(dbc)
717 self.close_server_database(dbc)
718 if not valid:
719 dbc = self.open_server_repository(read_only = False,
720 no_upload = True, repo = repo, is_new = True)
721 valid = do_validate(dbc)
722 self.close_server_database(dbc)
723
724 return valid
725
727
728 if repo == None:
729 repo = self.default_repository
730
731
732 lock_file = self.MirrorsService.get_database_lockfile(repo)
733 if os.path.isfile(lock_file):
734 self.updateProgress(
735 red(_("Entropy database is already locked by you :-)")),
736 importance = 1,
737 type = "info",
738 header = red(" * ")
739 )
740 else:
741
742 mytxt = "%s ..." % (_("Locking and Syncing Entropy database"),)
743 self.updateProgress(
744 red(mytxt),
745 importance = 1,
746 type = "info",
747 header = red(" * "),
748 back = True
749 )
750 for uri in self.get_remote_mirrors(repo):
751 given_up = self.MirrorsService.mirror_lock_check(uri,
752 repo = repo)
753 if given_up:
754 crippled_uri = \
755 self.entropyTools.extract_ftp_host_from_uri(uri)
756 mytxt = "%s:" % (_("Mirrors status table"),)
757 self.updateProgress(
758 darkgreen(mytxt),
759 importance = 1,
760 type = "info",
761 header = brown(" * ")
762 )
763 dbstatus = self.MirrorsService.get_mirrors_lock(repo = repo)
764 for db_uri, db_st1, db_st2 in dbstatus:
765 db_st1_info = darkgreen(_("Unlocked"))
766 if db_st1:
767 db_st1_info = red(_("Locked"))
768 db_st2_info = darkgreen(_("Unlocked"))
769 if db_st2:
770 db_st2_info = red(_("Locked"))
771
772 crippled_uri = \
773 self.entropyTools.extract_ftp_host_from_uri(db_uri)
774 self.updateProgress(
775 "%s: [%s: %s] [%s: %s]" % (
776 bold(crippled_uri),
777 brown(_("database")),
778 db_st1_info,
779 brown(_("download")),
780 db_st2_info,
781 ),
782 importance = 1,
783 type = "info",
784 header = "\t"
785 )
786
787 raise OnlineMirrorError("OnlineMirrorError: %s %s" % (
788 _("cannot lock mirror"),
789 crippled_uri,
790 )
791 )
792
793
794 self.MirrorsService.lock_mirrors(True, repo = repo)
795 self.MirrorsService.sync_databases(no_upload, repo = repo)
796
797
800
801 if mirrors is None:
802 mirrors = []
803 product = self.SystemSettings['repositories']['product']
804 dbc = self.open_memory_database(dbname = etpConst['serverdbid']+repoid)
805 self._memory_db_instances[repoid] = dbc
806
807 eapi3_port = int(etpConst['socket_service']['port'])
808 eapi3_ssl_port = int(etpConst['socket_service']['ssl_port'])
809
810 repodata = {
811 'repoid': repoid,
812 'description': description,
813 'mirrors': mirrors,
814 'community': community_repo,
815 'service_port': eapi3_port,
816 'ssl_service_port': eapi3_ssl_port,
817 'service_url': service_url,
818 'handler': '',
819 'in_memory': True,
820 }
821
822 etpConst['server_repositories'][repoid] = repodata
823 self.SystemSettings.clear()
824
825 return dbc
826
828 if dbname == None:
829 dbname = etpConst['genericdbid']
830 dbc = EntropyRepository(
831 readOnly = False,
832 dbFile = ':memory:',
833 clientDatabase = True,
834 dbname = dbname,
835 xcache = False,
836 indexing = False,
837 OutputInterface = self,
838 skipChecks = True,
839 ServiceInterface = self
840 )
841 dbc.initializeDatabase()
842 return dbc
843
844 - def open_server_repository(
845 self,
846 read_only = True,
847 no_upload = True,
848 just_reading = False,
849 repo = None,
850 indexing = True,
851 warnings = True,
852 do_cache = True,
853 use_branch = None,
854 lock_remote = True,
855 is_new = False,
856 do_treeupdates = True
857 ):
858
859 if repo == None:
860 repo = self.default_repository
861
862 if repo == etpConst['clientserverrepoid'] and self.community_repo:
863 return self.ClientService.clientDbconn
864
865
866 if repo in self._memory_db_instances:
867 return self._memory_db_instances.get(repo)
868
869 if just_reading:
870 read_only = True
871 no_upload = True
872
873 local_dbfile = self.get_local_database_file(repo, use_branch)
874 if do_cache:
875 cached = self.__server_dbcache.get(
876 (repo, etpConst['systemroot'], local_dbfile, read_only,
877 no_upload, just_reading, use_branch, lock_remote,)
878 )
879 if cached != None:
880 return cached
881
882 local_dbfile_dir = os.path.dirname(local_dbfile)
883 if not os.path.isdir(local_dbfile_dir):
884 os.makedirs(local_dbfile_dir)
885
886 if (not read_only) and (lock_remote) and \
887 (repo not in self.__sync_lock_cache):
888 self.do_server_repository_sync_lock(repo, no_upload)
889 self.__sync_lock_cache.add(repo)
890
891 local_dbfile_exists = os.path.lexists(local_dbfile)
892 conn = EntropyRepository(
893 readOnly = read_only,
894 dbFile = local_dbfile,
895 noUpload = no_upload,
896 OutputInterface = self,
897 ServiceInterface = self,
898 dbname = etpConst['serverdbid']+repo,
899 useBranch = use_branch,
900 lockRemote = lock_remote
901 )
902 if not local_dbfile_exists:
903
904 conn.readOnly = False
905 conn.initializeDatabase()
906 conn.commitChanges()
907
908 valid = True
909 try:
910 conn.validateDatabase()
911 except SystemDatabaseError:
912 valid = False
913
914
915
916 if (repo not in etpConst['server_treeupdatescalled']) and \
917 (not just_reading):
918
919
920 if valid:
921 if do_treeupdates:
922 conn.serverUpdatePackagesData()
923 elif warnings and not is_new:
924 mytxt = _("Entropy database is corrupted!")
925 self.updateProgress(
926 darkred(mytxt),
927 importance = 1,
928 type = "warning",
929 header = bold(" !!! ")
930 )
931
932 if not read_only and valid and indexing:
933
934 self.updateProgress(
935 "[repo:%s|%s] %s" % (
936 blue(repo),
937 red(_("database")),
938 blue(_("indexing database")),
939 ),
940 importance = 1,
941 type = "info",
942 header = brown(" @@ "),
943 back = True
944 )
945 conn.createAllIndexes()
946
947 if do_cache:
948
949
950 self.__server_dbcache[
951 (repo, etpConst['systemroot'], local_dbfile, read_only,
952 no_upload, just_reading, use_branch, lock_remote,)] = conn
953
954
955 if (not read_only) and (not is_new):
956 cur_sets = conn.retrievePackageSets()
957 sys_sets = self.get_configured_package_sets(repo)
958 if cur_sets != sys_sets:
959 self.update_database_package_sets(repo, dbconn = conn)
960 conn.commitChanges()
961
962 return conn
963
965
966 sys_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
967 server_repos = sys_set['repositories'].keys()
968 installed_packages = set()
969
970 if default_repo:
971 server_repos = [default_repo]
972
973 for repo in server_repos:
974 dbconn = self.open_server_repository(read_only = True,
975 no_upload = True, repo = repo, do_treeupdates = False)
976 installed_packages |= set([(x, repo) for x in \
977 dbconn.listAllIdpackages()])
978
979
980 deps_not_satisfied = set()
981 length = str((len(installed_packages)))
982 count = 0
983 mytxt = _("Checking")
984
985 for idpackage, repo in installed_packages:
986 count += 1
987 dbconn = self.open_server_repository(read_only = True,
988 no_upload = True, repo = repo, do_treeupdates = False)
989
990 if (count%150 == 0) or (count == length) or (count == 1):
991 atom = dbconn.retrieveAtom(idpackage)
992 self.updateProgress(
993 darkgreen(mytxt)+" "+bold(atom),
994 importance = 0,
995 type = "info",
996 back = True,
997 count = (count, length),
998 header = darkred(" @@ ")
999 )
1000
1001 xdeps = dbconn.retrieveDependencies(idpackage)
1002 for xdep in xdeps:
1003 xid, xuseless = self.atom_match(xdep)
1004 if xid == -1:
1005 deps_not_satisfied.add(xdep)
1006
1007 return deps_not_satisfied
1008
1010
1011 mytxt = "%s %s" % (blue(_("Running dependencies test")), red("..."))
1012 self.updateProgress(
1013 mytxt,
1014 importance = 2,
1015 type = "info",
1016 header = red(" @@ ")
1017 )
1018
1019 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
1020 server_repos = srv_set['repositories'].keys()
1021 deps_not_matched = self.deps_tester(repo)
1022
1023 if deps_not_matched:
1024
1025 crying_atoms = {}
1026 for atom in deps_not_matched:
1027 for repo in server_repos:
1028 dbconn = self.open_server_repository(just_reading = True,
1029 repo = repo, do_treeupdates = False)
1030 riddep = dbconn.searchDependency(atom)
1031 if riddep == -1:
1032 continue
1033 ridpackages = dbconn.searchIdpackageFromIddependency(riddep)
1034 for i in ridpackages:
1035 iatom = dbconn.retrieveAtom(i)
1036 if not crying_atoms.has_key(atom):
1037 crying_atoms[atom] = set()
1038 crying_atoms[atom].add((iatom, repo))
1039
1040 mytxt = blue("%s:") % (_("These are the dependencies not found"),)
1041 self.updateProgress(
1042 mytxt,
1043 importance = 1,
1044 type = "info",
1045 header = red(" @@ ")
1046 )
1047 mytxt = "%s:" % (_("Needed by"),)
1048 for atom in deps_not_matched:
1049 self.updateProgress(
1050 red(atom),
1051 importance = 1,
1052 type = "info",
1053 header = blue(" # ")
1054 )
1055 if crying_atoms.has_key(atom):
1056 self.updateProgress(
1057 red(mytxt),
1058 importance = 0,
1059 type = "info",
1060 header = blue(" # ")
1061 )
1062 for my_dep, myrepo in crying_atoms[atom]:
1063 self.updateProgress(
1064 "[%s:%s] %s" % (
1065 blue(_("by repo")),
1066 darkred(myrepo),
1067 darkgreen(my_dep),
1068 ),
1069 importance = 0,
1070 type = "info",
1071 header = blue(" # ")
1072 )
1073 else:
1074
1075 mytxt = blue(_("Every dependency is satisfied. It's all fine."))
1076 self.updateProgress(
1077 mytxt,
1078 importance = 2,
1079 type = "info",
1080 header = red(" @@ ")
1081 )
1082
1083 return deps_not_matched
1084
1086
1087
1088 dbconn = self.open_server_repository(read_only = True,
1089 no_upload = True, repo = repo)
1090 QA = self.QA()
1091 packages_matched, brokenexecs, status = QA.test_shared_objects(dbconn,
1092 broken_symbols = True)
1093 if status != 0:
1094 return 1, None
1095
1096 if get_files:
1097 return 0, brokenexecs
1098
1099 if (not brokenexecs) and (not packages_matched):
1100 mytxt = "%s." % (_("System is healthy"),)
1101 self.updateProgress(
1102 blue(mytxt),
1103 importance = 2,
1104 type = "info",
1105 header = red(" @@ ")
1106 )
1107 return 0, None
1108
1109 mytxt = "%s..." % (_("Matching libraries with Spm, please wait"),)
1110 self.updateProgress(
1111 blue(mytxt),
1112 importance = 1,
1113 type = "info",
1114 header = red(" @@ ")
1115 )
1116
1117 packages = self.SpmService.query_belongs_multiple(brokenexecs)
1118
1119 if packages:
1120 mytxt = "%s:" % (_("These are the matched packages"),)
1121 self.updateProgress(
1122 red(mytxt),
1123 importance = 1,
1124 type = "info",
1125 header = red(" @@ ")
1126 )
1127 for my_atom, my_elf_id in packages:
1128 package_slot = my_atom, my_elf_id
1129 self.updateProgress(
1130 "%s [elf:%s]" % (
1131 purple(my_atom),
1132 darkgreen(str(my_elf_id)),
1133 ),
1134 importance = 0,
1135 type = "info",
1136 header = red(" # ")
1137 )
1138 for filename in sorted(list(packages[package_slot])):
1139 self.updateProgress(
1140 darkgreen(filename),
1141 importance = 0,
1142 type = "info",
1143 header = brown(" => ")
1144 )
1145
1146 pkgstring = ' '.join(["%s:%s" % (
1147 self.entropyTools.dep_getkey(x[0]), x[1],) for x \
1148 in sorted(packages)])
1149 mytxt = "%s: %s" % (darkgreen(_("Packages string")), pkgstring,)
1150 self.updateProgress(
1151 mytxt,
1152 importance = 1,
1153 type = "info",
1154 header = red(" @@ ")
1155 )
1156 else:
1157 self.updateProgress(
1158 red(_("No matched packages")),
1159 importance = 1,
1160 type = "info",
1161 header = red(" @@ ")
1162 )
1163
1164 return 0, packages
1165
1167
1168 mytxt = "%s %s" % (
1169 blue(_("Running orphaned SPM packages test")), red("..."),)
1170 self.updateProgress(
1171 mytxt,
1172 importance = 2,
1173 type = "info",
1174 header = red(" @@ ")
1175 )
1176 installed_packages, length = self.SpmService.get_installed_packages()
1177 not_found = {}
1178 count = 0
1179 for installed_package in installed_packages:
1180 count += 1
1181 self.updateProgress(
1182 "%s: %s" % (
1183 darkgreen(_("Scanning package")),
1184 brown(installed_package),),
1185 importance = 0,
1186 type = "info",
1187 back = True,
1188 count = (count, length),
1189 header = darkred(" @@ ")
1190 )
1191 key, slot = (self.entropyTools.dep_getkey(installed_package),
1192 self.SpmService.get_installed_package_slot(installed_package),)
1193 pkg_atom = "%s:%s" % (key, slot,)
1194 tree_atom = self.SpmService.get_best_atom(pkg_atom)
1195 if not tree_atom:
1196 not_found[installed_package] = pkg_atom
1197 self.updateProgress(
1198 "%s: %s" % (
1199 blue(pkg_atom),
1200 darkred(_("not found anymore")),
1201 ),
1202 importance = 0,
1203 type = "warning",
1204 count = (count, length),
1205 header = darkred(" @@ ")
1206 )
1207
1208 if not_found:
1209 not_found_list = ' '.join([not_found[x] for x in sorted(not_found)])
1210 self.updateProgress(
1211 "%s: %s" % (
1212 blue(_("Packages string")),
1213 not_found_list,
1214 ),
1215 importance = 0,
1216 type = "warning",
1217 count = (count, length),
1218 header = darkred(" @@ ")
1219 )
1220
1221 return not_found
1222
1229
1236
1238 if dbpath == None:
1239 dbpath = self.get_local_database_file(repo)
1240
1241 dbdir = os.path.dirname(dbpath)
1242 if not os.path.isdir(dbdir):
1243 os.makedirs(dbdir)
1244
1245 mytxt = red("%s ...") % (_("Initializing an empty database"),)
1246 self.updateProgress(
1247 mytxt,
1248 importance = 1,
1249 type = "info",
1250 header = darkgreen(" * "),
1251 back = True
1252 )
1253 dbconn = self.ClientService.open_generic_database(dbpath)
1254 dbconn.initializeDatabase()
1255 dbconn.commitChanges()
1256 dbconn.closeDB()
1257 mytxt = "%s %s %s." % (
1258 red(_("Entropy database file")),
1259 bold(dbpath),
1260 red(_("successfully initialized")),
1261 )
1262 self.updateProgress(
1263 mytxt,
1264 importance = 1,
1265 type = "info",
1266 header = darkgreen(" * ")
1267 )
1268
1269 - def tag_packages(self, package_tag, idpackages, repo = None, ask = True):
1270
1271
1272
1273 try:
1274 package_tag = str(package_tag)
1275 if " " in package_tag:
1276 raise ValueError
1277 except (UnicodeDecodeError, UnicodeEncodeError, ValueError,):
1278 self.updateProgress(
1279 "%s: %s" % (
1280 blue(_("Invalid tag specified")),
1281 package_tag,
1282 ),
1283 importance = 1, type = "error", header = darkred(" !! ")
1284 )
1285 return 1, package_tag
1286
1287 if repo == None:
1288 repo = self.default_repository
1289
1290
1291 invalid_atoms = []
1292 dbconn = self.open_server_repository(read_only = True,
1293 no_upload = True, repo = repo)
1294 for idpackage in idpackages:
1295 ver_tag = dbconn.retrieveVersionTag(idpackage)
1296 if ver_tag:
1297 invalid_atoms.append(dbconn.retrieveAtom(idpackage))
1298
1299 if invalid_atoms:
1300 self.updateProgress(
1301 "%s: %s" % (
1302 blue(_("Packages already tagged, action aborted")),
1303 ', '.join([darkred(unicode(x)) for x in invalid_atoms]),
1304 ),
1305 importance = 1, type = "error", header = darkred(" !! ")
1306 )
1307 return 2, invalid_atoms
1308
1309 matches = [(x, repo) for x in idpackages]
1310 status = 0
1311 data = self.move_packages(
1312 matches, to_repo = repo, from_repo = repo, ask = ask,
1313 do_copy = True, new_tag = package_tag
1314 )
1315 return status, data
1316
1318 """
1319 When creating a new branch, for space reasons, packages are not
1320 moved to a new location. This works fine until old branch is removed.
1321 To avoid inconsistences, before deciding to do that, all the packages
1322 in the old branch should be flushed back to the the currently configured
1323 branch.
1324
1325 @param from_branches -- list of branches to move packages from
1326 @type from_branches -- list
1327 @param repo -- repository to work on
1328 @type repo -- str
1329 @param ask -- user interactivity
1330 @type ask -- bool
1331
1332 @return status
1333 """
1334
1335 status = True
1336 if repo == None:
1337 repo = self.default_repository
1338 branch = self.SystemSettings['repositories']['branch']
1339
1340 if branch in from_branches:
1341 from_branches = [x for x in from_branches if x != branch]
1342
1343 self.updateProgress(
1344 "[%s=>%s|%s] %s" % (
1345 darkgreen(', '.join(from_branches)),
1346 darkred(branch),
1347 brown(repo),
1348 blue(_("flushing back selected packages from branches")),
1349 ),
1350 importance = 2,
1351 type = "info",
1352 header = red(" @@ ")
1353 )
1354
1355 dbconn = self.open_server_repository(read_only = True,
1356 no_upload = True, repo = repo)
1357
1358 idpackage_map = dict(((x, [],) for x in from_branches))
1359 idpackages = dbconn.listAllIdpackages(order_by = 'atom')
1360 for idpackage in idpackages:
1361 download_url = dbconn.retrieveDownloadURL(idpackage)
1362 url_br = self.ClientService.get_branch_from_download_relative_uri(
1363 download_url)
1364 if url_br in from_branches:
1365 idpackage_map[url_br].append(idpackage)
1366
1367 mapped_branches = [x for x in idpackage_map if idpackage_map[x]]
1368 if not mapped_branches:
1369 self.updateProgress(
1370 "[%s=>%s|%s] %s !" % (
1371 darkgreen(', '.join(from_branches)),
1372 darkred(branch),
1373 brown(repo),
1374 blue(_("nothing to do")),
1375 ),
1376 importance = 0,
1377 type = "warning",
1378 header = blue(" @@ ")
1379 )
1380 return status
1381
1382
1383 all_fine = True
1384 tmp_down_dir = self.entropyTools.get_random_temp_file()
1385 os.makedirs(tmp_down_dir)
1386
1387 download_queue = {}
1388 local_up_dir = self.get_local_upload_directory(repo)
1389 local_basedir = os.path.join(local_up_dir, branch)
1390 dbconn = self.open_server_repository(read_only = False,
1391 no_upload = True, repo = repo)
1392
1393 def generate_queue(branch, repo, from_branch, down_q, idpackage_map):
1394
1395 self.updateProgress(
1396 "[%s=>%s|%s] %s" % (
1397 darkgreen(from_branch),
1398 darkred(branch),
1399 brown(repo),
1400 brown(_("these are the packages that will be flushed")),
1401 ),
1402 importance = 1,
1403 type = "info",
1404 header = brown(" @@ ")
1405 )
1406
1407
1408 for idpackage in idpackage_map[from_branch]:
1409 atom = dbconn.retrieveAtom(idpackage)
1410 self.updateProgress(
1411 "[%s=>%s|%s] %s" % (
1412 darkgreen(from_branch),
1413 darkred(branch),
1414 brown(repo),
1415 purple(atom),
1416 ),
1417 importance = 0,
1418 type = "info",
1419 header = blue(" # ")
1420 )
1421 pkg_fp = os.path.basename(dbconn.retrieveDownloadURL(idpackage))
1422 pkg_fp = os.path.join(tmp_down_dir, pkg_fp)
1423 down_q.append((pkg_fp, idpackage,))
1424
1425
1426 for from_branch in sorted(mapped_branches):
1427
1428 download_queue[from_branch] = []
1429 all_fine = False
1430 generate_queue(branch, repo, from_branch,
1431 download_queue[from_branch], idpackage_map)
1432
1433 if ask:
1434 rc_question = self.askQuestion(
1435 _("Would you like to continue ?"))
1436 if rc_question == "No":
1437 continue
1438
1439 remote_relative_path = self.get_remote_packages_relative_path(repo)
1440
1441 for uri in self.get_remote_mirrors(repo):
1442
1443 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri)
1444 ftp_basedir = os.path.join(remote_relative_path, from_branch)
1445
1446 downloader_queue = [x[0] for x in download_queue[from_branch]]
1447 downloader = self.MirrorsService.FtpServerHandler(
1448 FtpInterface,
1449 self,
1450 [uri],
1451 downloader_queue,
1452 critical_files = downloader_queue,
1453 use_handlers = True,
1454 ftp_basedir = ftp_basedir,
1455 local_basedir = tmp_down_dir,
1456 download = True,
1457 repo = repo
1458 )
1459
1460 errors, m_fine_uris, m_broken_uris = downloader.go()
1461
1462 if not errors:
1463 for downloaded_path, idpackage in \
1464 download_queue[from_branch]:
1465
1466 self.updateProgress(
1467 "[%s=>%s|%s|%s] %s: %s" % (
1468 darkgreen(from_branch),
1469 darkred(branch),
1470 brown(repo),
1471 dbconn.retrieveAtom(idpackage),
1472 blue(_("checking package hash")),
1473 darkgreen(os.path.basename(downloaded_path)),
1474 ),
1475 importance = 0,
1476 type = "info",
1477 header = brown(" "),
1478 back = True
1479 )
1480
1481 md5hash = self.entropyTools.md5sum(downloaded_path)
1482 db_md5hash = dbconn.retrieveDigest(idpackage)
1483 if md5hash != db_md5hash:
1484 errors = True
1485 self.updateProgress(
1486 "[%s=>%s|%s|%s] %s: %s" % (
1487 darkgreen(from_branch),
1488 darkred(branch),
1489 brown(repo),
1490 dbconn.retrieveAtom(idpackage),
1491 blue(_("hash does not match for")),
1492 darkgreen(os.path.basename(downloaded_path)),
1493 ),
1494 importance = 0,
1495 type = "error",
1496 header = brown(" ")
1497 )
1498 continue
1499
1500
1501
1502 if errors:
1503 reason = _("wrong md5")
1504 if m_broken_uris:
1505 my_broken_uris = [
1506 (self.entropyTools.extract_ftp_host_from_uri(x), y,) \
1507 for x, y in m_broken_uris]
1508 reason = my_broken_uris[0][1]
1509
1510 self.updateProgress(
1511 "[%s=>%s|%s] %s, %s: %s" % (
1512 darkgreen(from_branch),
1513 darkred(branch),
1514 brown(repo),
1515 blue(_("download errors")),
1516 blue(_("reason")),
1517 reason,
1518 ),
1519 importance = 1,
1520 type = "error",
1521 header = darkred(" !!! ")
1522 )
1523
1524 continue
1525
1526 all_fine = True
1527
1528 self.updateProgress(
1529 "[%s=>%s|%s] %s: %s" % (
1530 darkgreen(from_branch),
1531 darkred(branch),
1532 brown(repo),
1533 blue(_("download completed successfully")),
1534 darkgreen(crippled_uri),
1535 ),
1536 importance = 1,
1537 type = "info",
1538 header = darkgreen(" * ")
1539 )
1540
1541 if not all_fine:
1542 self.updateProgress(
1543 "[%s=>%s|%s] %s" % (
1544 darkgreen(', '.join(from_branches)),
1545 darkred(branch),
1546 brown(repo),
1547 blue(_("error downloading packages from mirrors")),
1548 ),
1549 importance = 2,
1550 type = "error",
1551 header = darkred(" !!! ")
1552 )
1553 return False
1554
1555 tmp_db_path = self.entropyTools.get_random_temp_file()
1556 for from_branch in sorted(mapped_branches):
1557
1558 self.updateProgress(
1559 "[%s=>%s|%s] %s: %s" % (
1560 darkgreen(from_branch),
1561 darkred(branch),
1562 brown(repo),
1563 blue(_("working on branch")),
1564 darkgreen(from_branch),
1565 ),
1566 importance = 1,
1567 type = "info",
1568 header = brown(" @@ ")
1569 )
1570
1571 down_queue = download_queue[from_branch]
1572 for package_path, idpackage in down_queue:
1573
1574 self.updateProgress(
1575 "[%s=>%s|%s] %s: %s" % (
1576 darkgreen(from_branch),
1577 darkred(branch),
1578 brown(repo),
1579 blue(_("updating package")),
1580 darkgreen(os.path.basename(package_path)),
1581 ),
1582 importance = 1,
1583 type = "info",
1584 header = brown(" "),
1585 back = True
1586 )
1587
1588
1589 package_name = os.path.basename(package_path)
1590 new_package_path = os.path.join(local_basedir, package_name)
1591 try:
1592 os.rename(package_path, new_package_path)
1593 except OSError:
1594 shutil.move(package_path, new_package_path)
1595
1596
1597 self.entropyTools.create_md5_file(new_package_path)
1598
1599
1600 download_url = dbconn.retrieveDownloadURL(idpackage)
1601 download_url = \
1602 self.ClientService.swap_branch_in_download_relative_uri(
1603 branch, download_url)
1604 dbconn.setDownloadURL(idpackage, download_url)
1605 dbconn.switchBranch(idpackage, branch)
1606 dbconn.commitChanges()
1607
1608 self.updateProgress(
1609 "[%s=>%s|%s] %s: %s" % (
1610 darkgreen(from_branch),
1611 darkred(branch),
1612 brown(repo),
1613 blue(_("package flushed")),
1614 darkgreen(os.path.basename(package_path)),
1615 ),
1616 importance = 1,
1617 type = "info",
1618 header = brown(" ")
1619 )
1620
1621 try:
1622 os.rmdir(tmp_down_dir)
1623 except OSError:
1624 pass
1625
1626 return True
1627
1628 - def move_packages(self, matches, to_repo, from_repo = None, ask = True,
1629 do_copy = False, new_tag = None, pull_deps = False):
1630
1631 if from_repo == None:
1632 from_repo = self.default_repository
1633 switched = set()
1634
1635 my_matches = list(matches)
1636
1637
1638 if etpConst['clientserverrepoid'] in (to_repo, from_repo):
1639 self.updateProgress(
1640 "%s: %s" % (
1641 blue(_("Cannot touch system database")),
1642 red(etpConst['clientserverrepoid']),
1643 ),
1644 importance = 2, type = "warning", header = darkred(" @@ ")
1645 )
1646 return switched
1647
1648 if not my_matches and from_repo:
1649 dbconn = self.open_server_repository(read_only = True,
1650 no_upload = True, repo = from_repo)
1651 my_matches = set( \
1652 [(x, from_repo) for x in \
1653 dbconn.listAllIdpackages()]
1654 )
1655
1656 mytxt = _("Preparing to move selected packages to")
1657 if do_copy:
1658 mytxt = _("Preparing to copy selected packages to")
1659 self.updateProgress(
1660 "%s %s:" % (
1661 blue(mytxt),
1662 red(to_repo),
1663 ),
1664 importance = 2,
1665 type = "info",
1666 header = red(" @@ ")
1667 )
1668 self.updateProgress(
1669 "%s: %s" % (
1670 bold(_("Note")),
1671 red(_("all old packages with conflicting scope will be " \
1672 "removed from destination repo unless injected")),
1673 ),
1674 importance = 1,
1675 type = "info",
1676 header = red(" @@ ")
1677 )
1678
1679 new_tag_string = ''
1680 if new_tag != None:
1681 new_tag_string = "[%s: %s]" % (darkgreen(_("new tag")),
1682 brown(new_tag),)
1683
1684 my_qa = self.QA()
1685 branch = self.SystemSettings['repositories']['branch']
1686 pull_deps_matches = []
1687 for idpackage, repo in my_matches:
1688 dbconn = self.open_server_repository(read_only = True,
1689 no_upload = True, repo = repo)
1690 self.updateProgress(
1691 "[%s=>%s|%s] %s " % (
1692 darkgreen(repo),
1693 darkred(to_repo),
1694 brown(branch),
1695 blue(dbconn.retrieveAtom(idpackage)),
1696 ) + new_tag_string,
1697 importance = 0,
1698 type = "info",
1699 header = brown(" # ")
1700 )
1701
1702 if pull_deps:
1703 dep_idpackages = my_qa.get_deep_dependency_list(dbconn,
1704 idpackage)
1705 for dep_idpackage in dep_idpackages:
1706
1707 my_dep_match = (dep_idpackage, repo,)
1708 if my_dep_match in pull_deps_matches:
1709 continue
1710 if my_dep_match in my_matches:
1711 continue
1712
1713 pull_deps_matches.append(my_dep_match)
1714 self.updateProgress(
1715 "[%s|%s] %s" % (
1716 brown(branch),
1717 blue(_("dependency")),
1718 purple(dbconn.retrieveAtom(dep_idpackage)),
1719 ),
1720 importance = 0,
1721 type = "info",
1722 header = purple(" >> ")
1723 )
1724
1725 if pull_deps:
1726
1727 my_matches = pull_deps_matches + [x for x in my_matches if x not \
1728 in pull_deps_matches]
1729
1730 if ask:
1731 rc_question = self.askQuestion(_("Would you like to continue ?"))
1732 if rc_question == "No":
1733 return switched
1734
1735 for idpackage, repo in my_matches:
1736 dbconn = self.open_server_repository(read_only = False,
1737 no_upload = True, repo = repo)
1738 match_branch = dbconn.retrieveBranch(idpackage)
1739 match_atom = dbconn.retrieveAtom(idpackage)
1740 package_filename = os.path.basename(
1741 dbconn.retrieveDownloadURL(idpackage))
1742 self.updateProgress(
1743 "[%s=>%s|%s] %s: %s" % (
1744 darkgreen(repo),
1745 darkred(to_repo),
1746 brown(branch),
1747 blue(_("switching")),
1748 darkgreen(match_atom),
1749 ),
1750 importance = 0,
1751 type = "info",
1752 header = red(" @@ "),
1753 back = True
1754 )
1755
1756 from_file = os.path.join(self.get_local_packages_directory(repo),
1757 match_branch, package_filename)
1758 if not os.path.isfile(from_file):
1759 from_file = os.path.join(self.get_local_upload_directory(repo),
1760 match_branch, package_filename)
1761 if not os.path.isfile(from_file):
1762 self.updateProgress(
1763 "[%s=>%s|%s] %s: %s -> %s" % (
1764 darkgreen(repo),
1765 darkred(to_repo),
1766 brown(branch),
1767 bold(_("cannot switch, package not found, skipping")),
1768 darkgreen(),
1769 red(from_file),
1770 ),
1771 importance = 1,
1772 type = "warning",
1773 header = darkred(" !!! ")
1774 )
1775 continue
1776
1777 if new_tag != None:
1778 match_category = dbconn.retrieveCategory(idpackage)
1779 match_name = dbconn.retrieveName(idpackage)
1780 match_version = dbconn.retrieveVersion(idpackage)
1781 tagged_package_filename = \
1782 self.entropyTools.create_package_filename(
1783 match_category, match_name, match_version, new_tag)
1784 to_file = os.path.join(self.get_local_upload_directory(to_repo),
1785 match_branch, tagged_package_filename)
1786 else:
1787 to_file = os.path.join(self.get_local_upload_directory(to_repo),
1788 match_branch, package_filename)
1789 if not os.path.isdir(os.path.dirname(to_file)):
1790 os.makedirs(os.path.dirname(to_file))
1791
1792 copy_data = [
1793 (from_file, to_file,),
1794 (from_file + etpConst['packagesmd5fileext'],
1795 to_file + etpConst['packagesmd5fileext'],),
1796 (from_file + etpConst['packagesexpirationfileext'],
1797 to_file + etpConst['packagesexpirationfileext'],)
1798 ]
1799
1800 for from_item, to_item in copy_data:
1801 self.updateProgress(
1802 "[%s=>%s|%s] %s: %s" % (
1803 darkgreen(repo),
1804 darkred(to_repo),
1805 brown(branch),
1806 blue(_("moving file")),
1807 darkgreen(os.path.basename(from_item)),
1808 ),
1809 importance = 0,
1810 type = "info",
1811 header = red(" @@ "),
1812 back = True
1813 )
1814 if os.path.isfile(from_item):
1815 shutil.copy2(from_item, to_item)
1816
1817 self.updateProgress(
1818 "[%s=>%s|%s] %s: %s" % (
1819 darkgreen(repo),
1820 darkred(to_repo),
1821 brown(branch),
1822 blue(_("loading data from source database")),
1823 darkgreen(repo),
1824 ),
1825 importance = 0,
1826 type = "info",
1827 header = red(" @@ "),
1828 back = True
1829 )
1830
1831 data = dbconn.getPackageData(idpackage)
1832 if new_tag != None:
1833 data['versiontag'] = new_tag
1834
1835 todbconn = self.open_server_repository(read_only = False,
1836 no_upload = True, repo = to_repo)
1837
1838 self.updateProgress(
1839 "[%s=>%s|%s] %s: %s" % (
1840 darkgreen(repo),
1841 darkred(to_repo),
1842 brown(branch),
1843 blue(_("injecting data to destination database")),
1844 darkgreen(to_repo),
1845 ),
1846 importance = 0,
1847 type = "info",
1848 header = red(" @@ "),
1849 back = True
1850 )
1851 new_idpackage, new_revision, new_data = todbconn.handlePackage(data)
1852 del data
1853 todbconn.commitChanges()
1854
1855 if not do_copy:
1856 self.updateProgress(
1857 "[%s=>%s|%s] %s: %s" % (
1858 darkgreen(repo),
1859 darkred(to_repo),
1860 brown(branch),
1861 blue(_("removing entry from source database")),
1862 darkgreen(repo),
1863 ),
1864 importance = 0,
1865 type = "info",
1866 header = red(" @@ "),
1867 back = True
1868 )
1869
1870
1871 dbconn.removePackage(idpackage)
1872 dbconn.commitChanges()
1873
1874 self.updateProgress(
1875 "[%s=>%s|%s] %s: %s" % (
1876 darkgreen(repo),
1877 darkred(to_repo),
1878 brown(branch),
1879 blue(_("successfully handled atom")),
1880 darkgreen(match_atom),
1881 ),
1882 importance = 0,
1883 type = "info",
1884 header = blue(" @@ ")
1885 )
1886 switched.add((idpackage, repo,))
1887
1888
1889 self.dependencies_test(to_repo)
1890
1891 return switched
1892
1893
1895
1896 if repo == None:
1897 repo = self.default_repository
1898
1899 upload_dir = os.path.join(self.get_local_upload_directory(repo),
1900 self.SystemSettings['repositories']['branch'])
1901 if not os.path.isdir(upload_dir):
1902 os.makedirs(upload_dir)
1903
1904 dbconn = self.open_server_repository(read_only = False,
1905 no_upload = True, repo = repo)
1906 self.updateProgress(
1907 red("[repo: %s] %s: %s" % (
1908 darkgreen(repo),
1909 _("adding package"),
1910 bold(os.path.basename(package_file)),
1911 )
1912 ),
1913 importance = 1,
1914 type = "info",
1915 header = brown(" * "),
1916 back = True
1917 )
1918 mydata = self.SpmService.extract_pkg_metadata(package_file,
1919 inject = inject)
1920 idpackage, revision, mydata = dbconn.handlePackage(mydata)
1921
1922
1923 trashing_counters = set()
1924 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
1925 myserver_repos = srv_set['repositories'].keys()
1926 for myrepo in myserver_repos:
1927 mydbconn = self.open_server_repository(read_only = True,
1928 no_upload = True, repo = myrepo)
1929 mylist = mydbconn.retrieve_packages_to_remove(
1930 mydata['name'],
1931 mydata['category'],
1932 mydata['slot'],
1933 mydata['injected']
1934 )
1935 for myitem in mylist:
1936 trashing_counters.add(mydbconn.retrieveCounter(myitem))
1937
1938 for mycounter in trashing_counters:
1939 dbconn.setTrashedCounter(mycounter)
1940
1941
1942 dbconn.removePackageFromInstalledTable(idpackage)
1943 dbconn.addPackageToInstalledTable(idpackage, repo)
1944 atom = dbconn.retrieveAtom(idpackage)
1945
1946 self.updateProgress(
1947 "[repo:%s] %s: %s %s: %s" % (
1948 darkgreen(repo),
1949 blue(_("added package")),
1950 darkgreen(atom),
1951 blue(_("rev")),
1952 bold(str(revision)),
1953 ),
1954 importance = 1,
1955 type = "info",
1956 header = red(" @@ ")
1957 )
1958
1959 manual_deps = sorted(dbconn.retrieveManualDependencies(idpackage))
1960 if manual_deps:
1961 self.updateProgress(
1962 "[repo:%s] %s: %s" % (
1963 darkgreen(repo),
1964 blue(_("manual dependencies for")),
1965 darkgreen(atom),
1966 ),
1967 importance = 1,
1968 type = "warning",
1969 header = darkgreen(" ## ")
1970 )
1971 for m_dep in manual_deps:
1972 self.updateProgress(
1973 brown(m_dep),
1974 importance = 1,
1975 type = "warning",
1976 header = darkred(" # ")
1977 )
1978
1979 download_url = self._setup_repository_package_filename(idpackage,
1980 repo = repo)
1981 downloadfile = os.path.basename(download_url)
1982 destination_path = os.path.join(upload_dir, downloadfile)
1983 try:
1984 os.rename(package_file, destination_path)
1985 except OSError:
1986 shutil.move(package_file, destination_path)
1987
1988 dbconn.commitChanges()
1989 return idpackage, destination_path
1990
1991
1993
1994 dbconn = self.open_server_repository(read_only = False,
1995 no_upload = True, repo = repo)
1996
1997 downloadurl = dbconn.retrieveDownloadURL(idpackage)
1998 packagerev = dbconn.retrieveRevision(idpackage)
1999 downloaddir = os.path.dirname(downloadurl)
2000 downloadfile = os.path.basename(downloadurl)
2001
2002 downloadfile = downloadfile[:-5]+"~%s%s" % (packagerev,
2003 etpConst['packagesext'],)
2004 downloadurl = os.path.join(downloaddir, downloadfile)
2005
2006
2007 dbconn.setDownloadURL(idpackage, downloadurl)
2008
2009 return downloadurl
2010
2013
2014 if repo == None:
2015 repo = self.default_repository
2016
2017 mycount = 0
2018 maxcount = len(packages_data)
2019 idpackages_added = set()
2020 to_be_injected = set()
2021 my_qa = self.QA()
2022 missing_deps_taint = False
2023 for package_filepath, inject in packages_data:
2024
2025 mycount += 1
2026 self.updateProgress(
2027 "[repo:%s] %s: %s" % (
2028 darkgreen(repo),
2029 blue(_("adding package")),
2030 darkgreen(os.path.basename(package_filepath)),
2031 ),
2032 importance = 1,
2033 type = "info",
2034 header = blue(" @@ "),
2035 count = (mycount, maxcount,)
2036 )
2037
2038 try:
2039
2040 idpackage, destination_path = self.package_injector(
2041 package_filepath,
2042 inject = inject,
2043 repo = repo
2044 )
2045 idpackages_added.add(idpackage)
2046 to_be_injected.add((idpackage, destination_path))
2047 except Exception, err:
2048 self.entropyTools.print_traceback()
2049 self.updateProgress(
2050 "[repo:%s] %s: %s" % (
2051 darkgreen(repo),
2052 darkred(_("Exception caught, closing tasks")),
2053 darkgreen(unicode(err)),
2054 ),
2055 importance = 1,
2056 type = "error",
2057 header = bold(" !!! "),
2058 count = (mycount, maxcount,)
2059 )
2060
2061 self.depends_table_initialize(repo)
2062
2063 self.library_paths_table_initialize(repo)
2064 if idpackages_added:
2065 dbconn = self.open_server_repository(read_only = False,
2066 no_upload = True, repo = repo)
2067 missing_deps_taint = my_qa.test_missing_dependencies(
2068 idpackages_added,
2069 dbconn,
2070 ask = ask,
2071 repo = repo,
2072 self_check = True,
2073 black_list = \
2074 self.get_missing_dependencies_blacklist(
2075 repo = repo),
2076 black_list_adder = \
2077 self.add_missing_dependencies_blacklist_items
2078 )
2079 my_qa.test_depends_linking(idpackages_added, dbconn,
2080 repo = repo)
2081 if to_be_injected:
2082 self.inject_database_into_packages(to_be_injected,
2083 repo = repo)
2084
2085 if missing_deps_taint:
2086 self.depends_table_initialize(repo)
2087 self.close_server_databases()
2088 raise
2089
2090
2091 self.depends_table_initialize(repo)
2092
2093 self.library_paths_table_initialize(repo)
2094
2095 if idpackages_added:
2096 dbconn = self.open_server_repository(read_only = False,
2097 no_upload = True, repo = repo)
2098 missing_deps_taint = my_qa.test_missing_dependencies(
2099 idpackages_added,
2100 dbconn,
2101 ask = ask,
2102 repo = repo,
2103 self_check = True,
2104 black_list = \
2105 self.get_missing_dependencies_blacklist(repo = repo),
2106 black_list_adder = \
2107 self.add_missing_dependencies_blacklist_items
2108 )
2109 my_qa.test_depends_linking(idpackages_added, dbconn, repo = repo)
2110
2111
2112 if missing_deps_taint:
2113 self.depends_table_initialize(repo)
2114
2115
2116 self.inject_database_into_packages(to_be_injected, repo = repo)
2117
2118 return idpackages_added
2119
2120
2122
2123 if repo == None:
2124 repo = self.default_repository
2125
2126
2127 self.updateProgress(
2128 "[repo:%s] %s:" % (
2129 darkgreen(repo),
2130 blue(_("Injecting entropy metadata into built packages")),
2131 ),
2132 importance = 1,
2133 type = "info",
2134 header = red(" @@ ")
2135 )
2136
2137 dbconn = self.open_server_repository(read_only = False,
2138 no_upload = True, repo = repo)
2139 for idpackage, package_path in injection_data:
2140 self.updateProgress(
2141 "[repo:%s|%s] %s: %s" % (
2142 darkgreen(repo),
2143 brown(str(idpackage)),
2144 blue(_("injecting entropy metadata")),
2145 darkgreen(os.path.basename(package_path)),
2146 ),
2147 importance = 1,
2148 type = "info",
2149 header = blue(" @@ "),
2150 back = True
2151 )
2152 data = dbconn.getPackageData(idpackage)
2153 treeupdates_actions = dbconn.listAllTreeUpdatesActions()
2154 dbpath = self.ClientService.inject_entropy_database_into_package(
2155 package_path, data, treeupdates_actions)
2156 digest = self.entropyTools.md5sum(package_path)
2157
2158 dbconn.setDigest(idpackage, digest)
2159
2160 signatures = data['signatures'].copy()
2161 for hash_key in sorted(signatures):
2162 hash_func = getattr(self.entropyTools, hash_key)
2163 signatures[hash_key] = hash_func(package_path)
2164 dbconn.setSignatures(idpackage, signatures)
2165 self.entropyTools.create_md5_file(package_path)
2166
2167 os.remove(dbpath)
2168 self.updateProgress(
2169 "[repo:%s|%s] %s: %s" % (
2170 darkgreen(repo),
2171 brown(str(idpackage)),
2172 blue(_("injection complete")),
2173 darkgreen(os.path.basename(package_path)),
2174 ),
2175 importance = 1,
2176 type = "info",
2177 header = red(" @@ ")
2178 )
2179 dbconn.commitChanges()
2180
2182 self.updateProgress(
2183 "[%s] %s" % (
2184 red(_("config files")),
2185 blue(_("checking system")),
2186 ),
2187 importance = 1,
2188 type = "info",
2189 header = blue(" @@ "),
2190 back = True
2191 )
2192
2193 scandata = self.ClientService.FileUpdates.scanfs(dcache = False)
2194 if scandata:
2195 self.updateProgress(
2196 "[%s] %s" % (
2197 red(_("config files")),
2198 blue(_("there are configuration files not updated yet")),
2199 ),
2200 importance = 1,
2201 type = "error",
2202 header = darkred(" @@ ")
2203 )
2204 for key in scandata:
2205 self.updateProgress(
2206 "%s" % (brown(etpConst['systemroot'] + \
2207 scandata[key]['destination'])),
2208 importance = 1,
2209 type = "info",
2210 header = "\t"
2211 )
2212 return True
2213 return False
2214
2216 return self.SpmService.quickpkg(atom, storedir)
2217
2218
2220
2221 if repo == None:
2222 repo = self.default_repository
2223
2224 dbconn = self.open_server_repository(read_only = False,
2225 no_upload = True, repo = repo)
2226 for idpackage in idpackages:
2227 atom = dbconn.retrieveAtom(idpackage)
2228 self.updateProgress(
2229 "[repo:%s] %s: %s" % (
2230 darkgreen(repo),
2231 blue(_("removing package")),
2232 darkgreen(atom),
2233 ),
2234 importance = 1,
2235 type = "info",
2236 header = brown(" @@ ")
2237 )
2238 dbconn.removePackage(idpackage)
2239 self.close_server_database(dbconn)
2240 self.updateProgress(
2241 "[repo:%s] %s" % (
2242 darkgreen(repo),
2243 blue(_("removal complete")),
2244 ),
2245 importance = 1,
2246 type = "info",
2247 header = brown(" @@ ")
2248 )
2249
2250
2256
2258 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2259 if repo == None:
2260 repo = self.default_repository
2261 return srv_set['repositories'][repo]['mirrors'][:]
2262
2264 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2265 if repo == None:
2266 repo = self.default_repository
2267 return srv_set['repositories'][repo]['packages_relative_path']
2268
2270 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2271 if repo == None:
2272 repo = self.default_repository
2273 return srv_set['repositories'][repo]['database_relative_path']
2274
2276 if repo == None:
2277 repo = self.default_repository
2278 return os.path.join(self.get_local_database_dir(repo, branch),
2279 etpConst['etpdatabasefile'])
2280
2282 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2283 if repo == None:
2284 repo = self.default_repository
2285 return srv_set['repositories'][repo]['store_dir']
2286
2288 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2289 if repo == None:
2290 repo = self.default_repository
2291 return srv_set['repositories'][repo]['upload_dir']
2292
2294 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2295 if repo == None:
2296 repo = self.default_repository
2297 return srv_set['repositories'][repo]['packages_dir']
2298
2300 if repo == None:
2301 repo = self.default_repository
2302 return os.path.join(self.get_local_database_dir(repo, branch),
2303 etpConst['etpdatabasetaintfile'])
2304
2306 if repo == None:
2307 repo = self.default_repository
2308 return os.path.join(self.get_local_database_dir(repo, branch),
2309 etpConst['etpdatabaserevisionfile'])
2310
2312 if repo == None:
2313 repo = self.default_repository
2314 return os.path.join(self.get_local_database_dir(repo, branch),
2315 etpConst['etpdatabasetimestampfile'])
2316
2318 if repo == None:
2319 repo = self.default_repository
2320 return os.path.join(self.get_local_database_dir(repo, branch),
2321 etpConst['etpdatabasecacertfile'])
2322
2324 if repo == None:
2325 repo = self.default_repository
2326 return os.path.join(self.get_local_database_dir(repo, branch),
2327 etpConst['etpdatabaseservercertfile'])
2328
2330 if repo == None:
2331 repo = self.default_repository
2332 return os.path.join(self.get_local_database_dir(repo, branch),
2333 etpConst['etpdatabasemaskfile'])
2334
2336 if repo == None:
2337 repo = self.default_repository
2338 return os.path.join(self.get_local_database_dir(repo, branch),
2339 etpConst['etpdatabasesytemmaskfile'])
2340
2342 if repo == None:
2343 repo = self.default_repository
2344 return os.path.join(self.get_local_database_dir(repo, branch),
2345 etpConst['etpdatabaseconflictingtaggedfile'])
2346
2349
2350 if repo == None:
2351 repo = self.default_repository
2352 return os.path.join(self.get_local_database_dir(repo, branch),
2353 etpConst['etpdatabaselicwhitelistfile'])
2354
2356 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2357 if repo == None:
2358 repo = self.default_repository
2359 return os.path.join(self.get_local_database_dir(repo, branch),
2360 srv_set['rss']['name'])
2361
2363 if repo == None:
2364 repo = self.default_repository
2365 return os.path.join(self.get_local_database_dir(repo, branch),
2366 etpConst['rss-light-name'])
2367
2369 if repo == None:
2370 repo = self.default_repository
2371 return os.path.join(self.get_local_database_dir(repo, branch),
2372 etpConst['rss-notice-board'])
2373
2375 if repo == None:
2376 repo = self.default_repository
2377 return os.path.join(self.get_local_database_dir(repo, branch),
2378 etpConst['etpdatabaseupdatefile'])
2379
2387
2395
2398 if repo == None:
2399 repo = self.default_repository
2400 return os.path.join(self.get_local_database_dir(repo, branch),
2401 etpConst['etpdatabaseexpbasedpkgsrm'])
2402
2404 if repo == None:
2405 repo = self.default_repository
2406 return os.path.join(self.get_local_database_dir(repo, branch),
2407 etpConst['etpdatabasepkglist'])
2408
2410 if repo == None:
2411 repo = self.default_repository
2412 return os.path.join(self.get_local_database_dir(repo, branch),
2413 etpConst['confsetsdirname'])
2414
2415 - def get_local_post_branch_mig_script(self, repo = None, branch = None):
2416 if repo == None:
2417 repo = self.default_repository
2418 return os.path.join(self.get_local_database_dir(repo, branch),
2419 etpConst['etp_post_branch_hop_script'])
2420
2421 - def get_local_post_branch_upg_script(self, repo = None, branch = None):
2422 if repo == None:
2423 repo = self.default_repository
2424 return os.path.join(self.get_local_database_dir(repo, branch),
2425 etpConst['etp_post_branch_upgrade_script'])
2426
2428 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2429 if repo == None:
2430 repo = self.default_repository
2431 if branch == None:
2432 branch = self.SystemSettings['repositories']['branch']
2433 return os.path.join(srv_set['repositories'][repo]['database_dir'],
2434 branch)
2435
2438 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2439 if repo == None:
2440 repo = self.default_repository
2441 if branch == None:
2442 branch = self.SystemSettings['repositories']['branch']
2443 return os.path.join(srv_set['repositories'][repo]['database_dir'],
2444 branch, etpConst['etpdatabasemissingdepsblfile'])
2445
2447 if repo == None:
2448 repo = self.default_repository
2449 if branch == None:
2450 branch = self.SystemSettings['repositories']['branch']
2451 wl_file = self.get_missing_dependencies_blacklist_file(repo, branch)
2452 wl_data = []
2453 if os.path.isfile(wl_file) and os.access(wl_file, os.R_OK):
2454 f_wl = open(wl_file,"r")
2455 wl_data = [x.strip() for x in f_wl.readlines() if x.strip() and \
2456 not x.strip().startswith("#")]
2457 f_wl.close()
2458 return set(wl_data)
2459
2462
2463 if repo == None:
2464 repo = self.default_repository
2465 if branch == None:
2466 branch = self.SystemSettings['repositories']['branch']
2467 wl_file = self.get_missing_dependencies_blacklist_file(repo, branch)
2468 wl_dir = os.path.dirname(wl_file)
2469 if not (os.path.isdir(wl_dir) and os.access(wl_dir, os.W_OK)):
2470 return
2471 if os.path.isfile(wl_file) and not os.access(wl_file, os.W_OK):
2472 return
2473 f_wl = open(wl_file,"a+")
2474 f_wl.write('\n'.join(items)+'\n')
2475 f_wl.flush()
2476 f_wl.close()
2477
2479
2480 if repo == None:
2481 repo = self.default_repository
2482
2483 dbrev_file = self.get_local_database_revision_file(repo)
2484 if os.path.isfile(dbrev_file):
2485 f_rev = open(dbrev_file)
2486 rev = f_rev.readline().strip()
2487 f_rev.close()
2488 try:
2489 rev = int(rev)
2490 except ValueError:
2491 self.updateProgress(
2492 "[repo:%s] %s: %s - %s" % (
2493 darkgreen(repo),
2494 blue(_("invalid database revision")),
2495 bold(rev),
2496 blue(_("defaulting to 0")),
2497 ),
2498 importance = 2,
2499 type = "error",
2500 header = darkred(" !!! ")
2501 )
2502 rev = 0
2503 return rev
2504 else:
2505 return 0
2506
2508
2509 if repo == None:
2510 repo = self.default_repository
2511
2512 remote_status = self.MirrorsService.get_remote_databases_status(repo)
2513 if not [x for x in remote_status if x[1]]:
2514 remote_revision = 0
2515 else:
2516 remote_revision = max([x[1] for x in remote_status])
2517
2518 return remote_revision
2519
2522
2524 from datetime import datetime
2525 import time
2526 return "%s" % (datetime.fromtimestamp(time.time()),)
2527
2529 pkglist_file = self.get_local_pkglist_file(repo = repo, branch = branch)
2530
2531 tmp_pkglist_file = pkglist_file + ".tmp"
2532 dbconn = self.open_server_repository(repo = repo, just_reading = True,
2533 do_treeupdates = False)
2534 pkglist = dbconn.listAllDownloads(do_sort = True, full_path = True)
2535
2536 with open(tmp_pkglist_file, "w") as pkg_f:
2537 for pkg in pkglist:
2538 pkg_f.write(pkg + "\n")
2539 pkg_f.flush()
2540
2541 os.rename(tmp_pkglist_file, pkglist_file)
2542
2544 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2545 repos = srv_set['repositories'].keys()
2546 kwargs['server_repos'] = repos
2547 kwargs['serverInstance'] = self
2548 return self.ClientService.package_set_list(*args, **kwargs)
2549
2551 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2552 repos = srv_set['repositories'].keys()
2553 kwargs['server_repos'] = repos
2554 kwargs['serverInstance'] = self
2555 return self.ClientService.package_set_search(*args, **kwargs)
2556
2558 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2559 repos = srv_set['repositories'].keys()
2560 kwargs['server_repos'] = repos
2561 kwargs['serverInstance'] = self
2562 return self.ClientService.package_set_match(*args, **kwargs)
2563
2565 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2566 repos = srv_set['repositories'].keys()
2567 kwargs['server_repos'] = repos
2568 kwargs['serverInstance'] = self
2569 return self.ClientService.atom_match(*args, **kwargs)
2570
2572
2573 installed_packages = self.SpmService.get_installed_packages_counter()
2574 installed_counters = set()
2575 to_be_added = set()
2576 to_be_removed = set()
2577 to_be_injected = set()
2578 my_settings = self.SystemSettings[self.sys_settings_plugin_id]['server']
2579 exp_based_scope = my_settings['exp_based_scope']
2580
2581 server_repos = my_settings['repositories'].keys()
2582 exp_pkgs_cache = {}
2583
2584
2585 for spm_atom, spm_counter in installed_packages:
2586 found = False
2587 for server_repo in server_repos:
2588 installed_counters.add(spm_counter)
2589 server_dbconn = self.open_server_repository(read_only = True,
2590 no_upload = True, repo = server_repo)
2591 counter = server_dbconn.isCounterAvailable(spm_counter)
2592 if counter:
2593 found = True
2594 break
2595 if not found:
2596 to_be_added.add((spm_atom, spm_counter,))
2597
2598
2599 database_counters = {}
2600 for server_repo in server_repos:
2601 server_dbconn = self.open_server_repository(read_only = True,
2602 no_upload = True, repo = server_repo)
2603 database_counters[server_repo] = server_dbconn.listAllCounters(
2604 branch = self.SystemSettings['repositories']['branch'])
2605
2606 ordered_counters = set()
2607 for server_repo in database_counters:
2608 for data in database_counters[server_repo]:
2609 ordered_counters.add((data, server_repo))
2610 database_counters = ordered_counters
2611
2612 for (counter, idpackage,), xrepo in database_counters:
2613
2614 if counter < 0:
2615 continue
2616
2617 if counter in installed_counters:
2618 continue
2619
2620 dbconn = self.open_server_repository(read_only = True,
2621 no_upload = True, repo = xrepo)
2622
2623 dorm = True
2624
2625 if to_be_added:
2626
2627 dorm = False
2628 atom = dbconn.retrieveAtom(idpackage)
2629 atomkey = self.entropyTools.dep_getkey(atom)
2630 atomtag = self.entropyTools.dep_gettag(atom)
2631 atomslot = dbconn.retrieveSlot(idpackage)
2632
2633 add = True
2634 for spm_atom, spm_counter in to_be_added:
2635 addslot = self.SpmService.get_installed_package_slot(
2636 spm_atom)
2637 addkey = self.entropyTools.dep_getkey(spm_atom)
2638
2639 if addslot == None:
2640 addslot = '0'
2641
2642 if (atomkey == addkey) and \
2643 ((str(atomslot) == str(addslot)) or (atomtag != None)):
2644
2645 add = False
2646 break
2647
2648 if not add:
2649 continue
2650 dorm = True
2651
2652
2653
2654
2655 if dorm and (xrepo == self.default_repository):
2656 trashed = self.is_counter_trashed(counter)
2657 if trashed:
2658
2659 try:
2660 key, slot = dbconn.retrieveKeySlot(idpackage)
2661 trashed = self.SpmService.get_installed_atom(
2662 key+":"+slot)
2663 except TypeError:
2664 trashed = True
2665 if not trashed:
2666
2667 dbtag = dbconn.retrieveVersionTag(idpackage)
2668 if dbtag:
2669 is_injected = dbconn.isInjected(idpackage)
2670 if not is_injected:
2671 to_be_injected.add((idpackage, xrepo))
2672
2673 elif exp_based_scope:
2674
2675
2676 plg_id = self.sys_settings_fatscope_plugin_id
2677 exp_data = self.SystemSettings[plg_id]['repos'].get(
2678 xrepo, set())
2679
2680
2681
2682 if (idpackage not in exp_data) and (-1 not in exp_data):
2683 to_be_removed.add((idpackage, xrepo))
2684 continue
2685
2686 idpackage_expired = self.is_match_expired((idpackage,
2687 xrepo,))
2688
2689 if idpackage_expired:
2690
2691
2692 to_be_removed.add((idpackage, xrepo))
2693 for my_id in dbconn.retrieveDepends(idpackage):
2694 to_be_removed.add((my_id, xrepo))
2695
2696 else:
2697 to_be_removed.add((idpackage, xrepo))
2698
2699 return to_be_added, to_be_removed, to_be_injected
2700
2702
2703 idpackage, repoid = match
2704 dbconn = self.open_server_repository(repo = repoid, just_reading = True)
2705
2706 my_settings = self.SystemSettings[self.sys_settings_plugin_id]['server']
2707 pkg_exp_secs = my_settings['packages_expiration_days'] * 86400
2708 cur_unix_time = self.entropyTools.get_current_unix_time()
2709
2710
2711
2712 mydate = dbconn.retrieveDateCreation(idpackage)
2713
2714 mydelta = cur_unix_time - float(mydate)
2715 if mydelta > pkg_exp_secs:
2716 return True
2717 return False
2718
2720 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2721 server_repos = srv_set['repositories'].keys()
2722 for repo in server_repos:
2723 dbconn = self.open_server_repository(read_only = True,
2724 no_upload = True, repo = repo)
2725 if dbconn.isCounterTrashed(counter):
2726 return True
2727 return False
2728
2735
2738
2739 if repo == None:
2740 repo = self.default_repository
2741
2742 self.close_server_databases()
2743 revisions_match = {}
2744 treeupdates_actions = []
2745 injected_packages = set()
2746 idpackages = set()
2747 idpackages_added = set()
2748
2749 mytxt = red("%s ...") % (_("Initializing Entropy database"),)
2750 self.updateProgress(
2751 mytxt, importance = 1,
2752 type = "info", header = darkgreen(" * "),
2753 back = True
2754 )
2755
2756 if os.path.isfile(self.get_local_database_file(repo)):
2757
2758 dbconn = self.open_server_repository(read_only = True,
2759 no_upload = True, repo = repo, warnings = warnings)
2760
2761 if dbconn.doesTableExist("baseinfo") and \
2762 dbconn.doesTableExist("extrainfo"):
2763 idpackages = dbconn.listAllIdpackages()
2764
2765 if dbconn.doesTableExist("treeupdatesactions"):
2766 treeupdates_actions = dbconn.listAllTreeUpdatesActions()
2767
2768
2769 if dbconn.doesTableExist("injected") and \
2770 dbconn.doesTableExist("extrainfo"):
2771 injected_packages = dbconn.listAllInjectedPackages(
2772 justFiles = True)
2773 injected_packages = set([os.path.basename(x) for x \
2774 in injected_packages])
2775
2776 for idpackage in idpackages:
2777 url = dbconn.retrieveDownloadURL(idpackage)
2778 package = os.path.basename(url)
2779 branch = dbconn.retrieveBranch(idpackage)
2780 revision = dbconn.retrieveRevision(idpackage)
2781 revisions_match[package] = (branch, revision,)
2782
2783 self.close_server_database(dbconn)
2784
2785 mytxt = "%s: %s: %s" % (
2786 bold(_("WARNING")),
2787 red(_("database already exists")),
2788 self.get_local_database_file(repo),
2789 )
2790 self.updateProgress(
2791 mytxt,
2792 importance = 1,
2793 type = "warning",
2794 header = darkred(" !!! ")
2795 )
2796
2797 rc_question = self.askQuestion(_("Do you want to continue ?"))
2798 if rc_question == "No":
2799 return
2800 try:
2801 os.remove(self.get_local_database_file(repo))
2802 except OSError:
2803 pass
2804
2805
2806
2807 dbconn = self.open_server_repository(read_only = False,
2808 no_upload = True, repo = repo, is_new = True)
2809 dbconn.initializeDatabase()
2810
2811 if not empty:
2812
2813 revisions_file = "/entropy-revisions-dump.txt"
2814
2815 if revisions_match:
2816 self.updateProgress(
2817 "%s: %s" % (
2818 red(_("Dumping current revisions to file")),
2819 darkgreen(revisions_file),
2820 ),
2821 importance = 1,
2822 type = "info",
2823 header = darkgreen(" * ")
2824 )
2825 f_rev = open(revisions_file,"w")
2826 f_rev.write(str(revisions_match))
2827 f_rev.flush()
2828 f_rev.close()
2829
2830
2831 treeupdates_file = "/entropy-treeupdates-dump.txt"
2832 if treeupdates_actions:
2833 self.updateProgress(
2834 "%s: %s" % (
2835
2836 red(_("Dumping current 'treeupdates' actions to file")),
2837 bold(treeupdates_file),
2838 ),
2839 importance = 1,
2840 type = "info",
2841 header = darkgreen(" * ")
2842 )
2843 f_tree = open(treeupdates_file,"w")
2844 f_tree.write(str(treeupdates_actions))
2845 f_tree.flush()
2846 f_tree.close()
2847
2848 rc_question = self.askQuestion(
2849 _("Would you like to sync packages first (important!) ?"))
2850 if rc_question == "Yes":
2851 self.MirrorsService.sync_packages(repo = repo)
2852
2853
2854 if treeupdates_actions:
2855 dbconn.bumpTreeUpdatesActions(treeupdates_actions)
2856
2857
2858 pkg_branch_dir = os.path.join(
2859 self.get_local_packages_directory(repo),
2860 self.SystemSettings['repositories']['branch'])
2861 pkglist = os.listdir(pkg_branch_dir)
2862
2863 pkg_ext_len = len(etpConst['packagesext'])
2864 pkglist = [x for x in pkglist if (x[(pkg_ext_len*-1):] == \
2865 etpConst['packagesext']) and not \
2866 os.path.isfile(os.path.join(pkg_branch_dir,
2867 x + etpConst['packagesexpirationfileext']))]
2868
2869 if pkglist:
2870 self.updateProgress(
2871 "%s '%s' %s %s" % (
2872 red(_("Reinitializing Entropy database for branch")),
2873 bold(self.SystemSettings['repositories']['branch']),
2874 red(_("using Packages in the repository")),
2875 red("..."),
2876 ),
2877 importance = 1,
2878 type = "info",
2879 header = darkgreen(" * ")
2880 )
2881
2882 counter = 0
2883 maxcount = len(pkglist)
2884 branch = self.SystemSettings['repositories']['branch']
2885 for pkg in pkglist:
2886 counter += 1
2887
2888 self.updateProgress(
2889 "[repo:%s|%s] %s: %s" % (
2890 darkgreen(repo),
2891 brown(branch),
2892 blue(_("analyzing")),
2893 bold(pkg),
2894 ),
2895 importance = 1,
2896 type = "info",
2897 header = " ",
2898 back = True,
2899 count = (counter, maxcount,)
2900 )
2901
2902 doinject = False
2903 if pkg in injected_packages:
2904 doinject = True
2905
2906 pkg_path = os.path.join(self.get_local_packages_directory(repo),
2907 branch, pkg)
2908 mydata = self.SpmService.extract_pkg_metadata(pkg_path,
2909 inject = doinject)
2910
2911
2912 revision_avail = revisions_match.get(pkg)
2913 add_revision = 0
2914 if (revision_avail != None):
2915 if branch == revision_avail[0]:
2916 add_revision = revision_avail[1]
2917
2918 idpackage, revision, mydata_upd = dbconn.addPackage(mydata,
2919 revision = add_revision)
2920 idpackages_added.add(idpackage)
2921
2922 self.updateProgress(
2923 "[repo:%s] [%s:%s/%s] %s: %s, %s: %s" % (
2924 repo,
2925 brown(branch),
2926 darkgreen(str(counter)),
2927 blue(str(maxcount)),
2928 red(_("added package")),
2929 darkgreen(pkg),
2930 red(_("revision")),
2931 brown(str(revision)),
2932 ),
2933 importance = 1,
2934 type = "info",
2935 header = " ",
2936 back = True
2937 )
2938
2939 self.depends_table_initialize(repo)
2940 self.library_paths_table_initialize(repo)
2941
2942 my_qa = self.QA()
2943
2944 if idpackages_added:
2945 dbconn = self.open_server_repository(read_only = False,
2946 no_upload = True, repo = repo)
2947 my_qa.test_missing_dependencies(
2948 idpackages_added, dbconn, ask = True,
2949 repo = repo, self_check = True,
2950 black_list = \
2951 self.get_missing_dependencies_blacklist(repo = repo),
2952 black_list_adder = \
2953 self.add_missing_dependencies_blacklist_items
2954 )
2955
2956 dbconn.commitChanges()
2957 self.close_server_databases()
2958
2959 return 0
2960
2962
2963 dbconn = self.open_server_repository(read_only = True,
2964 no_upload = True, repo = repo)
2965 if ("world" in packages) or not packages:
2966 return dbconn.listAllIdpackages(), True
2967 else:
2968 idpackages = set()
2969 for package in packages:
2970 matches = dbconn.atomMatch(package, multiMatch = True)
2971 if matches[1] == 0:
2972 idpackages |= matches[0]
2973 else:
2974 mytxt = "%s: %s: %s" % (
2975 red(_("Attention")),
2976 blue(_("cannot match")),
2977 bold(package),
2978 )
2979 self.updateProgress(
2980 mytxt,
2981 importance = 1,
2982 type = "warning",
2983 header = darkred(" !!! ")
2984 )
2985 return idpackages, False
2986
2988
2989 import urllib2
2990 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2991 if not srv_set['repositories'][repo].has_key('handler'):
2992 return None
2993 url = srv_set['repositories'][repo]['handler']
2994
2995
2996 filename = filename.replace("#","%23")
2997
2998 filename = filename.replace("+","%2b")
2999 request = os.path.join(url, etpConst['handlers']['md5sum'])
3000 request += filename+"&branch="+branch
3001
3002 proxy_settings = self.SystemSettings['system']['proxy']
3003 try:
3004 mydict = {}
3005 if proxy_settings['ftp']:
3006 mydict['ftp'] = proxy_settings['ftp']
3007 if proxy_settings['http']:
3008 mydict['http'] = proxy_settings['http']
3009 if mydict:
3010 mydict['username'] = proxy_settings['username']
3011 mydict['password'] = proxy_settings['password']
3012 self.entropyTools.add_proxy_opener(urllib2, mydict)
3013 else:
3014
3015 urllib2._opener = None
3016 item = urllib2.urlopen(request)
3017 result = item.readline().strip()
3018 item.close()
3019 del item
3020 return result
3021 except (urllib2.URLError, urllib2.HTTPError,):
3022 return None
3023
3025
3026 if repo == None:
3027 repo = self.default_repository
3028
3029 self.updateProgress(
3030 "[%s] %s:" % (
3031 red("remote"),
3032 blue(_("Integrity verification of the selected packages")),
3033 ),
3034 importance = 1,
3035 type = "info",
3036 header = blue(" @@ ")
3037 )
3038
3039 idpackages, world = self.match_packages(packages)
3040 dbconn = self.open_server_repository(read_only = True, no_upload = True,
3041 repo = repo)
3042 branch = self.SystemSettings['repositories']['branch']
3043
3044 if world:
3045 self.updateProgress(
3046 blue(
3047 _("All the packages in repository will be checked.")),
3048 importance = 1,
3049 type = "info",
3050 header = " "
3051 )
3052 else:
3053 mytxt = red("%s:") % (
3054 _("This is the list of the packages that would be checked"),)
3055 self.updateProgress(
3056 mytxt,
3057 importance = 1,
3058 type = "info",
3059 header = " "
3060 )
3061 for idpackage in idpackages:
3062 pkgatom = dbconn.retrieveAtom(idpackage)
3063 down_url = dbconn.retrieveDownloadURL(idpackage)
3064 pkgfile = os.path.basename(down_url)
3065 self.updateProgress(
3066 red(pkgatom) + " -> " + bold(os.path.join(branch, pkgfile)),
3067 importance = 1,
3068 type = "info",
3069 header = darkgreen(" - ")
3070 )
3071
3072 if ask:
3073 rc_question = self.askQuestion(
3074 _("Would you like to continue ?"))
3075 if rc_question == "No":
3076 return set(), set(), {}
3077
3078 match = set()
3079 not_match = set()
3080 broken_packages = {}
3081
3082 for uri in self.get_remote_mirrors(repo):
3083
3084 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri)
3085 self.updateProgress(
3086 "[repo:%s] %s: %s" % (
3087 darkgreen(repo),
3088 blue(_("Working on mirror")),
3089 brown(crippled_uri),
3090 ),
3091 importance = 1,
3092 type = "info",
3093 header = red(" @@ ")
3094 )
3095
3096
3097 totalcounter = len(idpackages)
3098 currentcounter = 0
3099 for idpackage in idpackages:
3100
3101 currentcounter += 1
3102 pkgfile = dbconn.retrieveDownloadURL(idpackage)
3103 orig_branch = self.get_branch_from_download_relative_uri(
3104 pkgfile)
3105
3106 self.updateProgress(
3107 "[%s] %s: %s" % (
3108 brown(crippled_uri),
3109 blue(_("checking hash")),
3110 darkgreen(pkgfile),
3111 ),
3112 importance = 1,
3113 type = "info",
3114 header = blue(" @@ "),
3115 back = True,
3116 count = (currentcounter, totalcounter,)
3117 )
3118
3119 checksum_ok = False
3120 ck_remote = self.get_remote_package_checksum(repo,
3121 os.path.basename(pkgfile), orig_branch)
3122 if ck_remote == None:
3123 self.updateProgress(
3124 "[%s] %s: %s %s" % (
3125 brown(crippled_uri),
3126 blue(_("digest verification of")),
3127 bold(pkgfile),
3128 blue(_("not supported")),
3129 ),
3130 importance = 1,
3131 type = "info",
3132 header = blue(" @@ "),
3133 count = (currentcounter, totalcounter,)
3134 )
3135 elif len(ck_remote) == 32:
3136 pkghash = dbconn.retrieveDigest(idpackage)
3137 if ck_remote == pkghash:
3138 checksum_ok = True
3139 else:
3140 self.updateProgress(
3141 "[%s] %s: %s %s" % (
3142 brown(crippled_uri),
3143 blue(_("digest verification of")),
3144 bold(pkgfile),
3145 blue(_("failed for unknown reasons")),
3146 ),
3147 importance = 1,
3148 type = "info",
3149 header = blue(" @@ "),
3150 count = (currentcounter, totalcounter,)
3151 )
3152
3153 if checksum_ok:
3154 match.add(idpackage)
3155 else:
3156 not_match.add(idpackage)
3157 self.updateProgress(
3158 "[%s] %s: %s %s" % (
3159 brown(crippled_uri),
3160 blue(_("package")),
3161 bold(pkgfile),
3162 red(_("NOT healthy")),
3163 ),
3164 importance = 1,
3165 type = "warning",
3166 header = darkred(" !!! "),
3167 count = (currentcounter, totalcounter,)
3168 )
3169 if not broken_packages.has_key(crippled_uri):
3170 broken_packages[crippled_uri] = []
3171 broken_packages[crippled_uri].append(pkgfile)
3172
3173 if broken_packages:
3174 mytxt = blue("%s:") % (
3175 _("This is the list of broken packages"),)
3176 self.updateProgress(
3177 mytxt,
3178 importance = 1,
3179 type = "info",
3180 header = red(" * ")
3181 )
3182 for mirror in broken_packages.keys():
3183 mytxt = "%s: %s" % (
3184 brown(_("Mirror")),
3185 bold(mirror),
3186 )
3187 self.updateProgress(
3188 mytxt,
3189 importance = 1,
3190 type = "info",
3191 header = red(" <> ")
3192 )
3193 for broken_package in broken_packages[mirror]:
3194 self.updateProgress(
3195 blue(broken_package),
3196 importance = 1,
3197 type = "info",
3198 header = red(" - ")
3199 )
3200
3201 self.updateProgress(
3202 "%s:" % (
3203 blue(_("Statistics")),
3204 ),
3205 importance = 1,
3206 type = "info",
3207 header = red(" @@ ")
3208 )
3209 self.updateProgress(
3210 "[%s] %s:\t%s" % (
3211 red(crippled_uri),
3212 brown(_("Number of checked packages")),
3213 brown(str(len(match) + len(not_match))),
3214 ),
3215 importance = 1,
3216 type = "info",
3217 header = brown(" # ")
3218 )
3219 self.updateProgress(
3220 "[%s] %s:\t%s" % (
3221 red(crippled_uri),
3222 darkgreen(_("Number of healthy packages")),
3223 darkgreen(str(len(match))),
3224 ),
3225 importance = 1,
3226 type = "info",
3227 header = brown(" # ")
3228 )
3229 self.updateProgress(
3230 "[%s] %s:\t%s" % (
3231 red(crippled_uri),
3232 darkred(_("Number of broken packages")),
3233 darkred(str(len(not_match))),
3234 ),
3235 importance = 1,
3236 type = "info",
3237 header = brown(" # ")
3238 )
3239
3240 return match, not_match, broken_packages
3241
3242
3244
3245 if repo == None:
3246 repo = self.default_repository
3247
3248 self.updateProgress(
3249 "[%s] %s:" % (
3250 red(_("local")),
3251 blue(_("Integrity verification of the selected packages")),
3252 ),
3253 importance = 1,
3254 type = "info",
3255 header = darkgreen(" * ")
3256 )
3257
3258 idpackages, world = self.match_packages(packages)
3259 dbconn = self.open_server_repository(read_only = True,
3260 no_upload = True, repo = repo)
3261
3262 if world:
3263 self.updateProgress(
3264 blue(_("All the packages in repository will be checked.")),
3265 importance = 1,
3266 type = "info",
3267 header = " "
3268 )
3269
3270 to_download = set()
3271 available = set()
3272 for idpackage in idpackages:
3273
3274 pkgatom = dbconn.retrieveAtom(idpackage)
3275 pkg_path = dbconn.retrieveDownloadURL(idpackage)
3276 pkg_rel_path = '/'.join(pkg_path.split("/")[2:])
3277
3278 bindir_path = os.path.join(self.get_local_packages_directory(repo),
3279 pkg_rel_path)
3280 uploaddir_path = os.path.join(self.get_local_upload_directory(repo),
3281 pkg_rel_path)
3282
3283 if os.path.isfile(bindir_path):
3284 if not world:
3285 self.updateProgress(
3286 "[%s] %s :: %s" % (
3287 darkgreen(_("available")),
3288 blue(pkgatom),
3289 darkgreen(pkg_rel_path),
3290 ),
3291 importance = 0,
3292 type = "info",
3293 header = darkgreen(" # ")
3294 )
3295 available.add(idpackage)
3296 elif os.path.isfile(uploaddir_path):
3297 if not world:
3298 self.updateProgress(
3299 "[%s] %s :: %s" % (
3300 darkred(_("upload/ignored")),
3301 blue(pkgatom),
3302 darkgreen(pkg_rel_path),
3303 ),
3304 importance = 0,
3305 type = "info",
3306 header = darkgreen(" # ")
3307 )
3308 else:
3309 self.updateProgress(
3310 "[%s] %s :: %s" % (
3311 brown(_("download")),
3312 blue(pkgatom),
3313 darkgreen(pkg_rel_path),
3314 ),
3315 importance = 0,
3316 type = "info",
3317 header = darkgreen(" # ")
3318 )
3319 to_download.add((idpackage, pkg_path,))
3320
3321 if ask:
3322 rc_question = self.askQuestion(_("Would you like to continue ?"))
3323 if rc_question == "No":
3324 return set(), set(), set(), set()
3325
3326
3327 fine = set()
3328 failed = set()
3329 downloaded_fine = set()
3330 downloaded_errors = set()
3331
3332 if to_download:
3333
3334 not_downloaded = set()
3335 mytxt = blue("%s ...") % (_("Starting to download missing files"),)
3336 self.updateProgress(
3337 mytxt,
3338 importance = 1,
3339 type = "info",
3340 header = " "
3341 )
3342 for uri in self.get_remote_mirrors(repo):
3343
3344 if not_downloaded:
3345 mytxt = blue("%s ...") % (
3346 _("Searching missing/broken files on another mirror"),)
3347 self.updateProgress(
3348 mytxt,
3349 importance = 1,
3350 type = "info",
3351 header = " "
3352 )
3353 to_download = not_downloaded.copy()
3354 not_downloaded = set()
3355
3356 for idpackage, pkg_path in to_download:
3357 rc_down = self.MirrorsService.download_package(uri,
3358 pkg_path, repo = repo)
3359 if rc_down:
3360 downloaded_fine.add(idpackage)
3361 available.add(idpackage)
3362 else:
3363 not_downloaded.add(pkg_path)
3364
3365 if not not_downloaded:
3366 self.updateProgress(
3367 red(_("Binary packages downloaded successfully.")),
3368 importance = 1,
3369 type = "info",
3370 header = " "
3371 )
3372 break
3373
3374 if not_downloaded:
3375 mytxt = blue("%s:") % (
3376 _("These are the packages that cannot be found online"),)
3377 self.updateProgress(
3378 mytxt,
3379 importance = 1,
3380 type = "info",
3381 header = " "
3382 )
3383 for pkg_path in not_downloaded:
3384 downloaded_errors.add(pkg_path)
3385 self.updateProgress(
3386 brown(pkg_path),
3387 importance = 1,
3388 type = "warning",
3389 header = red(" * ")
3390 )
3391 downloaded_errors |= not_downloaded
3392 mytxt = "%s." % (_("They won't be checked"),)
3393 self.updateProgress(
3394 mytxt,
3395 importance = 1,
3396 type = "warning",
3397 header = " "
3398 )
3399
3400 totalcounter = str(len(available))
3401 currentcounter = 0
3402 for idpackage in available:
3403 currentcounter += 1
3404 pkg_path = dbconn.retrieveDownloadURL(idpackage)
3405 orig_branch = self.get_branch_from_download_relative_uri(pkg_path)
3406 pkgfile = os.path.basename(pkg_path)
3407
3408 self.updateProgress(
3409 "[branch:%s] %s %s" % (
3410 brown(orig_branch),
3411 blue(_("checking hash of")),
3412 darkgreen(pkgfile),
3413 ),
3414 importance = 1,
3415 type = "info",
3416 header = " ",
3417 back = True,
3418 count = (currentcounter, totalcounter,)
3419 )
3420
3421 storedmd5 = dbconn.retrieveDigest(idpackage)
3422 pkgpath = os.path.join(self.get_local_packages_directory(repo),
3423 orig_branch, pkgfile)
3424 result = self.entropyTools.compare_md5(pkgpath, storedmd5)
3425 if result:
3426 fine.add(idpackage)
3427 else:
3428 failed.add(idpackage)
3429 self.updateProgress(
3430 "[branch:%s] %s %s %s: %s" % (
3431 brown(orig_branch),
3432 blue(_("package")),
3433 darkgreen(pkg_path),
3434 blue(_("is corrupted, stored checksum")),
3435 brown(storedmd5),
3436 ),
3437 importance = 1,
3438 type = "info",
3439 header = " ",
3440 count = (currentcounter, totalcounter,)
3441 )
3442
3443 if failed:
3444 mytxt = blue("%s:") % (_("This is the list of broken packages"),)
3445 self.updateProgress(
3446 mytxt,
3447 importance = 1,
3448 type = "warning",
3449 header = darkred(" # ")
3450 )
3451 for idpackage in failed:
3452 atom = dbconn.retrieveAtom(idpackage)
3453 down_p = dbconn.retrieveDownloadURL(idpackage)
3454 self.updateProgress(
3455 blue("[atom:%s] %s" % (atom, down_p,)),
3456 importance = 0,
3457 type = "warning",
3458 header = brown(" # ")
3459 )
3460
3461
3462 self.updateProgress(
3463 red("Statistics:"),
3464 importance = 1,
3465 type = "info",
3466 header = blue(" * ")
3467 )
3468 self.updateProgress(
3469 brown("%s => %s" % (
3470 len(fine) + len(failed),
3471 _("checked packages"),
3472 )
3473 ),
3474 importance = 0,
3475 type = "info",
3476 header = brown(" # ")
3477 )
3478 self.updateProgress(
3479 darkgreen("%s => %s" % (
3480 len(fine),
3481 _("healthy packages"),
3482 )
3483 ),
3484 importance = 0,
3485 type = "info",
3486 header = brown(" # ")
3487 )
3488 self.updateProgress(
3489 darkred("%s => %s" % (
3490 len(failed),
3491 _("broken packages"),
3492 )
3493 ),
3494 importance = 0,
3495 type = "info",
3496 header = brown(" # ")
3497 )
3498 self.updateProgress(
3499 blue("%s => %s" % (
3500 len(downloaded_fine),
3501 _("downloaded packages"),
3502 )
3503 ),
3504 importance = 0,
3505 type = "info",
3506 header = brown(" # ")
3507 )
3508 self.updateProgress(
3509 bold("%s => %s" % (
3510 len(downloaded_errors),
3511 _("failed downloads"),
3512 )
3513 ),
3514 importance = 0,
3515 type = "info",
3516 header = brown(" # ")
3517 )
3518
3519 self.close_server_database(dbconn)
3520 return fine, failed, downloaded_fine, downloaded_errors
3521
3522
3524
3525 if repo == None:
3526 repo = self.default_repository
3527
3528 if to_branch != self.SystemSettings['repositories']['branch']:
3529 mytxt = "%s: %s %s" % (
3530 blue(_("Please setup your branch to")),
3531 bold(to_branch),
3532 blue(_("and retry")),
3533 )
3534 self.updateProgress(
3535 mytxt,
3536 importance = 1,
3537 type = "error",
3538 header = darkred(" !! ")
3539 )
3540 return None
3541
3542 mytxt = red("%s ...") % (_("Copying database (if not exists)"),)
3543 self.updateProgress(
3544 mytxt,
3545 importance = 1,
3546 type = "info",
3547 header = darkgreen(" @@ ")
3548 )
3549 branch_dbdir = self.get_local_database_dir(repo)
3550 old_branch_dbdir = self.get_local_database_dir(repo, from_branch)
3551
3552
3553 self.close_server_databases()
3554
3555
3556
3557 branch_dbfile = self.get_local_database_file(repo)
3558 if os.path.isfile(branch_dbfile):
3559 if self.entropyTools.get_file_size(branch_dbfile) == 0:
3560 shutil.rmtree(branch_dbdir, True)
3561
3562 if os.path.isdir(branch_dbdir):
3563
3564 while 1:
3565 rnd_num = self.entropyTools.get_random_number()
3566 backup_dbdir = branch_dbdir + str(rnd_num)
3567 if not os.path.isdir(backup_dbdir):
3568 break
3569 os.rename(branch_dbdir, backup_dbdir)
3570
3571 if os.path.isdir(old_branch_dbdir):
3572 shutil.copytree(old_branch_dbdir, branch_dbdir)
3573
3574 mytxt = red("%s ...") % (_("Switching packages"),)
3575 self.updateProgress(
3576 mytxt,
3577 importance = 1,
3578 type = "info",
3579 header = darkgreen(" @@ ")
3580 )
3581
3582 dbconn = self.open_server_repository(read_only = False,
3583 no_upload = True, repo = repo, lock_remote = False)
3584 try:
3585 dbconn.validateDatabase()
3586 except SystemDatabaseError:
3587 self.handle_uninitialized_repository(repo)
3588 dbconn = self.open_server_repository(read_only = False,
3589 no_upload = True, repo = repo, lock_remote = False)
3590
3591 idpackages = dbconn.listAllIdpackages()
3592 already_switched = set()
3593 not_found = set()
3594 switched = set()
3595 ignored = set()
3596 no_checksum = set()
3597
3598 maxcount = len(idpackages)
3599 count = 0
3600 for idpackage in idpackages:
3601 count += 1
3602
3603 cur_branch = dbconn.retrieveBranch(idpackage)
3604 atom = dbconn.retrieveAtom(idpackage)
3605 if cur_branch == to_branch:
3606 already_switched.add(idpackage)
3607 self.updateProgress(
3608 red("%s %s, %s %s" % (
3609 _("Ignoring"),
3610 bold(atom),
3611 _("already in branch"),
3612 cur_branch,
3613 )
3614 ),
3615 importance = 0,
3616 type = "info",
3617 header = darkgreen(" @@ "),
3618 count = (count, maxcount,)
3619 )
3620 ignored.add(idpackage)
3621 continue
3622
3623 self.updateProgress(
3624 "[%s=>%s] %s" % (
3625 brown(cur_branch),
3626 bold(to_branch),
3627 darkgreen(atom),
3628 ),
3629 importance = 0,
3630 type = "info",
3631 header = darkgreen(" @@ "),
3632 back = True,
3633 count = (count, maxcount,)
3634 )
3635 dbconn.switchBranch(idpackage, to_branch)
3636 dbconn.commitChanges()
3637 switched.add(idpackage)
3638
3639 dbconn.commitChanges()
3640
3641
3642 dbconn.moveCountersToBranch(to_branch, from_branch = from_branch)
3643
3644 self.close_server_database(dbconn)
3645 mytxt = blue("%s.") % (_("migration loop completed"),)
3646 self.updateProgress(
3647 "[%s=>%s] %s" % (
3648 brown(from_branch),
3649 bold(to_branch),
3650 mytxt,
3651 ),
3652 importance = 1,
3653 type = "info",
3654 header = darkgreen(" * ")
3655 )
3656
3657 return switched, already_switched, ignored, not_found, no_checksum
3658
3660
3661 if branch == None:
3662 branch = self.SystemSettings['repositories']['branch']
3663 if repo == None:
3664 repo = self.default_repository
3665
3666 sets_dir = self.get_local_database_sets_dir(repo, branch)
3667 if not (os.path.isdir(sets_dir) and os.access(sets_dir, os.R_OK)):
3668 return {}
3669
3670 mydata = {}
3671 items = os.listdir(sets_dir)
3672 for item in items:
3673
3674 try:
3675 item_clean = str(item)
3676 except (UnicodeEncodeError, UnicodeDecodeError,):
3677 continue
3678 item_path = os.path.join(sets_dir, item)
3679 if not (os.path.isfile(item_path) and \
3680 os.access(item_path, os.R_OK)):
3681 continue
3682 item_elements = self.entropyTools.extract_packages_from_set_file(
3683 item_path)
3684 if item_elements:
3685 mydata[item_clean] = item_elements.copy()
3686
3687 return mydata
3688
3719
3733