1
2 '''
3 # DESCRIPTION:
4 # Entropy Object Oriented Interface
5
6 Copyright (C) 2007-2009 Fabio Erculiani
7
8 This program is free software; you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation; either version 2 of the License, or
11 (at your option) any later version.
12
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 '''
22
23 from __future__ import with_statement
24 import os
25 import shutil
26 from entropy.core import Singleton
27 from entropy.exceptions import OnlineMirrorError, PermissionDenied, \
28 SystemDatabaseError
29 from entropy.const import etpConst, etpSys, const_setup_perms, \
30 const_create_working_dirs, const_extract_srv_repo_params, etpUi
31 from entropy.output import TextInterface, purple, red, darkgreen, \
32 bold, brown, blue, darkred
33 from entropy.server.interfaces.mirrors import Server as MirrorsServer
34 from entropy.i18n import _
35 from entropy.core import SystemSettings, SystemSettingsPlugin
36 from entropy.transceivers import FtpInterface
37 from entropy.db import LocalRepository
38
40
42
43 """
44 Parses Entropy server system configuration file.
45
46 @return dict data
47 """
48
49 data = {
50 'repositories': etpConst['server_repositories'].copy(),
51 'default_repository_id': etpConst['officialserverrepositoryid'],
52 'packages_expiration_days': etpConst['packagesexpirationdays'],
53 'database_file_format': etpConst['etpdatabasefileformat'],
54 'disabled_eapis': set(),
55 'exp_based_scope': etpConst['expiration_based_scope'],
56 'rss': {
57 'enabled': etpConst['rss-feed'],
58 'name': etpConst['rss-name'],
59 'base_url': etpConst['rss-base-url'],
60 'website_url': etpConst['rss-website-url'],
61 'editor': etpConst['rss-managing-editor'],
62 'max_entries': etpConst['rss-max-entries'],
63 'light_max_entries': etpConst['rss-light-max-entries'],
64 },
65 }
66
67 fake_instance = self._helper.fake_default_repo
68
69 if not os.access(etpConst['serverconf'], os.R_OK):
70 return data
71
72 with open(etpConst['serverconf'],"r") as server_f:
73 serverconf = [x.strip() for x in server_f.readlines() if x.strip()]
74
75 for line in serverconf:
76
77 split_line = line.split("|")
78 split_line_len = len(split_line)
79
80 if (line.find("officialserverrepositoryid|") != -1) and \
81 (not line.startswith("#")) and (split_line_len == 2):
82
83 if not fake_instance:
84 data['default_repository_id'] = split_line[1].strip()
85
86 elif line.startswith("expiration-days|") and (split_line_len == 2):
87
88 mydays = split_line[1].strip()
89 try:
90 mydays = int(mydays)
91 data['packages_expiration_days'] = mydays
92 except ValueError:
93 continue
94
95 elif line.startswith("expiration-based-scope|") and \
96 (split_line_len == 2):
97
98 exp_opt = split_line[1].strip().lower()
99 if exp_opt in ("enable", "enabled", "true", "1", "yes"):
100 data['exp_based_scope'] = True
101 else:
102 data['exp_based_scope'] = False
103
104 elif line.startswith("disabled-eapis|") and (split_line_len == 2):
105
106 mydis = split_line[1].strip().split(",")
107 try:
108 mydis = [int(x) for x in mydis]
109 mydis = set([x for x in mydis if x in (1, 2, 3,)])
110 except ValueError:
111 continue
112 if (len(mydis) < 3) and mydis:
113 data['disabled_eapis'] = mydis
114
115
116 elif line.startswith("repository|") and (split_line_len in [5, 6]) \
117 and (not fake_instance):
118
119 repoid, repodata = const_extract_srv_repo_params(line,
120 product = sys_set['repositories']['product'])
121 if repoid in data['repositories']:
122
123 data['repositories'][repoid]['mirrors'].extend(
124 repodata['mirrors'])
125 else:
126 data['repositories'][repoid] = repodata.copy()
127
128 elif line.startswith("database-format|") and (split_line_len == 2):
129
130 fmt = split_line[1]
131 if fmt in etpConst['etpdatabasesupportedcformats']:
132 data['database_file_format'] = fmt
133
134 elif line.startswith("rss-feed|") and (split_line_len == 2):
135
136 feed = split_line[1]
137 if feed in ("enable", "enabled", "true", "1"):
138 data['rss']['enabled'] = True
139 elif feed in ("disable", "disabled", "false", "0", "no",):
140 data['rss']['enabled'] = False
141
142 elif line.startswith("rss-name|") and (split_line_len == 2):
143
144 feedname = line.split("rss-name|")[1].strip()
145 data['rss']['name'] = feedname
146
147 elif line.startswith("rss-base-url|") and (split_line_len == 2):
148
149 data['rss']['base_url'] = line.split("rss-base-url|")[1].strip()
150 if not data['rss']['base_url'][-1] == "/":
151 data['rss']['base_url'] += "/"
152
153 elif line.startswith("rss-website-url|") and (split_line_len == 2):
154
155 data['rss']['website_url'] = split_line[1].strip()
156
157 elif line.startswith("managing-editor|") and (split_line_len == 2):
158
159 data['rss']['editor'] = split_line[1].strip()
160
161 elif line.startswith("max-rss-entries|") and (split_line_len == 2):
162
163 try:
164 entries = int(split_line[1].strip())
165 data['rss']['max_entries'] = entries
166 except (ValueError, IndexError,):
167 continue
168
169 elif line.startswith("max-rss-light-entries|") and \
170 (split_line_len == 2):
171
172 try:
173 entries = int(split_line[1].strip())
174 data['rss']['light_max_entries'] = entries
175 except (ValueError, IndexError,):
176 continue
177
178
179 if self._helper.community_repo:
180 data['repositories'][etpConst['clientserverrepoid']] = {}
181 mydata = {}
182 mydata['description'] = "Community Repositories System Database"
183 mydata['mirrors'] = []
184 mydata['community'] = False
185 data['repositories'][etpConst['clientserverrepoid']].update(mydata)
186
187
188 for repoid in data['repositories']:
189 data['repositories'][repoid]['packages_dir'] = \
190 os.path.join( etpConst['entropyworkdir'],
191 "server",
192 repoid,
193 "packages",
194 etpSys['arch']
195 )
196 data['repositories'][repoid]['store_dir'] = \
197 os.path.join( etpConst['entropyworkdir'],
198 "server",
199 repoid,
200 "store",
201 etpSys['arch']
202 )
203 data['repositories'][repoid]['upload_dir'] = \
204 os.path.join( etpConst['entropyworkdir'],
205 "server",
206 repoid,
207 "upload",
208 etpSys['arch']
209 )
210 data['repositories'][repoid]['database_dir'] = \
211 os.path.join( etpConst['entropyworkdir'],
212 "server",
213 repoid,
214 "database",
215 etpSys['arch']
216 )
217 data['repositories'][repoid]['packages_relative_path'] = \
218 os.path.join( sys_set['repositories']['product'],
219 repoid,
220 "packages",
221 etpSys['arch']
222 )+"/"
223 data['repositories'][repoid]['database_relative_path'] = \
224 os.path.join( sys_set['repositories']['product'],
225 repoid,
226 "database",
227 etpSys['arch']
228 )+"/"
229
230
231 shell_repoid = os.getenv('ETP_REPO')
232 if shell_repoid:
233 data['default_repository_id'] = shell_repoid
234
235 expiration_days = os.getenv('ETP_EXPIRATION_DAYS')
236 if expiration_days:
237 try:
238 expiration_days = int(expiration_days)
239 data['packages_expiration_days'] = expiration_days
240 except ValueError:
241 pass
242
243 return data
244
246
247 import entropy.tools as entropyTools
248
250
251 data = {}
252 srv_plug_id = etpConst['system_settings_plugins_ids']['server_plugin']
253
254 srv_parser_data = sys_set[srv_plug_id]['server']
255 if not srv_parser_data['exp_based_scope']:
256 return data
257
258
259 for repoid in srv_parser_data['repositories']:
260
261
262
263 if repoid == etpConst['clientserverrepoid']:
264 continue
265
266 idpackages = set()
267 exp_fp = self._helper.get_local_exp_based_pkgs_rm_whitelist_file(
268 repo = repoid)
269 dbconn = self._helper.open_server_repository(
270 just_reading = True, repo = repoid)
271
272 if os.access(exp_fp, os.R_OK | os.F_OK):
273 pkgs = self.entropyTools.generic_file_content_parser(exp_fp)
274 if '*' in pkgs:
275 idpackages.add(-1)
276 else:
277 for pkg in pkgs:
278 idpackage, rc_match = dbconn.atomMatch(pkg)
279 if rc_match:
280 continue
281 idpackages.add(idpackage)
282
283 data[repoid] = idpackages
284
285 return data
286
287
288 -class Server(Singleton, TextInterface):
289
290 - def init_singleton(self, default_repository = None, save_repository = False,
291 community_repo = False, fake_default_repo = False,
292 fake_default_repo_id = '::fake::',
293 fake_default_repo_desc = 'this is a fake repository'):
294
295 self.__instance_destroyed = False
296 if etpConst['uid'] != 0:
297 mytxt = _("Entropy Server interface must be run as root")
298 raise PermissionDenied("PermissionDenied: %s" % (mytxt,))
299
300
301 from entropy.misc import LogFile
302 self.SystemSettings = SystemSettings()
303 self.community_repo = community_repo
304 from entropy.db import dbapi2
305 self.dbapi2 = dbapi2
306 etpSys['serverside'] = True
307 self._memory_db_instances = {}
308 self.fake_default_repo = fake_default_repo
309 self.indexing = False
310 self.xcache = False
311 self.MirrorsService = None
312 self.__server_dbcache = {}
313 self.repository_treeupdate_digests = {}
314 self.__settings_to_backup = []
315 self.__do_save_repository = save_repository
316 self.__sync_lock_cache = set()
317 self.rssMessages = {
318 'added': {},
319 'removed': {},
320 'commitmessage': "",
321 'light': {},
322 }
323
324 if fake_default_repo:
325 default_repository = fake_default_repo_id
326 etpConst['officialserverrepositoryid'] = fake_default_repo_id
327 self.init_generic_memory_server_repository(fake_default_repo_id,
328 fake_default_repo_desc)
329
330
331 self.sys_settings_plugin_id = \
332 etpConst['system_settings_plugins_ids']['server_plugin']
333 self.sys_settings_plugin = ServerSystemSettingsPlugin(
334 self.sys_settings_plugin_id, self)
335 self.SystemSettings.add_plugin(self.sys_settings_plugin)
336
337
338 self.sys_settings_fatscope_plugin_id = \
339 etpConst['system_settings_plugins_ids']['server_plugin_fatscope']
340 self.sys_settings_fatscope_plugin = ServerFatscopeSystemSettingsPlugin(
341 self.sys_settings_fatscope_plugin_id, self)
342 self.SystemSettings.add_plugin(self.sys_settings_fatscope_plugin)
343
344 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
345 self.default_repository = default_repository
346 if self.default_repository == None:
347 self.default_repository = srv_set['default_repository_id']
348
349 if self.default_repository in srv_set['repositories']:
350 self.ensure_paths(self.default_repository)
351
352
353
354
355 if self.default_repository not in srv_set['repositories']:
356 raise PermissionDenied("PermissionDenied: %s %s" % (
357 self.default_repository,
358 _("repository not configured"),
359 )
360 )
361 if etpConst['clientserverrepoid'] == self.default_repository:
362 raise PermissionDenied("PermissionDenied: %s %s" % (
363 etpConst['clientserverrepoid'],
364 _("protected repository id, can't use this, sorry dude..."),
365 )
366 )
367
368 self.switch_default_repository(self.default_repository)
369
371 self.__instance_destroyed = True
372 if hasattr(self,'ClientService'):
373 self.ClientService.destroy()
374 if hasattr(self,'sys_settings_server_plugin'):
375 try:
376 self.SystemSettings.remove_plugin(self.sys_settings_plugin_id)
377 except KeyError:
378 pass
379 if hasattr(self,'sys_settings_fatscope_plugin'):
380 try:
381 self.SystemSettings.remove_plugin(
382 self.sys_settings_fatscope_plugin)
383 except KeyError:
384 pass
385 self.close_server_databases()
386
388 return self.__instance_destroyed
389
392
401
402
404 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
405 migrated_filename = '.branch_migrated'
406 for repoid in srv_set['repositories'].keys():
407
408 if repoid == etpConst['clientserverrepoid']: continue
409 mydir = srv_set['repositories'][repoid]['database_dir']
410 if not os.path.isdir(mydir):
411 continue
412
413 migrated_filepath = os.path.join(mydir, migrated_filename)
414 if os.path.isfile(migrated_filepath):
415 continue
416
417 my_branched_dir = self.get_local_database_dir(repoid)
418 if os.path.isdir(my_branched_dir):
419 continue
420
421 self.updateProgress(
422 "[%s:%s] %s: %s, %s: %s" % (
423 brown("repo"),
424 purple(repoid),
425 _("migrating database path from"),
426 brown(mydir),
427 _('to'),
428 brown(my_branched_dir),
429 ),
430 importance = 1,
431 type = "info",
432 header = bold(" @@ ")
433 )
434
435 repo_files = [os.path.join(mydir, x) for x in os.listdir(mydir) if \
436 (os.path.isfile(os.path.join(mydir, x)) and \
437 os.access(os.path.join(mydir, x), os.W_OK))
438 ]
439 os.makedirs(my_branched_dir)
440 const_setup_perms(my_branched_dir, etpConst['entropygid'])
441
442 for repo_file in repo_files:
443 repo_filename = os.path.basename(repo_file)
444 shutil.move(repo_file,
445 os.path.join(my_branched_dir, repo_filename))
446
447 f_migrated = open(migrated_filepath,"w")
448 f_migrated.write("done\n")
449 f_migrated.flush()
450 f_migrated.close()
451
453 self.setup_entropy_settings()
454 cs_name = 'ClientService'
455 if hasattr(self, cs_name):
456 obj = getattr(self, cs_name)
457 obj.destroy()
458 from entropy.client.interfaces import Client
459 self.ClientService = Client(
460 indexing = self.indexing,
461 xcache = self.xcache,
462 repo_validation = False,
463 noclientdb = 1
464 )
465 from entropy.cache import EntropyCacher
466 self.Cacher = EntropyCacher()
467 self.ClientService.updateProgress = self.updateProgress
468 self.validRepositories = self.ClientService.validRepositories
469 self.entropyTools = self.ClientService.entropyTools
470 self.dumpTools = self.ClientService.dumpTools
471 self.QA = self.ClientService.QA
472 self.backup_entropy_settings()
473 self.SpmService = self.ClientService.Spm()
474 self.MirrorsService = MirrorsServer(self)
475
477 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
478 curr_repoid = srv_set['default_repository_id']
479 backup_list = [
480 'etpdatabaseclientfilepath',
481 'clientdbid',
482 {'server': srv_set.copy()},
483 ]
484 for setting in backup_list:
485 if setting not in self.__settings_to_backup:
486 self.__settings_to_backup.append(setting)
487
488 if not self.community_repo:
489 etpConst['etpdatabaseclientfilepath'] = \
490 self.get_local_database_file(repo)
491 etpConst['clientdbid'] = etpConst['serverdbid']
492 const_create_working_dirs()
493
495 if hasattr(self,'serverDbCache'):
496 for item in self.__server_dbcache:
497 try:
498 self.__server_dbcache[item].closeDB()
499 except self.dbapi2.ProgrammingError:
500 pass
501 self.__server_dbcache.clear()
502
504 found = None
505 for item in self.__server_dbcache:
506 if dbinstance == self.__server_dbcache[item]:
507 found = item
508 break
509 if found:
510 instance = self.__server_dbcache.pop(found)
511 instance.closeDB()
512
514 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
515 return srv_set['repositories'].copy()
516
544
546 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
547 if self.community_repo:
548 for repoid in srv_set['repositories']:
549 srv_set['repositories'][repoid]['community'] = True
550
551
553
554 if not self.is_repository_initialized(repoid):
555 mytxt = blue("%s.") % (
556 _("Your default repository is not initialized"),)
557 self.updateProgress(
558 "[%s:%s] %s" % (
559 brown("repo"),
560 purple(repoid),
561 mytxt,
562 ),
563 importance = 1,
564 type = "warning",
565 header = darkred(" !!! ")
566 )
567 answer = self.askQuestion(
568 _("Do you want to initialize your default repository ?"))
569 if answer == "No":
570 mytxt = red("%s.") % (
571 _("Continuing with an uninitialized repository"),)
572 self.updateProgress(
573 "[%s:%s] %s" % (
574 brown("repo"),
575 purple(repoid),
576 mytxt,
577 ),
578 importance = 1,
579 type = "warning",
580 header = darkred(" !!! ")
581 )
582 else:
583
584 dbfile = self.get_local_database_file(repoid)
585 if os.path.isfile(dbfile):
586 shutil.move(dbfile, dbfile+".backup")
587 self.initialize_server_database(empty = True,
588 repo = repoid, warnings = False)
589
590
592 type_txt = _("server-side repository")
593 if self.community_repo:
594 type_txt = _("community repository")
595
596 mytxt = _("Entropy Server Interface Instance on repository")
597 self.updateProgress(
598 blue("%s: %s, %s: %s (%s: %s)" % (
599 mytxt,
600 red(self.default_repository),
601 _("current branch"),
602 darkgreen(self.SystemSettings['repositories']['branch']),
603 purple(_("type")),
604 bold(type_txt),
605 )
606 ),
607 importance = 2,
608 type = "info",
609 header = red(" @@ ")
610 )
611 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
612 repos = srv_set['repositories'].keys()
613 mytxt = blue("%s:") % (_("Currently configured repositories"),)
614 self.updateProgress(
615 mytxt,
616 importance = 1,
617 type = "info",
618 header = red(" @@ ")
619 )
620 for repo in repos:
621 self.updateProgress(
622 darkgreen(repo),
623 importance = 0,
624 type = "info",
625 header = brown(" # ")
626 )
627
628
630
631
632 if repoid == etpConst['clientserverrepoid']:
633 return
634
635 if os.path.isfile(etpConst['serverconf']):
636 f_srv = open(etpConst['serverconf'],"r")
637 content = f_srv.readlines()
638 f_srv.close()
639 content = [x.strip() for x in content]
640 found = False
641 new_content = []
642 for line in content:
643 if line.strip().startswith("officialserverrepositoryid|"):
644 line = "officialserverrepositoryid|%s" % (repoid,)
645 found = True
646 new_content.append(line)
647 if not found:
648 new_content.append("officialserverrepositoryid|%s" % (repoid,))
649 f_srv_t = open(etpConst['serverconf']+".save_default_repo_tmp","w")
650 for line in new_content:
651 f_srv_t.write(line+"\n")
652 f_srv_t.flush()
653 f_srv_t.close()
654 os.rename(etpConst['serverconf']+".save_default_repo_tmp",
655 etpConst['serverconf'])
656 else:
657 f_srv = open(etpConst['serverconf'],"w")
658 f_srv.write("officialserverrepositoryid|%s\n" % (repoid,))
659 f_srv.flush()
660 f_srv.close()
661
663
664
665 if repoid == etpConst['clientserverrepoid']:
666 return False
667
668 if not os.path.isfile(etpConst['serverconf']):
669 return None
670 f_srv = open(etpConst['serverconf'])
671 tmpfile = etpConst['serverconf']+".switch"
672 mycontent = [x.strip() for x in f_srv.readlines()]
673 f_srv.close()
674 f_tmp = open(tmpfile,"w")
675 st = "repository|%s" % (repoid,)
676 status = False
677 for line in mycontent:
678 if enable:
679 if (line.find(st) != -1) and line.startswith("#") and \
680 (len(line.split("|")) == 5):
681 line = line[1:]
682 status = True
683 else:
684 if (line.find(st) != -1) and not line.startswith("#") and \
685 (len(line.split("|")) == 5):
686 line = "#"+line
687 status = True
688 f_tmp.write(line+"\n")
689 f_tmp.flush()
690 f_tmp.close()
691 shutil.move(tmpfile, etpConst['serverconf'])
692 if status:
693 self.close_server_databases()
694 self.SystemSettings.clear()
695 self.setup_services()
696 self.show_interface_status()
697 return status
698
705
714
715 dbc = self.open_server_repository(just_reading = True, repo = repo)
716 valid = do_validate(dbc)
717 self.close_server_database(dbc)
718 if not valid:
719 dbc = self.open_server_repository(read_only = False,
720 no_upload = True, repo = repo, is_new = True)
721 valid = do_validate(dbc)
722 self.close_server_database(dbc)
723
724 return valid
725
727
728 if repo == None:
729 repo = self.default_repository
730
731
732 lock_file = self.MirrorsService.get_database_lockfile(repo)
733 if os.path.isfile(lock_file):
734 self.updateProgress(
735 red(_("Entropy database is already locked by you :-)")),
736 importance = 1,
737 type = "info",
738 header = red(" * ")
739 )
740 else:
741
742 mytxt = "%s ..." % (_("Locking and Syncing Entropy database"),)
743 self.updateProgress(
744 red(mytxt),
745 importance = 1,
746 type = "info",
747 header = red(" * "),
748 back = True
749 )
750 for uri in self.get_remote_mirrors(repo):
751 given_up = self.MirrorsService.mirror_lock_check(uri,
752 repo = repo)
753 if given_up:
754 crippled_uri = \
755 self.entropyTools.extract_ftp_host_from_uri(uri)
756 mytxt = "%s:" % (_("Mirrors status table"),)
757 self.updateProgress(
758 darkgreen(mytxt),
759 importance = 1,
760 type = "info",
761 header = brown(" * ")
762 )
763 dbstatus = self.MirrorsService.get_mirrors_lock(repo = repo)
764 for db_uri, db_st1, db_st2 in dbstatus:
765 db_st1_info = darkgreen(_("Unlocked"))
766 if db_st1:
767 db_st1_info = red(_("Locked"))
768 db_st2_info = darkgreen(_("Unlocked"))
769 if db_st2:
770 db_st2_info = red(_("Locked"))
771
772 crippled_uri = \
773 self.entropyTools.extract_ftp_host_from_uri(db_uri)
774 self.updateProgress(
775 "%s: [%s: %s] [%s: %s]" % (
776 bold(crippled_uri),
777 brown(_("database")),
778 db_st1_info,
779 brown(_("download")),
780 db_st2_info,
781 ),
782 importance = 1,
783 type = "info",
784 header = "\t"
785 )
786
787 raise OnlineMirrorError("OnlineMirrorError: %s %s" % (
788 _("cannot lock mirror"),
789 crippled_uri,
790 )
791 )
792
793
794 self.MirrorsService.lock_mirrors(True, repo = repo)
795 self.MirrorsService.sync_databases(no_upload, repo = repo)
796
797
800
801 if mirrors is None:
802 mirrors = []
803 product = self.SystemSettings['repositories']['product']
804 dbc = self.open_memory_database(dbname = etpConst['serverdbid']+repoid)
805 self._memory_db_instances[repoid] = dbc
806
807 eapi3_port = int(etpConst['socket_service']['port'])
808 eapi3_ssl_port = int(etpConst['socket_service']['ssl_port'])
809
810 repodata = {
811 'repoid': repoid,
812 'description': description,
813 'mirrors': mirrors,
814 'community': community_repo,
815 'service_port': eapi3_port,
816 'ssl_service_port': eapi3_ssl_port,
817 'service_url': service_url,
818 'handler': '',
819 'in_memory': True,
820 }
821
822 etpConst['server_repositories'][repoid] = repodata
823 self.SystemSettings.clear()
824
825 return dbc
826
828 if dbname == None:
829 dbname = etpConst['genericdbid']
830 dbc = LocalRepository(
831 readOnly = False,
832 dbFile = ':memory:',
833 clientDatabase = True,
834 dbname = dbname,
835 xcache = False,
836 indexing = False,
837 OutputInterface = self,
838 skipChecks = True,
839 ServiceInterface = self
840 )
841 dbc.initializeDatabase()
842 return dbc
843
844 - def open_server_repository(
845 self,
846 read_only = True,
847 no_upload = True,
848 just_reading = False,
849 repo = None,
850 indexing = True,
851 warnings = True,
852 do_cache = True,
853 use_branch = None,
854 lock_remote = True,
855 is_new = False,
856 do_treeupdates = True
857 ):
858
859 if repo == None:
860 repo = self.default_repository
861
862 if repo == etpConst['clientserverrepoid'] and self.community_repo:
863 return self.ClientService.clientDbconn
864
865
866 if repo in self._memory_db_instances:
867 return self._memory_db_instances.get(repo)
868
869 if just_reading:
870 read_only = True
871 no_upload = True
872
873 local_dbfile = self.get_local_database_file(repo, use_branch)
874 if do_cache:
875 cached = self.__server_dbcache.get(
876 (repo, etpConst['systemroot'], local_dbfile, read_only,
877 no_upload, just_reading, use_branch, lock_remote,)
878 )
879 if cached != None:
880 return cached
881
882 local_dbfile_dir = os.path.dirname(local_dbfile)
883 if not os.path.isdir(local_dbfile_dir):
884 os.makedirs(local_dbfile_dir)
885
886 if (not read_only) and (lock_remote) and \
887 (repo not in self.__sync_lock_cache):
888 self.do_server_repository_sync_lock(repo, no_upload)
889 self.__sync_lock_cache.add(repo)
890
891 local_dbfile_exists = os.path.lexists(local_dbfile)
892 conn = LocalRepository(
893 readOnly = read_only,
894 dbFile = local_dbfile,
895 noUpload = no_upload,
896 OutputInterface = self,
897 ServiceInterface = self,
898 dbname = etpConst['serverdbid']+repo,
899 useBranch = use_branch,
900 lockRemote = lock_remote
901 )
902 if not local_dbfile_exists:
903
904 conn.readOnly = False
905 conn.initializeDatabase()
906 conn.commitChanges()
907
908 valid = True
909 try:
910 conn.validateDatabase()
911 except SystemDatabaseError:
912 valid = False
913
914
915
916 if (repo not in etpConst['server_treeupdatescalled']) and \
917 (not just_reading):
918
919
920 if valid:
921 if do_treeupdates:
922 conn.serverUpdatePackagesData()
923 elif warnings and not is_new:
924 mytxt = _("Entropy database is corrupted!")
925 self.updateProgress(
926 darkred(mytxt),
927 importance = 1,
928 type = "warning",
929 header = bold(" !!! ")
930 )
931
932 if not read_only and valid and indexing:
933
934 self.updateProgress(
935 "[repo:%s|%s] %s" % (
936 blue(repo),
937 red(_("database")),
938 blue(_("indexing database")),
939 ),
940 importance = 1,
941 type = "info",
942 header = brown(" @@ "),
943 back = True
944 )
945 conn.createAllIndexes()
946
947 if do_cache:
948
949
950 self.__server_dbcache[
951 (repo, etpConst['systemroot'], local_dbfile, read_only,
952 no_upload, just_reading, use_branch, lock_remote,)] = conn
953
954
955 if (not read_only) and (not is_new):
956 cur_sets = conn.retrievePackageSets()
957 sys_sets = self.get_configured_package_sets(repo)
958 if cur_sets != sys_sets:
959 self.update_database_package_sets(repo, dbconn = conn)
960 conn.commitChanges()
961
962 return conn
963
965
966 sys_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
967 server_repos = sys_set['repositories'].keys()
968 installed_packages = set()
969
970 if default_repo:
971 server_repos = [default_repo]
972
973 for repo in server_repos:
974 dbconn = self.open_server_repository(read_only = True,
975 no_upload = True, repo = repo, do_treeupdates = False)
976 installed_packages |= set([(x, repo) for x in \
977 dbconn.listAllIdpackages()])
978
979
980 deps_not_satisfied = set()
981 length = str((len(installed_packages)))
982 count = 0
983 mytxt = _("Checking")
984
985 for idpackage, repo in installed_packages:
986 count += 1
987 dbconn = self.open_server_repository(read_only = True,
988 no_upload = True, repo = repo, do_treeupdates = False)
989
990 if (count%150 == 0) or (count == length) or (count == 1):
991 atom = dbconn.retrieveAtom(idpackage)
992 self.updateProgress(
993 darkgreen(mytxt)+" "+bold(atom),
994 importance = 0,
995 type = "info",
996 back = True,
997 count = (count, length),
998 header = darkred(" @@ ")
999 )
1000
1001 xdeps = dbconn.retrieveDependencies(idpackage)
1002 for xdep in xdeps:
1003 xid, xuseless = self.atom_match(xdep)
1004 if xid == -1:
1005 deps_not_satisfied.add(xdep)
1006
1007 return deps_not_satisfied
1008
1010
1011 mytxt = "%s %s" % (blue(_("Running dependencies test")), red("..."))
1012 self.updateProgress(
1013 mytxt,
1014 importance = 2,
1015 type = "info",
1016 header = red(" @@ ")
1017 )
1018
1019 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
1020 server_repos = srv_set['repositories'].keys()
1021 deps_not_matched = self.deps_tester(repo)
1022
1023 if deps_not_matched:
1024
1025 crying_atoms = {}
1026 for atom in deps_not_matched:
1027 for repo in server_repos:
1028 dbconn = self.open_server_repository(just_reading = True,
1029 repo = repo, do_treeupdates = False)
1030 riddep = dbconn.searchDependency(atom)
1031 if riddep == -1:
1032 continue
1033 ridpackages = dbconn.searchIdpackageFromIddependency(riddep)
1034 for i in ridpackages:
1035 iatom = dbconn.retrieveAtom(i)
1036 if not crying_atoms.has_key(atom):
1037 crying_atoms[atom] = set()
1038 crying_atoms[atom].add((iatom, repo))
1039
1040 mytxt = blue("%s:") % (_("These are the dependencies not found"),)
1041 self.updateProgress(
1042 mytxt,
1043 importance = 1,
1044 type = "info",
1045 header = red(" @@ ")
1046 )
1047 mytxt = "%s:" % (_("Needed by"),)
1048 for atom in deps_not_matched:
1049 self.updateProgress(
1050 red(atom),
1051 importance = 1,
1052 type = "info",
1053 header = blue(" # ")
1054 )
1055 if crying_atoms.has_key(atom):
1056 self.updateProgress(
1057 red(mytxt),
1058 importance = 0,
1059 type = "info",
1060 header = blue(" # ")
1061 )
1062 for my_dep, myrepo in crying_atoms[atom]:
1063 self.updateProgress(
1064 "[%s:%s] %s" % (
1065 blue(_("by repo")),
1066 darkred(myrepo),
1067 darkgreen(my_dep),
1068 ),
1069 importance = 0,
1070 type = "info",
1071 header = blue(" # ")
1072 )
1073 else:
1074
1075 mytxt = blue(_("Every dependency is satisfied. It's all fine."))
1076 self.updateProgress(
1077 mytxt,
1078 importance = 2,
1079 type = "info",
1080 header = red(" @@ ")
1081 )
1082
1083 return deps_not_matched
1084
1086
1087
1088 dbconn = self.open_server_repository(read_only = True,
1089 no_upload = True, repo = repo)
1090 QA = self.QA()
1091 packages_matched, brokenexecs, status = QA.libraries_test(dbconn,
1092 broken_symbols = True)
1093 if status != 0:
1094 return 1, None
1095
1096 if get_files:
1097 return 0, brokenexecs
1098
1099 if (not brokenexecs) and (not packages_matched):
1100 mytxt = "%s." % (_("System is healthy"),)
1101 self.updateProgress(
1102 blue(mytxt),
1103 importance = 2,
1104 type = "info",
1105 header = red(" @@ ")
1106 )
1107 return 0, None
1108
1109 mytxt = "%s..." % (_("Matching libraries with Spm, please wait"),)
1110 self.updateProgress(
1111 blue(mytxt),
1112 importance = 1,
1113 type = "info",
1114 header = red(" @@ ")
1115 )
1116
1117 packages = self.SpmService.query_belongs_multiple(brokenexecs)
1118
1119 if packages:
1120 mytxt = "%s:" % (_("These are the matched packages"),)
1121 self.updateProgress(
1122 red(mytxt),
1123 importance = 1,
1124 type = "info",
1125 header = red(" @@ ")
1126 )
1127 for my_atom, my_elf_id in packages:
1128 package_slot = my_atom, my_elf_id
1129 self.updateProgress(
1130 "%s [elf:%s]" % (
1131 purple(my_atom),
1132 darkgreen(str(my_elf_id)),
1133 ),
1134 importance = 0,
1135 type = "info",
1136 header = red(" # ")
1137 )
1138 for filename in sorted(list(packages[package_slot])):
1139 self.updateProgress(
1140 darkgreen(filename),
1141 importance = 0,
1142 type = "info",
1143 header = brown(" => ")
1144 )
1145
1146 pkgstring = ' '.join(["%s:%s" % (
1147 self.entropyTools.dep_getkey(x[0]), x[1],) for x \
1148 in sorted(packages)])
1149 mytxt = "%s: %s" % (darkgreen(_("Packages string")), pkgstring,)
1150 self.updateProgress(
1151 mytxt,
1152 importance = 1,
1153 type = "info",
1154 header = red(" @@ ")
1155 )
1156 else:
1157 self.updateProgress(
1158 red(_("No matched packages")),
1159 importance = 1,
1160 type = "info",
1161 header = red(" @@ ")
1162 )
1163
1164 return 0, packages
1165
1167
1168 mytxt = "%s %s" % (
1169 blue(_("Running orphaned SPM packages test")), red("..."),)
1170 self.updateProgress(
1171 mytxt,
1172 importance = 2,
1173 type = "info",
1174 header = red(" @@ ")
1175 )
1176 installed_packages, length = self.SpmService.get_installed_packages()
1177 not_found = {}
1178 count = 0
1179 for installed_package in installed_packages:
1180 count += 1
1181 self.updateProgress(
1182 "%s: %s" % (
1183 darkgreen(_("Scanning package")),
1184 brown(installed_package),),
1185 importance = 0,
1186 type = "info",
1187 back = True,
1188 count = (count, length),
1189 header = darkred(" @@ ")
1190 )
1191 key, slot = (self.entropyTools.dep_getkey(installed_package),
1192 self.SpmService.get_installed_package_slot(installed_package),)
1193 pkg_atom = "%s:%s" % (key, slot,)
1194 tree_atom = self.SpmService.get_best_atom(pkg_atom)
1195 if not tree_atom:
1196 not_found[installed_package] = pkg_atom
1197 self.updateProgress(
1198 "%s: %s" % (
1199 blue(pkg_atom),
1200 darkred(_("not found anymore")),
1201 ),
1202 importance = 0,
1203 type = "warning",
1204 count = (count, length),
1205 header = darkred(" @@ ")
1206 )
1207
1208 if not_found:
1209 not_found_list = ' '.join([not_found[x] for x in sorted(not_found)])
1210 self.updateProgress(
1211 "%s: %s" % (
1212 blue(_("Packages string")),
1213 not_found_list,
1214 ),
1215 importance = 0,
1216 type = "warning",
1217 count = (count, length),
1218 header = darkred(" @@ ")
1219 )
1220
1221 return not_found
1222
1229
1236
1238 if dbpath == None:
1239 dbpath = self.get_local_database_file(repo)
1240
1241 dbdir = os.path.dirname(dbpath)
1242 if not os.path.isdir(dbdir):
1243 os.makedirs(dbdir)
1244
1245 mytxt = red("%s ...") % (_("Initializing an empty database"),)
1246 self.updateProgress(
1247 mytxt,
1248 importance = 1,
1249 type = "info",
1250 header = darkgreen(" * "),
1251 back = True
1252 )
1253 dbconn = self.ClientService.open_generic_database(dbpath)
1254 dbconn.initializeDatabase()
1255 dbconn.commitChanges()
1256 dbconn.closeDB()
1257 mytxt = "%s %s %s." % (
1258 red(_("Entropy database file")),
1259 bold(dbpath),
1260 red(_("successfully initialized")),
1261 )
1262 self.updateProgress(
1263 mytxt,
1264 importance = 1,
1265 type = "info",
1266 header = darkgreen(" * ")
1267 )
1268
1269 - def tag_packages(self, package_tag, idpackages, repo = None, ask = True):
1270
1271
1272
1273 try:
1274 package_tag = str(package_tag)
1275 if " " in package_tag:
1276 raise ValueError
1277 except (UnicodeDecodeError, UnicodeEncodeError, ValueError,):
1278 self.updateProgress(
1279 "%s: %s" % (
1280 blue(_("Invalid tag specified")),
1281 package_tag,
1282 ),
1283 importance = 1, type = "error", header = darkred(" !! ")
1284 )
1285 return 1, package_tag
1286
1287 if repo == None:
1288 repo = self.default_repository
1289
1290
1291 invalid_atoms = []
1292 dbconn = self.open_server_repository(read_only = True,
1293 no_upload = True, repo = repo)
1294 for idpackage in idpackages:
1295 ver_tag = dbconn.retrieveVersionTag(idpackage)
1296 if ver_tag:
1297 invalid_atoms.append(dbconn.retrieveAtom(idpackage))
1298
1299 if invalid_atoms:
1300 self.updateProgress(
1301 "%s: %s" % (
1302 blue(_("Packages already tagged, action aborted")),
1303 ', '.join([darkred(unicode(x)) for x in invalid_atoms]),
1304 ),
1305 importance = 1, type = "error", header = darkred(" !! ")
1306 )
1307 return 2, invalid_atoms
1308
1309 matches = [(x, repo) for x in idpackages]
1310 status = 0
1311 data = self.move_packages(
1312 matches, to_repo = repo, from_repo = repo, ask = ask,
1313 do_copy = True, new_tag = package_tag
1314 )
1315 return status, data
1316
1318 """
1319 When creating a new branch, for space reasons, packages are not
1320 moved to a new location. This works fine until old branch is removed.
1321 To avoid inconsistences, before deciding to do that, all the packages
1322 in the old branch should be flushed back to the the currently configured
1323 branch.
1324
1325 @param from_branches -- list of branches to move packages from
1326 @type from_branches -- list
1327 @param repo -- repository to work on
1328 @type repo -- str
1329 @param ask -- user interactivity
1330 @type ask -- bool
1331
1332 @return status
1333 """
1334
1335 status = True
1336 if repo == None:
1337 repo = self.default_repository
1338 branch = self.SystemSettings['repositories']['branch']
1339
1340 if branch in from_branches:
1341 from_branches = [x for x in from_branches if x != branch]
1342
1343 self.updateProgress(
1344 "[%s=>%s|%s] %s" % (
1345 darkgreen(', '.join(from_branches)),
1346 darkred(branch),
1347 brown(repo),
1348 blue(_("flushing back selected packages from branches")),
1349 ),
1350 importance = 2,
1351 type = "info",
1352 header = red(" @@ ")
1353 )
1354
1355 dbconn = self.open_server_repository(read_only = True,
1356 no_upload = True, repo = repo)
1357
1358 idpackage_map = dict(((x, [],) for x in from_branches))
1359 idpackages = dbconn.listAllIdpackages(order_by = 'atom')
1360 for idpackage in idpackages:
1361 download_url = dbconn.retrieveDownloadURL(idpackage)
1362 url_br = self.ClientService.get_branch_from_download_relative_uri(
1363 download_url)
1364 if url_br in from_branches:
1365 idpackage_map[url_br].append(idpackage)
1366
1367 mapped_branches = [x for x in idpackage_map if idpackage_map[x]]
1368 if not mapped_branches:
1369 self.updateProgress(
1370 "[%s=>%s|%s] %s !" % (
1371 darkgreen(', '.join(from_branches)),
1372 darkred(branch),
1373 brown(repo),
1374 blue(_("nothing to do")),
1375 ),
1376 importance = 0,
1377 type = "warning",
1378 header = blue(" @@ ")
1379 )
1380 return status
1381
1382
1383 all_fine = True
1384 tmp_down_dir = self.entropyTools.get_random_temp_file()
1385 os.makedirs(tmp_down_dir)
1386
1387 download_queue = {}
1388 local_up_dir = self.get_local_upload_directory(repo)
1389 local_basedir = os.path.join(local_up_dir, branch)
1390 dbconn = self.open_server_repository(read_only = False,
1391 no_upload = True, repo = repo)
1392
1393 def generate_queue(branch, repo, from_branch, down_q, idpackage_map):
1394
1395 self.updateProgress(
1396 "[%s=>%s|%s] %s" % (
1397 darkgreen(from_branch),
1398 darkred(branch),
1399 brown(repo),
1400 brown(_("these are the packages that will be flushed")),
1401 ),
1402 importance = 1,
1403 type = "info",
1404 header = brown(" @@ ")
1405 )
1406
1407
1408 for idpackage in idpackage_map[from_branch]:
1409 atom = dbconn.retrieveAtom(idpackage)
1410 self.updateProgress(
1411 "[%s=>%s|%s] %s" % (
1412 darkgreen(from_branch),
1413 darkred(branch),
1414 brown(repo),
1415 purple(atom),
1416 ),
1417 importance = 0,
1418 type = "info",
1419 header = blue(" # ")
1420 )
1421 pkg_fp = os.path.basename(dbconn.retrieveDownloadURL(idpackage))
1422 pkg_fp = os.path.join(tmp_down_dir, pkg_fp)
1423 down_q.append((pkg_fp, idpackage,))
1424
1425
1426 for from_branch in sorted(mapped_branches):
1427
1428 download_queue[from_branch] = []
1429 all_fine = False
1430 generate_queue(branch, repo, from_branch,
1431 download_queue[from_branch], idpackage_map)
1432
1433 if ask:
1434 rc_question = self.askQuestion(
1435 _("Would you like to continue ?"))
1436 if rc_question == "No":
1437 continue
1438
1439 remote_relative_path = self.get_remote_packages_relative_path(repo)
1440
1441 for uri in self.get_remote_mirrors(repo):
1442
1443 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri)
1444 ftp_basedir = os.path.join(remote_relative_path, from_branch)
1445
1446 downloader_queue = [x[0] for x in download_queue[from_branch]]
1447 downloader = self.MirrorsService.FtpServerHandler(
1448 FtpInterface,
1449 self,
1450 [uri],
1451 downloader_queue,
1452 critical_files = downloader_queue,
1453 use_handlers = True,
1454 ftp_basedir = ftp_basedir,
1455 local_basedir = tmp_down_dir,
1456 download = True,
1457 repo = repo
1458 )
1459
1460 errors, m_fine_uris, m_broken_uris = downloader.go()
1461
1462 if not errors:
1463 for downloaded_path, idpackage in \
1464 download_queue[from_branch]:
1465
1466 self.updateProgress(
1467 "[%s=>%s|%s|%s] %s: %s" % (
1468 darkgreen(from_branch),
1469 darkred(branch),
1470 brown(repo),
1471 dbconn.retrieveAtom(idpackage),
1472 blue(_("checking package hash")),
1473 darkgreen(os.path.basename(downloaded_path)),
1474 ),
1475 importance = 0,
1476 type = "info",
1477 header = brown(" "),
1478 back = True
1479 )
1480
1481 md5hash = self.entropyTools.md5sum(downloaded_path)
1482 db_md5hash = dbconn.retrieveDigest(idpackage)
1483 if md5hash != db_md5hash:
1484 errors = True
1485 self.updateProgress(
1486 "[%s=>%s|%s|%s] %s: %s" % (
1487 darkgreen(from_branch),
1488 darkred(branch),
1489 brown(repo),
1490 dbconn.retrieveAtom(idpackage),
1491 blue(_("hash does not match for")),
1492 darkgreen(os.path.basename(downloaded_path)),
1493 ),
1494 importance = 0,
1495 type = "error",
1496 header = brown(" ")
1497 )
1498 continue
1499
1500
1501
1502 if errors:
1503 reason = _("wrong md5")
1504 if m_broken_uris:
1505 my_broken_uris = [
1506 (self.entropyTools.extract_ftp_host_from_uri(x), y,) \
1507 for x, y in m_broken_uris]
1508 reason = my_broken_uris[0][1]
1509
1510 self.updateProgress(
1511 "[%s=>%s|%s] %s, %s: %s" % (
1512 darkgreen(from_branch),
1513 darkred(branch),
1514 brown(repo),
1515 blue(_("download errors")),
1516 blue(_("reason")),
1517 reason,
1518 ),
1519 importance = 1,
1520 type = "error",
1521 header = darkred(" !!! ")
1522 )
1523
1524 continue
1525
1526 all_fine = True
1527
1528 self.updateProgress(
1529 "[%s=>%s|%s] %s: %s" % (
1530 darkgreen(from_branch),
1531 darkred(branch),
1532 brown(repo),
1533 blue(_("download completed successfully")),
1534 darkgreen(crippled_uri),
1535 ),
1536 importance = 1,
1537 type = "info",
1538 header = darkgreen(" * ")
1539 )
1540
1541 if not all_fine:
1542 self.updateProgress(
1543 "[%s=>%s|%s] %s" % (
1544 darkgreen(', '.join(from_branches)),
1545 darkred(branch),
1546 brown(repo),
1547 blue(_("error downloading packages from mirrors")),
1548 ),
1549 importance = 2,
1550 type = "error",
1551 header = darkred(" !!! ")
1552 )
1553 return False
1554
1555 tmp_db_path = self.entropyTools.get_random_temp_file()
1556 for from_branch in sorted(mapped_branches):
1557
1558 self.updateProgress(
1559 "[%s=>%s|%s] %s: %s" % (
1560 darkgreen(from_branch),
1561 darkred(branch),
1562 brown(repo),
1563 blue(_("working on branch")),
1564 darkgreen(from_branch),
1565 ),
1566 importance = 1,
1567 type = "info",
1568 header = brown(" @@ ")
1569 )
1570
1571 down_queue = download_queue[from_branch]
1572 for package_path, idpackage in down_queue:
1573
1574 self.updateProgress(
1575 "[%s=>%s|%s] %s: %s" % (
1576 darkgreen(from_branch),
1577 darkred(branch),
1578 brown(repo),
1579 blue(_("updating package")),
1580 darkgreen(os.path.basename(package_path)),
1581 ),
1582 importance = 1,
1583 type = "info",
1584 header = brown(" "),
1585 back = True
1586 )
1587
1588
1589 package_name = os.path.basename(package_path)
1590 new_package_path = os.path.join(local_basedir, package_name)
1591 try:
1592 os.rename(package_path, new_package_path)
1593 except OSError:
1594 shutil.move(package_path, new_package_path)
1595
1596
1597 self.entropyTools.create_md5_file(new_package_path)
1598
1599
1600 download_url = dbconn.retrieveDownloadURL(idpackage)
1601 download_url = \
1602 self.ClientService.swap_branch_in_download_relative_uri(
1603 branch, download_url)
1604 dbconn.setDownloadURL(idpackage, download_url)
1605 dbconn.switchBranch(idpackage, branch)
1606 dbconn.commitChanges()
1607
1608 self.updateProgress(
1609 "[%s=>%s|%s] %s: %s" % (
1610 darkgreen(from_branch),
1611 darkred(branch),
1612 brown(repo),
1613 blue(_("package flushed")),
1614 darkgreen(os.path.basename(package_path)),
1615 ),
1616 importance = 1,
1617 type = "info",
1618 header = brown(" ")
1619 )
1620
1621 try:
1622 os.rmdir(tmp_down_dir)
1623 except OSError:
1624 pass
1625
1626 return True
1627
1628 - def move_packages(self, matches, to_repo, from_repo = None, ask = True,
1629 do_copy = False, new_tag = None, pull_deps = False):
1630
1631 if from_repo == None:
1632 from_repo = self.default_repository
1633 switched = set()
1634
1635 my_matches = list(matches)
1636
1637
1638 if etpConst['clientserverrepoid'] in (to_repo, from_repo):
1639 self.updateProgress(
1640 "%s: %s" % (
1641 blue(_("Cannot touch system database")),
1642 red(etpConst['clientserverrepoid']),
1643 ),
1644 importance = 2, type = "warning", header = darkred(" @@ ")
1645 )
1646 return switched
1647
1648 if not my_matches and from_repo:
1649 dbconn = self.open_server_repository(read_only = True,
1650 no_upload = True, repo = from_repo)
1651 my_matches = set( \
1652 [(x, from_repo) for x in \
1653 dbconn.listAllIdpackages()]
1654 )
1655
1656 mytxt = _("Preparing to move selected packages to")
1657 if do_copy:
1658 mytxt = _("Preparing to copy selected packages to")
1659 self.updateProgress(
1660 "%s %s:" % (
1661 blue(mytxt),
1662 red(to_repo),
1663 ),
1664 importance = 2,
1665 type = "info",
1666 header = red(" @@ ")
1667 )
1668 self.updateProgress(
1669 "%s: %s" % (
1670 bold(_("Note")),
1671 red(_("all old packages with conflicting scope will be " \
1672 "removed from destination repo unless injected")),
1673 ),
1674 importance = 1,
1675 type = "info",
1676 header = red(" @@ ")
1677 )
1678
1679 new_tag_string = ''
1680 if new_tag != None:
1681 new_tag_string = "[%s: %s]" % (darkgreen(_("new tag")),
1682 brown(new_tag),)
1683
1684 my_qa = self.QA()
1685 branch = self.SystemSettings['repositories']['branch']
1686 pull_deps_matches = []
1687 for idpackage, repo in my_matches:
1688 dbconn = self.open_server_repository(read_only = True,
1689 no_upload = True, repo = repo)
1690 self.updateProgress(
1691 "[%s=>%s|%s] %s " % (
1692 darkgreen(repo),
1693 darkred(to_repo),
1694 brown(branch),
1695 blue(dbconn.retrieveAtom(idpackage)),
1696 ) + new_tag_string,
1697 importance = 0,
1698 type = "info",
1699 header = brown(" # ")
1700 )
1701
1702 if pull_deps:
1703 dep_idpackages = my_qa.get_deep_dependency_list(dbconn,
1704 idpackage)
1705 for dep_idpackage in dep_idpackages:
1706
1707 my_dep_match = (dep_idpackage, repo,)
1708 if my_dep_match in pull_deps_matches:
1709 continue
1710 if my_dep_match in my_matches:
1711 continue
1712
1713 pull_deps_matches.append(my_dep_match)
1714 self.updateProgress(
1715 "[%s|%s] %s" % (
1716 brown(branch),
1717 blue(_("dependency")),
1718 purple(dbconn.retrieveAtom(dep_idpackage)),
1719 ),
1720 importance = 0,
1721 type = "info",
1722 header = purple(" >> ")
1723 )
1724
1725 if pull_deps:
1726
1727 my_matches = pull_deps_matches + [x for x in my_matches if x not \
1728 in pull_deps_matches]
1729
1730 if ask:
1731 rc_question = self.askQuestion(_("Would you like to continue ?"))
1732 if rc_question == "No":
1733 return switched
1734
1735 for idpackage, repo in my_matches:
1736 dbconn = self.open_server_repository(read_only = False,
1737 no_upload = True, repo = repo)
1738 match_branch = dbconn.retrieveBranch(idpackage)
1739 match_atom = dbconn.retrieveAtom(idpackage)
1740 package_filename = os.path.basename(
1741 dbconn.retrieveDownloadURL(idpackage))
1742 self.updateProgress(
1743 "[%s=>%s|%s] %s: %s" % (
1744 darkgreen(repo),
1745 darkred(to_repo),
1746 brown(branch),
1747 blue(_("switching")),
1748 darkgreen(match_atom),
1749 ),
1750 importance = 0,
1751 type = "info",
1752 header = red(" @@ "),
1753 back = True
1754 )
1755
1756 from_file = os.path.join(self.get_local_packages_directory(repo),
1757 match_branch, package_filename)
1758 if not os.path.isfile(from_file):
1759 from_file = os.path.join(self.get_local_upload_directory(repo),
1760 match_branch, package_filename)
1761 if not os.path.isfile(from_file):
1762 self.updateProgress(
1763 "[%s=>%s|%s] %s: %s -> %s" % (
1764 darkgreen(repo),
1765 darkred(to_repo),
1766 brown(branch),
1767 bold(_("cannot switch, package not found, skipping")),
1768 darkgreen(),
1769 red(from_file),
1770 ),
1771 importance = 1,
1772 type = "warning",
1773 header = darkred(" !!! ")
1774 )
1775 continue
1776
1777 if new_tag != None:
1778 match_category = dbconn.retrieveCategory(idpackage)
1779 match_name = dbconn.retrieveName(idpackage)
1780 match_version = dbconn.retrieveVersion(idpackage)
1781 tagged_package_filename = \
1782 self.entropyTools.create_package_filename(
1783 match_category, match_name, match_version, new_tag)
1784 to_file = os.path.join(self.get_local_upload_directory(to_repo),
1785 match_branch, tagged_package_filename)
1786 else:
1787 to_file = os.path.join(self.get_local_upload_directory(to_repo),
1788 match_branch, package_filename)
1789 if not os.path.isdir(os.path.dirname(to_file)):
1790 os.makedirs(os.path.dirname(to_file))
1791
1792 copy_data = [
1793 (from_file, to_file,),
1794 (from_file + etpConst['packagesmd5fileext'],
1795 to_file + etpConst['packagesmd5fileext'],),
1796 (from_file + etpConst['packagesexpirationfileext'],
1797 to_file + etpConst['packagesexpirationfileext'],)
1798 ]
1799
1800 for from_item, to_item in copy_data:
1801 self.updateProgress(
1802 "[%s=>%s|%s] %s: %s" % (
1803 darkgreen(repo),
1804 darkred(to_repo),
1805 brown(branch),
1806 blue(_("moving file")),
1807 darkgreen(os.path.basename(from_item)),
1808 ),
1809 importance = 0,
1810 type = "info",
1811 header = red(" @@ "),
1812 back = True
1813 )
1814 if os.path.isfile(from_item):
1815 shutil.copy2(from_item, to_item)
1816
1817 self.updateProgress(
1818 "[%s=>%s|%s] %s: %s" % (
1819 darkgreen(repo),
1820 darkred(to_repo),
1821 brown(branch),
1822 blue(_("loading data from source database")),
1823 darkgreen(repo),
1824 ),
1825 importance = 0,
1826 type = "info",
1827 header = red(" @@ "),
1828 back = True
1829 )
1830
1831 data = dbconn.getPackageData(idpackage)
1832 if new_tag != None:
1833 data['versiontag'] = new_tag
1834
1835 todbconn = self.open_server_repository(read_only = False,
1836 no_upload = True, repo = to_repo)
1837
1838 self.updateProgress(
1839 "[%s=>%s|%s] %s: %s" % (
1840 darkgreen(repo),
1841 darkred(to_repo),
1842 brown(branch),
1843 blue(_("injecting data to destination database")),
1844 darkgreen(to_repo),
1845 ),
1846 importance = 0,
1847 type = "info",
1848 header = red(" @@ "),
1849 back = True
1850 )
1851 new_idpackage, new_revision, new_data = todbconn.handlePackage(data)
1852 del data
1853 todbconn.commitChanges()
1854
1855 if not do_copy:
1856 self.updateProgress(
1857 "[%s=>%s|%s] %s: %s" % (
1858 darkgreen(repo),
1859 darkred(to_repo),
1860 brown(branch),
1861 blue(_("removing entry from source database")),
1862 darkgreen(repo),
1863 ),
1864 importance = 0,
1865 type = "info",
1866 header = red(" @@ "),
1867 back = True
1868 )
1869
1870
1871 dbconn.removePackage(idpackage)
1872 dbconn.commitChanges()
1873
1874 self.updateProgress(
1875 "[%s=>%s|%s] %s: %s" % (
1876 darkgreen(repo),
1877 darkred(to_repo),
1878 brown(branch),
1879 blue(_("successfully handled atom")),
1880 darkgreen(match_atom),
1881 ),
1882 importance = 0,
1883 type = "info",
1884 header = blue(" @@ ")
1885 )
1886 switched.add((idpackage, repo,))
1887
1888
1889 self.dependencies_test(to_repo)
1890
1891 return switched
1892
1893
1895
1896 if repo == None:
1897 repo = self.default_repository
1898
1899 upload_dir = os.path.join(self.get_local_upload_directory(repo),
1900 self.SystemSettings['repositories']['branch'])
1901 if not os.path.isdir(upload_dir):
1902 os.makedirs(upload_dir)
1903
1904 dbconn = self.open_server_repository(read_only = False,
1905 no_upload = True, repo = repo)
1906 self.updateProgress(
1907 red("[repo: %s] %s: %s" % (
1908 darkgreen(repo),
1909 _("adding package"),
1910 bold(os.path.basename(package_file)),
1911 )
1912 ),
1913 importance = 1,
1914 type = "info",
1915 header = brown(" * "),
1916 back = True
1917 )
1918 mydata = self.SpmService.extract_pkg_metadata(package_file,
1919 inject = inject)
1920 idpackage, revision, mydata = dbconn.handlePackage(mydata)
1921
1922
1923 trashing_counters = set()
1924 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
1925 myserver_repos = srv_set['repositories'].keys()
1926 for myrepo in myserver_repos:
1927 mydbconn = self.open_server_repository(read_only = True,
1928 no_upload = True, repo = myrepo)
1929 mylist = mydbconn.retrieve_packages_to_remove(
1930 mydata['name'],
1931 mydata['category'],
1932 mydata['slot'],
1933 mydata['injected']
1934 )
1935 for myitem in mylist:
1936 trashing_counters.add(mydbconn.retrieveCounter(myitem))
1937
1938 for mycounter in trashing_counters:
1939 dbconn.setTrashedCounter(mycounter)
1940
1941
1942 dbconn.removePackageFromInstalledTable(idpackage)
1943 dbconn.addPackageToInstalledTable(idpackage, repo)
1944 atom = dbconn.retrieveAtom(idpackage)
1945
1946 self.updateProgress(
1947 "[repo:%s] %s: %s %s: %s" % (
1948 darkgreen(repo),
1949 blue(_("added package")),
1950 darkgreen(atom),
1951 blue(_("rev")),
1952 bold(str(revision)),
1953 ),
1954 importance = 1,
1955 type = "info",
1956 header = red(" @@ ")
1957 )
1958
1959 manual_deps = sorted(dbconn.retrieveManualDependencies(idpackage))
1960 if manual_deps:
1961 self.updateProgress(
1962 "[repo:%s] %s: %s" % (
1963 darkgreen(repo),
1964 blue(_("manual dependencies for")),
1965 darkgreen(atom),
1966 ),
1967 importance = 1,
1968 type = "warning",
1969 header = darkgreen(" ## ")
1970 )
1971 for m_dep in manual_deps:
1972 self.updateProgress(
1973 brown(m_dep),
1974 importance = 1,
1975 type = "warning",
1976 header = darkred(" # ")
1977 )
1978
1979 download_url = self._setup_repository_package_filename(idpackage,
1980 repo = repo)
1981 downloadfile = os.path.basename(download_url)
1982 destination_path = os.path.join(upload_dir, downloadfile)
1983 try:
1984 os.rename(package_file, destination_path)
1985 except OSError:
1986 shutil.move(package_file, destination_path)
1987
1988 dbconn.commitChanges()
1989 return idpackage, destination_path
1990
1991
1993
1994 dbconn = self.open_server_repository(read_only = False,
1995 no_upload = True, repo = repo)
1996
1997 downloadurl = dbconn.retrieveDownloadURL(idpackage)
1998 packagerev = dbconn.retrieveRevision(idpackage)
1999 downloaddir = os.path.dirname(downloadurl)
2000 downloadfile = os.path.basename(downloadurl)
2001
2002 downloadfile = downloadfile[:-5]+"~%s%s" % (packagerev,
2003 etpConst['packagesext'],)
2004 downloadurl = os.path.join(downloaddir, downloadfile)
2005
2006
2007 dbconn.setDownloadURL(idpackage, downloadurl)
2008
2009 return downloadurl
2010
2013
2014 if repo == None:
2015 repo = self.default_repository
2016
2017 mycount = 0
2018 maxcount = len(packages_data)
2019 idpackages_added = set()
2020 to_be_injected = set()
2021 my_qa = self.QA()
2022 missing_deps_taint = False
2023 for package_filepath, inject in packages_data:
2024
2025 mycount += 1
2026 self.updateProgress(
2027 "[repo:%s] %s: %s" % (
2028 darkgreen(repo),
2029 blue(_("adding package")),
2030 darkgreen(os.path.basename(package_filepath)),
2031 ),
2032 importance = 1,
2033 type = "info",
2034 header = blue(" @@ "),
2035 count = (mycount, maxcount,)
2036 )
2037
2038 try:
2039
2040 idpackage, destination_path = self.package_injector(
2041 package_filepath,
2042 inject = inject,
2043 repo = repo
2044 )
2045 idpackages_added.add(idpackage)
2046 to_be_injected.add((idpackage, destination_path))
2047 except Exception, err:
2048 self.entropyTools.print_traceback()
2049 self.updateProgress(
2050 "[repo:%s] %s: %s" % (
2051 darkgreen(repo),
2052 darkred(_("Exception caught, closing tasks")),
2053 darkgreen(unicode(err)),
2054 ),
2055 importance = 1,
2056 type = "error",
2057 header = bold(" !!! "),
2058 count = (mycount, maxcount,)
2059 )
2060
2061 self.depends_table_initialize(repo)
2062
2063 self.library_paths_table_initialize(repo)
2064 if idpackages_added:
2065 dbconn = self.open_server_repository(read_only = False,
2066 no_upload = True, repo = repo)
2067 missing_deps_taint = my_qa.scan_missing_dependencies(
2068 idpackages_added,
2069 dbconn,
2070 ask = ask,
2071 repo = repo,
2072 self_check = True,
2073 black_list = \
2074 self.get_missing_dependencies_blacklist(
2075 repo = repo),
2076 black_list_adder = \
2077 self.add_missing_dependencies_blacklist_items
2078 )
2079 my_qa.test_depends_linking(idpackages_added, dbconn,
2080 repo = repo)
2081 if to_be_injected:
2082 self.inject_database_into_packages(to_be_injected,
2083 repo = repo)
2084
2085 if missing_deps_taint:
2086 self.depends_table_initialize(repo)
2087 self.close_server_databases()
2088 raise
2089
2090
2091 self.depends_table_initialize(repo)
2092
2093 self.library_paths_table_initialize(repo)
2094
2095 if idpackages_added:
2096 dbconn = self.open_server_repository(read_only = False,
2097 no_upload = True, repo = repo)
2098 missing_deps_taint = my_qa.scan_missing_dependencies(
2099 idpackages_added,
2100 dbconn,
2101 ask = ask,
2102 repo = repo,
2103 self_check = True,
2104 black_list = \
2105 self.get_missing_dependencies_blacklist(repo = repo),
2106 black_list_adder = \
2107 self.add_missing_dependencies_blacklist_items
2108 )
2109 my_qa.test_depends_linking(idpackages_added, dbconn, repo = repo)
2110
2111
2112 if missing_deps_taint:
2113 self.depends_table_initialize(repo)
2114
2115
2116 self.inject_database_into_packages(to_be_injected, repo = repo)
2117
2118 return idpackages_added
2119
2120
2122
2123 if repo == None:
2124 repo = self.default_repository
2125
2126
2127 self.updateProgress(
2128 "[repo:%s] %s:" % (
2129 darkgreen(repo),
2130 blue(_("Injecting entropy metadata into built packages")),
2131 ),
2132 importance = 1,
2133 type = "info",
2134 header = red(" @@ ")
2135 )
2136
2137 dbconn = self.open_server_repository(read_only = False,
2138 no_upload = True, repo = repo)
2139 for idpackage, package_path in injection_data:
2140 self.updateProgress(
2141 "[repo:%s|%s] %s: %s" % (
2142 darkgreen(repo),
2143 brown(str(idpackage)),
2144 blue(_("injecting entropy metadata")),
2145 darkgreen(os.path.basename(package_path)),
2146 ),
2147 importance = 1,
2148 type = "info",
2149 header = blue(" @@ "),
2150 back = True
2151 )
2152 data = dbconn.getPackageData(idpackage)
2153 treeupdates_actions = dbconn.listAllTreeUpdatesActions()
2154 dbpath = self.ClientService.inject_entropy_database_into_package(
2155 package_path, data, treeupdates_actions)
2156 digest = self.entropyTools.md5sum(package_path)
2157
2158 dbconn.setDigest(idpackage, digest)
2159
2160 signatures = data['signatures'].copy()
2161 for hash_key in sorted(signatures):
2162 hash_func = getattr(self.entropyTools, hash_key)
2163 signatures[hash_key] = hash_func(package_path)
2164 dbconn.setSignatures(idpackage, signatures)
2165 self.entropyTools.create_md5_file(package_path)
2166
2167 os.remove(dbpath)
2168 self.updateProgress(
2169 "[repo:%s|%s] %s: %s" % (
2170 darkgreen(repo),
2171 brown(str(idpackage)),
2172 blue(_("injection complete")),
2173 darkgreen(os.path.basename(package_path)),
2174 ),
2175 importance = 1,
2176 type = "info",
2177 header = red(" @@ ")
2178 )
2179 dbconn.commitChanges()
2180
2182 self.updateProgress(
2183 "[%s] %s" % (
2184 red(_("config files")),
2185 blue(_("checking system")),
2186 ),
2187 importance = 1,
2188 type = "info",
2189 header = blue(" @@ "),
2190 back = True
2191 )
2192
2193 scandata = self.ClientService.FileUpdates.scanfs(dcache = False)
2194 if scandata:
2195 self.updateProgress(
2196 "[%s] %s" % (
2197 red(_("config files")),
2198 blue(_("there are configuration files not updated yet")),
2199 ),
2200 importance = 1,
2201 type = "error",
2202 header = darkred(" @@ ")
2203 )
2204 for key in scandata:
2205 self.updateProgress(
2206 "%s" % (brown(etpConst['systemroot'] + \
2207 scandata[key]['destination'])),
2208 importance = 1,
2209 type = "info",
2210 header = "\t"
2211 )
2212 return True
2213 return False
2214
2216 return self.SpmService.quickpkg(atom, storedir)
2217
2218
2220
2221 if repo == None:
2222 repo = self.default_repository
2223
2224 dbconn = self.open_server_repository(read_only = False,
2225 no_upload = True, repo = repo)
2226 for idpackage in idpackages:
2227 atom = dbconn.retrieveAtom(idpackage)
2228 self.updateProgress(
2229 "[repo:%s] %s: %s" % (
2230 darkgreen(repo),
2231 blue(_("removing package")),
2232 darkgreen(atom),
2233 ),
2234 importance = 1,
2235 type = "info",
2236 header = brown(" @@ ")
2237 )
2238 dbconn.removePackage(idpackage)
2239 self.close_server_database(dbconn)
2240 self.updateProgress(
2241 "[repo:%s] %s" % (
2242 darkgreen(repo),
2243 blue(_("removal complete")),
2244 ),
2245 importance = 1,
2246 type = "info",
2247 header = brown(" @@ ")
2248 )
2249
2250
2256
2258 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2259 if repo == None:
2260 repo = self.default_repository
2261 return srv_set['repositories'][repo]['mirrors'][:]
2262
2264 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2265 if repo == None:
2266 repo = self.default_repository
2267 return srv_set['repositories'][repo]['packages_relative_path']
2268
2270 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2271 if repo == None:
2272 repo = self.default_repository
2273 return srv_set['repositories'][repo]['database_relative_path']
2274
2276 if repo == None:
2277 repo = self.default_repository
2278 return os.path.join(self.get_local_database_dir(repo, branch),
2279 etpConst['etpdatabasefile'])
2280
2282 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2283 if repo == None:
2284 repo = self.default_repository
2285 return srv_set['repositories'][repo]['store_dir']
2286
2288 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2289 if repo == None:
2290 repo = self.default_repository
2291 return srv_set['repositories'][repo]['upload_dir']
2292
2294 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2295 if repo == None:
2296 repo = self.default_repository
2297 return srv_set['repositories'][repo]['packages_dir']
2298
2300 if repo == None:
2301 repo = self.default_repository
2302 return os.path.join(self.get_local_database_dir(repo, branch),
2303 etpConst['etpdatabasetaintfile'])
2304
2306 if repo == None:
2307 repo = self.default_repository
2308 return os.path.join(self.get_local_database_dir(repo, branch),
2309 etpConst['etpdatabaserevisionfile'])
2310
2312 if repo == None:
2313 repo = self.default_repository
2314 return os.path.join(self.get_local_database_dir(repo, branch),
2315 etpConst['etpdatabasetimestampfile'])
2316
2318 if repo == None:
2319 repo = self.default_repository
2320 return os.path.join(self.get_local_database_dir(repo, branch),
2321 etpConst['etpdatabasecacertfile'])
2322
2324 if repo == None:
2325 repo = self.default_repository
2326 return os.path.join(self.get_local_database_dir(repo, branch),
2327 etpConst['etpdatabaseservercertfile'])
2328
2330 if repo == None:
2331 repo = self.default_repository
2332 return os.path.join(self.get_local_database_dir(repo, branch),
2333 etpConst['etpdatabasemaskfile'])
2334
2336 if repo == None:
2337 repo = self.default_repository
2338 return os.path.join(self.get_local_database_dir(repo, branch),
2339 etpConst['etpdatabasesytemmaskfile'])
2340
2342 if repo == None:
2343 repo = self.default_repository
2344 return os.path.join(self.get_local_database_dir(repo, branch),
2345 etpConst['etpdatabaseconflictingtaggedfile'])
2346
2349
2350 if repo == None:
2351 repo = self.default_repository
2352 return os.path.join(self.get_local_database_dir(repo, branch),
2353 etpConst['etpdatabaselicwhitelistfile'])
2354
2356 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2357 if repo == None:
2358 repo = self.default_repository
2359 return os.path.join(self.get_local_database_dir(repo, branch),
2360 srv_set['rss']['name'])
2361
2363 if repo == None:
2364 repo = self.default_repository
2365 return os.path.join(self.get_local_database_dir(repo, branch),
2366 etpConst['rss-light-name'])
2367
2369 if repo == None:
2370 repo = self.default_repository
2371 return os.path.join(self.get_local_database_dir(repo, branch),
2372 etpConst['rss-notice-board'])
2373
2375 if repo == None:
2376 repo = self.default_repository
2377 return os.path.join(self.get_local_database_dir(repo, branch),
2378 etpConst['etpdatabaseupdatefile'])
2379
2387
2395
2398 if repo == None:
2399 repo = self.default_repository
2400 return os.path.join(self.get_local_database_dir(repo, branch),
2401 etpConst['etpdatabaseexpbasedpkgsrm'])
2402
2404 if repo == None:
2405 repo = self.default_repository
2406 return os.path.join(self.get_local_database_dir(repo, branch),
2407 etpConst['etpdatabasepkglist'])
2408
2410 if repo == None:
2411 repo = self.default_repository
2412 return os.path.join(self.get_local_database_dir(repo, branch),
2413 etpConst['confsetsdirname'])
2414
2416 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2417 if repo == None:
2418 repo = self.default_repository
2419 if branch == None:
2420 branch = self.SystemSettings['repositories']['branch']
2421 return os.path.join(srv_set['repositories'][repo]['database_dir'],
2422 branch)
2423
2426 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2427 if repo == None:
2428 repo = self.default_repository
2429 if branch == None:
2430 branch = self.SystemSettings['repositories']['branch']
2431 return os.path.join(srv_set['repositories'][repo]['database_dir'],
2432 branch, etpConst['etpdatabasemissingdepsblfile'])
2433
2435 if repo == None:
2436 repo = self.default_repository
2437 if branch == None:
2438 branch = self.SystemSettings['repositories']['branch']
2439 wl_file = self.get_missing_dependencies_blacklist_file(repo, branch)
2440 wl_data = []
2441 if os.path.isfile(wl_file) and os.access(wl_file, os.R_OK):
2442 f_wl = open(wl_file,"r")
2443 wl_data = [x.strip() for x in f_wl.readlines() if x.strip() and \
2444 not x.strip().startswith("#")]
2445 f_wl.close()
2446 return set(wl_data)
2447
2450
2451 if repo == None:
2452 repo = self.default_repository
2453 if branch == None:
2454 branch = self.SystemSettings['repositories']['branch']
2455 wl_file = self.get_missing_dependencies_blacklist_file(repo, branch)
2456 wl_dir = os.path.dirname(wl_file)
2457 if not (os.path.isdir(wl_dir) and os.access(wl_dir, os.W_OK)):
2458 return
2459 if os.path.isfile(wl_file) and not os.access(wl_file, os.W_OK):
2460 return
2461 f_wl = open(wl_file,"a+")
2462 f_wl.write('\n'.join(items)+'\n')
2463 f_wl.flush()
2464 f_wl.close()
2465
2467
2468 if repo == None:
2469 repo = self.default_repository
2470
2471 dbrev_file = self.get_local_database_revision_file(repo)
2472 if os.path.isfile(dbrev_file):
2473 f_rev = open(dbrev_file)
2474 rev = f_rev.readline().strip()
2475 f_rev.close()
2476 try:
2477 rev = int(rev)
2478 except ValueError:
2479 self.updateProgress(
2480 "[repo:%s] %s: %s - %s" % (
2481 darkgreen(repo),
2482 blue(_("invalid database revision")),
2483 bold(rev),
2484 blue(_("defaulting to 0")),
2485 ),
2486 importance = 2,
2487 type = "error",
2488 header = darkred(" !!! ")
2489 )
2490 rev = 0
2491 return rev
2492 else:
2493 return 0
2494
2496
2497 if repo == None:
2498 repo = self.default_repository
2499
2500 remote_status = self.MirrorsService.get_remote_databases_status(repo)
2501 if not [x for x in remote_status if x[1]]:
2502 remote_revision = 0
2503 else:
2504 remote_revision = max([x[1] for x in remote_status])
2505
2506 return remote_revision
2507
2510
2512 from datetime import datetime
2513 import time
2514 return "%s" % (datetime.fromtimestamp(time.time()),)
2515
2517 pkglist_file = self.get_local_pkglist_file(repo = repo, branch = branch)
2518
2519 tmp_pkglist_file = pkglist_file + ".tmp"
2520 dbconn = self.open_server_repository(repo = repo, just_reading = True,
2521 do_treeupdates = False)
2522 pkglist = dbconn.listAllDownloads(do_sort = True, full_path = True)
2523
2524 with open(tmp_pkglist_file, "w") as pkg_f:
2525 for pkg in pkglist:
2526 pkg_f.write(pkg + "\n")
2527 pkg_f.flush()
2528
2529 os.rename(tmp_pkglist_file, pkglist_file)
2530
2532 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2533 repos = srv_set['repositories'].keys()
2534 kwargs['server_repos'] = repos
2535 kwargs['serverInstance'] = self
2536 return self.ClientService.package_set_list(*args, **kwargs)
2537
2539 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2540 repos = srv_set['repositories'].keys()
2541 kwargs['server_repos'] = repos
2542 kwargs['serverInstance'] = self
2543 return self.ClientService.package_set_search(*args, **kwargs)
2544
2546 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2547 repos = srv_set['repositories'].keys()
2548 kwargs['server_repos'] = repos
2549 kwargs['serverInstance'] = self
2550 return self.ClientService.package_set_match(*args, **kwargs)
2551
2553 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2554 repos = srv_set['repositories'].keys()
2555 kwargs['server_repos'] = repos
2556 kwargs['serverInstance'] = self
2557 return self.ClientService.atom_match(*args, **kwargs)
2558
2560
2561 installed_packages = self.SpmService.get_installed_packages_counter()
2562 installed_counters = set()
2563 to_be_added = set()
2564 to_be_removed = set()
2565 to_be_injected = set()
2566 my_settings = self.SystemSettings[self.sys_settings_plugin_id]['server']
2567 exp_based_scope = my_settings['exp_based_scope']
2568
2569 server_repos = my_settings['repositories'].keys()
2570 exp_pkgs_cache = {}
2571
2572
2573 for spm_atom, spm_counter in installed_packages:
2574 found = False
2575 for server_repo in server_repos:
2576 installed_counters.add(spm_counter)
2577 server_dbconn = self.open_server_repository(read_only = True,
2578 no_upload = True, repo = server_repo)
2579 counter = server_dbconn.isCounterAvailable(spm_counter)
2580 if counter:
2581 found = True
2582 break
2583 if not found:
2584 to_be_added.add((spm_atom, spm_counter,))
2585
2586
2587 database_counters = {}
2588 for server_repo in server_repos:
2589 server_dbconn = self.open_server_repository(read_only = True,
2590 no_upload = True, repo = server_repo)
2591 database_counters[server_repo] = server_dbconn.listAllCounters(
2592 branch = self.SystemSettings['repositories']['branch'])
2593
2594 ordered_counters = set()
2595 for server_repo in database_counters:
2596 for data in database_counters[server_repo]:
2597 ordered_counters.add((data, server_repo))
2598 database_counters = ordered_counters
2599
2600 for (counter, idpackage,), xrepo in database_counters:
2601
2602 if counter < 0:
2603 continue
2604
2605 if counter in installed_counters:
2606 continue
2607
2608 dbconn = self.open_server_repository(read_only = True,
2609 no_upload = True, repo = xrepo)
2610
2611 dorm = True
2612
2613 if to_be_added:
2614
2615 dorm = False
2616 atom = dbconn.retrieveAtom(idpackage)
2617 atomkey = self.entropyTools.dep_getkey(atom)
2618 atomtag = self.entropyTools.dep_gettag(atom)
2619 atomslot = dbconn.retrieveSlot(idpackage)
2620
2621 add = True
2622 for spm_atom, spm_counter in to_be_added:
2623 addslot = self.SpmService.get_installed_package_slot(
2624 spm_atom)
2625 addkey = self.entropyTools.dep_getkey(spm_atom)
2626
2627 if addslot == None:
2628 addslot = '0'
2629
2630 if (atomkey == addkey) and \
2631 ((str(atomslot) == str(addslot)) or (atomtag != None)):
2632
2633 add = False
2634 break
2635
2636 if not add:
2637 continue
2638 dorm = True
2639
2640
2641
2642
2643 if dorm and (xrepo == self.default_repository):
2644 trashed = self.is_counter_trashed(counter)
2645 if trashed:
2646
2647 try:
2648 key, slot = dbconn.retrieveKeySlot(idpackage)
2649 trashed = self.SpmService.get_installed_atom(
2650 key+":"+slot)
2651 except TypeError:
2652 trashed = True
2653 if not trashed:
2654
2655 dbtag = dbconn.retrieveVersionTag(idpackage)
2656 if dbtag:
2657 is_injected = dbconn.isInjected(idpackage)
2658 if not is_injected:
2659 to_be_injected.add((idpackage, xrepo))
2660
2661 elif exp_based_scope:
2662
2663
2664 plg_id = self.sys_settings_fatscope_plugin_id
2665 exp_data = self.SystemSettings[plg_id]['repos'].get(
2666 xrepo, set())
2667
2668
2669
2670 if (idpackage not in exp_data) and (-1 not in exp_data):
2671 to_be_removed.add((idpackage, xrepo))
2672 continue
2673
2674 idpackage_expired = self.is_match_expired((idpackage,
2675 xrepo,))
2676
2677 if idpackage_expired:
2678
2679
2680 to_be_removed.add((idpackage, xrepo))
2681 for my_id in dbconn.retrieveDepends(idpackage):
2682 to_be_removed.add((my_id, xrepo))
2683
2684 else:
2685 to_be_removed.add((idpackage, xrepo))
2686
2687 return to_be_added, to_be_removed, to_be_injected
2688
2690
2691 idpackage, repoid = match
2692 dbconn = self.open_server_repository(repo = repoid, just_reading = True)
2693
2694 my_settings = self.SystemSettings[self.sys_settings_plugin_id]['server']
2695 pkg_exp_secs = my_settings['packages_expiration_days'] * 86400
2696 cur_unix_time = self.entropyTools.get_current_unix_time()
2697
2698
2699
2700 mydate = dbconn.retrieveDateCreation(idpackage)
2701
2702 mydelta = cur_unix_time - float(mydate)
2703 if mydelta > pkg_exp_secs:
2704 return True
2705 return False
2706
2708 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2709 server_repos = srv_set['repositories'].keys()
2710 for repo in server_repos:
2711 dbconn = self.open_server_repository(read_only = True,
2712 no_upload = True, repo = repo)
2713 if dbconn.isCounterTrashed(counter):
2714 return True
2715 return False
2716
2723
2726
2727 if repo == None:
2728 repo = self.default_repository
2729
2730 self.close_server_databases()
2731 revisions_match = {}
2732 treeupdates_actions = []
2733 injected_packages = set()
2734 idpackages = set()
2735 idpackages_added = set()
2736
2737 mytxt = red("%s ...") % (_("Initializing Entropy database"),)
2738 self.updateProgress(
2739 mytxt, importance = 1,
2740 type = "info", header = darkgreen(" * "),
2741 back = True
2742 )
2743
2744 if os.path.isfile(self.get_local_database_file(repo)):
2745
2746 dbconn = self.open_server_repository(read_only = True,
2747 no_upload = True, repo = repo, warnings = warnings)
2748
2749 if dbconn.doesTableExist("baseinfo") and \
2750 dbconn.doesTableExist("extrainfo"):
2751 idpackages = dbconn.listAllIdpackages()
2752
2753 if dbconn.doesTableExist("treeupdatesactions"):
2754 treeupdates_actions = dbconn.listAllTreeUpdatesActions()
2755
2756
2757 if dbconn.doesTableExist("injected") and \
2758 dbconn.doesTableExist("extrainfo"):
2759 injected_packages = dbconn.listAllInjectedPackages(
2760 justFiles = True)
2761 injected_packages = set([os.path.basename(x) for x \
2762 in injected_packages])
2763
2764 for idpackage in idpackages:
2765 url = dbconn.retrieveDownloadURL(idpackage)
2766 package = os.path.basename(url)
2767 branch = dbconn.retrieveBranch(idpackage)
2768 revision = dbconn.retrieveRevision(idpackage)
2769 revisions_match[package] = (branch, revision,)
2770
2771 self.close_server_database(dbconn)
2772
2773 mytxt = "%s: %s: %s" % (
2774 bold(_("WARNING")),
2775 red(_("database already exists")),
2776 self.get_local_database_file(repo),
2777 )
2778 self.updateProgress(
2779 mytxt,
2780 importance = 1,
2781 type = "warning",
2782 header = darkred(" !!! ")
2783 )
2784
2785 rc_question = self.askQuestion(_("Do you want to continue ?"))
2786 if rc_question == "No":
2787 return
2788 try:
2789 os.remove(self.get_local_database_file(repo))
2790 except OSError:
2791 pass
2792
2793
2794
2795 dbconn = self.open_server_repository(read_only = False,
2796 no_upload = True, repo = repo, is_new = True)
2797 dbconn.initializeDatabase()
2798
2799 if not empty:
2800
2801 revisions_file = "/entropy-revisions-dump.txt"
2802
2803 if revisions_match:
2804 self.updateProgress(
2805 "%s: %s" % (
2806 red(_("Dumping current revisions to file")),
2807 darkgreen(revisions_file),
2808 ),
2809 importance = 1,
2810 type = "info",
2811 header = darkgreen(" * ")
2812 )
2813 f_rev = open(revisions_file,"w")
2814 f_rev.write(str(revisions_match))
2815 f_rev.flush()
2816 f_rev.close()
2817
2818
2819 treeupdates_file = "/entropy-treeupdates-dump.txt"
2820 if treeupdates_actions:
2821 self.updateProgress(
2822 "%s: %s" % (
2823
2824 red(_("Dumping current 'treeupdates' actions to file")),
2825 bold(treeupdates_file),
2826 ),
2827 importance = 1,
2828 type = "info",
2829 header = darkgreen(" * ")
2830 )
2831 f_tree = open(treeupdates_file,"w")
2832 f_tree.write(str(treeupdates_actions))
2833 f_tree.flush()
2834 f_tree.close()
2835
2836 rc_question = self.askQuestion(
2837 _("Would you like to sync packages first (important!) ?"))
2838 if rc_question == "Yes":
2839 self.MirrorsService.sync_packages(repo = repo)
2840
2841
2842 if treeupdates_actions:
2843 dbconn.bumpTreeUpdatesActions(treeupdates_actions)
2844
2845
2846 pkg_branch_dir = os.path.join(
2847 self.get_local_packages_directory(repo),
2848 self.SystemSettings['repositories']['branch'])
2849 pkglist = os.listdir(pkg_branch_dir)
2850
2851 pkg_ext_len = len(etpConst['packagesext'])
2852 pkglist = [x for x in pkglist if (x[(pkg_ext_len*-1):] == \
2853 etpConst['packagesext']) and not \
2854 os.path.isfile(os.path.join(pkg_branch_dir,
2855 x + etpConst['packagesexpirationfileext']))]
2856
2857 if pkglist:
2858 self.updateProgress(
2859 "%s '%s' %s %s" % (
2860 red(_("Reinitializing Entropy database for branch")),
2861 bold(self.SystemSettings['repositories']['branch']),
2862 red(_("using Packages in the repository")),
2863 red("..."),
2864 ),
2865 importance = 1,
2866 type = "info",
2867 header = darkgreen(" * ")
2868 )
2869
2870 counter = 0
2871 maxcount = len(pkglist)
2872 branch = self.SystemSettings['repositories']['branch']
2873 for pkg in pkglist:
2874 counter += 1
2875
2876 self.updateProgress(
2877 "[repo:%s|%s] %s: %s" % (
2878 darkgreen(repo),
2879 brown(branch),
2880 blue(_("analyzing")),
2881 bold(pkg),
2882 ),
2883 importance = 1,
2884 type = "info",
2885 header = " ",
2886 back = True,
2887 count = (counter, maxcount,)
2888 )
2889
2890 doinject = False
2891 if pkg in injected_packages:
2892 doinject = True
2893
2894 pkg_path = os.path.join(self.get_local_packages_directory(repo),
2895 branch, pkg)
2896 mydata = self.SpmService.extract_pkg_metadata(pkg_path,
2897 inject = doinject)
2898
2899
2900 revision_avail = revisions_match.get(pkg)
2901 add_revision = 0
2902 if (revision_avail != None):
2903 if branch == revision_avail[0]:
2904 add_revision = revision_avail[1]
2905
2906 idpackage, revision, mydata_upd = dbconn.addPackage(mydata,
2907 revision = add_revision)
2908 idpackages_added.add(idpackage)
2909
2910 self.updateProgress(
2911 "[repo:%s] [%s:%s/%s] %s: %s, %s: %s" % (
2912 repo,
2913 brown(branch),
2914 darkgreen(str(counter)),
2915 blue(str(maxcount)),
2916 red(_("added package")),
2917 darkgreen(pkg),
2918 red(_("revision")),
2919 brown(str(revision)),
2920 ),
2921 importance = 1,
2922 type = "info",
2923 header = " ",
2924 back = True
2925 )
2926
2927 self.depends_table_initialize(repo)
2928 self.library_paths_table_initialize(repo)
2929
2930 my_qa = self.QA()
2931
2932 if idpackages_added:
2933 dbconn = self.open_server_repository(read_only = False,
2934 no_upload = True, repo = repo)
2935 my_qa.scan_missing_dependencies(
2936 idpackages_added, dbconn, ask = True,
2937 repo = repo, self_check = True,
2938 black_list = \
2939 self.get_missing_dependencies_blacklist(repo = repo),
2940 black_list_adder = \
2941 self.add_missing_dependencies_blacklist_items
2942 )
2943
2944 dbconn.commitChanges()
2945 self.close_server_databases()
2946
2947 return 0
2948
2950
2951 dbconn = self.open_server_repository(read_only = True,
2952 no_upload = True, repo = repo)
2953 if ("world" in packages) or not packages:
2954 return dbconn.listAllIdpackages(), True
2955 else:
2956 idpackages = set()
2957 for package in packages:
2958 matches = dbconn.atomMatch(package, multiMatch = True)
2959 if matches[1] == 0:
2960 idpackages |= matches[0]
2961 else:
2962 mytxt = "%s: %s: %s" % (
2963 red(_("Attention")),
2964 blue(_("cannot match")),
2965 bold(package),
2966 )
2967 self.updateProgress(
2968 mytxt,
2969 importance = 1,
2970 type = "warning",
2971 header = darkred(" !!! ")
2972 )
2973 return idpackages, False
2974
2976
2977 import urllib2
2978 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2979 if not srv_set['repositories'][repo].has_key('handler'):
2980 return None
2981 url = srv_set['repositories'][repo]['handler']
2982
2983
2984 filename = filename.replace("#","%23")
2985
2986 filename = filename.replace("+","%2b")
2987 request = os.path.join(url, etpConst['handlers']['md5sum'])
2988 request += filename+"&branch="+branch
2989
2990 proxy_settings = self.SystemSettings['system']['proxy']
2991 try:
2992 mydict = {}
2993 if proxy_settings['ftp']:
2994 mydict['ftp'] = proxy_settings['ftp']
2995 if proxy_settings['http']:
2996 mydict['http'] = proxy_settings['http']
2997 if mydict:
2998 mydict['username'] = proxy_settings['username']
2999 mydict['password'] = proxy_settings['password']
3000 self.entropyTools.add_proxy_opener(urllib2, mydict)
3001 else:
3002
3003 urllib2._opener = None
3004 item = urllib2.urlopen(request)
3005 result = item.readline().strip()
3006 item.close()
3007 del item
3008 return result
3009 except (urllib2.URLError, urllib2.HTTPError,):
3010 return None
3011
3013
3014 if repo == None:
3015 repo = self.default_repository
3016
3017 self.updateProgress(
3018 "[%s] %s:" % (
3019 red("remote"),
3020 blue(_("Integrity verification of the selected packages")),
3021 ),
3022 importance = 1,
3023 type = "info",
3024 header = blue(" @@ ")
3025 )
3026
3027 idpackages, world = self.match_packages(packages)
3028 dbconn = self.open_server_repository(read_only = True, no_upload = True,
3029 repo = repo)
3030 branch = self.SystemSettings['repositories']['branch']
3031
3032 if world:
3033 self.updateProgress(
3034 blue(
3035 _("All the packages in repository will be checked.")),
3036 importance = 1,
3037 type = "info",
3038 header = " "
3039 )
3040 else:
3041 mytxt = red("%s:") % (
3042 _("This is the list of the packages that would be checked"),)
3043 self.updateProgress(
3044 mytxt,
3045 importance = 1,
3046 type = "info",
3047 header = " "
3048 )
3049 for idpackage in idpackages:
3050 pkgatom = dbconn.retrieveAtom(idpackage)
3051 down_url = dbconn.retrieveDownloadURL(idpackage)
3052 pkgfile = os.path.basename(down_url)
3053 self.updateProgress(
3054 red(pkgatom) + " -> " + bold(os.path.join(branch, pkgfile)),
3055 importance = 1,
3056 type = "info",
3057 header = darkgreen(" - ")
3058 )
3059
3060 if ask:
3061 rc_question = self.askQuestion(
3062 _("Would you like to continue ?"))
3063 if rc_question == "No":
3064 return set(), set(), {}
3065
3066 match = set()
3067 not_match = set()
3068 broken_packages = {}
3069
3070 for uri in self.get_remote_mirrors(repo):
3071
3072 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri)
3073 self.updateProgress(
3074 "[repo:%s] %s: %s" % (
3075 darkgreen(repo),
3076 blue(_("Working on mirror")),
3077 brown(crippled_uri),
3078 ),
3079 importance = 1,
3080 type = "info",
3081 header = red(" @@ ")
3082 )
3083
3084
3085 totalcounter = len(idpackages)
3086 currentcounter = 0
3087 for idpackage in idpackages:
3088
3089 currentcounter += 1
3090 pkgfile = dbconn.retrieveDownloadURL(idpackage)
3091 orig_branch = self.get_branch_from_download_relative_uri(
3092 pkgfile)
3093
3094 self.updateProgress(
3095 "[%s] %s: %s" % (
3096 brown(crippled_uri),
3097 blue(_("checking hash")),
3098 darkgreen(pkgfile),
3099 ),
3100 importance = 1,
3101 type = "info",
3102 header = blue(" @@ "),
3103 back = True,
3104 count = (currentcounter, totalcounter,)
3105 )
3106
3107 checksum_ok = False
3108 ck_remote = self.get_remote_package_checksum(repo,
3109 os.path.basename(pkgfile), orig_branch)
3110 if ck_remote == None:
3111 self.updateProgress(
3112 "[%s] %s: %s %s" % (
3113 brown(crippled_uri),
3114 blue(_("digest verification of")),
3115 bold(pkgfile),
3116 blue(_("not supported")),
3117 ),
3118 importance = 1,
3119 type = "info",
3120 header = blue(" @@ "),
3121 count = (currentcounter, totalcounter,)
3122 )
3123 elif len(ck_remote) == 32:
3124 pkghash = dbconn.retrieveDigest(idpackage)
3125 if ck_remote == pkghash:
3126 checksum_ok = True
3127 else:
3128 self.updateProgress(
3129 "[%s] %s: %s %s" % (
3130 brown(crippled_uri),
3131 blue(_("digest verification of")),
3132 bold(pkgfile),
3133 blue(_("failed for unknown reasons")),
3134 ),
3135 importance = 1,
3136 type = "info",
3137 header = blue(" @@ "),
3138 count = (currentcounter, totalcounter,)
3139 )
3140
3141 if checksum_ok:
3142 match.add(idpackage)
3143 else:
3144 not_match.add(idpackage)
3145 self.updateProgress(
3146 "[%s] %s: %s %s" % (
3147 brown(crippled_uri),
3148 blue(_("package")),
3149 bold(pkgfile),
3150 red(_("NOT healthy")),
3151 ),
3152 importance = 1,
3153 type = "warning",
3154 header = darkred(" !!! "),
3155 count = (currentcounter, totalcounter,)
3156 )
3157 if not broken_packages.has_key(crippled_uri):
3158 broken_packages[crippled_uri] = []
3159 broken_packages[crippled_uri].append(pkgfile)
3160
3161 if broken_packages:
3162 mytxt = blue("%s:") % (
3163 _("This is the list of broken packages"),)
3164 self.updateProgress(
3165 mytxt,
3166 importance = 1,
3167 type = "info",
3168 header = red(" * ")
3169 )
3170 for mirror in broken_packages.keys():
3171 mytxt = "%s: %s" % (
3172 brown(_("Mirror")),
3173 bold(mirror),
3174 )
3175 self.updateProgress(
3176 mytxt,
3177 importance = 1,
3178 type = "info",
3179 header = red(" <> ")
3180 )
3181 for broken_package in broken_packages[mirror]:
3182 self.updateProgress(
3183 blue(broken_package),
3184 importance = 1,
3185 type = "info",
3186 header = red(" - ")
3187 )
3188
3189 self.updateProgress(
3190 "%s:" % (
3191 blue(_("Statistics")),
3192 ),
3193 importance = 1,
3194 type = "info",
3195 header = red(" @@ ")
3196 )
3197 self.updateProgress(
3198 "[%s] %s:\t%s" % (
3199 red(crippled_uri),
3200 brown(_("Number of checked packages")),
3201 brown(str(len(match) + len(not_match))),
3202 ),
3203 importance = 1,
3204 type = "info",
3205 header = brown(" # ")
3206 )
3207 self.updateProgress(
3208 "[%s] %s:\t%s" % (
3209 red(crippled_uri),
3210 darkgreen(_("Number of healthy packages")),
3211 darkgreen(str(len(match))),
3212 ),
3213 importance = 1,
3214 type = "info",
3215 header = brown(" # ")
3216 )
3217 self.updateProgress(
3218 "[%s] %s:\t%s" % (
3219 red(crippled_uri),
3220 darkred(_("Number of broken packages")),
3221 darkred(str(len(not_match))),
3222 ),
3223 importance = 1,
3224 type = "info",
3225 header = brown(" # ")
3226 )
3227
3228 return match, not_match, broken_packages
3229
3230
3232
3233 if repo == None:
3234 repo = self.default_repository
3235
3236 self.updateProgress(
3237 "[%s] %s:" % (
3238 red(_("local")),
3239 blue(_("Integrity verification of the selected packages")),
3240 ),
3241 importance = 1,
3242 type = "info",
3243 header = darkgreen(" * ")
3244 )
3245
3246 idpackages, world = self.match_packages(packages)
3247 dbconn = self.open_server_repository(read_only = True,
3248 no_upload = True, repo = repo)
3249
3250 if world:
3251 self.updateProgress(
3252 blue(_("All the packages in repository will be checked.")),
3253 importance = 1,
3254 type = "info",
3255 header = " "
3256 )
3257
3258 to_download = set()
3259 available = set()
3260 for idpackage in idpackages:
3261
3262 pkgatom = dbconn.retrieveAtom(idpackage)
3263 pkg_path = dbconn.retrieveDownloadURL(idpackage)
3264 pkg_rel_path = '/'.join(pkg_path.split("/")[2:])
3265
3266 bindir_path = os.path.join(self.get_local_packages_directory(repo),
3267 pkg_rel_path)
3268 uploaddir_path = os.path.join(self.get_local_upload_directory(repo),
3269 pkg_rel_path)
3270
3271 if os.path.isfile(bindir_path):
3272 if not world:
3273 self.updateProgress(
3274 "[%s] %s :: %s" % (
3275 darkgreen(_("available")),
3276 blue(pkgatom),
3277 darkgreen(pkg_rel_path),
3278 ),
3279 importance = 0,
3280 type = "info",
3281 header = darkgreen(" # ")
3282 )
3283 available.add(idpackage)
3284 elif os.path.isfile(uploaddir_path):
3285 if not world:
3286 self.updateProgress(
3287 "[%s] %s :: %s" % (
3288 darkred(_("upload/ignored")),
3289 blue(pkgatom),
3290 darkgreen(pkg_rel_path),
3291 ),
3292 importance = 0,
3293 type = "info",
3294 header = darkgreen(" # ")
3295 )
3296 else:
3297 self.updateProgress(
3298 "[%s] %s :: %s" % (
3299 brown(_("download")),
3300 blue(pkgatom),
3301 darkgreen(pkg_rel_path),
3302 ),
3303 importance = 0,
3304 type = "info",
3305 header = darkgreen(" # ")
3306 )
3307 to_download.add((idpackage, pkg_path,))
3308
3309 if ask:
3310 rc_question = self.askQuestion(_("Would you like to continue ?"))
3311 if rc_question == "No":
3312 return set(), set(), set(), set()
3313
3314
3315 fine = set()
3316 failed = set()
3317 downloaded_fine = set()
3318 downloaded_errors = set()
3319
3320 if to_download:
3321
3322 not_downloaded = set()
3323 mytxt = blue("%s ...") % (_("Starting to download missing files"),)
3324 self.updateProgress(
3325 mytxt,
3326 importance = 1,
3327 type = "info",
3328 header = " "
3329 )
3330 for uri in self.get_remote_mirrors(repo):
3331
3332 if not_downloaded:
3333 mytxt = blue("%s ...") % (
3334 _("Searching missing/broken files on another mirror"),)
3335 self.updateProgress(
3336 mytxt,
3337 importance = 1,
3338 type = "info",
3339 header = " "
3340 )
3341 to_download = not_downloaded.copy()
3342 not_downloaded = set()
3343
3344 for idpackage, pkg_path in to_download:
3345 rc_down = self.MirrorsService.download_package(uri,
3346 pkg_path, repo = repo)
3347 if rc_down:
3348 downloaded_fine.add(idpackage)
3349 available.add(idpackage)
3350 else:
3351 not_downloaded.add(pkg_path)
3352
3353 if not not_downloaded:
3354 self.updateProgress(
3355 red(_("Binary packages downloaded successfully.")),
3356 importance = 1,
3357 type = "info",
3358 header = " "
3359 )
3360 break
3361
3362 if not_downloaded:
3363 mytxt = blue("%s:") % (
3364 _("These are the packages that cannot be found online"),)
3365 self.updateProgress(
3366 mytxt,
3367 importance = 1,
3368 type = "info",
3369 header = " "
3370 )
3371 for pkg_path in not_downloaded:
3372 downloaded_errors.add(pkg_path)
3373 self.updateProgress(
3374 brown(pkg_path),
3375 importance = 1,
3376 type = "warning",
3377 header = red(" * ")
3378 )
3379 downloaded_errors |= not_downloaded
3380 mytxt = "%s." % (_("They won't be checked"),)
3381 self.updateProgress(
3382 mytxt,
3383 importance = 1,
3384 type = "warning",
3385 header = " "
3386 )
3387
3388 totalcounter = str(len(available))
3389 currentcounter = 0
3390 for idpackage in available:
3391 currentcounter += 1
3392 pkg_path = dbconn.retrieveDownloadURL(idpackage)
3393 orig_branch = self.get_branch_from_download_relative_uri(pkg_path)
3394 pkgfile = os.path.basename(pkg_path)
3395
3396 self.updateProgress(
3397 "[branch:%s] %s %s" % (
3398 brown(orig_branch),
3399 blue(_("checking hash of")),
3400 darkgreen(pkgfile),
3401 ),
3402 importance = 1,
3403 type = "info",
3404 header = " ",
3405 back = True,
3406 count = (currentcounter, totalcounter,)
3407 )
3408
3409 storedmd5 = dbconn.retrieveDigest(idpackage)
3410 pkgpath = os.path.join(self.get_local_packages_directory(repo),
3411 orig_branch, pkgfile)
3412 result = self.entropyTools.compare_md5(pkgpath, storedmd5)
3413 if result:
3414 fine.add(idpackage)
3415 else:
3416 failed.add(idpackage)
3417 self.updateProgress(
3418 "[branch:%s] %s %s %s: %s" % (
3419 brown(orig_branch),
3420 blue(_("package")),
3421 darkgreen(pkg_path),
3422 blue(_("is corrupted, stored checksum")),
3423 brown(storedmd5),
3424 ),
3425 importance = 1,
3426 type = "info",
3427 header = " ",
3428 count = (currentcounter, totalcounter,)
3429 )
3430
3431 if failed:
3432 mytxt = blue("%s:") % (_("This is the list of broken packages"),)
3433 self.updateProgress(
3434 mytxt,
3435 importance = 1,
3436 type = "warning",
3437 header = darkred(" # ")
3438 )
3439 for idpackage in failed:
3440 atom = dbconn.retrieveAtom(idpackage)
3441 down_p = dbconn.retrieveDownloadURL(idpackage)
3442 self.updateProgress(
3443 blue("[atom:%s] %s" % (atom, down_p,)),
3444 importance = 0,
3445 type = "warning",
3446 header = brown(" # ")
3447 )
3448
3449
3450 self.updateProgress(
3451 red("Statistics:"),
3452 importance = 1,
3453 type = "info",
3454 header = blue(" * ")
3455 )
3456 self.updateProgress(
3457 brown("%s => %s" % (
3458 len(fine) + len(failed),
3459 _("checked packages"),
3460 )
3461 ),
3462 importance = 0,
3463 type = "info",
3464 header = brown(" # ")
3465 )
3466 self.updateProgress(
3467 darkgreen("%s => %s" % (
3468 len(fine),
3469 _("healthy packages"),
3470 )
3471 ),
3472 importance = 0,
3473 type = "info",
3474 header = brown(" # ")
3475 )
3476 self.updateProgress(
3477 darkred("%s => %s" % (
3478 len(failed),
3479 _("broken packages"),
3480 )
3481 ),
3482 importance = 0,
3483 type = "info",
3484 header = brown(" # ")
3485 )
3486 self.updateProgress(
3487 blue("%s => %s" % (
3488 len(downloaded_fine),
3489 _("downloaded packages"),
3490 )
3491 ),
3492 importance = 0,
3493 type = "info",
3494 header = brown(" # ")
3495 )
3496 self.updateProgress(
3497 bold("%s => %s" % (
3498 len(downloaded_errors),
3499 _("failed downloads"),
3500 )
3501 ),
3502 importance = 0,
3503 type = "info",
3504 header = brown(" # ")
3505 )
3506
3507 self.close_server_database(dbconn)
3508 return fine, failed, downloaded_fine, downloaded_errors
3509
3510
3512
3513 if repo == None:
3514 repo = self.default_repository
3515
3516 if to_branch != self.SystemSettings['repositories']['branch']:
3517 mytxt = "%s: %s %s" % (
3518 blue(_("Please setup your branch to")),
3519 bold(to_branch),
3520 blue(_("and retry")),
3521 )
3522 self.updateProgress(
3523 mytxt,
3524 importance = 1,
3525 type = "error",
3526 header = darkred(" !! ")
3527 )
3528 return None
3529
3530 mytxt = red("%s ...") % (_("Copying database (if not exists)"),)
3531 self.updateProgress(
3532 mytxt,
3533 importance = 1,
3534 type = "info",
3535 header = darkgreen(" @@ ")
3536 )
3537 branch_dbdir = self.get_local_database_dir(repo)
3538 old_branch_dbdir = self.get_local_database_dir(repo, from_branch)
3539
3540
3541 self.close_server_databases()
3542
3543
3544
3545 branch_dbfile = self.get_local_database_file(repo)
3546 if os.path.isfile(branch_dbfile):
3547 if self.entropyTools.get_file_size(branch_dbfile) == 0:
3548 shutil.rmtree(branch_dbdir, True)
3549
3550 if os.path.isdir(branch_dbdir):
3551
3552 while 1:
3553 rnd_num = self.entropyTools.get_random_number()
3554 backup_dbdir = branch_dbdir + str(rnd_num)
3555 if not os.path.isdir(backup_dbdir):
3556 break
3557 os.rename(branch_dbdir, backup_dbdir)
3558
3559 if os.path.isdir(old_branch_dbdir):
3560 shutil.copytree(old_branch_dbdir, branch_dbdir)
3561
3562 mytxt = red("%s ...") % (_("Switching packages"),)
3563 self.updateProgress(
3564 mytxt,
3565 importance = 1,
3566 type = "info",
3567 header = darkgreen(" @@ ")
3568 )
3569
3570 dbconn = self.open_server_repository(read_only = False,
3571 no_upload = True, repo = repo, lock_remote = False)
3572 try:
3573 dbconn.validateDatabase()
3574 except SystemDatabaseError:
3575 self.handle_uninitialized_repository(repo)
3576 dbconn = self.open_server_repository(read_only = False,
3577 no_upload = True, repo = repo, lock_remote = False)
3578
3579 idpackages = dbconn.listAllIdpackages()
3580 already_switched = set()
3581 not_found = set()
3582 switched = set()
3583 ignored = set()
3584 no_checksum = set()
3585
3586 maxcount = len(idpackages)
3587 count = 0
3588 for idpackage in idpackages:
3589 count += 1
3590
3591 cur_branch = dbconn.retrieveBranch(idpackage)
3592 atom = dbconn.retrieveAtom(idpackage)
3593 if cur_branch == to_branch:
3594 already_switched.add(idpackage)
3595 self.updateProgress(
3596 red("%s %s, %s %s" % (
3597 _("Ignoring"),
3598 bold(atom),
3599 _("already in branch"),
3600 cur_branch,
3601 )
3602 ),
3603 importance = 0,
3604 type = "info",
3605 header = darkgreen(" @@ "),
3606 count = (count, maxcount,)
3607 )
3608 ignored.add(idpackage)
3609 continue
3610
3611 self.updateProgress(
3612 "[%s=>%s] %s" % (
3613 brown(cur_branch),
3614 bold(to_branch),
3615 darkgreen(atom),
3616 ),
3617 importance = 0,
3618 type = "info",
3619 header = darkgreen(" @@ "),
3620 back = True,
3621 count = (count, maxcount,)
3622 )
3623 dbconn.switchBranch(idpackage, to_branch)
3624 dbconn.commitChanges()
3625 switched.add(idpackage)
3626
3627 dbconn.commitChanges()
3628
3629
3630 dbconn.moveCountersToBranch(to_branch, from_branch = from_branch)
3631
3632 self.close_server_database(dbconn)
3633 mytxt = blue("%s.") % (_("migration loop completed"),)
3634 self.updateProgress(
3635 "[%s=>%s] %s" % (
3636 brown(from_branch),
3637 bold(to_branch),
3638 mytxt,
3639 ),
3640 importance = 1,
3641 type = "info",
3642 header = darkgreen(" * ")
3643 )
3644
3645 return switched, already_switched, ignored, not_found, no_checksum
3646
3648
3649 if branch == None:
3650 branch = self.SystemSettings['repositories']['branch']
3651 if repo == None:
3652 repo = self.default_repository
3653
3654 sets_dir = self.get_local_database_sets_dir(repo, branch)
3655 if not (os.path.isdir(sets_dir) and os.access(sets_dir, os.R_OK)):
3656 return {}
3657
3658 mydata = {}
3659 items = os.listdir(sets_dir)
3660 for item in items:
3661
3662 try:
3663 item_clean = str(item)
3664 except (UnicodeEncodeError, UnicodeDecodeError,):
3665 continue
3666 item_path = os.path.join(sets_dir, item)
3667 if not (os.path.isfile(item_path) and \
3668 os.access(item_path, os.R_OK)):
3669 continue
3670 item_elements = self.entropyTools.extract_packages_from_set_file(
3671 item_path)
3672 if item_elements:
3673 mydata[item_clean] = item_elements.copy()
3674
3675 return mydata
3676
3707
3721