1
2 '''
3 # DESCRIPTION:
4 # Entropy Object Oriented Interface
5
6 Copyright (C) 2007-2009 Fabio Erculiani
7
8 This program is free software; you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation; either version 2 of the License, or
11 (at your option) any later version.
12
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 '''
22
23 from __future__ import with_statement
24 import os
25 import shutil
26 from entropy.core import Singleton
27 from entropy.exceptions import OnlineMirrorError, PermissionDenied, \
28 SystemDatabaseError
29 from entropy.const import etpConst, etpSys, const_setup_perms, \
30 const_create_working_dirs, const_extract_srv_repo_params, etpUi
31 from entropy.output import TextInterface, purple, red, darkgreen, \
32 bold, brown, blue, darkred
33 from entropy.server.interfaces.mirrors import Server as MirrorsServer
34 from entropy.i18n import _
35 from entropy.core import SystemSettings, SystemSettingsPlugin
36 from entropy.transceivers import FtpInterface
37 from entropy.db import LocalRepository
38
40
42
43 """
44 Parses Entropy server system configuration file.
45
46 @return dict data
47 """
48
49 data = {
50 'repositories': etpConst['server_repositories'].copy(),
51 'default_repository_id': etpConst['officialserverrepositoryid'],
52 'packages_expiration_days': etpConst['packagesexpirationdays'],
53 'database_file_format': etpConst['etpdatabasefileformat'],
54 'disabled_eapis': set(),
55 'exp_based_scope': etpConst['expiration_based_scope'],
56 'rss': {
57 'enabled': etpConst['rss-feed'],
58 'name': etpConst['rss-name'],
59 'base_url': etpConst['rss-base-url'],
60 'website_url': etpConst['rss-website-url'],
61 'editor': etpConst['rss-managing-editor'],
62 'max_entries': etpConst['rss-max-entries'],
63 'light_max_entries': etpConst['rss-light-max-entries'],
64 },
65 }
66
67 fake_instance = self._helper.fake_default_repo
68
69 if not os.access(etpConst['serverconf'], os.R_OK):
70 return data
71
72 with open(etpConst['serverconf'],"r") as server_f:
73 serverconf = [x.strip() for x in server_f.readlines() if x.strip()]
74
75 for line in serverconf:
76
77 split_line = line.split("|")
78 split_line_len = len(split_line)
79
80 if (line.find("officialserverrepositoryid|") != -1) and \
81 (not line.startswith("#")) and (split_line_len == 2):
82
83 if not fake_instance:
84 data['default_repository_id'] = split_line[1].strip()
85
86 elif line.startswith("expiration-days|") and (split_line_len == 2):
87
88 mydays = split_line[1].strip()
89 try:
90 mydays = int(mydays)
91 data['packages_expiration_days'] = mydays
92 except ValueError:
93 continue
94
95 elif line.startswith("expiration-based-scope|") and \
96 (split_line_len == 2):
97
98 exp_opt = split_line[1].strip().lower()
99 if exp_opt in ("enable", "enabled", "true", "1", "yes"):
100 data['exp_based_scope'] = True
101 else:
102 data['exp_based_scope'] = False
103
104 elif line.startswith("disabled-eapis|") and (split_line_len == 2):
105
106 mydis = split_line[1].strip().split(",")
107 try:
108 mydis = [int(x) for x in mydis]
109 mydis = set([x for x in mydis if x in (1, 2, 3,)])
110 except ValueError:
111 continue
112 if (len(mydis) < 3) and mydis:
113 data['disabled_eapis'] = mydis
114
115
116 elif line.startswith("repository|") and (split_line_len in [5, 6]) \
117 and (not fake_instance):
118
119 repoid, repodata = const_extract_srv_repo_params(line,
120 product = sys_set['repositories']['product'])
121 if repoid in data['repositories']:
122
123 data['repositories'][repoid]['mirrors'].extend(
124 repodata['mirrors'])
125 else:
126 data['repositories'][repoid] = repodata.copy()
127
128 elif line.startswith("database-format|") and (split_line_len == 2):
129
130 fmt = split_line[1]
131 if fmt in etpConst['etpdatabasesupportedcformats']:
132 data['database_file_format'] = fmt
133
134 elif line.startswith("rss-feed|") and (split_line_len == 2):
135
136 feed = split_line[1]
137 if feed in ("enable", "enabled", "true", "1"):
138 data['rss']['enabled'] = True
139 elif feed in ("disable", "disabled", "false", "0", "no",):
140 data['rss']['enabled'] = False
141
142 elif line.startswith("rss-name|") and (split_line_len == 2):
143
144 feedname = line.split("rss-name|")[1].strip()
145 data['rss']['name'] = feedname
146
147 elif line.startswith("rss-base-url|") and (split_line_len == 2):
148
149 data['rss']['base_url'] = line.split("rss-base-url|")[1].strip()
150 if not data['rss']['base_url'][-1] == "/":
151 data['rss']['base_url'] += "/"
152
153 elif line.startswith("rss-website-url|") and (split_line_len == 2):
154
155 data['rss']['website_url'] = split_line[1].strip()
156
157 elif line.startswith("managing-editor|") and (split_line_len == 2):
158
159 data['rss']['editor'] = split_line[1].strip()
160
161 elif line.startswith("max-rss-entries|") and (split_line_len == 2):
162
163 try:
164 entries = int(split_line[1].strip())
165 data['rss']['max_entries'] = entries
166 except (ValueError, IndexError,):
167 continue
168
169 elif line.startswith("max-rss-light-entries|") and \
170 (split_line_len == 2):
171
172 try:
173 entries = int(split_line[1].strip())
174 data['rss']['light_max_entries'] = entries
175 except (ValueError, IndexError,):
176 continue
177
178
179 if self._helper.community_repo:
180 data['repositories'][etpConst['clientserverrepoid']] = {}
181 mydata = {}
182 mydata['description'] = "Community Repositories System Database"
183 mydata['mirrors'] = []
184 mydata['community'] = False
185 data['repositories'][etpConst['clientserverrepoid']].update(mydata)
186
187
188 for repoid in data['repositories']:
189 data['repositories'][repoid]['packages_dir'] = \
190 os.path.join( etpConst['entropyworkdir'],
191 "server",
192 repoid,
193 "packages",
194 etpSys['arch']
195 )
196 data['repositories'][repoid]['store_dir'] = \
197 os.path.join( etpConst['entropyworkdir'],
198 "server",
199 repoid,
200 "store",
201 etpSys['arch']
202 )
203 data['repositories'][repoid]['upload_dir'] = \
204 os.path.join( etpConst['entropyworkdir'],
205 "server",
206 repoid,
207 "upload",
208 etpSys['arch']
209 )
210 data['repositories'][repoid]['database_dir'] = \
211 os.path.join( etpConst['entropyworkdir'],
212 "server",
213 repoid,
214 "database",
215 etpSys['arch']
216 )
217 data['repositories'][repoid]['packages_relative_path'] = \
218 os.path.join( sys_set['repositories']['product'],
219 repoid,
220 "packages",
221 etpSys['arch']
222 )+"/"
223 data['repositories'][repoid]['database_relative_path'] = \
224 os.path.join( sys_set['repositories']['product'],
225 repoid,
226 "database",
227 etpSys['arch']
228 )+"/"
229
230
231 shell_repoid = os.getenv('ETP_REPO')
232 if shell_repoid:
233 data['default_repository_id'] = shell_repoid
234
235 expiration_days = os.getenv('ETP_EXPIRATION_DAYS')
236 if expiration_days:
237 try:
238 expiration_days = int(expiration_days)
239 data['packages_expiration_days'] = expiration_days
240 except ValueError:
241 pass
242
243 return data
244
246
247 import entropy.tools as entropyTools
248
250
251 data = {}
252 srv_plug_id = etpConst['system_settings_plugins_ids']['server_plugin']
253
254 srv_parser_data = sys_set[srv_plug_id]['server']
255 if not srv_parser_data['exp_based_scope']:
256 return data
257
258
259 for repoid in srv_parser_data['repositories']:
260
261
262
263 if repoid == etpConst['clientserverrepoid']:
264 continue
265
266 idpackages = set()
267 exp_fp = self._helper.get_local_exp_based_pkgs_rm_whitelist_file(
268 repo = repoid)
269 dbconn = self._helper.open_server_repository(
270 just_reading = True, repo = repoid)
271
272 if os.access(exp_fp, os.R_OK | os.F_OK):
273 pkgs = self.entropyTools.generic_file_content_parser(exp_fp)
274 if '*' in pkgs:
275 idpackages.add(-1)
276 else:
277 for pkg in pkgs:
278 idpackage, rc_match = dbconn.atomMatch(pkg)
279 if rc_match:
280 continue
281 idpackages.add(idpackage)
282
283 data[repoid] = idpackages
284
285 return data
286
287
288 -class Server(Singleton, TextInterface):
289
290 - def init_singleton(self, default_repository = None, save_repository = False,
291 community_repo = False, fake_default_repo = False,
292 fake_default_repo_id = '::fake::',
293 fake_default_repo_desc = 'this is a fake repository'):
294
295 self.__instance_destroyed = False
296 if etpConst['uid'] != 0:
297 mytxt = _("Entropy Server interface must be run as root")
298 raise PermissionDenied("PermissionDenied: %s" % (mytxt,))
299
300
301 from entropy.misc import LogFile
302 self.SystemSettings = SystemSettings()
303 self.community_repo = community_repo
304 from entropy.db import dbapi2
305 self.dbapi2 = dbapi2
306 etpSys['serverside'] = True
307 self._memory_db_instances = {}
308 self.fake_default_repo = fake_default_repo
309 self.indexing = False
310 self.xcache = False
311 self.MirrorsService = None
312 self.__server_dbcache = {}
313 self.repository_treeupdate_digests = {}
314 self.__settings_to_backup = []
315 self.__do_save_repository = save_repository
316 self.__sync_lock_cache = set()
317 self.rssMessages = {
318 'added': {},
319 'removed': {},
320 'commitmessage': "",
321 'light': {},
322 }
323
324 if fake_default_repo:
325 default_repository = fake_default_repo_id
326 etpConst['officialserverrepositoryid'] = fake_default_repo_id
327 self.init_generic_memory_server_repository(fake_default_repo_id,
328 fake_default_repo_desc)
329
330
331 self.sys_settings_plugin_id = \
332 etpConst['system_settings_plugins_ids']['server_plugin']
333 self.sys_settings_plugin = ServerSystemSettingsPlugin(
334 self.sys_settings_plugin_id, self)
335 self.SystemSettings.add_plugin(self.sys_settings_plugin)
336
337
338 self.sys_settings_fatscope_plugin_id = \
339 etpConst['system_settings_plugins_ids']['server_plugin_fatscope']
340 self.sys_settings_fatscope_plugin = ServerFatscopeSystemSettingsPlugin(
341 self.sys_settings_fatscope_plugin_id, self)
342 self.SystemSettings.add_plugin(self.sys_settings_fatscope_plugin)
343
344 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
345 self.default_repository = default_repository
346 if self.default_repository == None:
347 self.default_repository = srv_set['default_repository_id']
348
349 if self.default_repository in srv_set['repositories']:
350 self.ensure_paths(self.default_repository)
351
352
353
354
355 if self.default_repository not in srv_set['repositories']:
356 raise PermissionDenied("PermissionDenied: %s %s" % (
357 self.default_repository,
358 _("repository not configured"),
359 )
360 )
361 if etpConst['clientserverrepoid'] == self.default_repository:
362 raise PermissionDenied("PermissionDenied: %s %s" % (
363 etpConst['clientserverrepoid'],
364 _("protected repository id, can't use this, sorry dude..."),
365 )
366 )
367
368 self.switch_default_repository(self.default_repository)
369
371 self.__instance_destroyed = True
372 if hasattr(self,'ClientService'):
373 self.ClientService.destroy()
374 if hasattr(self,'sys_settings_server_plugin'):
375 try:
376 self.SystemSettings.remove_plugin(self.sys_settings_plugin_id)
377 except KeyError:
378 pass
379 if hasattr(self,'sys_settings_fatscope_plugin'):
380 try:
381 self.SystemSettings.remove_plugin(
382 self.sys_settings_fatscope_plugin)
383 except KeyError:
384 pass
385 self.close_server_databases()
386
388 return self.__instance_destroyed
389
392
401
402
404 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
405 migrated_filename = '.branch_migrated'
406 for repoid in srv_set['repositories'].keys():
407
408 if repoid == etpConst['clientserverrepoid']: continue
409 mydir = srv_set['repositories'][repoid]['database_dir']
410 if not os.path.isdir(mydir):
411 continue
412
413 migrated_filepath = os.path.join(mydir, migrated_filename)
414 if os.path.isfile(migrated_filepath):
415 continue
416
417 my_branched_dir = self.get_local_database_dir(repoid)
418 if os.path.isdir(my_branched_dir):
419 continue
420
421 self.updateProgress(
422 "[%s:%s] %s: %s, %s: %s" % (
423 brown("repo"),
424 purple(repoid),
425 _("migrating database path from"),
426 brown(mydir),
427 _('to'),
428 brown(my_branched_dir),
429 ),
430 importance = 1,
431 type = "info",
432 header = bold(" @@ ")
433 )
434
435 repo_files = [os.path.join(mydir, x) for x in os.listdir(mydir) if \
436 (os.path.isfile(os.path.join(mydir, x)) and \
437 os.access(os.path.join(mydir, x), os.W_OK))
438 ]
439 os.makedirs(my_branched_dir)
440 const_setup_perms(my_branched_dir, etpConst['entropygid'])
441
442 for repo_file in repo_files:
443 repo_filename = os.path.basename(repo_file)
444 shutil.move(repo_file,
445 os.path.join(my_branched_dir, repo_filename))
446
447 f_migrated = open(migrated_filepath,"w")
448 f_migrated.write("done\n")
449 f_migrated.flush()
450 f_migrated.close()
451
453 self.setup_entropy_settings()
454 cs_name = 'ClientService'
455 if hasattr(self, cs_name):
456 obj = getattr(self, cs_name)
457 obj.destroy()
458 from entropy.client.interfaces import Client
459 self.ClientService = Client(
460 indexing = self.indexing,
461 xcache = self.xcache,
462 repo_validation = False,
463 noclientdb = 1
464 )
465 from entropy.cache import EntropyCacher
466 self.Cacher = EntropyCacher()
467 self.ClientService.updateProgress = self.updateProgress
468 self.validRepositories = self.ClientService.validRepositories
469 self.entropyTools = self.ClientService.entropyTools
470 self.dumpTools = self.ClientService.dumpTools
471 self.QA = self.ClientService.QA
472 self.backup_entropy_settings()
473 self.SpmService = self.ClientService.Spm()
474 self.MirrorsService = MirrorsServer(self)
475
477 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
478 curr_repoid = srv_set['default_repository_id']
479 backup_list = [
480 'etpdatabaseclientfilepath',
481 'clientdbid',
482 {'server': srv_set.copy()},
483 ]
484 for setting in backup_list:
485 if setting not in self.__settings_to_backup:
486 self.__settings_to_backup.append(setting)
487
488 if not self.community_repo:
489 etpConst['etpdatabaseclientfilepath'] = \
490 self.get_local_database_file(repo)
491 etpConst['clientdbid'] = etpConst['serverdbid']
492 const_create_working_dirs()
493
495 if hasattr(self,'serverDbCache'):
496 for item in self.__server_dbcache:
497 try:
498 self.__server_dbcache[item].closeDB()
499 except self.dbapi2.ProgrammingError:
500 pass
501 self.__server_dbcache.clear()
502
504 found = None
505 for item in self.__server_dbcache:
506 if dbinstance == self.__server_dbcache[item]:
507 found = item
508 break
509 if found:
510 instance = self.__server_dbcache.pop(found)
511 instance.closeDB()
512
514 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
515 return srv_set['repositories'].copy()
516
544
546 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
547 if self.community_repo:
548 for repoid in srv_set['repositories']:
549 srv_set['repositories'][repoid]['community'] = True
550
551
553
554 if not self.is_repository_initialized(repoid):
555 mytxt = blue("%s.") % (
556 _("Your default repository is not initialized"),)
557 self.updateProgress(
558 "[%s:%s] %s" % (
559 brown("repo"),
560 purple(repoid),
561 mytxt,
562 ),
563 importance = 1,
564 type = "warning",
565 header = darkred(" !!! ")
566 )
567 answer = self.askQuestion(
568 _("Do you want to initialize your default repository ?"))
569 if answer == "No":
570 mytxt = red("%s.") % (
571 _("Continuing with an uninitialized repository"),)
572 self.updateProgress(
573 "[%s:%s] %s" % (
574 brown("repo"),
575 purple(repoid),
576 mytxt,
577 ),
578 importance = 1,
579 type = "warning",
580 header = darkred(" !!! ")
581 )
582 else:
583
584 dbfile = self.get_local_database_file(repoid)
585 if os.path.isfile(dbfile):
586 shutil.move(dbfile, dbfile+".backup")
587 self.initialize_server_database(empty = True,
588 repo = repoid, warnings = False)
589
590
592 type_txt = _("server-side repository")
593 if self.community_repo:
594 type_txt = _("community repository")
595
596 mytxt = _("Entropy Server Interface Instance on repository")
597 self.updateProgress(
598 blue("%s: %s, %s: %s (%s: %s)" % (
599 mytxt,
600 red(self.default_repository),
601 _("current branch"),
602 darkgreen(self.SystemSettings['repositories']['branch']),
603 purple(_("type")),
604 bold(type_txt),
605 )
606 ),
607 importance = 2,
608 type = "info",
609 header = red(" @@ ")
610 )
611 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
612 repos = srv_set['repositories'].keys()
613 mytxt = blue("%s:") % (_("Currently configured repositories"),)
614 self.updateProgress(
615 mytxt,
616 importance = 1,
617 type = "info",
618 header = red(" @@ ")
619 )
620 for repo in repos:
621 self.updateProgress(
622 darkgreen(repo),
623 importance = 0,
624 type = "info",
625 header = brown(" # ")
626 )
627
628
630
631
632 if repoid == etpConst['clientserverrepoid']:
633 return
634
635 if os.path.isfile(etpConst['serverconf']):
636 f_srv = open(etpConst['serverconf'],"r")
637 content = f_srv.readlines()
638 f_srv.close()
639 content = [x.strip() for x in content]
640 found = False
641 new_content = []
642 for line in content:
643 if line.strip().startswith("officialserverrepositoryid|"):
644 line = "officialserverrepositoryid|%s" % (repoid,)
645 found = True
646 new_content.append(line)
647 if not found:
648 new_content.append("officialserverrepositoryid|%s" % (repoid,))
649 f_srv_t = open(etpConst['serverconf']+".save_default_repo_tmp","w")
650 for line in new_content:
651 f_srv_t.write(line+"\n")
652 f_srv_t.flush()
653 f_srv_t.close()
654 os.rename(etpConst['serverconf']+".save_default_repo_tmp",
655 etpConst['serverconf'])
656 else:
657 f_srv = open(etpConst['serverconf'],"w")
658 f_srv.write("officialserverrepositoryid|%s\n" % (repoid,))
659 f_srv.flush()
660 f_srv.close()
661
663
664
665 if repoid == etpConst['clientserverrepoid']:
666 return False
667
668 if not os.path.isfile(etpConst['serverconf']):
669 return None
670 f_srv = open(etpConst['serverconf'])
671 tmpfile = etpConst['serverconf']+".switch"
672 mycontent = [x.strip() for x in f_srv.readlines()]
673 f_srv.close()
674 f_tmp = open(tmpfile,"w")
675 st = "repository|%s" % (repoid,)
676 status = False
677 for line in mycontent:
678 if enable:
679 if (line.find(st) != -1) and line.startswith("#") and \
680 (len(line.split("|")) == 5):
681 line = line[1:]
682 status = True
683 else:
684 if (line.find(st) != -1) and not line.startswith("#") and \
685 (len(line.split("|")) == 5):
686 line = "#"+line
687 status = True
688 f_tmp.write(line+"\n")
689 f_tmp.flush()
690 f_tmp.close()
691 shutil.move(tmpfile, etpConst['serverconf'])
692 if status:
693 self.close_server_databases()
694 self.SystemSettings.clear()
695 self.setup_services()
696 self.show_interface_status()
697 return status
698
705
714
715 dbc = self.open_server_repository(just_reading = True, repo = repo)
716 valid = do_validate(dbc)
717 self.close_server_database(dbc)
718 if not valid:
719 dbc = self.open_server_repository(read_only = False,
720 no_upload = True, repo = repo, is_new = True)
721 valid = do_validate(dbc)
722 self.close_server_database(dbc)
723
724 return valid
725
727
728 if repo == None:
729 repo = self.default_repository
730
731
732 lock_file = self.MirrorsService.get_database_lockfile(repo)
733 if os.path.isfile(lock_file):
734 self.updateProgress(
735 red(_("Entropy database is already locked by you :-)")),
736 importance = 1,
737 type = "info",
738 header = red(" * ")
739 )
740 else:
741
742 mytxt = "%s ..." % (_("Locking and Syncing Entropy database"),)
743 self.updateProgress(
744 red(mytxt),
745 importance = 1,
746 type = "info",
747 header = red(" * "),
748 back = True
749 )
750 for uri in self.get_remote_mirrors(repo):
751 given_up = self.MirrorsService.mirror_lock_check(uri,
752 repo = repo)
753 if given_up:
754 crippled_uri = \
755 self.entropyTools.extract_ftp_host_from_uri(uri)
756 mytxt = "%s:" % (_("Mirrors status table"),)
757 self.updateProgress(
758 darkgreen(mytxt),
759 importance = 1,
760 type = "info",
761 header = brown(" * ")
762 )
763 dbstatus = self.MirrorsService.get_mirrors_lock(repo = repo)
764 for db_uri, db_st1, db_st2 in dbstatus:
765 db_st1_info = darkgreen(_("Unlocked"))
766 if db_st1:
767 db_st1_info = red(_("Locked"))
768 db_st2_info = darkgreen(_("Unlocked"))
769 if db_st2:
770 db_st2_info = red(_("Locked"))
771
772 crippled_uri = \
773 self.entropyTools.extract_ftp_host_from_uri(db_uri)
774 self.updateProgress(
775 "%s: [%s: %s] [%s: %s]" % (
776 bold(crippled_uri),
777 brown(_("database")),
778 db_st1_info,
779 brown(_("download")),
780 db_st2_info,
781 ),
782 importance = 1,
783 type = "info",
784 header = "\t"
785 )
786
787 raise OnlineMirrorError("OnlineMirrorError: %s %s" % (
788 _("cannot lock mirror"),
789 crippled_uri,
790 )
791 )
792
793
794 self.MirrorsService.lock_mirrors(True, repo = repo)
795 self.MirrorsService.sync_databases(no_upload, repo = repo)
796
797
800
801 if mirrors is None:
802 mirrors = []
803 product = self.SystemSettings['repositories']['product']
804 dbc = self.open_memory_database(dbname = etpConst['serverdbid']+repoid)
805 self._memory_db_instances[repoid] = dbc
806
807 eapi3_port = int(etpConst['socket_service']['port'])
808 eapi3_ssl_port = int(etpConst['socket_service']['ssl_port'])
809
810 repodata = {
811 'repoid': repoid,
812 'description': description,
813 'mirrors': mirrors,
814 'community': community_repo,
815 'service_port': eapi3_port,
816 'ssl_service_port': eapi3_ssl_port,
817 'service_url': service_url,
818 'handler': '',
819 'in_memory': True,
820 }
821
822 etpConst['server_repositories'][repoid] = repodata
823 self.SystemSettings.clear()
824
825 return dbc
826
828 if dbname == None:
829 dbname = etpConst['genericdbid']
830 dbc = LocalRepository(
831 readOnly = False,
832 dbFile = ':memory:',
833 clientDatabase = True,
834 dbname = dbname,
835 xcache = False,
836 indexing = False,
837 OutputInterface = self,
838 skipChecks = True,
839 ServiceInterface = self
840 )
841 dbc.initializeDatabase()
842 return dbc
843
844 - def open_server_repository(
845 self,
846 read_only = True,
847 no_upload = True,
848 just_reading = False,
849 repo = None,
850 indexing = True,
851 warnings = True,
852 do_cache = True,
853 use_branch = None,
854 lock_remote = True,
855 is_new = False,
856 do_treeupdates = True
857 ):
858
859 if repo == None:
860 repo = self.default_repository
861
862 if repo == etpConst['clientserverrepoid'] and self.community_repo:
863 return self.ClientService.clientDbconn
864
865
866 if repo in self._memory_db_instances:
867 return self._memory_db_instances.get(repo)
868
869 if just_reading:
870 read_only = True
871 no_upload = True
872
873 local_dbfile = self.get_local_database_file(repo, use_branch)
874 if do_cache:
875 cached = self.__server_dbcache.get(
876 (repo, etpConst['systemroot'], local_dbfile, read_only,
877 no_upload, just_reading, use_branch, lock_remote,)
878 )
879 if cached != None:
880 return cached
881
882 local_dbfile_dir = os.path.dirname(local_dbfile)
883 if not os.path.isdir(local_dbfile_dir):
884 os.makedirs(local_dbfile_dir)
885
886 if (not read_only) and (lock_remote) and \
887 (repo not in self.__sync_lock_cache):
888 self.do_server_repository_sync_lock(repo, no_upload)
889 self.__sync_lock_cache.add(repo)
890
891 local_dbfile_exists = os.path.lexists(local_dbfile)
892 conn = LocalRepository(
893 readOnly = read_only,
894 dbFile = local_dbfile,
895 noUpload = no_upload,
896 OutputInterface = self,
897 ServiceInterface = self,
898 dbname = etpConst['serverdbid']+repo,
899 useBranch = use_branch,
900 lockRemote = lock_remote
901 )
902 if not local_dbfile_exists:
903
904 conn.readOnly = False
905 conn.initializeDatabase()
906 conn.commitChanges()
907
908 valid = True
909 try:
910 conn.validateDatabase()
911 except SystemDatabaseError:
912 valid = False
913
914
915
916 if (repo not in etpConst['server_treeupdatescalled']) and \
917 (not just_reading):
918
919
920 if valid:
921 if do_treeupdates:
922 conn.serverUpdatePackagesData()
923 elif warnings and not is_new:
924 mytxt = _("Entropy database is corrupted!")
925 self.updateProgress(
926 darkred(mytxt),
927 importance = 1,
928 type = "warning",
929 header = bold(" !!! ")
930 )
931
932 if not read_only and valid and indexing:
933
934 self.updateProgress(
935 "[repo:%s|%s] %s" % (
936 blue(repo),
937 red(_("database")),
938 blue(_("indexing database")),
939 ),
940 importance = 1,
941 type = "info",
942 header = brown(" @@ "),
943 back = True
944 )
945 conn.createAllIndexes()
946
947 if do_cache:
948
949
950 self.__server_dbcache[
951 (repo, etpConst['systemroot'], local_dbfile, read_only,
952 no_upload, just_reading, use_branch, lock_remote,)] = conn
953
954
955 if (not read_only) and (not is_new):
956 cur_sets = conn.retrievePackageSets()
957 sys_sets = self.get_configured_package_sets(repo)
958 if cur_sets != sys_sets:
959 self.update_database_package_sets(repo, dbconn = conn)
960 conn.commitChanges()
961
962 return conn
963
965
966 sys_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
967 server_repos = sys_set['repositories'].keys()
968 installed_packages = set()
969
970 if default_repo:
971 server_repos = [default_repo]
972
973 for repo in server_repos:
974 dbconn = self.open_server_repository(read_only = True,
975 no_upload = True, repo = repo, do_treeupdates = False)
976 installed_packages |= set([(x, repo) for x in \
977 dbconn.listAllIdpackages()])
978
979
980 deps_not_satisfied = set()
981 length = str((len(installed_packages)))
982 count = 0
983 mytxt = _("Checking")
984
985 for idpackage, repo in installed_packages:
986 count += 1
987 dbconn = self.open_server_repository(read_only = True,
988 no_upload = True, repo = repo, do_treeupdates = False)
989
990 if (count%150 == 0) or (count == length) or (count == 1):
991 atom = dbconn.retrieveAtom(idpackage)
992 self.updateProgress(
993 darkgreen(mytxt)+" "+bold(atom),
994 importance = 0,
995 type = "info",
996 back = True,
997 count = (count, length),
998 header = darkred(" @@ ")
999 )
1000
1001 xdeps = dbconn.retrieveDependencies(idpackage)
1002 for xdep in xdeps:
1003 xid, xuseless = self.atom_match(xdep)
1004 if xid == -1:
1005 deps_not_satisfied.add(xdep)
1006
1007 return deps_not_satisfied
1008
1010
1011 mytxt = "%s %s" % (blue(_("Running dependencies test")), red("..."))
1012 self.updateProgress(
1013 mytxt,
1014 importance = 2,
1015 type = "info",
1016 header = red(" @@ ")
1017 )
1018
1019 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
1020 server_repos = srv_set['repositories'].keys()
1021 deps_not_matched = self.deps_tester(repo)
1022
1023 if deps_not_matched:
1024
1025 crying_atoms = {}
1026 for atom in deps_not_matched:
1027 for repo in server_repos:
1028 dbconn = self.open_server_repository(just_reading = True,
1029 repo = repo, do_treeupdates = False)
1030 riddep = dbconn.searchDependency(atom)
1031 if riddep == -1:
1032 continue
1033 ridpackages = dbconn.searchIdpackageFromIddependency(riddep)
1034 for i in ridpackages:
1035 iatom = dbconn.retrieveAtom(i)
1036 if not crying_atoms.has_key(atom):
1037 crying_atoms[atom] = set()
1038 crying_atoms[atom].add((iatom, repo))
1039
1040 mytxt = blue("%s:") % (_("These are the dependencies not found"),)
1041 self.updateProgress(
1042 mytxt,
1043 importance = 1,
1044 type = "info",
1045 header = red(" @@ ")
1046 )
1047 mytxt = "%s:" % (_("Needed by"),)
1048 for atom in deps_not_matched:
1049 self.updateProgress(
1050 red(atom),
1051 importance = 1,
1052 type = "info",
1053 header = blue(" # ")
1054 )
1055 if crying_atoms.has_key(atom):
1056 self.updateProgress(
1057 red(mytxt),
1058 importance = 0,
1059 type = "info",
1060 header = blue(" # ")
1061 )
1062 for my_dep, myrepo in crying_atoms[atom]:
1063 self.updateProgress(
1064 "[%s:%s] %s" % (
1065 blue(_("by repo")),
1066 darkred(myrepo),
1067 darkgreen(my_dep),
1068 ),
1069 importance = 0,
1070 type = "info",
1071 header = blue(" # ")
1072 )
1073 else:
1074
1075 mytxt = blue(_("Every dependency is satisfied. It's all fine."))
1076 self.updateProgress(
1077 mytxt,
1078 importance = 2,
1079 type = "info",
1080 header = red(" @@ ")
1081 )
1082
1083 return deps_not_matched
1084
1086
1087
1088 dbconn = self.open_server_repository(read_only = True,
1089 no_upload = True, repo = repo)
1090 packages_matched, brokenexecs, status = \
1091 self.ClientService.libraries_test(dbconn = dbconn,
1092 broken_symbols = False)
1093 if status != 0:
1094 return 1, None
1095
1096 if get_files:
1097 return 0, brokenexecs
1098
1099 if (not brokenexecs) and (not packages_matched):
1100 mytxt = "%s." % (_("System is healthy"),)
1101 self.updateProgress(
1102 blue(mytxt),
1103 importance = 2,
1104 type = "info",
1105 header = red(" @@ ")
1106 )
1107 return 0, None
1108
1109 mytxt = "%s..." % (_("Matching libraries with Spm, please wait"),)
1110 self.updateProgress(
1111 blue(mytxt),
1112 importance = 1,
1113 type = "info",
1114 header = red(" @@ ")
1115 )
1116
1117 packages = self.SpmService.query_belongs_multiple(brokenexecs)
1118
1119 if packages:
1120 mytxt = "%s:" % (_("These are the matched packages"),)
1121 self.updateProgress(
1122 red(mytxt),
1123 importance = 1,
1124 type = "info",
1125 header = red(" @@ ")
1126 )
1127 for package_slot in packages:
1128 self.updateProgress(
1129 blue(unicode(package_slot)),
1130 importance = 0,
1131 type = "info",
1132 header = red(" # ")
1133 )
1134 for filename in sorted(list(packages[package_slot])):
1135 self.updateProgress(
1136 blue(filename),
1137 importance = 0,
1138 type = "info",
1139 header = brown(" => ")
1140 )
1141
1142 pkgstring = ' '.join(["%s:%s" % (
1143 self.entropyTools.dep_getkey(x[0]), x[1],) for x \
1144 in sorted(packages)])
1145 mytxt = "%s: %s" % (darkgreen(_("Packages string")), pkgstring,)
1146 self.updateProgress(
1147 mytxt,
1148 importance = 1,
1149 type = "info",
1150 header = red(" @@ ")
1151 )
1152 else:
1153 self.updateProgress(
1154 red(_("No matched packages")),
1155 importance = 1,
1156 type = "info",
1157 header = red(" @@ ")
1158 )
1159
1160 return 0, packages
1161
1163
1164 mytxt = "%s %s" % (
1165 blue(_("Running orphaned SPM packages test")), red("..."),)
1166 self.updateProgress(
1167 mytxt,
1168 importance = 2,
1169 type = "info",
1170 header = red(" @@ ")
1171 )
1172 installed_packages, length = self.SpmService.get_installed_packages()
1173 not_found = {}
1174 count = 0
1175 for installed_package in installed_packages:
1176 count += 1
1177 self.updateProgress(
1178 "%s: %s" % (
1179 darkgreen(_("Scanning package")),
1180 brown(installed_package),),
1181 importance = 0,
1182 type = "info",
1183 back = True,
1184 count = (count, length),
1185 header = darkred(" @@ ")
1186 )
1187 key, slot = (self.entropyTools.dep_getkey(installed_package),
1188 self.SpmService.get_installed_package_slot(installed_package),)
1189 pkg_atom = "%s:%s" % (key, slot,)
1190 tree_atom = self.SpmService.get_best_atom(pkg_atom)
1191 if not tree_atom:
1192 not_found[installed_package] = pkg_atom
1193 self.updateProgress(
1194 "%s: %s" % (
1195 blue(pkg_atom),
1196 darkred(_("not found anymore")),
1197 ),
1198 importance = 0,
1199 type = "warning",
1200 count = (count, length),
1201 header = darkred(" @@ ")
1202 )
1203
1204 if not_found:
1205 not_found_list = ' '.join([not_found[x] for x in sorted(not_found)])
1206 self.updateProgress(
1207 "%s: %s" % (
1208 blue(_("Packages string")),
1209 not_found_list,
1210 ),
1211 importance = 0,
1212 type = "warning",
1213 count = (count, length),
1214 header = darkred(" @@ ")
1215 )
1216
1217 return not_found
1218
1225
1232
1234 if dbpath == None:
1235 dbpath = self.get_local_database_file(repo)
1236
1237 dbdir = os.path.dirname(dbpath)
1238 if not os.path.isdir(dbdir):
1239 os.makedirs(dbdir)
1240
1241 mytxt = red("%s ...") % (_("Initializing an empty database"),)
1242 self.updateProgress(
1243 mytxt,
1244 importance = 1,
1245 type = "info",
1246 header = darkgreen(" * "),
1247 back = True
1248 )
1249 dbconn = self.ClientService.open_generic_database(dbpath)
1250 dbconn.initializeDatabase()
1251 dbconn.commitChanges()
1252 dbconn.closeDB()
1253 mytxt = "%s %s %s." % (
1254 red(_("Entropy database file")),
1255 bold(dbpath),
1256 red(_("successfully initialized")),
1257 )
1258 self.updateProgress(
1259 mytxt,
1260 importance = 1,
1261 type = "info",
1262 header = darkgreen(" * ")
1263 )
1264
1265 - def tag_packages(self, package_tag, idpackages, repo = None, ask = True):
1266
1267
1268
1269 try:
1270 package_tag = str(package_tag)
1271 if " " in package_tag:
1272 raise ValueError
1273 except (UnicodeDecodeError, UnicodeEncodeError, ValueError,):
1274 self.updateProgress(
1275 "%s: %s" % (
1276 blue(_("Invalid tag specified")),
1277 package_tag,
1278 ),
1279 importance = 1, type = "error", header = darkred(" !! ")
1280 )
1281 return 1, package_tag
1282
1283 if repo == None:
1284 repo = self.default_repository
1285
1286
1287 invalid_atoms = []
1288 dbconn = self.open_server_repository(read_only = True,
1289 no_upload = True, repo = repo)
1290 for idpackage in idpackages:
1291 ver_tag = dbconn.retrieveVersionTag(idpackage)
1292 if ver_tag:
1293 invalid_atoms.append(dbconn.retrieveAtom(idpackage))
1294
1295 if invalid_atoms:
1296 self.updateProgress(
1297 "%s: %s" % (
1298 blue(_("Packages already tagged, action aborted")),
1299 ', '.join([darkred(unicode(x)) for x in invalid_atoms]),
1300 ),
1301 importance = 1, type = "error", header = darkred(" !! ")
1302 )
1303 return 2, invalid_atoms
1304
1305 matches = [(x, repo) for x in idpackages]
1306 status = 0
1307 data = self.move_packages(
1308 matches, to_repo = repo, from_repo = repo, ask = ask,
1309 do_copy = True, new_tag = package_tag
1310 )
1311 return status, data
1312
1314 """
1315 When creating a new branch, for space reasons, packages are not
1316 moved to a new location. This works fine until old branch is removed.
1317 To avoid inconsistences, before deciding to do that, all the packages
1318 in the old branch should be flushed back to the the currently configured
1319 branch.
1320
1321 @param from_branches -- list of branches to move packages from
1322 @type from_branches -- list
1323 @param repo -- repository to work on
1324 @type repo -- str
1325 @param ask -- user interactivity
1326 @type ask -- bool
1327
1328 @return status
1329 """
1330
1331 status = True
1332 if repo == None:
1333 repo = self.default_repository
1334 branch = self.SystemSettings['repositories']['branch']
1335
1336 if branch in from_branches:
1337 from_branches = [x for x in from_branches if x != branch]
1338
1339 self.updateProgress(
1340 "[%s=>%s|%s] %s" % (
1341 darkgreen(', '.join(from_branches)),
1342 darkred(branch),
1343 brown(repo),
1344 blue(_("flushing back selected packages from branches")),
1345 ),
1346 importance = 2,
1347 type = "info",
1348 header = red(" @@ ")
1349 )
1350
1351 dbconn = self.open_server_repository(read_only = True,
1352 no_upload = True, repo = repo)
1353
1354 idpackage_map = dict(((x, [],) for x in from_branches))
1355 idpackages = dbconn.listAllIdpackages(order_by = 'atom')
1356 for idpackage in idpackages:
1357 download_url = dbconn.retrieveDownloadURL(idpackage)
1358 url_br = self.ClientService.get_branch_from_download_relative_uri(
1359 download_url)
1360 if url_br in from_branches:
1361 idpackage_map[url_br].append(idpackage)
1362
1363 mapped_branches = [x for x in idpackage_map if idpackage_map[x]]
1364 if not mapped_branches:
1365 self.updateProgress(
1366 "[%s=>%s|%s] %s !" % (
1367 darkgreen(', '.join(from_branches)),
1368 darkred(branch),
1369 brown(repo),
1370 blue(_("nothing to do")),
1371 ),
1372 importance = 0,
1373 type = "warning",
1374 header = blue(" @@ ")
1375 )
1376 return status
1377
1378
1379 all_fine = True
1380 tmp_down_dir = self.entropyTools.get_random_temp_file()
1381 os.makedirs(tmp_down_dir)
1382
1383 download_queue = {}
1384 local_up_dir = self.get_local_upload_directory(repo)
1385 local_basedir = os.path.join(local_up_dir, branch)
1386 dbconn = self.open_server_repository(read_only = False,
1387 no_upload = True, repo = repo)
1388
1389 def generate_queue(branch, repo, from_branch, down_q, idpackage_map):
1390
1391 self.updateProgress(
1392 "[%s=>%s|%s] %s" % (
1393 darkgreen(from_branch),
1394 darkred(branch),
1395 brown(repo),
1396 brown(_("these are the packages that will be flushed")),
1397 ),
1398 importance = 1,
1399 type = "info",
1400 header = brown(" @@ ")
1401 )
1402
1403
1404 for idpackage in idpackage_map[from_branch]:
1405 atom = dbconn.retrieveAtom(idpackage)
1406 self.updateProgress(
1407 "[%s=>%s|%s] %s" % (
1408 darkgreen(from_branch),
1409 darkred(branch),
1410 brown(repo),
1411 purple(atom),
1412 ),
1413 importance = 0,
1414 type = "info",
1415 header = blue(" # ")
1416 )
1417 pkg_fp = os.path.basename(dbconn.retrieveDownloadURL(idpackage))
1418 pkg_fp = os.path.join(tmp_down_dir, pkg_fp)
1419 down_q.append((pkg_fp, idpackage,))
1420
1421
1422 for from_branch in sorted(mapped_branches):
1423
1424 download_queue[from_branch] = []
1425 all_fine = False
1426 generate_queue(branch, repo, from_branch,
1427 download_queue[from_branch], idpackage_map)
1428
1429 if ask:
1430 rc_question = self.askQuestion(
1431 _("Would you like to continue ?"))
1432 if rc_question == "No":
1433 continue
1434
1435 remote_relative_path = self.get_remote_packages_relative_path(repo)
1436
1437 for uri in self.get_remote_mirrors(repo):
1438
1439 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri)
1440 ftp_basedir = os.path.join(remote_relative_path, from_branch)
1441
1442 downloader_queue = [x[0] for x in download_queue[from_branch]]
1443 downloader = self.MirrorsService.FtpServerHandler(
1444 FtpInterface,
1445 self,
1446 [uri],
1447 downloader_queue,
1448 critical_files = downloader_queue,
1449 use_handlers = True,
1450 ftp_basedir = ftp_basedir,
1451 local_basedir = tmp_down_dir,
1452 download = True,
1453 repo = repo
1454 )
1455
1456 errors, m_fine_uris, m_broken_uris = downloader.go()
1457
1458 if not errors:
1459 for downloaded_path, idpackage in \
1460 download_queue[from_branch]:
1461
1462 self.updateProgress(
1463 "[%s=>%s|%s|%s] %s: %s" % (
1464 darkgreen(from_branch),
1465 darkred(branch),
1466 brown(repo),
1467 dbconn.retrieveAtom(idpackage),
1468 blue(_("checking package hash")),
1469 darkgreen(os.path.basename(downloaded_path)),
1470 ),
1471 importance = 0,
1472 type = "info",
1473 header = brown(" "),
1474 back = True
1475 )
1476
1477 md5hash = self.entropyTools.md5sum(downloaded_path)
1478 db_md5hash = dbconn.retrieveDigest(idpackage)
1479 if md5hash != db_md5hash:
1480 errors = True
1481 self.updateProgress(
1482 "[%s=>%s|%s|%s] %s: %s" % (
1483 darkgreen(from_branch),
1484 darkred(branch),
1485 brown(repo),
1486 dbconn.retrieveAtom(idpackage),
1487 blue(_("hash does not match for")),
1488 darkgreen(os.path.basename(downloaded_path)),
1489 ),
1490 importance = 0,
1491 type = "error",
1492 header = brown(" ")
1493 )
1494 continue
1495
1496
1497
1498 if errors:
1499 reason = _("wrong md5")
1500 if m_broken_uris:
1501 my_broken_uris = [
1502 (self.entropyTools.extract_ftp_host_from_uri(x), y,) \
1503 for x, y in m_broken_uris]
1504 reason = my_broken_uris[0][1]
1505
1506 self.updateProgress(
1507 "[%s=>%s|%s] %s, %s: %s" % (
1508 darkgreen(from_branch),
1509 darkred(branch),
1510 brown(repo),
1511 blue(_("download errors")),
1512 blue(_("reason")),
1513 reason,
1514 ),
1515 importance = 1,
1516 type = "error",
1517 header = darkred(" !!! ")
1518 )
1519
1520 continue
1521
1522 all_fine = True
1523
1524 self.updateProgress(
1525 "[%s=>%s|%s] %s: %s" % (
1526 darkgreen(from_branch),
1527 darkred(branch),
1528 brown(repo),
1529 blue(_("download completed successfully")),
1530 darkgreen(crippled_uri),
1531 ),
1532 importance = 1,
1533 type = "info",
1534 header = darkgreen(" * ")
1535 )
1536
1537 if not all_fine:
1538 self.updateProgress(
1539 "[%s=>%s|%s] %s" % (
1540 darkgreen(', '.join(from_branches)),
1541 darkred(branch),
1542 brown(repo),
1543 blue(_("error downloading packages from mirrors")),
1544 ),
1545 importance = 2,
1546 type = "error",
1547 header = darkred(" !!! ")
1548 )
1549 return False
1550
1551 tmp_db_path = self.entropyTools.get_random_temp_file()
1552 for from_branch in sorted(mapped_branches):
1553
1554 self.updateProgress(
1555 "[%s=>%s|%s] %s: %s" % (
1556 darkgreen(from_branch),
1557 darkred(branch),
1558 brown(repo),
1559 blue(_("working on branch")),
1560 darkgreen(from_branch),
1561 ),
1562 importance = 1,
1563 type = "info",
1564 header = brown(" @@ ")
1565 )
1566
1567 down_queue = download_queue[from_branch]
1568 for package_path, idpackage in down_queue:
1569
1570 self.updateProgress(
1571 "[%s=>%s|%s] %s: %s" % (
1572 darkgreen(from_branch),
1573 darkred(branch),
1574 brown(repo),
1575 blue(_("updating package")),
1576 darkgreen(os.path.basename(package_path)),
1577 ),
1578 importance = 1,
1579 type = "info",
1580 header = brown(" "),
1581 back = True
1582 )
1583
1584
1585 package_name = os.path.basename(package_path)
1586 new_package_path = os.path.join(local_basedir, package_name)
1587 try:
1588 os.rename(package_path, new_package_path)
1589 except OSError:
1590 shutil.move(package_path, new_package_path)
1591
1592
1593 self.entropyTools.create_md5_file(new_package_path)
1594
1595
1596 download_url = dbconn.retrieveDownloadURL(idpackage)
1597 download_url = \
1598 self.ClientService.swap_branch_in_download_relative_uri(
1599 branch, download_url)
1600 dbconn.setDownloadURL(idpackage, download_url)
1601 dbconn.switchBranch(idpackage, branch)
1602 dbconn.commitChanges()
1603
1604 self.updateProgress(
1605 "[%s=>%s|%s] %s: %s" % (
1606 darkgreen(from_branch),
1607 darkred(branch),
1608 brown(repo),
1609 blue(_("package flushed")),
1610 darkgreen(os.path.basename(package_path)),
1611 ),
1612 importance = 1,
1613 type = "info",
1614 header = brown(" ")
1615 )
1616
1617 try:
1618 os.rmdir(tmp_down_dir)
1619 except OSError:
1620 pass
1621
1622 return True
1623
1624 - def move_packages(self, matches, to_repo, from_repo = None, ask = True,
1625 do_copy = False, new_tag = None, pull_deps = False):
1626
1627 if from_repo == None:
1628 from_repo = self.default_repository
1629 switched = set()
1630
1631 my_matches = list(matches)
1632
1633
1634 if etpConst['clientserverrepoid'] in (to_repo, from_repo):
1635 self.updateProgress(
1636 "%s: %s" % (
1637 blue(_("Cannot touch system database")),
1638 red(etpConst['clientserverrepoid']),
1639 ),
1640 importance = 2, type = "warning", header = darkred(" @@ ")
1641 )
1642 return switched
1643
1644 if not my_matches and from_repo:
1645 dbconn = self.open_server_repository(read_only = True,
1646 no_upload = True, repo = from_repo)
1647 my_matches = set( \
1648 [(x, from_repo) for x in \
1649 dbconn.listAllIdpackages()]
1650 )
1651
1652 mytxt = _("Preparing to move selected packages to")
1653 if do_copy:
1654 mytxt = _("Preparing to copy selected packages to")
1655 self.updateProgress(
1656 "%s %s:" % (
1657 blue(mytxt),
1658 red(to_repo),
1659 ),
1660 importance = 2,
1661 type = "info",
1662 header = red(" @@ ")
1663 )
1664 self.updateProgress(
1665 "%s: %s" % (
1666 bold(_("Note")),
1667 red(_("all old packages with conflicting scope will be " \
1668 "removed from destination repo unless injected")),
1669 ),
1670 importance = 1,
1671 type = "info",
1672 header = red(" @@ ")
1673 )
1674
1675 new_tag_string = ''
1676 if new_tag != None:
1677 new_tag_string = "[%s: %s]" % (darkgreen(_("new tag")),
1678 brown(new_tag),)
1679
1680 my_qa = self.QA()
1681 branch = self.SystemSettings['repositories']['branch']
1682 pull_deps_matches = []
1683 for idpackage, repo in my_matches:
1684 dbconn = self.open_server_repository(read_only = True,
1685 no_upload = True, repo = repo)
1686 self.updateProgress(
1687 "[%s=>%s|%s] %s " % (
1688 darkgreen(repo),
1689 darkred(to_repo),
1690 brown(branch),
1691 blue(dbconn.retrieveAtom(idpackage)),
1692 ) + new_tag_string,
1693 importance = 0,
1694 type = "info",
1695 header = brown(" # ")
1696 )
1697
1698 if pull_deps:
1699 dep_idpackages = my_qa.get_deep_dependency_list(dbconn,
1700 idpackage)
1701 for dep_idpackage in dep_idpackages:
1702
1703 my_dep_match = (dep_idpackage, repo,)
1704 if my_dep_match in pull_deps_matches:
1705 continue
1706 if my_dep_match in my_matches:
1707 continue
1708
1709 pull_deps_matches.append(my_dep_match)
1710 self.updateProgress(
1711 "[%s|%s] %s" % (
1712 brown(branch),
1713 blue(_("dependency")),
1714 purple(dbconn.retrieveAtom(dep_idpackage)),
1715 ),
1716 importance = 0,
1717 type = "info",
1718 header = purple(" >> ")
1719 )
1720
1721 if pull_deps:
1722
1723 my_matches = pull_deps_matches + [x for x in my_matches if x not \
1724 in pull_deps_matches]
1725
1726 if ask:
1727 rc_question = self.askQuestion(_("Would you like to continue ?"))
1728 if rc_question == "No":
1729 return switched
1730
1731 for idpackage, repo in my_matches:
1732 dbconn = self.open_server_repository(read_only = False,
1733 no_upload = True, repo = repo)
1734 match_branch = dbconn.retrieveBranch(idpackage)
1735 match_atom = dbconn.retrieveAtom(idpackage)
1736 package_filename = os.path.basename(
1737 dbconn.retrieveDownloadURL(idpackage))
1738 self.updateProgress(
1739 "[%s=>%s|%s] %s: %s" % (
1740 darkgreen(repo),
1741 darkred(to_repo),
1742 brown(branch),
1743 blue(_("switching")),
1744 darkgreen(match_atom),
1745 ),
1746 importance = 0,
1747 type = "info",
1748 header = red(" @@ "),
1749 back = True
1750 )
1751
1752 from_file = os.path.join(self.get_local_packages_directory(repo),
1753 match_branch, package_filename)
1754 if not os.path.isfile(from_file):
1755 from_file = os.path.join(self.get_local_upload_directory(repo),
1756 match_branch, package_filename)
1757 if not os.path.isfile(from_file):
1758 self.updateProgress(
1759 "[%s=>%s|%s] %s: %s -> %s" % (
1760 darkgreen(repo),
1761 darkred(to_repo),
1762 brown(branch),
1763 bold(_("cannot switch, package not found, skipping")),
1764 darkgreen(),
1765 red(from_file),
1766 ),
1767 importance = 1,
1768 type = "warning",
1769 header = darkred(" !!! ")
1770 )
1771 continue
1772
1773 if new_tag != None:
1774 match_category = dbconn.retrieveCategory(idpackage)
1775 match_name = dbconn.retrieveName(idpackage)
1776 match_version = dbconn.retrieveVersion(idpackage)
1777 tagged_package_filename = \
1778 self.entropyTools.create_package_filename(
1779 match_category, match_name, match_version, new_tag)
1780 to_file = os.path.join(self.get_local_upload_directory(to_repo),
1781 match_branch, tagged_package_filename)
1782 else:
1783 to_file = os.path.join(self.get_local_upload_directory(to_repo),
1784 match_branch, package_filename)
1785 if not os.path.isdir(os.path.dirname(to_file)):
1786 os.makedirs(os.path.dirname(to_file))
1787
1788 copy_data = [
1789 (from_file, to_file,),
1790 (from_file + etpConst['packagesmd5fileext'],
1791 to_file + etpConst['packagesmd5fileext'],),
1792 (from_file + etpConst['packagesexpirationfileext'],
1793 to_file + etpConst['packagesexpirationfileext'],)
1794 ]
1795
1796 for from_item, to_item in copy_data:
1797 self.updateProgress(
1798 "[%s=>%s|%s] %s: %s" % (
1799 darkgreen(repo),
1800 darkred(to_repo),
1801 brown(branch),
1802 blue(_("moving file")),
1803 darkgreen(os.path.basename(from_item)),
1804 ),
1805 importance = 0,
1806 type = "info",
1807 header = red(" @@ "),
1808 back = True
1809 )
1810 if os.path.isfile(from_item):
1811 shutil.copy2(from_item, to_item)
1812
1813 self.updateProgress(
1814 "[%s=>%s|%s] %s: %s" % (
1815 darkgreen(repo),
1816 darkred(to_repo),
1817 brown(branch),
1818 blue(_("loading data from source database")),
1819 darkgreen(repo),
1820 ),
1821 importance = 0,
1822 type = "info",
1823 header = red(" @@ "),
1824 back = True
1825 )
1826
1827 data = dbconn.getPackageData(idpackage)
1828 if new_tag != None:
1829 data['versiontag'] = new_tag
1830
1831 todbconn = self.open_server_repository(read_only = False,
1832 no_upload = True, repo = to_repo)
1833
1834 self.updateProgress(
1835 "[%s=>%s|%s] %s: %s" % (
1836 darkgreen(repo),
1837 darkred(to_repo),
1838 brown(branch),
1839 blue(_("injecting data to destination database")),
1840 darkgreen(to_repo),
1841 ),
1842 importance = 0,
1843 type = "info",
1844 header = red(" @@ "),
1845 back = True
1846 )
1847 new_idpackage, new_revision, new_data = todbconn.handlePackage(data)
1848 del data
1849 todbconn.commitChanges()
1850
1851 if not do_copy:
1852 self.updateProgress(
1853 "[%s=>%s|%s] %s: %s" % (
1854 darkgreen(repo),
1855 darkred(to_repo),
1856 brown(branch),
1857 blue(_("removing entry from source database")),
1858 darkgreen(repo),
1859 ),
1860 importance = 0,
1861 type = "info",
1862 header = red(" @@ "),
1863 back = True
1864 )
1865
1866
1867 dbconn.removePackage(idpackage)
1868 dbconn.commitChanges()
1869
1870 self.updateProgress(
1871 "[%s=>%s|%s] %s: %s" % (
1872 darkgreen(repo),
1873 darkred(to_repo),
1874 brown(branch),
1875 blue(_("successfully handled atom")),
1876 darkgreen(match_atom),
1877 ),
1878 importance = 0,
1879 type = "info",
1880 header = blue(" @@ ")
1881 )
1882 switched.add((idpackage, repo,))
1883
1884
1885 self.dependencies_test(to_repo)
1886
1887 return switched
1888
1889
1891
1892 if repo == None:
1893 repo = self.default_repository
1894
1895 upload_dir = os.path.join(self.get_local_upload_directory(repo),
1896 self.SystemSettings['repositories']['branch'])
1897 if not os.path.isdir(upload_dir):
1898 os.makedirs(upload_dir)
1899
1900 dbconn = self.open_server_repository(read_only = False,
1901 no_upload = True, repo = repo)
1902 self.updateProgress(
1903 red("[repo: %s] %s: %s" % (
1904 darkgreen(repo),
1905 _("adding package"),
1906 bold(os.path.basename(package_file)),
1907 )
1908 ),
1909 importance = 1,
1910 type = "info",
1911 header = brown(" * "),
1912 back = True
1913 )
1914 mydata = self.SpmService.extract_pkg_metadata(package_file,
1915 inject = inject)
1916 idpackage, revision, mydata = dbconn.handlePackage(mydata)
1917
1918
1919 trashing_counters = set()
1920 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
1921 myserver_repos = srv_set['repositories'].keys()
1922 for myrepo in myserver_repos:
1923 mydbconn = self.open_server_repository(read_only = True,
1924 no_upload = True, repo = myrepo)
1925 mylist = mydbconn.retrieve_packages_to_remove(
1926 mydata['name'],
1927 mydata['category'],
1928 mydata['slot'],
1929 mydata['injected']
1930 )
1931 for myitem in mylist:
1932 trashing_counters.add(mydbconn.retrieveCounter(myitem))
1933
1934 for mycounter in trashing_counters:
1935 dbconn.setTrashedCounter(mycounter)
1936
1937
1938 dbconn.removePackageFromInstalledTable(idpackage)
1939 dbconn.addPackageToInstalledTable(idpackage, repo)
1940 atom = dbconn.retrieveAtom(idpackage)
1941
1942 self.updateProgress(
1943 "[repo:%s] %s: %s %s: %s" % (
1944 darkgreen(repo),
1945 blue(_("added package")),
1946 darkgreen(atom),
1947 blue(_("rev")),
1948 bold(str(revision)),
1949 ),
1950 importance = 1,
1951 type = "info",
1952 header = red(" @@ ")
1953 )
1954
1955 manual_deps = sorted(dbconn.retrieveManualDependencies(idpackage))
1956 if manual_deps:
1957 self.updateProgress(
1958 "[repo:%s] %s: %s" % (
1959 darkgreen(repo),
1960 blue(_("manual dependencies for")),
1961 darkgreen(atom),
1962 ),
1963 importance = 1,
1964 type = "warning",
1965 header = darkgreen(" ## ")
1966 )
1967 for m_dep in manual_deps:
1968 self.updateProgress(
1969 brown(m_dep),
1970 importance = 1,
1971 type = "warning",
1972 header = darkred(" # ")
1973 )
1974
1975 download_url = self._setup_repository_package_filename(idpackage,
1976 repo = repo)
1977 downloadfile = os.path.basename(download_url)
1978 destination_path = os.path.join(upload_dir, downloadfile)
1979 try:
1980 os.rename(package_file, destination_path)
1981 except OSError:
1982 shutil.move(package_file, destination_path)
1983
1984 dbconn.commitChanges()
1985 return idpackage, destination_path
1986
1987
1989
1990 dbconn = self.open_server_repository(read_only = False,
1991 no_upload = True, repo = repo)
1992
1993 downloadurl = dbconn.retrieveDownloadURL(idpackage)
1994 packagerev = dbconn.retrieveRevision(idpackage)
1995 downloaddir = os.path.dirname(downloadurl)
1996 downloadfile = os.path.basename(downloadurl)
1997
1998 downloadfile = downloadfile[:-5]+"~%s%s" % (packagerev,
1999 etpConst['packagesext'],)
2000 downloadurl = os.path.join(downloaddir, downloadfile)
2001
2002
2003 dbconn.setDownloadURL(idpackage, downloadurl)
2004
2005 return downloadurl
2006
2009
2010 if repo == None:
2011 repo = self.default_repository
2012
2013 mycount = 0
2014 maxcount = len(packages_data)
2015 idpackages_added = set()
2016 to_be_injected = set()
2017 my_qa = self.QA()
2018 missing_deps_taint = False
2019 for package_filepath, inject in packages_data:
2020
2021 mycount += 1
2022 self.updateProgress(
2023 "[repo:%s] %s: %s" % (
2024 darkgreen(repo),
2025 blue(_("adding package")),
2026 darkgreen(os.path.basename(package_filepath)),
2027 ),
2028 importance = 1,
2029 type = "info",
2030 header = blue(" @@ "),
2031 count = (mycount, maxcount,)
2032 )
2033
2034 try:
2035
2036 idpackage, destination_path = self.package_injector(
2037 package_filepath,
2038 inject = inject,
2039 repo = repo
2040 )
2041 idpackages_added.add(idpackage)
2042 to_be_injected.add((idpackage, destination_path))
2043 except Exception, err:
2044 self.entropyTools.print_traceback()
2045 self.updateProgress(
2046 "[repo:%s] %s: %s" % (
2047 darkgreen(repo),
2048 darkred(_("Exception caught, closing tasks")),
2049 darkgreen(unicode(err)),
2050 ),
2051 importance = 1,
2052 type = "error",
2053 header = bold(" !!! "),
2054 count = (mycount, maxcount,)
2055 )
2056
2057 self.depends_table_initialize(repo)
2058
2059 self.library_paths_table_initialize(repo)
2060 if idpackages_added:
2061 dbconn = self.open_server_repository(read_only = False,
2062 no_upload = True, repo = repo)
2063 missing_deps_taint = my_qa.scan_missing_dependencies(
2064 idpackages_added,
2065 dbconn,
2066 ask = ask,
2067 repo = repo,
2068 self_check = True,
2069 black_list = \
2070 self.get_missing_dependencies_blacklist(
2071 repo = repo),
2072 black_list_adder = \
2073 self.add_missing_dependencies_blacklist_items
2074 )
2075 my_qa.test_depends_linking(idpackages_added, dbconn,
2076 repo = repo)
2077 if to_be_injected:
2078 self.inject_database_into_packages(to_be_injected,
2079 repo = repo)
2080
2081 if missing_deps_taint:
2082 self.depends_table_initialize(repo)
2083 self.close_server_databases()
2084 raise
2085
2086
2087 self.depends_table_initialize(repo)
2088
2089 self.library_paths_table_initialize(repo)
2090
2091 if idpackages_added:
2092 dbconn = self.open_server_repository(read_only = False,
2093 no_upload = True, repo = repo)
2094 missing_deps_taint = my_qa.scan_missing_dependencies(
2095 idpackages_added,
2096 dbconn,
2097 ask = ask,
2098 repo = repo,
2099 self_check = True,
2100 black_list = \
2101 self.get_missing_dependencies_blacklist(repo = repo),
2102 black_list_adder = \
2103 self.add_missing_dependencies_blacklist_items
2104 )
2105 my_qa.test_depends_linking(idpackages_added, dbconn, repo = repo)
2106
2107
2108 if missing_deps_taint:
2109 self.depends_table_initialize(repo)
2110
2111
2112 self.inject_database_into_packages(to_be_injected, repo = repo)
2113
2114 return idpackages_added
2115
2116
2118
2119 if repo == None:
2120 repo = self.default_repository
2121
2122
2123 self.updateProgress(
2124 "[repo:%s] %s:" % (
2125 darkgreen(repo),
2126 blue(_("Injecting entropy metadata into built packages")),
2127 ),
2128 importance = 1,
2129 type = "info",
2130 header = red(" @@ ")
2131 )
2132
2133 dbconn = self.open_server_repository(read_only = False,
2134 no_upload = True, repo = repo)
2135 for idpackage, package_path in injection_data:
2136 self.updateProgress(
2137 "[repo:%s|%s] %s: %s" % (
2138 darkgreen(repo),
2139 brown(str(idpackage)),
2140 blue(_("injecting entropy metadata")),
2141 darkgreen(os.path.basename(package_path)),
2142 ),
2143 importance = 1,
2144 type = "info",
2145 header = blue(" @@ "),
2146 back = True
2147 )
2148 data = dbconn.getPackageData(idpackage)
2149 treeupdates_actions = dbconn.listAllTreeUpdatesActions()
2150 dbpath = self.ClientService.inject_entropy_database_into_package(
2151 package_path, data, treeupdates_actions)
2152 digest = self.entropyTools.md5sum(package_path)
2153
2154 dbconn.setDigest(idpackage, digest)
2155
2156 signatures = data['signatures'].copy()
2157 for hash_key in sorted(signatures):
2158 hash_func = getattr(self.entropyTools, hash_key)
2159 signatures[hash_key] = hash_func(package_path)
2160 dbconn.setSignatures(idpackage, signatures)
2161 self.entropyTools.create_md5_file(package_path)
2162
2163 os.remove(dbpath)
2164 self.updateProgress(
2165 "[repo:%s|%s] %s: %s" % (
2166 darkgreen(repo),
2167 brown(str(idpackage)),
2168 blue(_("injection complete")),
2169 darkgreen(os.path.basename(package_path)),
2170 ),
2171 importance = 1,
2172 type = "info",
2173 header = red(" @@ ")
2174 )
2175 dbconn.commitChanges()
2176
2178 self.updateProgress(
2179 "[%s] %s" % (
2180 red(_("config files")),
2181 blue(_("checking system")),
2182 ),
2183 importance = 1,
2184 type = "info",
2185 header = blue(" @@ "),
2186 back = True
2187 )
2188
2189 scandata = self.ClientService.FileUpdates.scanfs(dcache = False)
2190 if scandata:
2191 self.updateProgress(
2192 "[%s] %s" % (
2193 red(_("config files")),
2194 blue(_("there are configuration files not updated yet")),
2195 ),
2196 importance = 1,
2197 type = "error",
2198 header = darkred(" @@ ")
2199 )
2200 for key in scandata:
2201 self.updateProgress(
2202 "%s" % (brown(etpConst['systemroot'] + \
2203 scandata[key]['destination'])),
2204 importance = 1,
2205 type = "info",
2206 header = "\t"
2207 )
2208 return True
2209 return False
2210
2212 return self.SpmService.quickpkg(atom, storedir)
2213
2214
2216
2217 if repo == None:
2218 repo = self.default_repository
2219
2220 dbconn = self.open_server_repository(read_only = False,
2221 no_upload = True, repo = repo)
2222 for idpackage in idpackages:
2223 atom = dbconn.retrieveAtom(idpackage)
2224 self.updateProgress(
2225 "[repo:%s] %s: %s" % (
2226 darkgreen(repo),
2227 blue(_("removing package")),
2228 darkgreen(atom),
2229 ),
2230 importance = 1,
2231 type = "info",
2232 header = brown(" @@ ")
2233 )
2234 dbconn.removePackage(idpackage)
2235 self.close_server_database(dbconn)
2236 self.updateProgress(
2237 "[repo:%s] %s" % (
2238 darkgreen(repo),
2239 blue(_("removal complete")),
2240 ),
2241 importance = 1,
2242 type = "info",
2243 header = brown(" @@ ")
2244 )
2245
2246
2252
2254 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2255 if repo == None:
2256 repo = self.default_repository
2257 return srv_set['repositories'][repo]['mirrors'][:]
2258
2260 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2261 if repo == None:
2262 repo = self.default_repository
2263 return srv_set['repositories'][repo]['packages_relative_path']
2264
2266 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2267 if repo == None:
2268 repo = self.default_repository
2269 return srv_set['repositories'][repo]['database_relative_path']
2270
2272 if repo == None:
2273 repo = self.default_repository
2274 return os.path.join(self.get_local_database_dir(repo, branch),
2275 etpConst['etpdatabasefile'])
2276
2278 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2279 if repo == None:
2280 repo = self.default_repository
2281 return srv_set['repositories'][repo]['store_dir']
2282
2284 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2285 if repo == None:
2286 repo = self.default_repository
2287 return srv_set['repositories'][repo]['upload_dir']
2288
2290 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2291 if repo == None:
2292 repo = self.default_repository
2293 return srv_set['repositories'][repo]['packages_dir']
2294
2296 if repo == None:
2297 repo = self.default_repository
2298 return os.path.join(self.get_local_database_dir(repo, branch),
2299 etpConst['etpdatabasetaintfile'])
2300
2302 if repo == None:
2303 repo = self.default_repository
2304 return os.path.join(self.get_local_database_dir(repo, branch),
2305 etpConst['etpdatabaserevisionfile'])
2306
2308 if repo == None:
2309 repo = self.default_repository
2310 return os.path.join(self.get_local_database_dir(repo, branch),
2311 etpConst['etpdatabasetimestampfile'])
2312
2314 if repo == None:
2315 repo = self.default_repository
2316 return os.path.join(self.get_local_database_dir(repo, branch),
2317 etpConst['etpdatabasecacertfile'])
2318
2320 if repo == None:
2321 repo = self.default_repository
2322 return os.path.join(self.get_local_database_dir(repo, branch),
2323 etpConst['etpdatabaseservercertfile'])
2324
2326 if repo == None:
2327 repo = self.default_repository
2328 return os.path.join(self.get_local_database_dir(repo, branch),
2329 etpConst['etpdatabasemaskfile'])
2330
2332 if repo == None:
2333 repo = self.default_repository
2334 return os.path.join(self.get_local_database_dir(repo, branch),
2335 etpConst['etpdatabasesytemmaskfile'])
2336
2338 if repo == None:
2339 repo = self.default_repository
2340 return os.path.join(self.get_local_database_dir(repo, branch),
2341 etpConst['etpdatabaseconflictingtaggedfile'])
2342
2345
2346 if repo == None:
2347 repo = self.default_repository
2348 return os.path.join(self.get_local_database_dir(repo, branch),
2349 etpConst['etpdatabaselicwhitelistfile'])
2350
2352 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2353 if repo == None:
2354 repo = self.default_repository
2355 return os.path.join(self.get_local_database_dir(repo, branch),
2356 srv_set['rss']['name'])
2357
2359 if repo == None:
2360 repo = self.default_repository
2361 return os.path.join(self.get_local_database_dir(repo, branch),
2362 etpConst['rss-light-name'])
2363
2365 if repo == None:
2366 repo = self.default_repository
2367 return os.path.join(self.get_local_database_dir(repo, branch),
2368 etpConst['rss-notice-board'])
2369
2371 if repo == None:
2372 repo = self.default_repository
2373 return os.path.join(self.get_local_database_dir(repo, branch),
2374 etpConst['etpdatabaseupdatefile'])
2375
2383
2391
2394 if repo == None:
2395 repo = self.default_repository
2396 return os.path.join(self.get_local_database_dir(repo, branch),
2397 etpConst['etpdatabaseexpbasedpkgsrm'])
2398
2400 if repo == None:
2401 repo = self.default_repository
2402 return os.path.join(self.get_local_database_dir(repo, branch),
2403 etpConst['etpdatabasepkglist'])
2404
2406 if repo == None:
2407 repo = self.default_repository
2408 return os.path.join(self.get_local_database_dir(repo, branch),
2409 etpConst['confsetsdirname'])
2410
2412 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2413 if repo == None:
2414 repo = self.default_repository
2415 if branch == None:
2416 branch = self.SystemSettings['repositories']['branch']
2417 return os.path.join(srv_set['repositories'][repo]['database_dir'],
2418 branch)
2419
2422 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2423 if repo == None:
2424 repo = self.default_repository
2425 if branch == None:
2426 branch = self.SystemSettings['repositories']['branch']
2427 return os.path.join(srv_set['repositories'][repo]['database_dir'],
2428 branch, etpConst['etpdatabasemissingdepsblfile'])
2429
2431 if repo == None:
2432 repo = self.default_repository
2433 if branch == None:
2434 branch = self.SystemSettings['repositories']['branch']
2435 wl_file = self.get_missing_dependencies_blacklist_file(repo, branch)
2436 wl_data = []
2437 if os.path.isfile(wl_file) and os.access(wl_file, os.R_OK):
2438 f_wl = open(wl_file,"r")
2439 wl_data = [x.strip() for x in f_wl.readlines() if x.strip() and \
2440 not x.strip().startswith("#")]
2441 f_wl.close()
2442 return set(wl_data)
2443
2446
2447 if repo == None:
2448 repo = self.default_repository
2449 if branch == None:
2450 branch = self.SystemSettings['repositories']['branch']
2451 wl_file = self.get_missing_dependencies_blacklist_file(repo, branch)
2452 wl_dir = os.path.dirname(wl_file)
2453 if not (os.path.isdir(wl_dir) and os.access(wl_dir, os.W_OK)):
2454 return
2455 if os.path.isfile(wl_file) and not os.access(wl_file, os.W_OK):
2456 return
2457 f_wl = open(wl_file,"a+")
2458 f_wl.write('\n'.join(items)+'\n')
2459 f_wl.flush()
2460 f_wl.close()
2461
2463
2464 if repo == None:
2465 repo = self.default_repository
2466
2467 dbrev_file = self.get_local_database_revision_file(repo)
2468 if os.path.isfile(dbrev_file):
2469 f_rev = open(dbrev_file)
2470 rev = f_rev.readline().strip()
2471 f_rev.close()
2472 try:
2473 rev = int(rev)
2474 except ValueError:
2475 self.updateProgress(
2476 "[repo:%s] %s: %s - %s" % (
2477 darkgreen(repo),
2478 blue(_("invalid database revision")),
2479 bold(rev),
2480 blue(_("defaulting to 0")),
2481 ),
2482 importance = 2,
2483 type = "error",
2484 header = darkred(" !!! ")
2485 )
2486 rev = 0
2487 return rev
2488 else:
2489 return 0
2490
2492
2493 if repo == None:
2494 repo = self.default_repository
2495
2496 remote_status = self.MirrorsService.get_remote_databases_status(repo)
2497 if not [x for x in remote_status if x[1]]:
2498 remote_revision = 0
2499 else:
2500 remote_revision = max([x[1] for x in remote_status])
2501
2502 return remote_revision
2503
2506
2508 from datetime import datetime
2509 import time
2510 return "%s" % (datetime.fromtimestamp(time.time()),)
2511
2513 pkglist_file = self.get_local_pkglist_file(repo = repo, branch = branch)
2514
2515 tmp_pkglist_file = pkglist_file + ".tmp"
2516 dbconn = self.open_server_repository(repo = repo, just_reading = True,
2517 do_treeupdates = False)
2518 pkglist = dbconn.listAllDownloads(do_sort = True, full_path = True)
2519
2520 with open(tmp_pkglist_file, "w") as pkg_f:
2521 for pkg in pkglist:
2522 pkg_f.write(pkg + "\n")
2523 pkg_f.flush()
2524
2525 os.rename(tmp_pkglist_file, pkglist_file)
2526
2528 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2529 repos = srv_set['repositories'].keys()
2530 kwargs['server_repos'] = repos
2531 kwargs['serverInstance'] = self
2532 return self.ClientService.package_set_list(*args, **kwargs)
2533
2535 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2536 repos = srv_set['repositories'].keys()
2537 kwargs['server_repos'] = repos
2538 kwargs['serverInstance'] = self
2539 return self.ClientService.package_set_search(*args, **kwargs)
2540
2542 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2543 repos = srv_set['repositories'].keys()
2544 kwargs['server_repos'] = repos
2545 kwargs['serverInstance'] = self
2546 return self.ClientService.package_set_match(*args, **kwargs)
2547
2549 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2550 repos = srv_set['repositories'].keys()
2551 kwargs['server_repos'] = repos
2552 kwargs['serverInstance'] = self
2553 return self.ClientService.atom_match(*args, **kwargs)
2554
2556
2557 installed_packages = self.SpmService.get_installed_packages_counter()
2558 installed_counters = set()
2559 to_be_added = set()
2560 to_be_removed = set()
2561 to_be_injected = set()
2562 my_settings = self.SystemSettings[self.sys_settings_plugin_id]['server']
2563 exp_based_scope = my_settings['exp_based_scope']
2564
2565 server_repos = my_settings['repositories'].keys()
2566 exp_pkgs_cache = {}
2567
2568
2569 for spm_atom, spm_counter in installed_packages:
2570 found = False
2571 for server_repo in server_repos:
2572 installed_counters.add(spm_counter)
2573 server_dbconn = self.open_server_repository(read_only = True,
2574 no_upload = True, repo = server_repo)
2575 counter = server_dbconn.isCounterAvailable(spm_counter)
2576 if counter:
2577 found = True
2578 break
2579 if not found:
2580 to_be_added.add((spm_atom, spm_counter,))
2581
2582
2583 database_counters = {}
2584 for server_repo in server_repos:
2585 server_dbconn = self.open_server_repository(read_only = True,
2586 no_upload = True, repo = server_repo)
2587 database_counters[server_repo] = server_dbconn.listAllCounters(
2588 branch = self.SystemSettings['repositories']['branch'])
2589
2590 ordered_counters = set()
2591 for server_repo in database_counters:
2592 for data in database_counters[server_repo]:
2593 ordered_counters.add((data, server_repo))
2594 database_counters = ordered_counters
2595
2596 for (counter, idpackage,), xrepo in database_counters:
2597
2598 if counter < 0:
2599 continue
2600
2601 if counter in installed_counters:
2602 continue
2603
2604 dbconn = self.open_server_repository(read_only = True,
2605 no_upload = True, repo = xrepo)
2606
2607 dorm = True
2608
2609 if to_be_added:
2610
2611 dorm = False
2612 atom = dbconn.retrieveAtom(idpackage)
2613 atomkey = self.entropyTools.dep_getkey(atom)
2614 atomtag = self.entropyTools.dep_gettag(atom)
2615 atomslot = dbconn.retrieveSlot(idpackage)
2616
2617 add = True
2618 for spm_atom, spm_counter in to_be_added:
2619 addslot = self.SpmService.get_installed_package_slot(
2620 spm_atom)
2621 addkey = self.entropyTools.dep_getkey(spm_atom)
2622
2623 if addslot == None:
2624 addslot = '0'
2625
2626 if (atomkey == addkey) and \
2627 ((str(atomslot) == str(addslot)) or (atomtag != None)):
2628
2629 add = False
2630 break
2631
2632 if not add:
2633 continue
2634 dorm = True
2635
2636
2637
2638
2639 if dorm and (xrepo == self.default_repository):
2640 trashed = self.is_counter_trashed(counter)
2641 if trashed:
2642
2643 try:
2644 key, slot = dbconn.retrieveKeySlot(idpackage)
2645 trashed = self.SpmService.get_installed_atom(
2646 key+":"+slot)
2647 except TypeError:
2648 trashed = True
2649 if not trashed:
2650
2651 dbtag = dbconn.retrieveVersionTag(idpackage)
2652 if dbtag:
2653 is_injected = dbconn.isInjected(idpackage)
2654 if not is_injected:
2655 to_be_injected.add((idpackage, xrepo))
2656
2657 elif exp_based_scope:
2658
2659
2660 plg_id = self.sys_settings_fatscope_plugin_id
2661 exp_data = self.SystemSettings[plg_id]['repos'].get(
2662 xrepo, set())
2663
2664
2665
2666 if (idpackage not in exp_data) and (-1 not in exp_data):
2667 to_be_removed.add((idpackage, xrepo))
2668 continue
2669
2670 idpackage_expired = self.is_match_expired((idpackage,
2671 xrepo,))
2672
2673 if idpackage_expired:
2674
2675
2676 to_be_removed.add((idpackage, xrepo))
2677 for my_id in dbconn.retrieveDepends(idpackage):
2678 to_be_removed.add((my_id, xrepo))
2679
2680 else:
2681 to_be_removed.add((idpackage, xrepo))
2682
2683 return to_be_added, to_be_removed, to_be_injected
2684
2686
2687 idpackage, repoid = match
2688 dbconn = self.open_server_repository(repo = repoid, just_reading = True)
2689
2690 my_settings = self.SystemSettings[self.sys_settings_plugin_id]['server']
2691 pkg_exp_secs = my_settings['packages_expiration_days'] * 86400
2692 cur_unix_time = self.entropyTools.get_current_unix_time()
2693
2694
2695
2696 mydate = dbconn.retrieveDateCreation(idpackage)
2697
2698 mydelta = cur_unix_time - float(mydate)
2699 if mydelta > pkg_exp_secs:
2700 return True
2701 return False
2702
2704 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2705 server_repos = srv_set['repositories'].keys()
2706 for repo in server_repos:
2707 dbconn = self.open_server_repository(read_only = True,
2708 no_upload = True, repo = repo)
2709 if dbconn.isCounterTrashed(counter):
2710 return True
2711 return False
2712
2719
2722
2723 if repo == None:
2724 repo = self.default_repository
2725
2726 self.close_server_databases()
2727 revisions_match = {}
2728 treeupdates_actions = []
2729 injected_packages = set()
2730 idpackages = set()
2731 idpackages_added = set()
2732
2733 mytxt = red("%s ...") % (_("Initializing Entropy database"),)
2734 self.updateProgress(
2735 mytxt, importance = 1,
2736 type = "info", header = darkgreen(" * "),
2737 back = True
2738 )
2739
2740 if os.path.isfile(self.get_local_database_file(repo)):
2741
2742 dbconn = self.open_server_repository(read_only = True,
2743 no_upload = True, repo = repo, warnings = warnings)
2744
2745 if dbconn.doesTableExist("baseinfo") and \
2746 dbconn.doesTableExist("extrainfo"):
2747 idpackages = dbconn.listAllIdpackages()
2748
2749 if dbconn.doesTableExist("treeupdatesactions"):
2750 treeupdates_actions = dbconn.listAllTreeUpdatesActions()
2751
2752
2753 if dbconn.doesTableExist("injected") and \
2754 dbconn.doesTableExist("extrainfo"):
2755 injected_packages = dbconn.listAllInjectedPackages(
2756 justFiles = True)
2757 injected_packages = set([os.path.basename(x) for x \
2758 in injected_packages])
2759
2760 for idpackage in idpackages:
2761 url = dbconn.retrieveDownloadURL(idpackage)
2762 package = os.path.basename(url)
2763 branch = dbconn.retrieveBranch(idpackage)
2764 revision = dbconn.retrieveRevision(idpackage)
2765 revisions_match[package] = (branch, revision,)
2766
2767 self.close_server_database(dbconn)
2768
2769 mytxt = "%s: %s: %s" % (
2770 bold(_("WARNING")),
2771 red(_("database already exists")),
2772 self.get_local_database_file(repo),
2773 )
2774 self.updateProgress(
2775 mytxt,
2776 importance = 1,
2777 type = "warning",
2778 header = darkred(" !!! ")
2779 )
2780
2781 rc_question = self.askQuestion(_("Do you want to continue ?"))
2782 if rc_question == "No":
2783 return
2784 try:
2785 os.remove(self.get_local_database_file(repo))
2786 except OSError:
2787 pass
2788
2789
2790
2791 dbconn = self.open_server_repository(read_only = False,
2792 no_upload = True, repo = repo, is_new = True)
2793 dbconn.initializeDatabase()
2794
2795 if not empty:
2796
2797 revisions_file = "/entropy-revisions-dump.txt"
2798
2799 if revisions_match:
2800 self.updateProgress(
2801 "%s: %s" % (
2802 red(_("Dumping current revisions to file")),
2803 darkgreen(revisions_file),
2804 ),
2805 importance = 1,
2806 type = "info",
2807 header = darkgreen(" * ")
2808 )
2809 f_rev = open(revisions_file,"w")
2810 f_rev.write(str(revisions_match))
2811 f_rev.flush()
2812 f_rev.close()
2813
2814
2815 treeupdates_file = "/entropy-treeupdates-dump.txt"
2816 if treeupdates_actions:
2817 self.updateProgress(
2818 "%s: %s" % (
2819
2820 red(_("Dumping current 'treeupdates' actions to file")),
2821 bold(treeupdates_file),
2822 ),
2823 importance = 1,
2824 type = "info",
2825 header = darkgreen(" * ")
2826 )
2827 f_tree = open(treeupdates_file,"w")
2828 f_tree.write(str(treeupdates_actions))
2829 f_tree.flush()
2830 f_tree.close()
2831
2832 rc_question = self.askQuestion(
2833 _("Would you like to sync packages first (important!) ?"))
2834 if rc_question == "Yes":
2835 self.MirrorsService.sync_packages(repo = repo)
2836
2837
2838 if treeupdates_actions:
2839 dbconn.bumpTreeUpdatesActions(treeupdates_actions)
2840
2841
2842 pkg_branch_dir = os.path.join(
2843 self.get_local_packages_directory(repo),
2844 self.SystemSettings['repositories']['branch'])
2845 pkglist = os.listdir(pkg_branch_dir)
2846
2847 pkg_ext_len = len(etpConst['packagesext'])
2848 pkglist = [x for x in pkglist if (x[(pkg_ext_len*-1):] == \
2849 etpConst['packagesext']) and not \
2850 os.path.isfile(os.path.join(pkg_branch_dir,
2851 x + etpConst['packagesexpirationfileext']))]
2852
2853 if pkglist:
2854 self.updateProgress(
2855 "%s '%s' %s %s" % (
2856 red(_("Reinitializing Entropy database for branch")),
2857 bold(self.SystemSettings['repositories']['branch']),
2858 red(_("using Packages in the repository")),
2859 red("..."),
2860 ),
2861 importance = 1,
2862 type = "info",
2863 header = darkgreen(" * ")
2864 )
2865
2866 counter = 0
2867 maxcount = len(pkglist)
2868 branch = self.SystemSettings['repositories']['branch']
2869 for pkg in pkglist:
2870 counter += 1
2871
2872 self.updateProgress(
2873 "[repo:%s|%s] %s: %s" % (
2874 darkgreen(repo),
2875 brown(branch),
2876 blue(_("analyzing")),
2877 bold(pkg),
2878 ),
2879 importance = 1,
2880 type = "info",
2881 header = " ",
2882 back = True,
2883 count = (counter, maxcount,)
2884 )
2885
2886 doinject = False
2887 if pkg in injected_packages:
2888 doinject = True
2889
2890 pkg_path = os.path.join(self.get_local_packages_directory(repo),
2891 branch, pkg)
2892 mydata = self.SpmService.extract_pkg_metadata(pkg_path,
2893 inject = doinject)
2894
2895
2896 revision_avail = revisions_match.get(pkg)
2897 add_revision = 0
2898 if (revision_avail != None):
2899 if branch == revision_avail[0]:
2900 add_revision = revision_avail[1]
2901
2902 idpackage, revision, mydata_upd = dbconn.addPackage(mydata,
2903 revision = add_revision)
2904 idpackages_added.add(idpackage)
2905
2906 self.updateProgress(
2907 "[repo:%s] [%s:%s/%s] %s: %s, %s: %s" % (
2908 repo,
2909 brown(branch),
2910 darkgreen(str(counter)),
2911 blue(str(maxcount)),
2912 red(_("added package")),
2913 darkgreen(pkg),
2914 red(_("revision")),
2915 brown(str(revision)),
2916 ),
2917 importance = 1,
2918 type = "info",
2919 header = " ",
2920 back = True
2921 )
2922
2923 self.depends_table_initialize(repo)
2924 self.library_paths_table_initialize(repo)
2925
2926 my_qa = self.QA()
2927
2928 if idpackages_added:
2929 dbconn = self.open_server_repository(read_only = False,
2930 no_upload = True, repo = repo)
2931 my_qa.scan_missing_dependencies(
2932 idpackages_added, dbconn, ask = True,
2933 repo = repo, self_check = True,
2934 black_list = \
2935 self.get_missing_dependencies_blacklist(repo = repo),
2936 black_list_adder = \
2937 self.add_missing_dependencies_blacklist_items
2938 )
2939
2940 dbconn.commitChanges()
2941 self.close_server_databases()
2942
2943 return 0
2944
2946
2947 dbconn = self.open_server_repository(read_only = True,
2948 no_upload = True, repo = repo)
2949 if ("world" in packages) or not packages:
2950 return dbconn.listAllIdpackages(), True
2951 else:
2952 idpackages = set()
2953 for package in packages:
2954 matches = dbconn.atomMatch(package, multiMatch = True)
2955 if matches[1] == 0:
2956 idpackages |= matches[0]
2957 else:
2958 mytxt = "%s: %s: %s" % (
2959 red(_("Attention")),
2960 blue(_("cannot match")),
2961 bold(package),
2962 )
2963 self.updateProgress(
2964 mytxt,
2965 importance = 1,
2966 type = "warning",
2967 header = darkred(" !!! ")
2968 )
2969 return idpackages, False
2970
2972
2973 import urllib2
2974 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
2975 if not srv_set['repositories'][repo].has_key('handler'):
2976 return None
2977 url = srv_set['repositories'][repo]['handler']
2978
2979
2980 filename = filename.replace("#","%23")
2981
2982 filename = filename.replace("+","%2b")
2983 request = os.path.join(url, etpConst['handlers']['md5sum'])
2984 request += filename+"&branch="+branch
2985
2986 proxy_settings = self.SystemSettings['system']['proxy']
2987 try:
2988 mydict = {}
2989 if proxy_settings['ftp']:
2990 mydict['ftp'] = proxy_settings['ftp']
2991 if proxy_settings['http']:
2992 mydict['http'] = proxy_settings['http']
2993 if mydict:
2994 mydict['username'] = proxy_settings['username']
2995 mydict['password'] = proxy_settings['password']
2996 self.entropyTools.add_proxy_opener(urllib2, mydict)
2997 else:
2998
2999 urllib2._opener = None
3000 item = urllib2.urlopen(request)
3001 result = item.readline().strip()
3002 item.close()
3003 del item
3004 return result
3005 except (urllib2.URLError, urllib2.HTTPError,):
3006 return None
3007
3009
3010 if repo == None:
3011 repo = self.default_repository
3012
3013 self.updateProgress(
3014 "[%s] %s:" % (
3015 red("remote"),
3016 blue(_("Integrity verification of the selected packages")),
3017 ),
3018 importance = 1,
3019 type = "info",
3020 header = blue(" @@ ")
3021 )
3022
3023 idpackages, world = self.match_packages(packages)
3024 dbconn = self.open_server_repository(read_only = True, no_upload = True,
3025 repo = repo)
3026 branch = self.SystemSettings['repositories']['branch']
3027
3028 if world:
3029 self.updateProgress(
3030 blue(
3031 _("All the packages in repository will be checked.")),
3032 importance = 1,
3033 type = "info",
3034 header = " "
3035 )
3036 else:
3037 mytxt = red("%s:") % (
3038 _("This is the list of the packages that would be checked"),)
3039 self.updateProgress(
3040 mytxt,
3041 importance = 1,
3042 type = "info",
3043 header = " "
3044 )
3045 for idpackage in idpackages:
3046 pkgatom = dbconn.retrieveAtom(idpackage)
3047 down_url = dbconn.retrieveDownloadURL(idpackage)
3048 pkgfile = os.path.basename(down_url)
3049 self.updateProgress(
3050 red(pkgatom) + " -> " + bold(os.path.join(branch, pkgfile)),
3051 importance = 1,
3052 type = "info",
3053 header = darkgreen(" - ")
3054 )
3055
3056 if ask:
3057 rc_question = self.askQuestion(
3058 _("Would you like to continue ?"))
3059 if rc_question == "No":
3060 return set(), set(), {}
3061
3062 match = set()
3063 not_match = set()
3064 broken_packages = {}
3065
3066 for uri in self.get_remote_mirrors(repo):
3067
3068 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri)
3069 self.updateProgress(
3070 "[repo:%s] %s: %s" % (
3071 darkgreen(repo),
3072 blue(_("Working on mirror")),
3073 brown(crippled_uri),
3074 ),
3075 importance = 1,
3076 type = "info",
3077 header = red(" @@ ")
3078 )
3079
3080
3081 totalcounter = len(idpackages)
3082 currentcounter = 0
3083 for idpackage in idpackages:
3084
3085 currentcounter += 1
3086 pkgfile = dbconn.retrieveDownloadURL(idpackage)
3087 orig_branch = self.get_branch_from_download_relative_uri(
3088 pkgfile)
3089
3090 self.updateProgress(
3091 "[%s] %s: %s" % (
3092 brown(crippled_uri),
3093 blue(_("checking hash")),
3094 darkgreen(pkgfile),
3095 ),
3096 importance = 1,
3097 type = "info",
3098 header = blue(" @@ "),
3099 back = True,
3100 count = (currentcounter, totalcounter,)
3101 )
3102
3103 checksum_ok = False
3104 ck_remote = self.get_remote_package_checksum(repo,
3105 os.path.basename(pkgfile), orig_branch)
3106 if ck_remote == None:
3107 self.updateProgress(
3108 "[%s] %s: %s %s" % (
3109 brown(crippled_uri),
3110 blue(_("digest verification of")),
3111 bold(pkgfile),
3112 blue(_("not supported")),
3113 ),
3114 importance = 1,
3115 type = "info",
3116 header = blue(" @@ "),
3117 count = (currentcounter, totalcounter,)
3118 )
3119 elif len(ck_remote) == 32:
3120 pkghash = dbconn.retrieveDigest(idpackage)
3121 if ck_remote == pkghash:
3122 checksum_ok = True
3123 else:
3124 self.updateProgress(
3125 "[%s] %s: %s %s" % (
3126 brown(crippled_uri),
3127 blue(_("digest verification of")),
3128 bold(pkgfile),
3129 blue(_("failed for unknown reasons")),
3130 ),
3131 importance = 1,
3132 type = "info",
3133 header = blue(" @@ "),
3134 count = (currentcounter, totalcounter,)
3135 )
3136
3137 if checksum_ok:
3138 match.add(idpackage)
3139 else:
3140 not_match.add(idpackage)
3141 self.updateProgress(
3142 "[%s] %s: %s %s" % (
3143 brown(crippled_uri),
3144 blue(_("package")),
3145 bold(pkgfile),
3146 red(_("NOT healthy")),
3147 ),
3148 importance = 1,
3149 type = "warning",
3150 header = darkred(" !!! "),
3151 count = (currentcounter, totalcounter,)
3152 )
3153 if not broken_packages.has_key(crippled_uri):
3154 broken_packages[crippled_uri] = []
3155 broken_packages[crippled_uri].append(pkgfile)
3156
3157 if broken_packages:
3158 mytxt = blue("%s:") % (
3159 _("This is the list of broken packages"),)
3160 self.updateProgress(
3161 mytxt,
3162 importance = 1,
3163 type = "info",
3164 header = red(" * ")
3165 )
3166 for mirror in broken_packages.keys():
3167 mytxt = "%s: %s" % (
3168 brown(_("Mirror")),
3169 bold(mirror),
3170 )
3171 self.updateProgress(
3172 mytxt,
3173 importance = 1,
3174 type = "info",
3175 header = red(" <> ")
3176 )
3177 for broken_package in broken_packages[mirror]:
3178 self.updateProgress(
3179 blue(broken_package),
3180 importance = 1,
3181 type = "info",
3182 header = red(" - ")
3183 )
3184
3185 self.updateProgress(
3186 "%s:" % (
3187 blue(_("Statistics")),
3188 ),
3189 importance = 1,
3190 type = "info",
3191 header = red(" @@ ")
3192 )
3193 self.updateProgress(
3194 "[%s] %s:\t%s" % (
3195 red(crippled_uri),
3196 brown(_("Number of checked packages")),
3197 brown(str(len(match) + len(not_match))),
3198 ),
3199 importance = 1,
3200 type = "info",
3201 header = brown(" # ")
3202 )
3203 self.updateProgress(
3204 "[%s] %s:\t%s" % (
3205 red(crippled_uri),
3206 darkgreen(_("Number of healthy packages")),
3207 darkgreen(str(len(match))),
3208 ),
3209 importance = 1,
3210 type = "info",
3211 header = brown(" # ")
3212 )
3213 self.updateProgress(
3214 "[%s] %s:\t%s" % (
3215 red(crippled_uri),
3216 darkred(_("Number of broken packages")),
3217 darkred(str(len(not_match))),
3218 ),
3219 importance = 1,
3220 type = "info",
3221 header = brown(" # ")
3222 )
3223
3224 return match, not_match, broken_packages
3225
3226
3228
3229 if repo == None:
3230 repo = self.default_repository
3231
3232 self.updateProgress(
3233 "[%s] %s:" % (
3234 red(_("local")),
3235 blue(_("Integrity verification of the selected packages")),
3236 ),
3237 importance = 1,
3238 type = "info",
3239 header = darkgreen(" * ")
3240 )
3241
3242 idpackages, world = self.match_packages(packages)
3243 dbconn = self.open_server_repository(read_only = True,
3244 no_upload = True, repo = repo)
3245
3246 if world:
3247 self.updateProgress(
3248 blue(_("All the packages in repository will be checked.")),
3249 importance = 1,
3250 type = "info",
3251 header = " "
3252 )
3253
3254 to_download = set()
3255 available = set()
3256 for idpackage in idpackages:
3257
3258 pkgatom = dbconn.retrieveAtom(idpackage)
3259 pkg_path = dbconn.retrieveDownloadURL(idpackage)
3260 pkg_rel_path = '/'.join(pkg_path.split("/")[2:])
3261
3262 bindir_path = os.path.join(self.get_local_packages_directory(repo),
3263 pkg_rel_path)
3264 uploaddir_path = os.path.join(self.get_local_upload_directory(repo),
3265 pkg_rel_path)
3266
3267 if os.path.isfile(bindir_path):
3268 if not world:
3269 self.updateProgress(
3270 "[%s] %s :: %s" % (
3271 darkgreen(_("available")),
3272 blue(pkgatom),
3273 darkgreen(pkg_rel_path),
3274 ),
3275 importance = 0,
3276 type = "info",
3277 header = darkgreen(" # ")
3278 )
3279 available.add(idpackage)
3280 elif os.path.isfile(uploaddir_path):
3281 if not world:
3282 self.updateProgress(
3283 "[%s] %s :: %s" % (
3284 darkred(_("upload/ignored")),
3285 blue(pkgatom),
3286 darkgreen(pkg_rel_path),
3287 ),
3288 importance = 0,
3289 type = "info",
3290 header = darkgreen(" # ")
3291 )
3292 else:
3293 self.updateProgress(
3294 "[%s] %s :: %s" % (
3295 brown(_("download")),
3296 blue(pkgatom),
3297 darkgreen(pkg_rel_path),
3298 ),
3299 importance = 0,
3300 type = "info",
3301 header = darkgreen(" # ")
3302 )
3303 to_download.add((idpackage, pkg_path,))
3304
3305 if ask:
3306 rc_question = self.askQuestion(_("Would you like to continue ?"))
3307 if rc_question == "No":
3308 return set(), set(), set(), set()
3309
3310
3311 fine = set()
3312 failed = set()
3313 downloaded_fine = set()
3314 downloaded_errors = set()
3315
3316 if to_download:
3317
3318 not_downloaded = set()
3319 mytxt = blue("%s ...") % (_("Starting to download missing files"),)
3320 self.updateProgress(
3321 mytxt,
3322 importance = 1,
3323 type = "info",
3324 header = " "
3325 )
3326 for uri in self.get_remote_mirrors(repo):
3327
3328 if not_downloaded:
3329 mytxt = blue("%s ...") % (
3330 _("Searching missing/broken files on another mirror"),)
3331 self.updateProgress(
3332 mytxt,
3333 importance = 1,
3334 type = "info",
3335 header = " "
3336 )
3337 to_download = not_downloaded.copy()
3338 not_downloaded = set()
3339
3340 for idpackage, pkg_path in to_download:
3341 rc_down = self.MirrorsService.download_package(uri,
3342 pkg_path, repo = repo)
3343 if rc_down:
3344 downloaded_fine.add(idpackage)
3345 available.add(idpackage)
3346 else:
3347 not_downloaded.add(pkg_path)
3348
3349 if not not_downloaded:
3350 self.updateProgress(
3351 red(_("Binary packages downloaded successfully.")),
3352 importance = 1,
3353 type = "info",
3354 header = " "
3355 )
3356 break
3357
3358 if not_downloaded:
3359 mytxt = blue("%s:") % (
3360 _("These are the packages that cannot be found online"),)
3361 self.updateProgress(
3362 mytxt,
3363 importance = 1,
3364 type = "info",
3365 header = " "
3366 )
3367 for pkg_path in not_downloaded:
3368 downloaded_errors.add(pkg_path)
3369 self.updateProgress(
3370 brown(pkg_path),
3371 importance = 1,
3372 type = "warning",
3373 header = red(" * ")
3374 )
3375 downloaded_errors |= not_downloaded
3376 mytxt = "%s." % (_("They won't be checked"),)
3377 self.updateProgress(
3378 mytxt,
3379 importance = 1,
3380 type = "warning",
3381 header = " "
3382 )
3383
3384 totalcounter = str(len(available))
3385 currentcounter = 0
3386 for idpackage in available:
3387 currentcounter += 1
3388 pkg_path = dbconn.retrieveDownloadURL(idpackage)
3389 orig_branch = self.get_branch_from_download_relative_uri(pkg_path)
3390 pkgfile = os.path.basename(pkg_path)
3391
3392 self.updateProgress(
3393 "[branch:%s] %s %s" % (
3394 brown(orig_branch),
3395 blue(_("checking hash of")),
3396 darkgreen(pkgfile),
3397 ),
3398 importance = 1,
3399 type = "info",
3400 header = " ",
3401 back = True,
3402 count = (currentcounter, totalcounter,)
3403 )
3404
3405 storedmd5 = dbconn.retrieveDigest(idpackage)
3406 pkgpath = os.path.join(self.get_local_packages_directory(repo),
3407 orig_branch, pkgfile)
3408 result = self.entropyTools.compare_md5(pkgpath, storedmd5)
3409 if result:
3410 fine.add(idpackage)
3411 else:
3412 failed.add(idpackage)
3413 self.updateProgress(
3414 "[branch:%s] %s %s %s: %s" % (
3415 brown(orig_branch),
3416 blue(_("package")),
3417 darkgreen(pkg_path),
3418 blue(_("is corrupted, stored checksum")),
3419 brown(storedmd5),
3420 ),
3421 importance = 1,
3422 type = "info",
3423 header = " ",
3424 count = (currentcounter, totalcounter,)
3425 )
3426
3427 if failed:
3428 mytxt = blue("%s:") % (_("This is the list of broken packages"),)
3429 self.updateProgress(
3430 mytxt,
3431 importance = 1,
3432 type = "warning",
3433 header = darkred(" # ")
3434 )
3435 for idpackage in failed:
3436 atom = dbconn.retrieveAtom(idpackage)
3437 down_p = dbconn.retrieveDownloadURL(idpackage)
3438 self.updateProgress(
3439 blue("[atom:%s] %s" % (atom, down_p,)),
3440 importance = 0,
3441 type = "warning",
3442 header = brown(" # ")
3443 )
3444
3445
3446 self.updateProgress(
3447 red("Statistics:"),
3448 importance = 1,
3449 type = "info",
3450 header = blue(" * ")
3451 )
3452 self.updateProgress(
3453 brown("%s => %s" % (
3454 len(fine) + len(failed),
3455 _("checked packages"),
3456 )
3457 ),
3458 importance = 0,
3459 type = "info",
3460 header = brown(" # ")
3461 )
3462 self.updateProgress(
3463 darkgreen("%s => %s" % (
3464 len(fine),
3465 _("healthy packages"),
3466 )
3467 ),
3468 importance = 0,
3469 type = "info",
3470 header = brown(" # ")
3471 )
3472 self.updateProgress(
3473 darkred("%s => %s" % (
3474 len(failed),
3475 _("broken packages"),
3476 )
3477 ),
3478 importance = 0,
3479 type = "info",
3480 header = brown(" # ")
3481 )
3482 self.updateProgress(
3483 blue("%s => %s" % (
3484 len(downloaded_fine),
3485 _("downloaded packages"),
3486 )
3487 ),
3488 importance = 0,
3489 type = "info",
3490 header = brown(" # ")
3491 )
3492 self.updateProgress(
3493 bold("%s => %s" % (
3494 len(downloaded_errors),
3495 _("failed downloads"),
3496 )
3497 ),
3498 importance = 0,
3499 type = "info",
3500 header = brown(" # ")
3501 )
3502
3503 self.close_server_database(dbconn)
3504 return fine, failed, downloaded_fine, downloaded_errors
3505
3506
3508
3509 if repo == None:
3510 repo = self.default_repository
3511
3512 if to_branch != self.SystemSettings['repositories']['branch']:
3513 mytxt = "%s: %s %s" % (
3514 blue(_("Please setup your branch to")),
3515 bold(to_branch),
3516 blue(_("and retry")),
3517 )
3518 self.updateProgress(
3519 mytxt,
3520 importance = 1,
3521 type = "error",
3522 header = darkred(" !! ")
3523 )
3524 return None
3525
3526 mytxt = red("%s ...") % (_("Copying database (if not exists)"),)
3527 self.updateProgress(
3528 mytxt,
3529 importance = 1,
3530 type = "info",
3531 header = darkgreen(" @@ ")
3532 )
3533 branch_dbdir = self.get_local_database_dir(repo)
3534 old_branch_dbdir = self.get_local_database_dir(repo, from_branch)
3535
3536
3537 self.close_server_databases()
3538
3539
3540
3541 branch_dbfile = self.get_local_database_file(repo)
3542 if os.path.isfile(branch_dbfile):
3543 if self.entropyTools.get_file_size(branch_dbfile) == 0:
3544 shutil.rmtree(branch_dbdir, True)
3545
3546 if os.path.isdir(branch_dbdir):
3547
3548 while 1:
3549 rnd_num = self.entropyTools.get_random_number()
3550 backup_dbdir = branch_dbdir + str(rnd_num)
3551 if not os.path.isdir(backup_dbdir):
3552 break
3553 os.rename(branch_dbdir, backup_dbdir)
3554
3555 if os.path.isdir(old_branch_dbdir):
3556 shutil.copytree(old_branch_dbdir, branch_dbdir)
3557
3558 mytxt = red("%s ...") % (_("Switching packages"),)
3559 self.updateProgress(
3560 mytxt,
3561 importance = 1,
3562 type = "info",
3563 header = darkgreen(" @@ ")
3564 )
3565
3566 dbconn = self.open_server_repository(read_only = False,
3567 no_upload = True, repo = repo, lock_remote = False)
3568 try:
3569 dbconn.validateDatabase()
3570 except SystemDatabaseError:
3571 self.handle_uninitialized_repository(repo)
3572 dbconn = self.open_server_repository(read_only = False,
3573 no_upload = True, repo = repo, lock_remote = False)
3574
3575 idpackages = dbconn.listAllIdpackages()
3576 already_switched = set()
3577 not_found = set()
3578 switched = set()
3579 ignored = set()
3580 no_checksum = set()
3581
3582 maxcount = len(idpackages)
3583 count = 0
3584 for idpackage in idpackages:
3585 count += 1
3586
3587 cur_branch = dbconn.retrieveBranch(idpackage)
3588 atom = dbconn.retrieveAtom(idpackage)
3589 if cur_branch == to_branch:
3590 already_switched.add(idpackage)
3591 self.updateProgress(
3592 red("%s %s, %s %s" % (
3593 _("Ignoring"),
3594 bold(atom),
3595 _("already in branch"),
3596 cur_branch,
3597 )
3598 ),
3599 importance = 0,
3600 type = "info",
3601 header = darkgreen(" @@ "),
3602 count = (count, maxcount,)
3603 )
3604 ignored.add(idpackage)
3605 continue
3606
3607 self.updateProgress(
3608 "[%s=>%s] %s" % (
3609 brown(cur_branch),
3610 bold(to_branch),
3611 darkgreen(atom),
3612 ),
3613 importance = 0,
3614 type = "info",
3615 header = darkgreen(" @@ "),
3616 back = True,
3617 count = (count, maxcount,)
3618 )
3619 dbconn.switchBranch(idpackage, to_branch)
3620 dbconn.commitChanges()
3621 switched.add(idpackage)
3622
3623 dbconn.commitChanges()
3624
3625
3626 dbconn.moveCountersToBranch(to_branch, from_branch = from_branch)
3627
3628 self.close_server_database(dbconn)
3629 mytxt = blue("%s.") % (_("migration loop completed"),)
3630 self.updateProgress(
3631 "[%s=>%s] %s" % (
3632 brown(from_branch),
3633 bold(to_branch),
3634 mytxt,
3635 ),
3636 importance = 1,
3637 type = "info",
3638 header = darkgreen(" * ")
3639 )
3640
3641 return switched, already_switched, ignored, not_found, no_checksum
3642
3644
3645 if branch == None:
3646 branch = self.SystemSettings['repositories']['branch']
3647 if repo == None:
3648 repo = self.default_repository
3649
3650 sets_dir = self.get_local_database_sets_dir(repo, branch)
3651 if not (os.path.isdir(sets_dir) and os.access(sets_dir, os.R_OK)):
3652 return {}
3653
3654 mydata = {}
3655 items = os.listdir(sets_dir)
3656 for item in items:
3657
3658 try:
3659 item_clean = str(item)
3660 except (UnicodeEncodeError, UnicodeDecodeError,):
3661 continue
3662 item_path = os.path.join(sets_dir, item)
3663 if not (os.path.isfile(item_path) and \
3664 os.access(item_path, os.R_OK)):
3665 continue
3666 item_elements = self.entropyTools.extract_packages_from_set_file(
3667 item_path)
3668 if item_elements:
3669 mydata[item_clean] = item_elements.copy()
3670
3671 return mydata
3672
3703
3717