1
2 """
3
4 @author: Fabio Erculiani <lxnay@sabayonlinux.org>
5 @contact: lxnay@sabayonlinux.org
6 @copyright: Fabio Erculiani
7 @license: GPL-2
8
9 B{Entropy Services System Management Executors Interface}.
10
11 """
12 import os
13 import sys
14 import subprocess
15 from entropy.const import etpConst
16 from entropy.output import blue, red
17 from entropy.exceptions import InvalidAtom
18 from entropy.i18n import _
19 from entropy.transceivers import EntropyTransceiver
20
22
23 import entropy.tools as entropyTools
24 - def __init__(self, SystemManagerExecutorInstance, *args, **kwargs):
25
26 try:
27 import pickle as pickle
28 except ImportError:
29 import pickle
30 self.pickle = pickle
31
32 self.SystemManagerExecutor = SystemManagerExecutorInstance
33 self.args = args
34 self.kwargs = kwargs
35 self.available_commands = {
36 'sync_spm': {
37 'func': self.sync_portage,
38 'args': 1,
39 },
40 'compile_atoms': {
41 'func': self.compile_atoms,
42 'args': 2,
43 },
44 'spm_remove_atoms': {
45 'func': self.spm_remove_atoms,
46 'args': 2,
47 },
48 'get_spm_categories_updates': {
49 'func': self.get_spm_categories_updates,
50 'args': 2,
51 },
52 'get_spm_categories_installed': {
53 'func': self.get_spm_categories_installed,
54 'args': 2,
55 },
56 'enable_uses_for_atoms': {
57 'func': self.enable_uses_for_atoms,
58 'args': 3,
59 },
60 'disable_uses_for_atoms': {
61 'func': self.disable_uses_for_atoms,
62 'args': 3,
63 },
64 'get_spm_atoms_info': {
65 'func': self.get_spm_atoms_info,
66 'args': 2,
67 },
68 'run_spm_info': {
69 'func': self.run_spm_info,
70 'args': 1,
71 },
72 'run_custom_shell_command': {
73 'func': self.run_custom_shell_command,
74 'args': 1,
75 },
76 'get_spm_glsa_data': {
77 'func': self.get_spm_glsa_data,
78 'args': 1,
79 },
80 'move_entropy_packages_to_repository': {
81 'func': self.move_entropy_packages_to_repository,
82 'args': 5,
83 },
84 'scan_entropy_packages_database_changes': {
85 'func': self.scan_entropy_packages_database_changes,
86 'args': 1,
87 },
88 'run_entropy_database_updates': {
89 'func': self.run_entropy_database_updates,
90 'args': 4,
91 },
92 'run_entropy_dependency_test': {
93 'func': self.run_entropy_dependency_test,
94 'args': 1,
95 },
96 'run_entropy_library_test': {
97 'func': self.run_entropy_library_test,
98 'args': 1,
99 },
100 'run_entropy_treeupdates': {
101 'func': self.run_entropy_treeupdates,
102 'args': 2,
103 },
104 'scan_entropy_mirror_updates': {
105 'func': self.scan_entropy_mirror_updates,
106 'args': 2,
107 },
108 'run_entropy_mirror_updates': {
109 'func': self.run_entropy_mirror_updates,
110 'args': 2,
111 },
112 'run_entropy_checksum_test': {
113 'func': self.run_entropy_checksum_test,
114 'args': 3,
115 },
116 'get_notice_board': {
117 'func': self.get_notice_board,
118 'args': 2,
119 },
120 'remove_notice_board_entries': {
121 'func': self.remove_notice_board_entries,
122 'args': 3,
123 },
124 'add_notice_board_entry': {
125 'func': self.add_notice_board_entry,
126 'args': 5,
127 },
128 }
129
131 with self.SystemManagerExecutor.SystemInterface.QueueLock:
132 self.SystemManagerExecutor.SystemInterface.load_queue()
133 live_item, key = self.SystemManagerExecutor.SystemInterface._get_item_by_queue_id(queue_id)
134 if isinstance(live_item, dict):
135 live_item['processing_pid'] = process_pid
136
137 self.SystemManagerExecutor.SystemInterface.store_queue()
138
140
141 queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True)
142 if queue_data == None:
143 return False, 'no item in queue'
144
145 stdout_err = open(queue_data['stdout'], "a+")
146
147 cmd = ["emerge", "--sync"]
148 try:
149 p = subprocess.Popen(cmd, stdout = stdout_err, stderr = stdout_err, stdin = self._get_stdin(queue_id))
150 self._set_processing_pid(queue_id, p.pid)
151 rc = p.wait()
152 finally:
153 stdout_err.write("\n### Done ###\n")
154 stdout_err.flush()
155 stdout_err.close()
156 return True, rc
157
158 - def compile_atoms(
159 self,
160 queue_id, atoms,
161 pretend = False, oneshot = False,
162 verbose = True, nocolor = True,
163 fetchonly = False, buildonly = False,
164 nodeps = False, custom_use = '', ldflags = '', cflags = ''):
165
166 sys_intf = self.SystemManagerExecutor.SystemInterface
167 queue_data, key = sys_intf.get_item_by_queue_id(queue_id, copy = True)
168 if queue_data == None:
169 return False, 'no item in queue'
170
171 def set_proc_pid(pid):
172 self._set_processing_pid(queue_id, pid)
173
174 stdout_err = open(queue_data['stdout'], "a+")
175 stdout_err.write("Preparing to spawn compilation of: '%s'"
176 ". Good luck mate!\n" % (' '.join(atoms),))
177 stdout_err.flush()
178
179 env = {}
180 if ldflags:
181 env['LDFLAGS'] = ldflags
182 if cflags:
183 env['CFLAGS'] = cflags
184
185 try:
186
187 spm = sys_intf.Entropy.Spm()
188 status = spm.compile_packages(atoms,
189 stdin = self._get_stdin(queue_id),
190 stdout = stdour_err, stderr = stdout_err, environ = env,
191 pid_write_func = set_proc_pid, pretend = pretend,
192 verbose = verbose, fetch_only = fetchonly,
193 build_only = buildonly, no_dependencies = nodeps,
194 coloured_output = not nocolor, oneshot = oneshot)
195
196 finally:
197
198 stdout_err.write("\n### Done ###\n")
199 stdout_err.flush()
200 stdout_err.close()
201
202 return True, status
203
204 - def spm_remove_atoms(self, queue_id, atoms, pretend = True, verbose = True, nocolor = True):
205
206 sys_intf = self.SystemManagerExecutor.SystemInterface
207 queue_data, key = sys_intf.get_item_by_queue_id(queue_id, copy = True)
208 if queue_data == None:
209 return False, 'no item in queue'
210
211 def set_proc_pid(pid):
212 self._set_processing_pid(queue_id, pid)
213
214 stdout_err = open(queue_data['stdout'], "a+")
215 stdout_err.write("Preparing to spawn removal of: '%s'"
216 ". Good luck mate!\n" % (' '.join(atoms),))
217 stdout_err.flush()
218
219 try:
220
221 spm = sys_intf.Entropy.Spm()
222 status = spm.remove_packages(atoms,
223 stdin = self._get_stdin(queue_id),
224 stdout = stdour_err, stderr = stdout_err,
225 pid_write_func = set_proc_pid, pretend = pretend,
226 verbose = verbose, fetch_only = fetchonly,
227 build_only = buildonly, no_dependencies = nodeps,
228 coloured_output = not nocolor, oneshot = oneshot)
229
230 finally:
231
232 stdout_err.write("\n### Done ###\n")
233 stdout_err.flush()
234 stdout_err.close()
235
236 return True, status
237
239
240 queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True)
241 if queue_data == None:
242 return False, 'no item in queue'
243
244 use_data = {}
245 spm = self.SystemManagerExecutor.SystemInterface.Entropy.Spm()
246 for atom in atoms:
247 try:
248 status = spm.enable_package_compile_options(atom, useflags)
249 except:
250 continue
251 if status:
252 use_data[atom] = {}
253 matched_atom = spm.match_package(atom)
254 use_data[atom] = spm.get_package_compile_options(matched_atom)
255
256 return True, use_data
257
259
260 queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True)
261 if queue_data == None:
262 return False, 'no item in queue'
263
264 use_data = {}
265 spm = self.SystemManagerExecutor.SystemInterface.Entropy.Spm()
266 for atom in atoms:
267 try:
268 status = spm.disable_package_compile_options(atom, useflags)
269 except:
270 continue
271 if status:
272 use_data[atom] = {}
273 matched_atom = spm.match_package(atom)
274 use_data[atom] = spm.get_package_compile_options(matched_atom)
275
276 return True, use_data
277
279
280 queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True)
281 if queue_data == None:
282 return False, 'no item in queue'
283
284 atoms_data = {}
285 spm = self.SystemManagerExecutor.SystemInterface.Entropy.Spm()
286 for atom in atoms:
287
288 try:
289 key = self.entropyTools.dep_getkey(atom)
290 category = key.split("/")[0]
291 except:
292 continue
293 try:
294 matched_atom = spm.match_package(atom)
295 except InvalidAtom:
296 continue
297 if not matched_atom:
298 continue
299
300 if category not in atoms_data:
301 atoms_data[category] = {}
302
303 atoms_data[category][matched_atom] = self._get_spm_pkginfo(matched_atom)
304
305 return True, atoms_data
306
308
309 queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True)
310 if queue_data == None:
311 return False, 'no item in queue'
312
313 spm = self.SystemManagerExecutor.SystemInterface.Entropy.Spm()
314 packages = spm.get_packages(categories)
315 package_data = {}
316 for package in packages:
317 try:
318 key = self.entropyTools.dep_getkey(package)
319 category = key.split("/")[0]
320 except:
321 continue
322 if category not in package_data:
323 package_data[category] = {}
324 package_data[category][package] = self._get_spm_pkginfo(package)
325
326 return True, package_data
327
329
330 queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True)
331 if queue_data == None:
332 return False, 'no item in queue'
333
334 spm = self.SystemManagerExecutor.SystemInterface.Entropy.Spm()
335 packages = spm.get_installed_packages(categories = categories)
336 package_data = {}
337 for package in packages:
338 try:
339 key = self.entropyTools.dep_getkey(package)
340 category = key.split("/")[0]
341 except:
342 continue
343 if category not in package_data:
344 package_data[category] = {}
345 package_data[category][package] = self._get_spm_pkginfo(package, from_installed = True)
346
347 return True, package_data
348
350
351 sys_intf = self.SystemManagerExecutor.SystemInterface
352 queue_data, key = sys_intf.get_item_by_queue_id(queue_id, copy = True)
353 if queue_data == None:
354 return False, 'no item in queue'
355
356 def set_proc_pid(pid):
357 self._set_processing_pid(queue_id, pid)
358
359 stdout_err = open(queue_data['stdout'], "a+")
360 stdout_err.write("Preparing to spawn SPM info. Good luck mate!\n")
361 stdout_err.flush()
362
363 try:
364
365 spm = sys_intf.Entropy.Spm()
366 status = spm.print_build_environment_info(
367 stdin = self._get_stdin(queue_id),
368 stdout = stdour_err, stderr = stdout_err,
369 pid_write_func = set_proc_pid, coloured_output = False)
370
371 finally:
372
373 stdout_err.write("\n### Done ###\n")
374 stdout_err.flush()
375 stdout_err.close()
376
377 return True, status
378
380
381 queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True)
382 if queue_data == None:
383 return False, 'no item in queue'
384
385 stdout_err = open(queue_data['stdout'], "a+")
386
387 cmd = command.split()
388 cmd = ' '.join(cmd)
389 stdout_err.write("Preparing to spawn parameter: '%s'. Good luck mate!\n" % (cmd,))
390 stdout_err.flush()
391
392 try:
393 p = subprocess.Popen(cmd, stdout = stdout_err, stderr = stdout_err,
394 stdin = self._get_stdin(queue_id), shell = True)
395 self._set_processing_pid(queue_id, p.pid)
396 rc = p.wait()
397 finally:
398 stdout_err.write("\n### Done ###\n")
399 stdout_err.flush()
400 stdout_err.close()
401 return True, rc
402
404
405 queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True)
406 if queue_data == None:
407 return False, 'no item in queue'
408
409
410 matches = []
411 for idpackage in idpackages:
412 matches.append((idpackage, from_repo,))
413
414 stdout_err = open(queue_data['stdout'], "a+")
415
416 def myfunc():
417 sys.stdout = stdout_err
418 sys.stderr = stdout_err
419 mystdin = self._get_stdin(queue_id)
420 if mystdin: sys.stdin = os.fdopen(mystdin, 'rb')
421 try:
422 switched = self.SystemManagerExecutor.SystemInterface.Entropy.move_packages(
423 matches, to_repo,
424 from_repo = from_repo,
425 ask = False,
426 do_copy = do_copy
427 )
428 return switched
429 finally:
430 sys.stdout.write("\n### Done ###\n")
431 sys.stdout.flush()
432 sys.stdout = sys.__stdout__
433 sys.stderr = sys.__stderr__
434 sys.stdin = sys.__stdin__
435
436 def write_pid(pid):
437 self._set_processing_pid(queue_id, pid)
438
439 switched = self.entropyTools.spawn_function(myfunc, write_pid_func = write_pid)
440 stdout_err.flush()
441 stdout_err.close()
442
443 rc = 1
444 if len(switched) == len(idpackages):
445 rc = 0
446 return True, rc
447
449
450 queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True)
451 if queue_data == None:
452 return False, 'no item in queue'
453
454 stdout_err = open(queue_data['stdout'], "a+")
455 Entropy = self.SystemManagerExecutor.SystemInterface.Entropy
456
457 def myfunc():
458 sys.stdout = stdout_err
459 sys.stderr = stdout_err
460 mystdin = self._get_stdin(queue_id)
461 if mystdin: sys.stdin = os.fdopen(mystdin, 'rb')
462 try:
463
464 for repoid in Entropy.get_available_repositories():
465 self.run_entropy_treeupdates(queue_id, repoid)
466
467 stdout_err.write("\n"+_("Calculating updates...").encode('utf-8')+"\n")
468 stdout_err.flush()
469
470 to_add, to_remove, to_inject = Entropy.scan_package_changes()
471 mydict = { 'add': to_add, 'remove': to_remove, 'inject': to_inject }
472
473
474 mydict['add_data'] = {}
475 for portage_atom, portage_counter in to_add:
476 mydict['add_data'][(portage_atom, portage_counter,)] = self._get_spm_pkginfo(portage_atom, from_installed = True)
477
478 mydict['remove_data'] = {}
479 for idpackage, repoid in to_remove:
480 dbconn = Entropy.open_server_repository(repo = repoid, just_reading = True, warnings = False, do_cache = False)
481 mydict['remove_data'][(idpackage, repoid,)] = self._get_entropy_pkginfo(dbconn, idpackage, repoid)
482 dbconn.closeDB()
483
484 mydict['inject_data'] = {}
485 for idpackage, repoid in to_inject:
486 dbconn = Entropy.open_server_repository(repo = repoid, just_reading = True, warnings = False, do_cache = False)
487 mydict['inject_data'][(idpackage, repoid,)] = self._get_entropy_pkginfo(dbconn, idpackage, repoid)
488 dbconn.closeDB()
489
490 return True, mydict
491
492 finally:
493 sys.stdout.write("\n### Done ###\n")
494 sys.stdout.flush()
495 sys.stdout = sys.__stdout__
496 sys.stderr = sys.__stderr__
497 sys.stdin = sys.__stdin__
498
499 def write_pid(pid):
500 self._set_processing_pid(queue_id, pid)
501
502 data = self.entropyTools.spawn_function(myfunc, write_pid_func = write_pid)
503 stdout_err.flush()
504 stdout_err.close()
505 return data
506
508
509 queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True)
510 if queue_data == None:
511 return False, 'no item in queue'
512
513 stdout_err = open(queue_data['stdout'], "a+")
514 Entropy = self.SystemManagerExecutor.SystemInterface.Entropy
515
516 def myfunc():
517 sys.stdout = stdout_err
518 sys.stderr = stdout_err
519 mystdin = self._get_stdin(queue_id)
520 if mystdin: sys.stdin = os.fdopen(mystdin, 'rb')
521 try:
522
523 atoms_removed = []
524 matches_injected = set()
525
526 if to_inject: Entropy.updateProgress(_("Running package injection"))
527
528
529 for idpackage, repoid in to_inject:
530 matches_injected.add((idpackage, repoid,))
531 Entropy.transform_package_into_injected(idpackage, repo = repoid)
532
533 if to_remove: Entropy.updateProgress(_("Running package removal"))
534
535
536 remdata = {}
537 for idpackage, repoid in to_remove:
538 dbconn = Entropy.open_server_repository(repo = repoid, just_reading = True, warnings = False, do_cache = False)
539 atoms_removed.append(dbconn.retrieveAtom(idpackage))
540 dbconn.closeDB()
541 if repoid not in remdata:
542 remdata[repoid] = set()
543 remdata[repoid].add(idpackage)
544 for repoid in remdata:
545 Entropy.remove_packages(remdata[repoid], repo = repoid)
546
547 mydict = {
548 'added_data': {},
549 'remove_data': atoms_removed,
550 'inject_data': {}
551 }
552
553 if to_add:
554 problems = Entropy.check_config_file_updates()
555 if problems:
556 return False, mydict
557 Entropy.updateProgress(_("Running package quickpkg"))
558
559
560 for repoid in to_add:
561 store_dir = Entropy.get_local_store_directory(repo = repoid)
562 for atom in to_add[repoid]:
563 Entropy.Spm().generate_package(atom, store_dir)
564
565
566 avail_repos = Entropy.get_available_repositories()
567 if etpConst['clientserverrepoid'] in avail_repos:
568 avail_repos.pop(etpConst['clientserverrepoid'])
569 matches_added = set()
570 for repoid in avail_repos:
571 store_dir = Entropy.get_local_store_directory(repo = repoid)
572 package_files = os.listdir(store_dir)
573 if not package_files: continue
574 package_files = [(os.path.join(store_dir, x), False) for x in package_files]
575
576 Entropy.updateProgress( "[%s|%s] %s" % (
577 repoid,
578 Entropy.SystemSettings['repositories']['branch'],
579 _("Adding packages"),
580 )
581 )
582 for package_file, inject in package_files:
583 Entropy.updateProgress(" %s" % (package_file,))
584
585 idpackages = Entropy.add_packages_to_repository(package_files, ask = False, repo = repoid)
586 matches_added |= set([(x, repoid,) for x in idpackages])
587
588
589 Entropy.dependencies_test()
590
591 for idpackage, repoid in matches_added:
592 dbconn = Entropy.open_server_repository(repo = repoid, just_reading = True, warnings = False, do_cache = False)
593 mydict['added_data'][(idpackage, repoid,)] = self._get_entropy_pkginfo(dbconn, idpackage, repoid)
594 dbconn.closeDB()
595 for idpackage, repoid in matches_injected:
596 dbconn = Entropy.open_server_repository(repo = repoid, just_reading = True, warnings = False, do_cache = False)
597 mydict['inject_data'][(idpackage, repoid,)] = self._get_entropy_pkginfo(dbconn, idpackage, repoid)
598 dbconn.closeDB()
599 return True, mydict
600
601 finally:
602 sys.stdout.write("\n### Done ###\n")
603 sys.stdout.flush()
604 sys.stdout = sys.__stdout__
605 sys.stderr = sys.__stderr__
606 sys.stdin = sys.__stdin__
607
608 def write_pid(pid):
609 self._set_processing_pid(queue_id, pid)
610
611 data = self.entropyTools.spawn_function(myfunc, write_pid_func = write_pid)
612 stdout_err.flush()
613 stdout_err.close()
614 return data
615
617
618 queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True)
619 if queue_data == None:
620 return False, 'no item in queue'
621
622 stdout_err = open(queue_data['stdout'], "a+")
623
624 def myfunc():
625 sys.stdout = stdout_err
626 sys.stderr = stdout_err
627 mystdin = self._get_stdin(queue_id)
628 if mystdin: sys.stdin = os.fdopen(mystdin, 'rb')
629 try:
630 deps_not_matched = self.SystemManagerExecutor.SystemInterface.Entropy.dependencies_test()
631 return True, deps_not_matched
632 except Exception as e:
633 self.entropyTools.print_traceback()
634 return False, str(e)
635 finally:
636 sys.stdout.write("\n### Done ###\n")
637 sys.stdout.flush()
638 sys.stdout = sys.__stdout__
639 sys.stderr = sys.__stderr__
640 sys.stdin = sys.__stdin__
641
642 def write_pid(pid):
643 self._set_processing_pid(queue_id, pid)
644
645 data = self.entropyTools.spawn_function(myfunc, write_pid_func = write_pid)
646 stdout_err.flush()
647 stdout_err.close()
648 return data
649
651
652 queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True)
653 if queue_data == None:
654 return False, 'no item in queue'
655
656 stdout_err = open(queue_data['stdout'], "a+")
657
658 def myfunc():
659 sys.stdout = stdout_err
660 sys.stderr = stdout_err
661 mystdin = self._get_stdin(queue_id)
662 if mystdin: sys.stdin = os.fdopen(mystdin, 'rb')
663 try:
664 return self.SystemManagerExecutor.SystemInterface.Entropy.test_shared_objects()
665 except Exception as e:
666 self.entropyTools.print_traceback()
667 return False, str(e)
668 finally:
669 sys.stdout.write("\n### Done ###\n")
670 sys.stdout.flush()
671 sys.stdout = sys.__stdout__
672 sys.stderr = sys.__stderr__
673 sys.stdin = sys.__stdin__
674
675 def write_pid(pid):
676 self._set_processing_pid(queue_id, pid)
677
678 status, result = self.entropyTools.spawn_function(myfunc, write_pid_func = write_pid)
679 stdout_err.flush()
680 stdout_err.close()
681
682 mystatus = False
683 if status == 0: mystatus = True
684 if not result: result = set()
685 return mystatus, result
686
688
689 queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True)
690 if queue_data == None:
691 return False, 'no item in queue'
692
693 stdout_err = open(queue_data['stdout'], "a+")
694
695 def myfunc():
696 sys.stdout = stdout_err
697 sys.stderr = stdout_err
698 mystdin = self._get_stdin(queue_id)
699 if mystdin: sys.stdin = os.fdopen(mystdin, 'rb')
700 try:
701 if mode == "local":
702 data = self.SystemManagerExecutor.SystemInterface.Entropy.verify_local_packages([], ask = False, repo = repoid)
703 else:
704 data = self.SystemManagerExecutor.SystemInterface.Entropy.verify_remote_packages([], ask = False, repo = repoid)
705 return True, data
706 except Exception as e:
707 self.entropyTools.print_traceback()
708 return False, str(e)
709 finally:
710 sys.stdout.write("\n### Done ###\n")
711 sys.stdout.flush()
712 sys.stdout = sys.__stdout__
713 sys.stderr = sys.__stderr__
714 sys.stdin = sys.__stdin__
715
716 def write_pid(pid):
717 self._set_processing_pid(queue_id, pid)
718
719 mydata = self.entropyTools.spawn_function(myfunc, write_pid_func = write_pid)
720 stdout_err.flush()
721 stdout_err.close()
722 return mydata
723
725
726 queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True)
727 if queue_data == None:
728 return False, 'no item in queue'
729
730 stdout_err = open(queue_data['stdout'], "a+")
731
732 def myfunc():
733 sys.stdout = stdout_err
734 sys.stderr = stdout_err
735 mystdin = self._get_stdin(queue_id)
736 if mystdin: sys.stdin = os.fdopen(mystdin, 'rb')
737 try:
738 sys.stdout.write(_("Opening database to let it run treeupdates. If you won't see anything below, it's just fine.").encode('utf-8')+"\n")
739 dbconn = self.SystemManagerExecutor.SystemInterface.Entropy.open_server_repository(
740 repo = repoid, do_cache = False,
741 read_only = True
742 )
743 dbconn.closeDB()
744 except Exception as e:
745 self.entropyTools.print_traceback()
746 return False, str(e)
747 finally:
748 sys.stdout.write("\n### Done ###\n")
749 sys.stdout.flush()
750 sys.stdout = sys.__stdout__
751 sys.stderr = sys.__stderr__
752 sys.stdin = sys.__stdin__
753
754 def write_pid(pid):
755 self._set_processing_pid(queue_id, pid)
756
757 self.entropyTools.spawn_function(myfunc, write_pid_func = write_pid)
758 stdout_err.flush()
759 stdout_err.close()
760 return True, 0
761
763
764 queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True)
765 if queue_data == None:
766 return False, 'no item in queue'
767
768 stdout_err = open(queue_data['stdout'], "a+")
769 import socket
770 Entropy = self.SystemManagerExecutor.SystemInterface.Entropy
771
772 def myfunc():
773 sys.stdout = stdout_err
774 sys.stderr = stdout_err
775 mystdin = self._get_stdin(queue_id)
776 if mystdin: sys.stdin = os.fdopen(mystdin, 'rb')
777 try:
778
779 sys.stdout.write(_("Scanning").encode('utf-8')+"\n")
780 repo_data = {}
781 for repoid in repositories:
782
783 repo_data[repoid] = {}
784
785 for uri in Entropy.get_remote_mirrors(repoid):
786
787 crippled_uri = EntropyTransceiver.get_uri_name(uri)
788
789 repo_data[repoid][crippled_uri] = {}
790 repo_data[repoid][crippled_uri]['packages'] = {}
791
792 try:
793 upload_queue, download_queue, removal_queue, \
794 fine_queue, remote_packages_data = Entropy.MirrorsService.calculate_packages_to_sync(
795 uri, Entropy.SystemSettings['repositories']['branch'],
796 repoid)
797 except socket.error:
798 self.entropyTools.print_traceback(f = stdout_err)
799 stdout_err.write("\n"+_("Socket error, continuing...").encode('utf-8')+"\n")
800 continue
801
802 if (upload_queue or download_queue or removal_queue):
803 upload, download, removal, copy, metainfo = Entropy.MirrorsService.expand_queues(
804 upload_queue,
805 download_queue,
806 removal_queue,
807 remote_packages_data,
808 Entropy.SystemSettings['repositories']['branch'],
809 repoid
810 )
811 if len(upload)+len(download)+len(removal)+len(copy):
812 repo_data[repoid][crippled_uri]['packages'] = {
813 'upload': upload,
814 'download': download,
815 'removal': removal,
816 'copy': copy,
817 }
818
819
820 current_revision = Entropy.get_local_database_revision(repoid)
821 remote_revision = Entropy.get_remote_database_revision(repoid)
822 download_latest, upload_queue = Entropy.MirrorsService.calculate_database_sync_queues(repoid)
823
824 repo_data[repoid][crippled_uri]['database'] = {
825 'current_revision': current_revision,
826 'remote_revision': remote_revision,
827 'download_latest': download_latest,
828 'upload_queue': [(EntropyTransceiver.get_uri_name(x[0]), x[1],) for x in upload_queue]
829 }
830
831 return True, repo_data
832
833 except Exception as e:
834 self.entropyTools.print_traceback()
835 return False, str(e)
836 finally:
837 sys.stdout.write("\n### Done ###\n")
838 sys.stdout.flush()
839 sys.stdout = sys.__stdout__
840 sys.stderr = sys.__stderr__
841 sys.stdin = sys.__stdin__
842
843 def write_pid(pid):
844 self._set_processing_pid(queue_id, pid)
845
846 data = self.entropyTools.spawn_function(myfunc, write_pid_func = write_pid)
847 stdout_err.flush()
848 stdout_err.close()
849 return data
850
852
853 queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True)
854 if queue_data == None:
855 return False, 'no item in queue'
856
857 stdout_err = open(queue_data['stdout'], "a+")
858 Entropy = self.SystemManagerExecutor.SystemInterface.Entropy
859
860 def sync_remote_databases(repoid, pretend):
861
862 rdb_status = Entropy.MirrorsService.get_remote_databases_status()
863 Entropy.updateProgress(
864 "%s:" % (_("Remote Entropy Database Repository Status"),),
865 header = " * "
866 )
867 for myuri, myrev in rdb_status:
868 Entropy.updateProgress("\t %s:\t %s" % (_("Host"), EntropyTransceiver.get_uri_name(myuri),))
869 Entropy.updateProgress("\t * %s: %s" % (_("Database revision"), myrev,))
870 local_revision = Entropy.get_local_database_revision(repoid)
871 Entropy.updateProgress("\t * %s: %s" % (_("Database local revision currently at"), local_revision,))
872 if pretend:
873 return 0, set(), set()
874
875 errors, fine_uris, broken_uris = Entropy.MirrorsService.sync_databases(no_upload = False)
876 remote_status = Entropy.MirrorsService.get_remote_databases_status(repoid)
877 Entropy.updateProgress(" * %s: " % (_("Remote Entropy Database Repository Status"),))
878 for myuri, myrev in remote_status:
879 Entropy.updateProgress("\t %s:\t%s" % (_("Host"), EntropyTransceiver.get_uri_name(myuri),))
880 Entropy.updateProgress("\t * %s: %s" % (_("Database revision"), myrev,) )
881
882 return errors, fine_uris, broken_uris
883
884
885 def myfunc():
886 sys.stdout = stdout_err
887 sys.stderr = stdout_err
888 mystdin = self._get_stdin(queue_id)
889 if mystdin: sys.stdin = os.fdopen(mystdin, 'rb')
890 try:
891
892 repo_data = {}
893 sys_settings_srv_plugin_id = \
894 etpConst['system_settings_plugins_ids']['server_plugin']
895 for repoid in repository_data:
896
897
898 if repoid == etpConst['clientserverrepoid']:
899 continue
900
901 successfull_mirrors = set()
902 mirrors_errors = False
903 mirrors_tainted = False
904 broken_mirrors = set()
905 check_data = []
906
907 repo_data[repoid] = {
908 'mirrors_tainted': mirrors_tainted,
909 'mirrors_errors': mirrors_errors,
910 'successfull_mirrors': successfull_mirrors.copy(),
911 'broken_mirrors': broken_mirrors.copy(),
912 'check_data': check_data,
913 'db_errors': 0,
914 'db_fine': set(),
915 'db_broken': set(),
916 }
917
918 if repository_data[repoid]['pkg']:
919
920 mirrors_tainted, mirrors_errors, \
921 successfull_mirrors, broken_mirrors, \
922 check_data = Entropy.MirrorsService.sync_packages(
923 ask = False, pretend = repository_data[repoid]['pretend'],
924 packages_check = repository_data[repoid]['pkg_check'], repo = repoid)
925
926 repo_data[repoid]['mirrors_tainted'] = mirrors_tainted
927 repo_data[repoid]['mirrors_errors'] = mirrors_errors
928 repo_data[repoid]['successfull_mirrors'] = successfull_mirrors
929 repo_data[repoid]['broken_mirrors'] = broken_mirrors
930 repo_data[repoid]['check_data'] = check_data
931
932 if (not successfull_mirrors) and (not repository_data[repoid]['pretend']):
933 continue
934
935 if (not mirrors_errors) and repository_data[repoid]['db']:
936
937 if mirrors_tainted and Entropy.SystemSettings[sys_settings_srv_plugin_id]['server']['rss']['enabled']:
938 commit_msg = repository_data[repoid]['commit_msg']
939 if not commit_msg: commit_msg = "Autodriven update"
940 Entropy.rssMessages['commitmessage'] = commit_msg
941
942 errors, fine, broken = sync_remote_databases(repoid, repository_data[repoid]['pretend'])
943 repo_data[repoid]['db_errors'] = errors
944 repo_data[repoid]['db_fine'] = fine.copy()
945 repo_data[repoid]['db_broken'] = broken.copy()
946 if errors:
947 continue
948 Entropy.MirrorsService.lock_mirrors(lock = False, repo = repoid)
949 Entropy.MirrorsService.tidy_mirrors(
950 repo = repoid, ask = False,
951 pretend = repository_data[repoid]['pretend']
952 )
953
954 return True, repo_data
955
956 except Exception as e:
957 self.entropyTools.print_traceback()
958 return False, str(e)
959 finally:
960 sys.stdout.write("\n### Done ###\n")
961 sys.stdout.flush()
962 sys.stdout = sys.__stdout__
963 sys.stderr = sys.__stderr__
964 sys.stdin = sys.__stdin__
965
966 def write_pid(pid):
967 self._set_processing_pid(queue_id, pid)
968
969 data = self.entropyTools.spawn_function(myfunc, write_pid_func = write_pid)
970 stdout_err.flush()
971 stdout_err.close()
972 return data
973
975
976 queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True)
977 if queue_data == None:
978 return False, 'no item in queue'
979
980 spm = self.SystemManagerExecutor.SystemInterface.Entropy.Spm()
981 glsa_ids = spm.get_security_packages(list_type)
982 if not glsa_ids:
983 return True, []
984
985 data = {}
986 for myid in glsa_ids:
987 data[myid] = spm.get_security_advisory_metadata(myid)
988 return True, data
989
991
992 queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True)
993 if queue_data == None:
994 return False, 'no item in queue'
995
996 stdout_err = open(queue_data['stdout'], "a+")
997
998 def myfunc():
999 sys.stdout = stdout_err
1000 sys.stderr = stdout_err
1001 mystdin = self._get_stdin(queue_id)
1002 if mystdin: sys.stdin = os.fdopen(mystdin, 'rb')
1003 try:
1004 data = self.SystemManagerExecutor.SystemInterface.Entropy.MirrorsService.read_notice_board(repo = repoid)
1005 if data == None:
1006 return False, None
1007 return True, data
1008 except Exception as e:
1009 self.entropyTools.print_traceback()
1010 return False, str(e)
1011 finally:
1012 sys.stdout.write("\n### Done ###\n")
1013 sys.stdout.flush()
1014 sys.stdout = sys.__stdout__
1015 sys.stderr = sys.__stderr__
1016 sys.stdin = sys.__stdin__
1017
1018 def write_pid(pid):
1019 self._set_processing_pid(queue_id, pid)
1020
1021 mydata = self.entropyTools.spawn_function(myfunc, write_pid_func = write_pid)
1022 stdout_err.flush()
1023 stdout_err.close()
1024 return mydata
1025
1027
1028 queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True)
1029 if queue_data == None:
1030 return False, 'no item in queue'
1031
1032 stdout_err = open(queue_data['stdout'], "a+")
1033
1034 def myfunc():
1035 sys.stdout = stdout_err
1036 sys.stderr = stdout_err
1037 mystdin = self._get_stdin(queue_id)
1038 if mystdin: sys.stdin = os.fdopen(mystdin, 'rb')
1039 try:
1040 for entry_id in entry_ids:
1041 data = self.SystemManagerExecutor.SystemInterface.Entropy.MirrorsService.remove_from_notice_board(entry_id, repo = repoid)
1042 self.SystemManagerExecutor.SystemInterface.Entropy.MirrorsService.upload_notice_board(repo = repoid)
1043 return True, data
1044 except Exception as e:
1045 self.entropyTools.print_traceback()
1046 return False, str(e)
1047 finally:
1048 sys.stdout.write("\n### Done ###\n")
1049 sys.stdout.flush()
1050 sys.stdout = sys.__stdout__
1051 sys.stderr = sys.__stderr__
1052 sys.stdin = sys.__stdin__
1053
1054 def write_pid(pid):
1055 self._set_processing_pid(queue_id, pid)
1056
1057 mydata = self.entropyTools.spawn_function(myfunc, write_pid_func = write_pid)
1058 stdout_err.flush()
1059 stdout_err.close()
1060 return mydata
1061
1062 - def add_notice_board_entry(self, queue_id, repoid, title, notice_text, link):
1063
1064 queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id, copy = True)
1065 if queue_data == None:
1066 return False, 'no item in queue'
1067
1068 stdout_err = open(queue_data['stdout'], "a+")
1069
1070 def myfunc():
1071 sys.stdout = stdout_err
1072 sys.stderr = stdout_err
1073 mystdin = self._get_stdin(queue_id)
1074 if mystdin: sys.stdin = os.fdopen(mystdin, 'rb')
1075 try:
1076 data = self.SystemManagerExecutor.SystemInterface.Entropy.MirrorsService.update_notice_board(title, notice_text, link = link, repo = repoid)
1077 return True, data
1078 except Exception as e:
1079 self.entropyTools.print_traceback()
1080 return False, str(e)
1081 finally:
1082 sys.stdout.write("\n### Done ###\n")
1083 sys.stdout.flush()
1084 sys.stdout = sys.__stdout__
1085 sys.stderr = sys.__stderr__
1086 sys.stdin = sys.__stdin__
1087
1088 def write_pid(pid):
1089 self._set_processing_pid(queue_id, pid)
1090
1091 mydata = self.entropyTools.spawn_function(myfunc, write_pid_func = write_pid)
1092 stdout_err.flush()
1093 stdout_err.close()
1094 return mydata
1095
1097 mystdin = None
1098 std_data = self.SystemManagerExecutor.SystemInterface.ManagerQueueStdInOut.get(queue_id)
1099 if std_data != None: mystdin = std_data[0]
1100 return mystdin
1101
1103
1104 f.flush()
1105 back = mykwargs.get("back")
1106 count = mykwargs.get("count")
1107 header = mykwargs.get("header")
1108 percent = mykwargs.get("percent")
1109 text = myargs[0].encode('utf-8')
1110 if not header: header = ''
1111
1112 count_str = ""
1113 if count:
1114 if len(count) > 1:
1115 if percent:
1116 count_str = " ("+str(round((float(count[0])/count[1])*100, 1))+"%) "
1117 else:
1118 count_str = " (%s/%s) " % (red(str(count[0])), blue(str(count[1])),)
1119
1120 def is_last_newline(f):
1121 try:
1122 f.seek(-1, os.SEEK_END)
1123 last = f.read(1)
1124 if last == "\n":
1125 return True
1126 except IOError:
1127 pass
1128 return False
1129
1130 if back:
1131 self.entropyTools.seek_till_newline(f)
1132 txt = header+count_str+text
1133 else:
1134 if not is_last_newline(f): f.write("\n")
1135 txt = header+count_str+text+"\n"
1136 f.write(txt)
1137
1138 f.flush()
1139
1140
1142 data = {}
1143 try:
1144 data['atom'], data['name'], data['version'], data['versiontag'], \
1145 data['description'], data['category'], data['chost'], \
1146 data['cflags'], data['cxxflags'], data['homepage'], \
1147 data['license'], data['branch'], data['download'], \
1148 data['digest'], data['slot'], data['etpapi'], \
1149 data['datecreation'], data['size'], data['revision'] = dbconn.getBaseData(idpackage)
1150 except TypeError:
1151 return data
1152 data['injected'] = dbconn.isInjected(idpackage)
1153 data['repoid'] = repoid
1154 data['idpackage'] = idpackage
1155 return data
1156
1183