1
2 """
3
4 @author: Fabio Erculiani <lxnay@sabayonlinux.org>
5 @contact: lxnay@sabayonlinux.org
6 @copyright: Fabio Erculiani
7 @license: GPL-2
8
9 B{Entropy Server transceivers module}.
10
11 """
12 import os
13 import sys
14 import time
15 from entropy.const import const_isstring, const_isnumber, etpConst
16 from entropy.output import darkred, blue, brown, darkgreen, red, bold
17 from entropy.exceptions import *
18 from entropy.i18n import _
19 from entropy.core.settings.base import SystemSettings
20 from entropy.transceivers import EntropyTransceiver
21 from entropy.tools import print_traceback, is_valid_md5, compare_md5
22
24
25 - def __init__(self, entropy_interface, uris, files_to_upload,
26 download = False, remove = False, txc_basedir = None,
27 local_basedir = None, critical_files = None,
28 handlers_data = None, repo = None):
29
30 if critical_files is None:
31 critical_files = []
32 if handlers_data is None:
33 handlers_data = {}
34
35 self.Entropy = entropy_interface
36 if not isinstance(uris, list):
37 raise AttributeError("InvalidDataType: %s" % (
38 _("uris must be a list instance"),
39 )
40 )
41 if not isinstance(files_to_upload, (list, dict)):
42 raise AttributeError("InvalidDataType: %s" % (
43 _("files_to_upload must be a list or dict instance"),
44 )
45 )
46 self.uris = uris
47 if isinstance(files_to_upload, list):
48 self.myfiles = files_to_upload[:]
49 else:
50 self.myfiles = sorted([x for x in files_to_upload])
51
52 self.SystemSettings = SystemSettings()
53 self.sys_settings_plugin_id = \
54 etpConst['system_settings_plugins_ids']['server_plugin']
55 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
56
57
58 self.speed_limit = srv_set['sync_speed_limit']
59 self.download = download
60 self.remove = remove
61 self.repo = repo
62 if self.repo == None:
63 self.repo = self.Entropy.default_repository
64 if self.remove:
65 self.download = False
66
67 if not txc_basedir:
68
69 branch = self.SystemSettings['repositories']['branch']
70 my_path = os.path.join(
71 self.Entropy.get_remote_database_relative_path(repo), branch)
72 self.txc_basedir = my_path
73 else:
74 self.txc_basedir = txc_basedir
75
76 if not local_basedir:
77
78 self.local_basedir = os.path.dirname(
79 self.Entropy.get_local_database_file(self.repo))
80 else:
81 self.local_basedir = local_basedir
82
83 self.critical_files = critical_files
84 self.handlers_data = handlers_data.copy()
85
88
89 crippled_uri = EntropyTransceiver.get_uri_name(uri)
90
91 self.Entropy.updateProgress(
92 "[%s|#%s|(%s/%s)] %s: %s" % (
93 blue(crippled_uri),
94 darkgreen(str(tries)),
95 blue(str(counter)),
96 bold(str(maxcount)),
97 darkgreen(_("verifying upload (if supported)")),
98 blue(os.path.basename(local_filepath)),
99 ),
100 importance = 0,
101 type = "info",
102 header = red(" @@ "),
103 back = True
104 )
105
106 valid_remote_md5 = True
107
108 if const_isstring(remote_md5):
109 valid_md5 = is_valid_md5(remote_md5)
110 ckres = False
111 if valid_md5:
112 ckres = compare_md5(local_filepath, remote_md5)
113 if ckres:
114 self.Entropy.updateProgress(
115 "[%s|#%s|(%s/%s)] %s: %s: %s" % (
116 blue(crippled_uri),
117 darkgreen(str(tries)),
118 blue(str(counter)),
119 bold(str(maxcount)),
120 blue(_("digest verification")),
121 os.path.basename(local_filepath),
122 darkgreen(_("so far, so good!")),
123 ),
124 importance = 0,
125 type = "info",
126 header = red(" @@ ")
127 )
128 return True
129
130 elif not valid_md5:
131
132 self.Entropy.updateProgress(
133 "[%s|#%s|(%s/%s)] %s: %s: %s" % (
134 blue(crippled_uri),
135 darkgreen(str(tries)),
136 blue(str(counter)),
137 bold(str(maxcount)),
138 blue(_("digest verification")),
139 os.path.basename(local_filepath),
140 bold(_("malformed md5 provided to function")),
141 ),
142 importance = 0,
143 type = "warning",
144 header = brown(" @@ ")
145 )
146 else:
147 self.Entropy.updateProgress(
148 "[%s|#%s|(%s/%s)] %s: %s: %s" % (
149 blue(crippled_uri),
150 darkgreen(str(tries)),
151 blue(str(counter)),
152 bold(str(maxcount)),
153 blue(_("digest verification")),
154 os.path.basename(local_filepath),
155 bold(_("remote md5 is invalid")),
156 ),
157 importance = 0,
158 type = "warning",
159 header = brown(" @@ ")
160 )
161 valid_remote_md5 = False
162
163 return valid_remote_md5
164
166
167 fine = set()
168 broken = set()
169 fail = False
170 crippled_uri = EntropyTransceiver.get_uri_name(uri)
171 action = 'upload'
172 if self.download:
173 action = 'download'
174 elif self.remove:
175 action = 'remove'
176
177 try:
178 txc = EntropyTransceiver(uri)
179 if const_isnumber(self.speed_limit):
180 txc.set_speed_limit(self.speed_limit)
181 txc.set_output_interface(self.Entropy)
182 except ConnectionError:
183 print_traceback()
184 return True, fine, broken
185
186 maxcount = len(self.myfiles)
187 counter = 0
188
189 with txc as handler:
190
191 for mypath in self.myfiles:
192
193 base_dir = self.txc_basedir
194
195 mycwd = None
196 if isinstance(mypath, tuple):
197 if len(mypath) < 2:
198 continue
199 base_dir, mypath = mypath
200
201 if not handler.is_dir(base_dir):
202 handler.makedirs(base_dir)
203
204 mypath_fn = os.path.basename(mypath)
205 remote_path = os.path.join(base_dir, mypath_fn)
206
207 syncer = handler.upload
208 myargs = (mypath, remote_path)
209 if self.download:
210 syncer = handler.download
211 local_path = os.path.join(self.local_basedir, mypath_fn)
212 myargs = (remote_path, local_path)
213 elif self.remove:
214 syncer = handler.delete
215 myargs = (remote_path,)
216
217 counter += 1
218 tries = 0
219 done = False
220 lastrc = None
221
222 while tries < 5:
223 tries += 1
224 self.Entropy.updateProgress(
225 "[%s|#%s|(%s/%s)] %s: %s" % (
226 blue(crippled_uri),
227 darkgreen(str(tries)),
228 blue(str(counter)),
229 bold(str(maxcount)),
230 blue(action+"ing"),
231 red(os.path.basename(mypath)),
232 ),
233 importance = 0,
234 type = "info",
235 header = red(" @@ ")
236 )
237 rc = syncer(*myargs)
238 if rc and not (self.download or self.remove):
239 remote_md5 = handler.get_md5(remote_path)
240 rc = self.handler_verify_upload(mypath, uri,
241 counter, maxcount, tries, remote_md5 = remote_md5)
242 if rc:
243 self.Entropy.updateProgress(
244 "[%s|#%s|(%s/%s)] %s %s: %s" % (
245 blue(crippled_uri),
246 darkgreen(str(tries)),
247 blue(str(counter)),
248 bold(str(maxcount)),
249 blue(action),
250 _("successful"),
251 red(os.path.basename(mypath)),
252 ),
253 importance = 0,
254 type = "info",
255 header = darkgreen(" @@ ")
256 )
257 done = True
258 fine.add(uri)
259 break
260 else:
261 self.Entropy.updateProgress(
262 "[%s|#%s|(%s/%s)] %s %s: %s" % (
263 blue(crippled_uri),
264 darkgreen(str(tries)),
265 blue(str(counter)),
266 bold(str(maxcount)),
267 blue(action),
268 brown(_("failed, retrying")),
269 red(os.path.basename(mypath)),
270 ),
271 importance = 0,
272 type = "warning",
273 header = brown(" @@ ")
274 )
275 lastrc = rc
276 continue
277
278 if not done:
279
280 self.Entropy.updateProgress(
281 "[%s|(%s/%s)] %s %s: %s - %s: %s" % (
282 blue(crippled_uri),
283 blue(str(counter)),
284 bold(str(maxcount)),
285 blue(action),
286 darkred("failed, giving up"),
287 red(os.path.basename(mypath)),
288 _("error"),
289 lastrc,
290 ),
291 importance = 1,
292 type = "error",
293 header = darkred(" !!! ")
294 )
295
296 if mypath not in self.critical_files:
297 self.Entropy.updateProgress(
298 "[%s|(%s/%s)] %s: %s, %s..." % (
299 blue(crippled_uri),
300 blue(str(counter)),
301 bold(str(maxcount)),
302 blue(_("not critical")),
303 os.path.basename(mypath),
304 blue(_("continuing")),
305 ),
306 importance = 1,
307 type = "warning",
308 header = brown(" @@ ")
309 )
310 continue
311
312 fail = True
313 broken.add((uri, lastrc))
314
315 break
316
317 return fail, fine, broken
318
319
321
322 broken_uris = set()
323 fine_uris = set()
324 errors = False
325 action = 'upload'
326 if self.download:
327 action = 'download'
328 elif self.remove:
329 action = 'remove'
330
331 for uri in self.uris:
332
333 crippled_uri = EntropyTransceiver.get_uri_name(uri)
334 self.Entropy.updateProgress(
335 "[%s|%s] %s..." % (
336 blue(crippled_uri),
337 brown(action),
338 blue(_("connecting to mirror")),
339 ),
340 importance = 0,
341 type = "info",
342 header = blue(" @@ ")
343 )
344
345 branch = SystemSettings()['repositories']['branch']
346 my_path = os.path.join(
347 self.Entropy.get_remote_database_relative_path(self.repo),
348 branch)
349
350 self.Entropy.updateProgress(
351 "[%s|%s] %s %s..." % (
352 blue(crippled_uri),
353 brown(action),
354 blue(_("setting directory to")),
355 darkgreen(my_path),
356 ),
357 importance = 0,
358 type = "info",
359 header = blue(" @@ ")
360 )
361
362 fail, fine, broken = self._transceive(uri)
363 fine_uris |= fine
364 broken_uris |= broken
365 if fail:
366 errors = True
367
368 return errors, fine_uris, broken_uris
369