Source Code for Module entropy.transceivers.uri_handlers.plugins.interfaces.ftp_plugin

  1  # -*- coding: utf-8 -*- 
  2  """ 
  3   
  4      @author: Fabio Erculiani <lxnay@sabayonlinux.org> 
  5      @contact: lxnay@sabayonlinux.org 
  6      @copyright: Fabio Erculiani 
  7      @license: GPL-2 
  8   
  9      B{EntropyTransceiver FTP URI Handler module}. 
 10   
 11  """ 
 12  import os 
 13  import time 
 14   
 15  from entropy.tools import print_traceback, get_file_size, \ 
 16      convert_seconds_to_fancy_output, bytes_into_human, spliturl 
 17  from entropy.output import blue, brown, darkgreen, red 
 18  from entropy.i18n import _ 
 19  from entropy.exceptions import ConnectionError, TransceiverError 
 20  from entropy.transceivers.uri_handlers.skel import EntropyUriHandler 
 21   
 22 -class EntropyFtpUriHandler(EntropyUriHandler): 
 23   
 24      PLUGIN_API_VERSION = 0 
 25   
 26      """ 
 27      EntropyUriHandler based FTP transceiver plugin. 
 28      """ 
 29   
 30      _DEFAULT_TIMEOUT = 60 
 31   
 32      @staticmethod 
 33 -    def approve_uri(uri): 
 34          if uri.startswith("ftp://"): 
 35              return True 
 36          return False 
 37   
 38      @staticmethod 
 39 -    def get_uri_name(uri): 
 40          myuri = spliturl(uri)[1] 
 41          # remove username:pass@ 
 42          myuri = myuri.split("@")[-1] 
 43          return myuri 
 44   
 45      @staticmethod 
 46 -    def hide_sensible_data(uri): 
 47          ftppassword = uri.split("@")[:-1] 
 48          if not ftppassword: 
 49              return uri 
 50          ftppassword = '@'.join(ftppassword) 
 51          ftppassword = ftppassword.split(":")[-1] 
 52          if not ftppassword: 
 53              return uri 
 54          newuri = uri.replace(ftppassword, "xxxxxxxx") 
 55          return newuri 
 56   
 57 -    def __init__(self, uri): 
 58          EntropyUriHandler.__init__(self, uri) 
 59   
 60          import socket, ftplib 
 61          self.socket, self.ftplib = socket, ftplib 
 62          self.__connected = False 
 63          self.__ftpconn = None 
 64          self.__currentdir = '.' 
 65          self.__ftphost = EntropyFtpUriHandler.get_uri_name(self._uri) 
 66          self.__ftpuser, self.__ftppassword, self.__ftpport, self.__ftpdir = \ 
 67              self.__extract_ftp_data(self._uri) 
 68   
 69          self._init_vars() 
 70   
 71          # as documentation suggests 
 72          # test out connection first 
 73          self._connect() 
 74          self._disconnect() 
 75   
 76 -    def __enter__(self): 
 77          pass # self.__connect_if_not() 
 78   
 79 -    def __exit__(self, exc_type, exc_value, traceback): 
 80          self.close() 
 81   
 82 -    def __extract_ftp_data(self, ftpuri): 
 83          ftpuser = ftpuri.split("ftp://")[-1].split(":")[0] 
 84          if (not ftpuser): 
 85              ftpuser = "anonymous@" 
 86              ftppassword = "anonymous" 
 87          else: 
 88              ftppassword = ftpuri.split("@")[:-1] 
 89              if len(ftppassword) > 1: 
 90                  ftppassword = '@'.join(ftppassword) 
 91                  ftppassword = ftppassword.split(":")[-1] 
 92                  if (not ftppassword): 
 93                      ftppassword = "anonymous" 
 94              else: 
 95                  ftppassword = ftppassword[0] 
 96                  ftppassword = ftppassword.split(":")[-1] 
 97                  if not ftppassword: 
 98                      ftppassword = "anonymous" 
 99   
100          ftpport = ftpuri.split(":")[-1] 
101          try: 
102              ftpport = int(ftpport) 
103          except ValueError: 
104              ftpport = 21 
105   
106          ftpdir = '/' 
107          if ftpuri.count("/") > 2: 
108              ftpdir = ftpuri[6:] 
109              ftpdir = "/" + ftpdir.split("/", 1)[-1] 
110              ftpdir = ftpdir.split(":")[0] 
111              if not ftpdir: 
112                  ftpdir = '/' 
113              elif ftpdir.endswith("/") and (ftpdir != "/"): 
114                  ftpdir = ftpdir[:-1] 
115   
116          return ftpuser, ftppassword, ftpport, ftpdir 
117   
118 -    def __connect_if_not(self): 
119          """ 
120          Handy internal method. 
121          """ 
122          if not self.__connected: 
123              self._connect() 
124          try: 
125              self.keep_alive() 
126          except ConnectionError: 
127              self._connect() 
128   
129 -    def _connect(self): 
130          """ 
131          Connect to FTP host. 
132          """ 
133          timeout = self._timeout 
134          if timeout is None: 
135              # default timeout set to 60 seconds 
136              timeout = EntropyFtpUriHandler._DEFAULT_TIMEOUT 
137   
138          count = 10 
139          while True: 
140              count -= 1 
141              try: 
142                  self.__ftpconn = self.ftplib.FTP() 
143                  self.__ftpconn.connect(self.__ftphost, self.__ftpport, timeout) 
144                  break 
145              except (self.socket.gaierror,) as e: 
146                  raise ConnectionError('ConnectionError: %s' % (e,)) 
147              except (self.socket.error,) as e: 
148                  if not count: 
149                      raise ConnectionError('ConnectionError: %s' % (e,)) 
150              except: 
151                  if not count: 
152                      raise 
153   
154          if self._verbose: 
155              mytxt = _("connecting with user") 
156              self.updateProgress( 
157                  "[ftp:%s] %s: %s" % ( 
158                      darkgreen(self.__ftphost), mytxt, blue(self.__ftpuser), 
159                  ), 
160                  importance = 1, 
161                  type = "info", 
162                  header = darkgreen(" * ") 
163              ) 
164          try: 
165              self.__ftpconn.login(self.__ftpuser, self.__ftppassword) 
166          except self.ftplib.error_perm as e: 
167              raise ConnectionError('ConnectionError: %s' % (e,)) 
168   
169          if self._verbose: 
170              mytxt = _("switching to") 
171              self.updateProgress( 
172                  "[ftp:%s] %s: %s" % ( 
173                      darkgreen(self.__ftphost), mytxt, blue(self.__ftpdir), 
174                  ), 
175                  importance = 1, 
176                  type = "info", 
177                  header = darkgreen(" * ") 
178              ) 
179          # create dirs if they don't exist 
180          self._set_cwd(self.__ftpdir, dodir = True) 
181          self.__connected = True 
182   
183 -    def _disconnect(self): 
184          if self.__ftpconn is not None: 
185              # try to disconnect 
186              try: 
187                  self.__ftpconn.quit() 
188              except (EOFError, self.socket, self.socket.timeout, 
189                  self.ftplib.error_reply,): 
190                  # AttributeError is raised when socket gets trashed 
191                  # EOFError is raised when the connection breaks 
192                  # timeout, who cares! 
193                  pass 
194              self.__ftpconn = None 
195          self.__connected = False 
196   
197 -    def _reconnect(self): 
198          self._disconnect() 
199          self._connect() 
200   
201 -    def _init_vars(self): 
202          self.__oldprogress_t = time.time() 
203          self.__datatransfer = 0 
204          self.__filesize = 0 
205          self.__filekbcount = 0 
206          self.__transfersize = 0 
207          self.__startingposition = 0 
208          self.__elapsed = 0.0 
209          self.__time_remaining_secs = 0 
210          self.__time_remaining = "(%s)" % (_("infinite"),) 
211          self.__starttime = time.time() 
212   
213 -    def _get_cwd(self): 
214          return self.__ftpconn.pwd() 
215   
216 -    def _set_cwd(self, mydir, dodir = False): 
217          try: 
218              return self.__set_cwd(mydir, dodir) 
219          except self.ftplib.error_perm as e: 
220              raise TransceiverError('TransceiverError: %s' % (e,)) 
221   
222 -    def __set_cwd(self, mydir, dodir = False): 
223          if self._verbose: 
224              mytxt = _("switching to") 
225              self.updateProgress( 
226                  "[ftp:%s] %s: %s" % (darkgreen(self.__ftphost), 
227                      mytxt, blue(mydir),), 
228                  importance = 1, 
229                  type = "info", 
230                  header = darkgreen(" * ") 
231              ) 
232          try: 
233              self.__ftpconn.cwd(mydir) 
234          except self.ftplib.error_perm as e: 
235              if e[0][:3] == '550' and dodir: 
236                  self.makedirs(mydir) 
237                  self.__ftpconn.cwd(mydir) 
238              else: 
239                  raise 
240          self.__currentdir = self._get_cwd() 
241   
242 -    def _set_pasv(self, pasv): 
243          self.__ftpconn.set_pasv(pasv) 
244   
245 -    def _set_chmod(self, chmodvalue, filename): 
246          return self.__ftpconn.voidcmd("SITE CHMOD " + str(chmodvalue) + " " + \ 
247              str(filename)) 
248   
249 -    def _get_file_mtime(self, path): 
250          rc = self.__ftpconn.sendcmd("mdtm " + path) 
251          return rc.split()[-1] 
252   
253 -    def _send_cmd(self, cmd): 
254          return self.__ftpconn.sendcmd(cmd) 
255   
256 -    def _update_speed(self): 
257          current_time = time.time() 
258          self.__elapsed = current_time - self.__starttime 
259          # we have the diff size 
260          pos_diff = self.__transfersize - self.__startingposition 
261          self.__datatransfer = pos_diff / self.__elapsed 
262          if self.__datatransfer < 0: 
263              self.__datatransfer = 0 
264          try: 
265              round_fsize = int(round(self.__filesize*1024, 0)) 
266              round_rsize = int(round(self.__transfersize, 0)) 
267              self.__time_remaining_secs = int(round((round_fsize - \ 
268                  round_rsize)/self.__datatransfer, 0)) 
269              self.__time_remaining = \ 
270                  convert_seconds_to_fancy_output(self.__time_remaining_secs) 
271          except (ValueError, TypeError,): 
272              self.__time_remaining = "(%s)" % (_("infinite"),) 
273   
274 -    def _speed_limit_loop(self): 
275          if self._speed_limit: 
276              while self.__datatransfer > self._speed_limit * 1024: 
277                  time.sleep(0.1) 
278                  self._update_speed() 
279                  self._update_progress() 
280   
281 -    def _commit_buffer_update(self, buf_len): 
282          # get the buffer size 
283          self.__filekbcount += float(buf_len)/1024 
284          self.__transfersize += buf_len 
285   
286 -    def _update_progress(self, force = False): 
287   
288          upload_percent = 100.0 
289          upload_size = round(self.__filekbcount, 1) 
290   
291          if self.__filesize >= 1: 
292              kbcount_round = round(self.__filekbcount, 1) 
293              upload_percent = round((kbcount_round / self.__filesize) * 100, 1) 
294   
295          delta_secs = 0.5 
296          cur_t = time.time() 
297          if (cur_t > (self.__oldprogress_t + delta_secs)) or force: 
298   
299              upload_percent = str(upload_percent)+"%" 
300              # create text 
301              mytxt = _("Transfer status") 
302              current_txt = brown("    <-> %s: " % (mytxt,)) + \ 
303                  darkgreen(str(upload_size)) + "/" + \ 
304                  red(str(self.__filesize)) + " kB " + \ 
305                  brown("[") + str(upload_percent) + brown("]") + \ 
306                  " " + self.__time_remaining + " " + \ 
307                  bytes_into_human(self.__datatransfer) + \ 
308                  "/" + _("sec") 
309   
310              self.updateProgress(current_txt, back = True) 
311              self.__oldprogress_t = cur_t 
312   
313 -    def _get_file_size(self, filename): 
314          return self.__ftpconn.size(filename) 
315   
316 -    def _get_file_size_compat(self, filename): 
317          try: 
318              sc = self.__ftpconn.sendcmd 
319              data = [x.split() for x in sc("stat %s" % (filename,)).split("\n")] 
320          except self.ftplib.error_temp: 
321              return "" 
322          for item in data: 
323              if item[-1] == filename: 
324                  return item[4] 
325          return "" 
326   
327 -    def _mkdir(self, directory): 
328          return self.__ftpconn.mkd(directory) 
329   
330 -    def download(self, remote_path, save_path): 
331   
332          self.__connect_if_not() 
333          path = os.path.join(self.__ftpdir, remote_path) 
334          tmp_save_path = save_path + ".dtmp" 
335   
336          def writer(buf): 
337              # writing file buffer 
338              f.write(buf) 
339              self._commit_buffer_update(len(buf)) 
340              self._update_speed() 
341              self._update_progress() 
342              self._speed_limit_loop() 
343   
344          tries = 10 
345          while tries: 
346              tries -= 1 
347   
348              self._init_vars() 
349              try: 
350   
351                  self.__filekbcount = 0 
352                  # get the file size 
353                  self.__filesize = self._get_file_size_compat(path) 
354                  if (self.__filesize): 
355                      self.__filesize = round(float(int(self.__filesize))/1024, 1) 
356                      if (self.__filesize == 0): 
357                          self.__filesize = 1 
358                  elif not self.is_path_available(path): 
359                      return False 
360                  else: 
361                      self.__filesize = 0 
362   
363                  with open(tmp_save_path, "wb") as f: 
364                      rc = self.__ftpconn.retrbinary('RETR ' + path, writer, 8192) 
365                      f.flush() 
366                  self._update_progress(force = True) 
367   
368                  done = rc.find("226") != -1 
369                  if done: 
370                      # download complete, atomic mv 
371                      os.rename(tmp_save_path, save_path) 
372   
373                  return done 
374   
375              except Exception as e: # connection reset by peer 
376   
377                  print_traceback() 
378                  mytxt = red("%s: %s, %s... #%s") % ( 
379                      _("Download issue"), 
380                      e, 
381                      _("retrying"), 
382                      tries+1, 
383                  ) 
384                  self.updateProgress( 
385                      mytxt, 
386                      importance = 1, 
387                      type = "warning", 
388                      header = "  " 
389                      ) 
390                  self._reconnect() # reconnect 
391   
392 -    def upload(self, load_path, remote_path): 
393   
394          self.__connect_if_not() 
395          path = os.path.join(self.__ftpdir, remote_path) 
396   
397          tmp_path = path + ".dtmp" 
398          tries = 0 
399   
400          def updater(buf): 
401              self._commit_buffer_update(len(buf)) 
402              self._update_speed() 
403              self._update_progress() 
404              self._speed_limit_loop() 
405   
406          while tries < 10: 
407   
408              tries += 1 
409              self._init_vars() 
410   
411              try: 
412   
413                  file_size = get_file_size(load_path) 
414                  self.__filesize = round(float(file_size)/ 1024, 1) 
415                  self.__filekbcount = 0 
416   
417                  with open(load_path, "r") as f: 
418                      rc = self.__ftpconn.storbinary("STOR " + tmp_path, f, 
419                          8192, updater) 
420   
421                  self._update_progress(force = True) 
422                  # now we can rename the file with its original name 
423                  self.rename(tmp_path, path) 
424   
425                  done = rc.find("226") != -1 
426                  return done 
427   
428              except Exception as e: # connection reset by peer 
429   
430                  print_traceback() 
431                  mytxt = red("%s: %s, %s... #%s") % ( 
432                      _("Upload issue"), 
433                      e, 
434                      _("retrying"), 
435                      tries+1, 
436                  ) 
437                  self.updateProgress( 
438                      mytxt, 
439                      importance = 1, 
440                      type = "warning", 
441                      header = "  " 
442                      ) 
443                  self._reconnect() # reconnect 
444                  self.delete(tmp_path) 
445                  self.delete(path) 
446   
447 -    def rename(self, remote_path_old, remote_path_new): 
448   
449          self.__connect_if_not() 
450   
451          old = os.path.join(self.__ftpdir, remote_path_old) 
452          new = os.path.join(self.__ftpdir, remote_path_new) 
453          try: 
454              rc = self.__ftpconn.rename(old, new) 
455          except self.ftplib.error_perm as err: 
456              # if err[0][:3] in ('553',): 
457              # workaround for some servers 
458              # try to delete old file first, and then call rename again 
459              self.delete(remote_path_new) 
460              rc = self.__ftpconn.rename(old, new) 
461   
462          done = rc.find("250") != -1 
463          return done 
464   
465 -    def delete(self, remote_path): 
466   
467          self.__connect_if_not() 
468          path = os.path.join(self.__ftpdir, remote_path) 
469   
470          done = False 
471          try: 
472              rc = self.__ftpconn.delete(path) 
473              if rc.startswith("250"): 
474                  done = True 
475          except self.ftplib.error_perm as err: 
476              if err[0][:3] == '550': 
477                  done = True 
478              # otherwise not found 
479   
480          return done 
481   
482 -    def get_md5(self, remote_path): 
483          # PROFTPD with mod_md5 supports it! 
484          self.__connect_if_not() 
485          path = os.path.join(self.__ftpdir, remote_path) 
486   
487          try: 
488              rc_data = self.__ftpconn.sendcmd("SITE MD5 %s" % (path,)) 
489          except self.ftplib.error_perm: 
490              return None # not supported 
491   
492          try: 
493              return rc_data.split("\n")[0].split("\t")[0].split("-")[1] 
494          except (IndexError, TypeError,): # wrong output 
495              return None 
496   
497 -    def list_content(self, remote_path): 
498          self.__connect_if_not() 
499          path = os.path.join(self.__ftpdir, remote_path) 
500          try: 
501              return [os.path.basename(x) for x in self.__ftpconn.nlst(path)] 
502          except self.ftplib.error_temp: 
503              raise ValueError("No such file or directory") 
504   
505 -    def list_content_metadata(self, remote_path): 
506          self.__connect_if_not() 
507          path = os.path.join(self.__ftpdir, remote_path) 
508   
509          mybuffer = [] 
510          def bufferizer(buf): 
511              mybuffer.append(buf) 
512   
513          try: 
514              self.__ftpconn.dir(path, bufferizer) 
515          except self.ftplib.error_temp: 
516              raise ValueError("No such file or directory") 
517   
518          data = [] 
519          for item in mybuffer: 
520              item = item.split() 
521              name, size, owner, group, perms = item[8], item[4], item[2], \ 
522                  item[3], item[0] 
523              data.append((name, size, owner, group, perms,)) 
524   
525          return data 
526   
527 -    def is_dir(self, remote_path): 
528   
529          self.__connect_if_not() 
530          path = os.path.join(self.__ftpdir, remote_path) 
531   
532          cwd = self._get_cwd() 
533          data = True 
534          try: 
535              self.__set_cwd(path) 
536          except self.ftplib.error_perm: 
537              data = False 
538          finally: 
539              self.__set_cwd(cwd) 
540   
541          return data 
542   
543 -    def is_file(self, remote_path): 
544   
545          if self.is_dir(remote_path): 
546              return False 
547          return self.is_path_available(remote_path) 
548   
549 -    def _is_path_available(self, full_path): 
550   
551          path, fn = os.path.split(full_path) 
552          content = [] 
553          def cb(x): 
554              y = os.path.basename(x) 
555              if y == fn: 
556                  content.append(y) 
557   
558          try: 
559              self.__ftpconn.retrlines('NLST %s' % (path,), cb) 
560          except self.ftplib.error_temp as err: 
561              if not str(err).startswith("450"): 
562                  raise # wtf? 
563              # path does not exist if FTP error 450 is raised 
564   
565          if content: 
566              return True 
567          return False 
568   
569 -    def is_path_available(self, remote_path): 
570          self.__connect_if_not() 
571          path = os.path.join(self.__ftpdir, remote_path) 
572          return self._is_path_available(path) 
573   
574 -    def makedirs(self, remote_path): 
575          self.__connect_if_not() 
576          path = os.path.join(self.__ftpdir, remote_path) 
577          mydirs = [x for x in path.split("/") if x] 
578   
579          mycurpath = "" 
580          for mydir in mydirs: 
581              mycurpath = os.path.join(mycurpath, mydir) 
582              if not self._is_path_available(mycurpath): 
583                  try: 
584                      self._mkdir(mycurpath) 
585                  except self.ftplib.error_perm as e: 
586                      if e[0].lower().find("permission denied") != -1: 
587                          raise 
588                      elif e[0][:3] != '550': 
589                          raise 
590   
591 -    def keep_alive(self): 
592          """ 
593          Send a keep-alive ping to handler. 
594          """ 
595          if not self.__connected: 
596              raise ConnectionError("keep_alive when not connected") 
597          try: 
598              self.__ftpconn.sendcmd("NOOP") 
599          except (self.ftplib.error_temp, self.ftplib.error_reply,): 
600              raise ConnectionError("cannot execute keep_alive") 
601   
602 -    def close(self): 
603          """ just call our disconnect method """ 
604          self._disconnect() 
605