diff --git a/libraries/entropy/transceivers.py b/libraries/entropy/transceivers.py index 726d744cf..ff041c81b 100644 --- a/libraries/entropy/transceivers.py +++ b/libraries/entropy/transceivers.py @@ -28,7 +28,7 @@ from entropy.output import TextInterface, darkblue, darkred, purple, blue, \ brown, darkgreen, red, bold from entropy.i18n import _ -from entropy.misc import ParallelTask +from entropy.misc import ParallelTask, Lifo from entropy.core.settings.base import SystemSettings from entropy.exceptions import * @@ -741,6 +741,24 @@ class EntropyTransceiver(TextInterface): self._timeout = None self._silent = None self._output_interface = None + self.__with_stack = Lifo() + + def __enter__(self): + """ + Support for "with" statement, this method will execute swallow() and + return a valid EntropyUriHandler instance. + """ + handler = self.swallow() + self.__with_stack.push(handler) + return handler + + def __exit__(self, exc_type, exc_value, traceback): + """ + Support for "with" statement, this method will automagically close the + previously created EntropyUriHandler instance connection. + """ + handler = self.__with_stack.pop() # if this fails, it's not a good sign + handler.close() def set_output_interface(self, output_interface): """ @@ -853,6 +871,20 @@ class EntropyUriHandler(TextInterface): self._silent = False self._timeout = None + def __enter__(self): + """ + Support for "with" statement, this will trigger UriHandler connection + setup. + """ + raise NotImplementedError() + + def __exit__(self, exc_type, exc_value, traceback): + """ + Support for "with" statement, this will trigger UriHandler connection + hang up. + """ + raise NotImplementedError() + @staticmethod def approve_uri(uri): """ @@ -1033,6 +1065,7 @@ class EntropyUriHandler(TextInterface): @type remote_path: string @return: content @rtype: list + @raise ValueError: if remote_path does not exist """ raise NotImplementedError() @@ -1046,6 +1079,7 @@ class EntropyUriHandler(TextInterface): @type remote_path: string @return: content @rtype: list + @raise ValueError: if remote_path does not exist """ raise NotImplementedError() @@ -1061,6 +1095,30 @@ class EntropyUriHandler(TextInterface): """ raise NotImplementedError() + def is_dir(self, remote_path): + """ + Given a remote path (which can point to dir or file), determine whether + it's a directory. + + @param remote_path: URI to handle + @type remote_path: string + @return: True, if remote_path is a directory + @rtype: bool + """ + raise NotImplementedError() + + def is_file(self, remote_path): + """ + Given a remote path (which can point to dir or file), determine whether + it's a file. + + @param remote_path: URI to handle + @type remote_path: string + @return: True, if remote_path is a file + @rtype: bool + """ + raise NotImplementedError() + def makedirs(self, remote_path): """ Given a remote path, recursively create all the missing directories. @@ -1137,6 +1195,49 @@ class EntropyFtpUriHandler(EntropyUriHandler): self._connect() self._disconnect() + def __enter__(self): + pass # self.__connect_if_not() + + def __exit__(self, exc_type, exc_value, traceback): + self.close() + + def __extract_ftp_data(self, ftpuri): + ftpuser = ftpuri.split("ftp://")[-1].split(":")[0] + if (not ftpuser): + ftpuser = "anonymous@" + ftppassword = "anonymous" + else: + ftppassword = ftpuri.split("@")[:-1] + if len(ftppassword) > 1: + ftppassword = '@'.join(ftppassword) + ftppassword = ftppassword.split(":")[-1] + if (not ftppassword): + ftppassword = "anonymous" + else: + ftppassword = ftppassword[0] + ftppassword = ftppassword.split(":")[-1] + if not ftppassword: + ftppassword = "anonymous" + + ftpport = ftpuri.split(":")[-1] + try: + ftpport = int(ftpport) + except ValueError: + ftpport = 21 + + ftpdir = '/' + if ftpuri.count("/") > 2: + if ftpuri.startswith("ftp://"): + ftpdir = ftpuri[6:] + ftpdir = "/" + ftpdir.split("/", 1)[-1] + ftpdir = ftpdir.split(":")[0] + if not ftpdir: + ftpdir = '/' + elif ftpdir.endswith("/") and (ftpdir != "/"): + ftpdir = ftpdir[:-1] + + return ftpuser, ftppassword, ftpport, ftpdir + def __connect_if_not(self): """ Handy internal method. @@ -1385,7 +1486,7 @@ class EntropyFtpUriHandler(EntropyUriHandler): self.__filesize = 0 with open(tmp_save_path, "wb") as f: - rc = self.__ftpconn.retrbinary('RETR ' + path, writer, 1024) + rc = self.__ftpconn.retrbinary('RETR ' + path, writer, 8192) f.flush() done = rc.find("226") != -1 @@ -1412,32 +1513,6 @@ class EntropyFtpUriHandler(EntropyUriHandler): ) self._reconnect() # reconnect - def __ftp_advanced_stor(self, cmd, fp): - """ - this function also supports callback, because storbinary doesn't - """ - self.__ftpconn.voidcmd('TYPE I') - conn = self.__ftpconn.transfercmd(cmd) - while True: - buf = fp.readline() - if not buf: - break - conn.sendall(buf) - self._commit_buffer_update(len(buf)) - self._update_speed() - self._update_progress() - self._speed_limit_loop() - - # that's another workaround - #return "226" - try: - rc = self.__ftpconn.voidresp() - except: - self._reconnect() - return "226" - - return rc - def upload(self, load_path, remote_path): self.__connect_if_not() @@ -1446,6 +1521,12 @@ class EntropyFtpUriHandler(EntropyUriHandler): tmp_path = path + ".dtmp" tries = 0 + def updater(buf): + self._commit_buffer_update(len(buf)) + self._update_speed() + self._update_progress() + self._speed_limit_loop() + while tries < 10: tries += 1 @@ -1459,7 +1540,8 @@ class EntropyFtpUriHandler(EntropyUriHandler): self.__filesize = round(float(file_size)/ 1024, 1) self.__filekbcount = 0 - rc = self.__ftp_advanced_stor("STOR " + tmp_path, f) + rc = self.__ftpconn.storbinary("STOR " + tmp_path, f, + 8192, updater) # now we can rename the file with its original name self.rename(tmp_path, path) @@ -1531,7 +1613,10 @@ class EntropyFtpUriHandler(EntropyUriHandler): def list_content(self, remote_path): self.__connect_if_not() path = os.path.join(self.__ftpdir, remote_path) - return [os.path.basename(x) for x in self.__ftpconn.nlst(path)] + try: + return [os.path.basename(x) for x in self.__ftpconn.nlst(path)] + except self.ftplib.error_temp: + raise ValueError("No such file or directory") def list_content_metadata(self, remote_path): self.__connect_if_not() @@ -1540,7 +1625,11 @@ class EntropyFtpUriHandler(EntropyUriHandler): mybuffer = [] def bufferizer(buf): mybuffer.append(buf) - self.__ftpconn.dir(path, bufferizer) + + try: + self.__ftpconn.dir(path, bufferizer) + except self.ftplib.error_temp: + raise ValueError("No such file or directory") data = [] for item in mybuffer: @@ -1551,6 +1640,28 @@ class EntropyFtpUriHandler(EntropyUriHandler): return data + def is_dir(self, remote_path): + + self.__connect_if_not() + path = os.path.join(self.__ftpdir, remote_path) + + cwd = self._get_cwd() + data = True + try: + self.__set_cwd(path) + except self.ftplib.error_perm: + data = False + finally: + self.__set_cwd(cwd) + + return data + + def is_file(self, remote_path): + + if self.is_dir(remote_path): + return False + return self.is_path_available(remote_path) + def _is_path_available(self, full_path): path, fn = os.path.split(full_path) @@ -1560,7 +1671,12 @@ class EntropyFtpUriHandler(EntropyUriHandler): if y == fn: content.append(y) - self.__ftpconn.retrlines('NLST %s' % (path,), cb) + try: + self.__ftpconn.retrlines('NLST %s' % (path,), cb) + except self.ftplib.error_temp as err: + if not str(err).startswith("450"): + raise # wtf? + # path does not exist if FTP error 450 is raised if content: return True diff --git a/libraries/tests/standalone/test_ftp_urihandler.py b/libraries/tests/standalone/test_ftp_urihandler.py new file mode 100644 index 000000000..4c31bb214 --- /dev/null +++ b/libraries/tests/standalone/test_ftp_urihandler.py @@ -0,0 +1,21 @@ +import sys, os +sys.path.insert(0, "../../libraries") +from entropy.transceivers import EntropyTransceiver + +txc = EntropyTransceiver("uri://") +# txc.set_speed_limit(35) +txc.set_silent(False) +txc.set_verbosity(True) +handler = txc.swallow() +print handler +print "listing", handler.list_content(".") +print "listing meta", handler.list_content_metadata(".") +print "pushing keep alive", handler.keep_alive() +print "trying to delete test.rnd", handler.is_path_available("test.rnd"), handler.delete("test.rnd") +print "trying to upload test.rnd", handler.upload("test.rnd", "test.rnd") +print "getting md5 of test.rnd", handler.get_md5("test.rnd") +print "downloading test.rnd", handler.download("test.rnd", "test.rnd_down"), os.path.isfile("test.rnd_down") +print "renaming test.rnd", handler.rename("test.rnd", "test.rnd2") +print "checking if test.rnd2 is available", handler.is_path_available("test.rnd2"), "old one =>", handler.is_path_available("test.rnd") +print "deleting test.rnd2", handler.delete("test.rnd2"), handler.is_path_available("test.rnd2") +print "bye bye", handler.close()