Speeding up FTP upstreams

This is a multi-part message in MIME format.

------=_NextPart_000_0057_01C47645.C9677C80
Content-Type: text/plain;
	charset="US-ASCII"
Content-Transfer-Encoding: 7bit

I noticed that the FTP driver was changing directory to each and every
subdirectory on the way to a file for each and every file, so I fixed it. It
now keeps track of which directories it has asserted (either visited or
created), which greatly shortens the time taken to upstream everything again
to a distant FTP server. 

Regards,
Garth.

------=_NextPart_000_0057_01C47645.C9677C80
Content-Type: application/octet-stream;
	name="20040730-1456-ftpspeed.diff"
Content-Transfer-Encoding: quoted-printable
Content-Disposition: attachment;
	filename="20040730-1456-ftpspeed.diff"

Index: PyDS/FTPDriver.py
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D
RCS file: /pyds/PyDS/PyDS/FTPDriver.py,v
retrieving revision 1.7
diff -c -r1.7 FTPDriver.py
*** PyDS/FTPDriver.py	2 Jan 2004 11:07:15 -0000	1.7
--- PyDS/FTPDriver.py	30 Jul 2004 04:40:51 -0000
***************
*** 37,42 ****
--- 37,43 ----
  	from StringIO import StringIO
 =20
  from PyDS.DriverRegistry import registerDriver, BaseDriver, =
getDummyResult
+ import PyDS.Tool
 =20
  class FTPDriver(BaseDriver):
 =20
***************
*** 49,54 ****
--- 50,56 ----
  		if len(tool.driver) =3D=3D 0:
  			tool.driver.append({})
  		tool._commit()
+ 		self.checkedDirectories =3D {}
  =09
  	def _backup(self):
  		l =3D self.tool._backupTable('ftpdriver', exceptcols=3D['password'])
***************
*** 246,251 ****
--- 248,254 ----
  	# this method upstreams files with given contents without keeping =
track of
  	# their status (actually without the need of them to be in the =
filesystem!)
  	def upstreamMultipleFiles(self, files, contents):
+ 		debugOutput =3D PyDS.Tool.debugOutput
  		tool =3D self.tool
  		liste =3D []
  		server =3D FTP(tool.get('driver', 'server'))
***************
*** 253,277 ****
  			tool.get('driver', 'user'),
  			tool.get('driver', 'password')
  		)
  		for idx in range(len(files)):
  			fn =3D files[idx]
  			fullname =3D os.path.join(tool.get('driver', 'path'), =
fn).replace(os.sep, '/')
- 			curdir =3D server.pwd()
  			dirname =3D os.path.dirname(fullname)
! 			dirs =3D dirname.split('/')
! 			if dirname[0] =3D=3D '/':
! 				server.cwd('/')
! 			for d in dirs:
! 				try:
! 					server.cwd(d)
! 				except:
! 					server.mkd(d)
! 					server.cwd(d)
! 			server.cwd(curdir)
  			server.storbinary(
  				'STOR ' + fullname,
  				StringIO(contents[idx])
  			)
  			tool.set('status', 'upstreams', tool.get('status', 'upstreams') + =
1)
  			tool.set('status', 'bytesupstreamed', tool.get('status', =
'bytesupstreamed') + len(contents[idx]))
  			url =3D tool.get('prefs', 'cloudurl')
--- 256,302 ----
  			tool.get('driver', 'user'),
  			tool.get('driver', 'password')
  		)
+ 		debugOutput("FTP: logged in to %s" % tool.get('driver', 'server'))
+ 		curdir =3D server.pwd()
+ 		debugOutput("FTP: default login cwd is %s" % curdir)
+ 		checkedDirectories =3D self.checkedDirectories=20
+ 		lastDirectory =3D '\n'
  		for idx in range(len(files)):
  			fn =3D files[idx]
  			fullname =3D os.path.join(tool.get('driver', 'path'), =
fn).replace(os.sep, '/')
  			dirname =3D os.path.dirname(fullname)
! 			if not checkedDirectories.has_key(dirname):=20
! 				dirs =3D dirname.split('/')
! 				if dirname[0] =3D=3D '/':
! 					base =3D []
! 				else:=20
! 					base =3D curdir.split('/')
! 				stack =3D []
! 				for d in dirs:
! 					stack.append(d)
! 					thisdirname =3D '/'.join(base + stack)
! 					if checkedDirectories.has_key(thisdirname):=20
! 						continue
! 					debugOutput("FTP: haven't seen %s" % thisdirname)
! 					try:
! 						server.cwd(thisdirname)
! 					except:
! 						server.mkd(thisdirname)
! 						server.cwd(thisdirname)
! 					checkedDirectories[thisdirname] =3D 1
! 				server.cwd(curdir) # jump back for relative STOR
! 			debugOutput("FTP: writing %s..." % fullname)
! 			begin =3D time.time()
  			server.storbinary(
  				'STOR ' + fullname,
  				StringIO(contents[idx])
  			)
+ 			duration =3D time.time() - begin
+ 			if duration > 0:=20
+ 				size =3D len(contents[idx])/1024
+ 				debugOutput("     %0.2fKB in %0.2fs (%.2fKB/s)" % (size, duration, =
size/duration))
+ 			else:=20
+ 				debugOutput("     %0.2fKB in no time at all")
  			tool.set('status', 'upstreams', tool.get('status', 'upstreams') + =
1)
  			tool.set('status', 'bytesupstreamed', tool.get('status', =
'bytesupstreamed') + len(contents[idx]))
  			url =3D tool.get('prefs', 'cloudurl')
***************
*** 284,289 ****
--- 309,315 ----
  				url +=3D fn.replace(os.sep, '/')
  			liste.append(url)
  		tool.set('status', 'lastupstream', time.ctime())
+ 		debugOutput("FTP: done all.")
  		return getDummyResult(urlList=3Dliste)
 =20
  registerDriver('FTP', _('FTP Driver'), FTPDriver)

------=_NextPart_000_0057_01C47645.C9677C80--