mirror of https://github.com/Mikaela/Limnoria.git
Added a bit of extra protectedness in PeriodicFileDownloader._downloadFile.
This commit is contained in:
parent
332a1826d5
commit
8fc6502ed3
|
@ -199,32 +199,41 @@ class PeriodicFileDownloader(object):
|
||||||
self.getFile(filename)
|
self.getFile(filename)
|
||||||
|
|
||||||
def _downloadFile(self, filename, url, f):
|
def _downloadFile(self, filename, url, f):
|
||||||
infd = urllib2.urlopen(url)
|
try:
|
||||||
newFilename = os.path.join(conf.dataDir, utils.mktemp())
|
try:
|
||||||
outfd = file(newFilename, 'wb')
|
infd = urllib2.urlopen(url)
|
||||||
start = time.time()
|
except IOError, e:
|
||||||
s = infd.read(4096)
|
self.log.warning('Error downloading %s', url)
|
||||||
while s:
|
self.log.exception('Exception:')
|
||||||
outfd.write(s)
|
return
|
||||||
s = infd.read(4096)
|
newFilename = os.path.join(conf.dataDir, utils.mktemp())
|
||||||
infd.close()
|
outfd = file(newFilename, 'wb')
|
||||||
outfd.close()
|
|
||||||
self.log.info('Downloaded %s in %s seconds',filename,time.time()-start)
|
|
||||||
self.downloadedCounter[filename] += 1
|
|
||||||
self.lastDownloaded[filename] = time.time()
|
|
||||||
if f is None:
|
|
||||||
toFilename = os.path.join(conf.dataDir, filename)
|
|
||||||
if os.name == 'nt':
|
|
||||||
# Windows, grrr...
|
|
||||||
if os.path.exists(toFilename):
|
|
||||||
os.remove(toFilename)
|
|
||||||
os.rename(newFilename, toFilename)
|
|
||||||
else:
|
|
||||||
start = time.time()
|
start = time.time()
|
||||||
f(newFilename)
|
s = infd.read(4096)
|
||||||
total = time.time() - start
|
while s:
|
||||||
self.log.info('Function ran on %s in %s seconds', filename, total)
|
outfd.write(s)
|
||||||
self.currentlyDownloading.remove(filename)
|
s = infd.read(4096)
|
||||||
|
infd.close()
|
||||||
|
outfd.close()
|
||||||
|
self.log.info('Downloaded %s in %s seconds',
|
||||||
|
filename, time.time()-start)
|
||||||
|
self.downloadedCounter[filename] += 1
|
||||||
|
self.lastDownloaded[filename] = time.time()
|
||||||
|
if f is None:
|
||||||
|
toFilename = os.path.join(conf.dataDir, filename)
|
||||||
|
if os.name == 'nt':
|
||||||
|
# Windows, grrr...
|
||||||
|
if os.path.exists(toFilename):
|
||||||
|
os.remove(toFilename)
|
||||||
|
os.rename(newFilename, toFilename)
|
||||||
|
else:
|
||||||
|
start = time.time()
|
||||||
|
f(newFilename)
|
||||||
|
total = time.time() - start
|
||||||
|
self.log.info('Function ran on %s in %s seconds',
|
||||||
|
filename, total)
|
||||||
|
finally:
|
||||||
|
self.currentlyDownloading.remove(filename)
|
||||||
|
|
||||||
def getFile(self, filename):
|
def getFile(self, filename):
|
||||||
(url, timeLimit, f) = self.periodicFiles[filename]
|
(url, timeLimit, f) = self.periodicFiles[filename]
|
||||||
|
|
Loading…
Reference in New Issue