Build: split fetch into df-fetch and df-verify
- moved common python code to lib/hb_distfile.py - beautified tmpfile creation - added stack-style resource management to df-fetch - fixed contrib assumptions about single URL
This commit is contained in:
KonaBlend
Bradley Sepos
parent
dfd3d3c091
commit
8a3e309341
@ -14,7 +14,9 @@ import json
|
||||
import optparse
|
||||
import os
|
||||
import platform
|
||||
import random
|
||||
import re
|
||||
import string
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
@ -128,7 +130,7 @@ class Configure( object ):
|
||||
dir = os.path.dirname( args[0] )
|
||||
if len(args) > 1 and args[1].find('w') != -1:
|
||||
self.mkdirs( dir )
|
||||
m = re.match( '^(.*)\.tmp$', args[0] )
|
||||
m = re.match( '^(.*)\.tmp\..{8}$', args[0] )
|
||||
if m:
|
||||
self.infof( 'write: %s\n', m.group(1) )
|
||||
else:
|
||||
@ -210,6 +212,10 @@ class Configure( object ):
|
||||
if os.path.abspath( self.src_dir ) == os.path.abspath( self.build_dir ):
|
||||
self.build_dir = os.path.join( self.build_dir, 'build' )
|
||||
|
||||
## generate a temporary filename - not worried about race conditions
|
||||
def mktmpname( self, filename ):
|
||||
return filename + '.tmp.' + ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(8))
|
||||
|
||||
###############################################################################
|
||||
##
|
||||
## abstract action
|
||||
@ -1153,7 +1159,7 @@ class ConfigDocument:
|
||||
else:
|
||||
raise ValueError, 'unknown file type: ' + type
|
||||
|
||||
ftmp = fname + '.tmp'
|
||||
ftmp = cfg.mktmpname(fname)
|
||||
try:
|
||||
try:
|
||||
file = cfg.open( ftmp, 'w' )
|
||||
@ -1177,16 +1183,16 @@ class ConfigDocument:
|
||||
|
||||
###############################################################################
|
||||
|
||||
def encodeFetchConfig():
|
||||
fname = 'fetch.cfg'
|
||||
ftmp = fname + '.tmp'
|
||||
data = [
|
||||
options.verbose_fetch,
|
||||
options.disable_fetch,
|
||||
options.disable_fetch_md5,
|
||||
options.accept_fetch_url,
|
||||
options.deny_fetch_url,
|
||||
]
|
||||
def encodeDistfileConfig():
|
||||
fname = 'distfile.cfg'
|
||||
ftmp = cfg.mktmpname(fname)
|
||||
data = {
|
||||
'disable-fetch': options.disable_df_fetch,
|
||||
'disable-verify': options.disable_df_verify,
|
||||
'verbosity': options.df_verbosity,
|
||||
'accept-url': options.df_accept_url,
|
||||
'deny-url': options.df_deny_url,
|
||||
}
|
||||
try:
|
||||
try:
|
||||
file = cfg.open( ftmp, 'w' )
|
||||
@ -1252,13 +1258,13 @@ def createCLI():
|
||||
cli.add_option( '--force', default=False, action='store_true', help='overwrite existing build config' )
|
||||
cli.add_option( '--verbose', default=False, action='store_true', help='increase verbosity' )
|
||||
|
||||
## add fetch options
|
||||
grp = OptionGroup( cli, 'Fetch Options' )
|
||||
grp.add_option( '--verbose-fetch', default=False, action='store_true', help='increase fetch verbosity' )
|
||||
grp.add_option( '--disable-fetch', default=False, action='store_true', help='disable automatic downloads of 3rd-party distributions' )
|
||||
grp.add_option( '--disable-fetch-md5', default=False, action='store_true', help='disable MD5 data error detection' )
|
||||
grp.add_option( '--accept-fetch-url', default=[], action='append', metavar='SPEC', help='accept URL regex pattern' )
|
||||
grp.add_option( '--deny-fetch-url', default=[], action='append', metavar='SPEC', help='deny URL regex pattern' )
|
||||
## add distfile options
|
||||
grp = OptionGroup( cli, 'Distfile Options' )
|
||||
grp.add_option( '--disable-df-fetch', default=False, action='store_true', help='disable distfile downloads' )
|
||||
grp.add_option( '--disable-df-verify', default=False, action='store_true', help='disable distfile data verification' )
|
||||
grp.add_option( '--df-verbose', default=1, action='count', dest='df_verbosity', help='increase distfile tools verbosity' )
|
||||
grp.add_option( '--df-accept-url', default=[], action='append', metavar='SPEC', help='accept URLs matching regex pattern' )
|
||||
grp.add_option( '--df-deny-url', default=[], action='append', metavar='SPEC', help='deny URLs matching regex pattern' )
|
||||
cli.add_option_group( grp )
|
||||
|
||||
## add install options
|
||||
@ -1929,7 +1935,7 @@ int main ()
|
||||
## perform
|
||||
doc.write( 'make' )
|
||||
doc.write( 'm4' )
|
||||
encodeFetchConfig()
|
||||
encodeDistfileConfig()
|
||||
|
||||
if options.launch:
|
||||
Launcher( targets )
|
||||
|
||||
203
make/df-fetch.py
Normal file
203
make/df-fetch.py
Normal file
@ -0,0 +1,203 @@
|
||||
###############################################################################
|
||||
##
|
||||
## Coded for minimum version of Python 2.7 .
|
||||
##
|
||||
## Python3 is incompatible.
|
||||
##
|
||||
## Authors: konablend
|
||||
##
|
||||
###############################################################################
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
import os
|
||||
import sys
|
||||
import urllib2
|
||||
|
||||
sys.dont_write_bytecode = True
|
||||
sys.path.insert(0, os.path.join(sys.path[0], 'lib'))
|
||||
import hb_distfile
|
||||
|
||||
###############################################################################
|
||||
|
||||
## simple structure object
|
||||
class Struct(object):
|
||||
pass
|
||||
|
||||
## track resources and ensure cleanup
|
||||
##
|
||||
## - items are lambdas accepting no args
|
||||
## - item order of insertion is important
|
||||
## - cleanup will run in reverse order of insertion
|
||||
## - item update does not effect order
|
||||
##
|
||||
class Ensure(object):
|
||||
def __init__(self):
|
||||
super(Ensure, self).__setattr__('_items', [])
|
||||
|
||||
def __delattr__(self, key):
|
||||
if key in self.__dict__:
|
||||
self._items.remove(self.__dict__[key])
|
||||
super(Ensure, self).__delattr__(key)
|
||||
|
||||
def __setattr__(self, key, value):
|
||||
if not key in self.__dict__:
|
||||
self._items.insert(0, value)
|
||||
super(Ensure, self).__setattr__(key, value)
|
||||
|
||||
def run(self):
|
||||
for item in self._items:
|
||||
try:
|
||||
item()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
###############################################################################
|
||||
|
||||
class Tool(hb_distfile.Tool):
|
||||
def __init__(self):
|
||||
super(Tool, self).__init__()
|
||||
self.parser.prog = self.name
|
||||
self.parser.usage = '%prog [OPTIONS] URL...'
|
||||
self.parser.description = 'Fetch and verify distfile data integrity.'
|
||||
self.parser.add_option('--disable', default=False, action='store_true', help='do nothing and exit without error')
|
||||
self.parser.add_option('--md5', default=None, action='store', metavar='HASH', help='verify MD5 HASH against data')
|
||||
self.parser.add_option('--accept-url', default=[], action='append', metavar='SPEC', help='accept URL regex pattern')
|
||||
self.parser.add_option('--deny-url', default=[], action='append', metavar='SPEC', help='deny URL regex pattern')
|
||||
self.parser.add_option('--output', default=None, action='store', metavar='FILE', help='write to FILE')
|
||||
self._parse()
|
||||
|
||||
def _load_config2(self, parser, data):
|
||||
parser.values.disable = data['disable-fetch']
|
||||
parser.values.accept_url = data['accept-url']
|
||||
parser.values.deny_url = data['deny-url']
|
||||
|
||||
def _run(self, error):
|
||||
if self.options.disable:
|
||||
self.infof('%s disabled; stop.\n' % self.name)
|
||||
sys.exit(0)
|
||||
if len(self.args) < 1:
|
||||
self.parser.print_usage()
|
||||
sys.exit(1)
|
||||
## create URL objects and keep active
|
||||
urls = []
|
||||
i = 0
|
||||
for arg in self.args:
|
||||
url = URL(arg, i)
|
||||
if url.active:
|
||||
urls.append(url)
|
||||
i += 1
|
||||
## try each URL until first success
|
||||
error.op = 'download'
|
||||
while urls:
|
||||
url = urls.pop(0)
|
||||
try:
|
||||
url.download(error)
|
||||
break
|
||||
except Exception, x:
|
||||
## propagate exception if no remaining urls
|
||||
if not urls:
|
||||
raise
|
||||
self.errln('%s failure; %s' % (error.op,x))
|
||||
|
||||
def run(self):
|
||||
error = hb_distfile.ToolError('run')
|
||||
try:
|
||||
self._run(error)
|
||||
except Exception, x:
|
||||
self.debug_exception()
|
||||
self.errln('%s failure; %s' % (error.op,x), exit=1)
|
||||
|
||||
###############################################################################
|
||||
|
||||
class URL(object):
|
||||
def __init__(self, url, index):
|
||||
self.index = index
|
||||
self.url = url
|
||||
self.active = True
|
||||
self.rule = 'none'
|
||||
self._accept()
|
||||
self._deny()
|
||||
tool.verbosef('URL[%d]: %s\n' % (self.index,self.url))
|
||||
tool.verbosef(' active: %s\n' % ('yes' if self.active else 'no'))
|
||||
tool.verbosef(' rule: %s\n' % (self.rule))
|
||||
|
||||
def _accept(self):
|
||||
if not tool.options.accept_url:
|
||||
return
|
||||
index = 0
|
||||
for spec in tool.options.accept_url:
|
||||
if re.search(spec, self.url):
|
||||
self.rule = 'via accept rule %d: %s' % (index,spec)
|
||||
return
|
||||
index += 1
|
||||
self.active = False
|
||||
self.rule = 'no matching accept rule'
|
||||
|
||||
def _deny(self):
|
||||
index = 0
|
||||
for spec in tool.options.deny_url:
|
||||
if re.search(spec, self.url):
|
||||
self.active = False
|
||||
self.rule = 'via deny rule %d: %s' % (index,spec)
|
||||
return
|
||||
index += 1
|
||||
|
||||
def _download(self, error, ensure):
|
||||
filename = tool.options.output
|
||||
hasher = hashlib.md5()
|
||||
if filename:
|
||||
tool.infof('downloading %s to %s\n' % (self.url,filename))
|
||||
ftmp = tool.mktmpname(filename)
|
||||
hout = open(ftmp, 'w')
|
||||
ensure.unlink_ftmp = lambda: os.unlink(ftmp)
|
||||
ensure.close_hout = lambda: hout.close()
|
||||
else:
|
||||
tool.infof('downloading %s\n' % (self.url))
|
||||
hin = urllib2.urlopen(self.url, None, 30)
|
||||
ensure.close_hin = lambda: hin.close()
|
||||
info = hin.info()
|
||||
try:
|
||||
content_length = int(info.getheader('Content-Length'))
|
||||
except:
|
||||
content_length = None
|
||||
data_total = 0
|
||||
while True:
|
||||
data = hin.read(65536)
|
||||
if not data:
|
||||
break
|
||||
if filename:
|
||||
hout.write(data)
|
||||
hasher.update(data)
|
||||
data_total += len(data)
|
||||
if content_length and content_length != data_total:
|
||||
raise error('expected %d bytes, got %d bytes' % (content_length,data_total))
|
||||
s = 'downloaded %d bytes' % data_total
|
||||
if filename:
|
||||
s += '; MD5 (%s) = %s' % (filename,hasher.hexdigest())
|
||||
else:
|
||||
s += '; MD5 = %s' % (hasher.hexdigest())
|
||||
if tool.options.md5:
|
||||
md5_pass = tool.options.md5 == hasher.hexdigest()
|
||||
s += ' (%s)' % ('pass' if md5_pass else 'fail; expecting %s' % tool.options.md5)
|
||||
tool.infof('%s\n' % s)
|
||||
if filename and tool.options.md5:
|
||||
if md5_pass:
|
||||
if os.access(filename, os.F_OK) and not os.access(filename, os.W_OK):
|
||||
raise error("permission denied: '%s'" % filename)
|
||||
else:
|
||||
raise error("expected MD5 hash '%s', got '%s'" % (tool.options.md5, hasher.hexdigest()))
|
||||
os.rename(ftmp,filename)
|
||||
del ensure.unlink_ftmp
|
||||
|
||||
def download(self, error):
|
||||
ensure = Ensure()
|
||||
try:
|
||||
self._download(error, ensure)
|
||||
finally:
|
||||
ensure.run()
|
||||
|
||||
###############################################################################
|
||||
|
||||
tool = Tool()
|
||||
tool.run()
|
||||
91
make/df-verify.py
Normal file
91
make/df-verify.py
Normal file
@ -0,0 +1,91 @@
|
||||
###############################################################################
|
||||
##
|
||||
## Coded for minimum version of Python 2.7 .
|
||||
##
|
||||
## Python3 is incompatible.
|
||||
##
|
||||
## Authors: konablend
|
||||
##
|
||||
###############################################################################
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.dont_write_bytecode = True
|
||||
sys.path.insert(0, os.path.join(sys.path[0], 'lib'))
|
||||
import hb_distfile
|
||||
|
||||
###############################################################################
|
||||
|
||||
## simple structure object
|
||||
class Struct(object):
|
||||
pass
|
||||
|
||||
###############################################################################
|
||||
|
||||
class Tool(hb_distfile.Tool):
|
||||
def __init__(self):
|
||||
super(Tool, self).__init__()
|
||||
self.parser.prog = self.name
|
||||
self.parser.usage = '%prog [OPTIONS] FILE'
|
||||
self.parser.description = 'Verify distfile data integrity.'
|
||||
self.parser.add_option('--disable', default=False, action='store_true', help='do nothing and exit without error')
|
||||
self.parser.add_option('--md5', default=None, action='store', metavar='HASH', help='verify MD5 HASH against data')
|
||||
self._parse()
|
||||
|
||||
def _load_config2(self, parser, data):
|
||||
parser.values.disable = data['disable-verify']
|
||||
|
||||
def _scan(self, filename):
|
||||
self.verbosef('scanning %s\n' % filename)
|
||||
hasher = hashlib.md5()
|
||||
with open(filename, 'r') as o:
|
||||
data_total = 0
|
||||
while True:
|
||||
data = o.read(65536)
|
||||
if not data:
|
||||
break
|
||||
hasher.update(data)
|
||||
data_total += len(data)
|
||||
self.verbosef('scanned %d bytes\n' % data_total)
|
||||
r = Struct()
|
||||
r.md5 = hasher.hexdigest()
|
||||
r.size = data_total
|
||||
return r
|
||||
|
||||
def _verify(self, filename):
|
||||
r = Struct()
|
||||
r.scan = self._scan(filename)
|
||||
r.status = self.options.md5 == r.scan.md5
|
||||
return r
|
||||
|
||||
def _run(self, error):
|
||||
if self.options.disable:
|
||||
self.infof('%s disabled; stop.\n' % self.name)
|
||||
sys.exit(0)
|
||||
if len(self.args) != 1:
|
||||
self.parser.print_usage()
|
||||
sys.exit(1)
|
||||
filename = self.args[0]
|
||||
if self.options.md5:
|
||||
error.op = 'verify'
|
||||
r = self._verify(filename)
|
||||
self.infof('MD5 (%s) = %s (%s)\n', filename, r.scan.md5, 'pass' if r.status else 'fail; expecting %s' % self.options.md5)
|
||||
else:
|
||||
error.op = 'scan'
|
||||
r = self._scan(filename)
|
||||
self.infof('MD5 (%s) = %s (%d bytes)\n', filename, r.md5, r.size)
|
||||
|
||||
def run(self):
|
||||
error = hb_distfile.ToolError('run')
|
||||
try:
|
||||
self._run(error)
|
||||
except Exception, x:
|
||||
self.debug_exception()
|
||||
self.errln('%s failure; %s' % (error.op,x), exit=1)
|
||||
|
||||
###############################################################################
|
||||
|
||||
tool = Tool()
|
||||
tool.run()
|
||||
268
make/fetch.py
268
make/fetch.py
@ -1,268 +0,0 @@
|
||||
###############################################################################
|
||||
##
|
||||
## This script is coded for minimum version of Python 2.7 .
|
||||
##
|
||||
## Python3 is incompatible.
|
||||
##
|
||||
## Authors: konablend
|
||||
##
|
||||
###############################################################################
|
||||
|
||||
import errno
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
import traceback
|
||||
import urllib2
|
||||
|
||||
from optparse import OptionGroup
|
||||
from optparse import OptionParser
|
||||
from sys import stderr
|
||||
from sys import stdout
|
||||
from urlparse import urlparse
|
||||
|
||||
###############################################################################
|
||||
|
||||
class Fetch(object):
|
||||
def __init__(self, options, urls):
|
||||
if options.disable:
|
||||
self.errln('fetching files from the network is disabled.')
|
||||
self.options = options
|
||||
self.urls = urls
|
||||
if len(self.urls) > 1:
|
||||
self.infof('\n')
|
||||
self.verbosef('OPTIONS:\n')
|
||||
self.verbosef(' disable: %s\n' % self.options.disable)
|
||||
self.verbosef(' disable_md5: %s\n' % self.options.disable_md5)
|
||||
self.verbosef(' config: %s\n' % self.options.config)
|
||||
self.verbosef(' md5: %s\n' % self.options.md5)
|
||||
self.verbosef(' output_dir: %s\n' % self.options.output_dir)
|
||||
index = 0
|
||||
for spec in self.options.accept_url:
|
||||
self.verbosef(' accept_url[%d]: %s\n' % (index,spec))
|
||||
index += 1
|
||||
if not self.options.accept_url:
|
||||
self.verbosef(' accept_url: %s\n' % None)
|
||||
index = 0
|
||||
for spec in self.options.deny_url:
|
||||
self.verbosef(' deny_url[%d]: %s\n' % (index,spec))
|
||||
index += 1
|
||||
if not self.options.deny_url:
|
||||
self.verbosef(' deny_url: %s\n' % None)
|
||||
|
||||
def run(self):
|
||||
if not self.urls:
|
||||
self.errln('no URLs specified')
|
||||
files = []
|
||||
for url in self.urls:
|
||||
files.append(self._process_url(url))
|
||||
index = 0
|
||||
for file in files:
|
||||
file.dump(index)
|
||||
index += 1
|
||||
canon = files[0].filename
|
||||
for file in files:
|
||||
if file.filename != canon:
|
||||
self.errln('URL basename is not consistent')
|
||||
scan = os.access(canon, os.F_OK)
|
||||
for file in files:
|
||||
if file.run(scan):
|
||||
return
|
||||
self.errln('%s failed.' % ('scan' if scan else 'download'))
|
||||
|
||||
def errln(self, format, *args):
|
||||
s = (format % args)
|
||||
if re.match( '^.*[!?:;.]$', s ):
|
||||
stderr.write('ERROR: %s fetch stop.\n' % (s))
|
||||
else:
|
||||
stderr.write('ERROR: %s; fetch stop.\n' % (s))
|
||||
exit(1)
|
||||
|
||||
def warnln(self, format, *args):
|
||||
s = (format % args)
|
||||
if re.match( '^.*[!?:;.]$', s ):
|
||||
stderr.write('WARNING: %s fetch continuing.\n' % (s))
|
||||
else:
|
||||
stderr.write('WARNING: %s; fetch continuing.\n' % (s))
|
||||
|
||||
def infof(self, format, *args):
|
||||
stdout.write(format % args)
|
||||
|
||||
def verbosef(self, format, *args):
|
||||
if self.options.verbose:
|
||||
stdout.write(format % args)
|
||||
|
||||
def _process_url(self, url):
|
||||
props = {}
|
||||
index = 0
|
||||
while True:
|
||||
## optional per-URL properties
|
||||
## [key=value][...]URL
|
||||
m = re.match('(\[(\w+)=([^]]+)\])?(.*)', url[index:])
|
||||
if not m.group(1):
|
||||
break
|
||||
props[m.group(2)] = m.group(3)
|
||||
index += len(m.group(1))
|
||||
return File(url[index:], **props)
|
||||
|
||||
###############################################################################
|
||||
|
||||
class File(object):
|
||||
def __init__(self, url, **kwargs):
|
||||
self.url = url
|
||||
self.props = kwargs # not currently used
|
||||
self.filename = os.path.join(fetch.options.output_dir,os.path.basename(urlparse(self.url).path))
|
||||
self.active = True
|
||||
self.active_descr = 'default'
|
||||
self._accept()
|
||||
self._deny()
|
||||
|
||||
def dump(self, index):
|
||||
fetch.verbosef('URL[%d]: %s\n' % (index,self.url))
|
||||
fetch.verbosef(' filename: %s\n' % self.filename)
|
||||
fetch.verbosef(' active: %s (%s)\n' % ('yes' if self.active else 'no',self.active_descr))
|
||||
|
||||
def run(self, scan):
|
||||
if not self.active:
|
||||
return False
|
||||
try:
|
||||
if (self._scan() if scan else self._download()):
|
||||
return True
|
||||
except Exception, x:
|
||||
if fetch.options.verbose:
|
||||
traceback.print_exc()
|
||||
fetch.warnln('%s' % x)
|
||||
return False
|
||||
|
||||
def _accept(self):
|
||||
if not fetch.options.accept_url:
|
||||
return
|
||||
index = 0
|
||||
for spec in fetch.options.accept_url:
|
||||
if re.match(spec, self.url):
|
||||
self.active_descr = 'via accept rule %d: %s' % (index,spec)
|
||||
return
|
||||
index += 1
|
||||
self.active = False
|
||||
self.active_descr = 'no matching spec'
|
||||
|
||||
def _deny(self):
|
||||
index = 0
|
||||
for spec in fetch.options.deny_url:
|
||||
if re.match(spec, self.url):
|
||||
self.active = False
|
||||
self.active_descr = 'via deny rule %d: %s' % (index,spec)
|
||||
return
|
||||
index += 1
|
||||
|
||||
def _download(self):
|
||||
fetch.infof('downloading %s to %s\n' % (self.url,self.filename))
|
||||
hasher = hashlib.md5()
|
||||
ftmp = self.filename + '.' + os.path.basename(tempfile.mktemp())
|
||||
r = urllib2.urlopen(self.url, None, 30)
|
||||
try:
|
||||
o = open(ftmp, 'w')
|
||||
info = r.info()
|
||||
try:
|
||||
content_length = int(info.getheader('Content-Length'))
|
||||
except:
|
||||
content_length = None
|
||||
data_total = 0
|
||||
while True:
|
||||
data = r.read(65536)
|
||||
if not data:
|
||||
break
|
||||
o.write(data)
|
||||
hasher.update(data)
|
||||
data_total += len(data)
|
||||
except:
|
||||
os.unlink(ftmp)
|
||||
finally:
|
||||
for closeable in [r,o]:
|
||||
try:
|
||||
if not closeable:
|
||||
continue
|
||||
closeable.close()
|
||||
except:
|
||||
pass
|
||||
|
||||
if content_length and content_length != data_total:
|
||||
fetch.warnln('expected %d bytes, got %d bytes' % (content_length,data_total))
|
||||
os.unlink(ftmp)
|
||||
return False
|
||||
|
||||
if not fetch.options.disable_md5 and fetch.options.md5 and fetch.options.md5 == hasher.hexdigest():
|
||||
s = ' (verified)'
|
||||
else:
|
||||
s = ''
|
||||
|
||||
fetch.infof("downloaded '%s' - %d bytes - %s%s\n" % (self.filename,data_total,hasher.hexdigest(),s))
|
||||
if not fetch.options.disable_md5 and fetch.options.md5 and fetch.options.md5 != hasher.hexdigest():
|
||||
os.unlink(ftmp)
|
||||
raise RuntimeError("expected MD5 hash '%s', got '%s'" % (fetch.options.md5, hasher.hexdigest()))
|
||||
|
||||
if os.access(self.filename, os.F_OK) and not os.access(self.filename, os.W_OK):
|
||||
os.unlink(ftmp)
|
||||
raise IOError(errno.EACCES, "Permission denied: '%s'" % self.filename)
|
||||
|
||||
try:
|
||||
os.rename(ftmp,self.filename)
|
||||
except:
|
||||
os.unlink(ftmp)
|
||||
return True
|
||||
|
||||
def _scan(self):
|
||||
fetch.infof('scanning %s\n' % self.filename)
|
||||
hasher = hashlib.md5()
|
||||
try:
|
||||
o = open(self.filename, 'r')
|
||||
data_total = 0
|
||||
while True:
|
||||
data = o.read(65536)
|
||||
if not data:
|
||||
break
|
||||
hasher.update(data)
|
||||
data_total += len(data)
|
||||
finally:
|
||||
o.close()
|
||||
|
||||
if not fetch.options.disable_md5 and fetch.options.md5 and fetch.options.md5 == hasher.hexdigest():
|
||||
s = ' (verified)'
|
||||
else:
|
||||
s = ''
|
||||
|
||||
fetch.infof("scanned '%s' - %d bytes - %s%s\n" % (self.filename,data_total,hasher.hexdigest(),s))
|
||||
if not fetch.options.disable_md5 and fetch.options.md5 and fetch.options.md5 != hasher.hexdigest():
|
||||
raise RuntimeError("expected MD5 hash '%s', got '%s'" % (fetch.options.md5, hasher.hexdigest()))
|
||||
return True
|
||||
|
||||
###############################################################################
|
||||
|
||||
def load_config(option, opt, value, parser):
|
||||
with open(value, 'r') as file:
|
||||
data = json.load(file)
|
||||
parser.values.verbose = data[0]
|
||||
parser.values.disable = data[1]
|
||||
parser.values.disable_md5 = data[2]
|
||||
parser.values.accept_url = data[3]
|
||||
parser.values.deny_url = data[4]
|
||||
|
||||
###############################################################################
|
||||
|
||||
parser = OptionParser('usage: %prog [OPTIONS...] [URL...]')
|
||||
|
||||
parser.description = 'Fetch files from the network.'
|
||||
|
||||
parser.add_option('--verbose', default=False, action='store_true', help='increase verbosity')
|
||||
parser.add_option('--config', default=None, action='callback', metavar='FILE', type='str', callback=load_config, help='specify configuration file')
|
||||
parser.add_option('--disable', default=False, action='store_true', help='print disabled message and exit with error')
|
||||
parser.add_option('--disable-md5', default=False, action='store_true', help='disable MD5 data error detection')
|
||||
parser.add_option('--md5', default=None, action='store', metavar='HASH', help='set default MD5 hash value')
|
||||
parser.add_option('--accept-url', default=[], action='append', metavar='SPEC', help='accept URL regex pattern')
|
||||
parser.add_option('--deny-url', default=[], action='append', metavar='SPEC', help='deny URL regex pattern')
|
||||
parser.add_option('--output-dir', default='', action='store', help='specify output directory')
|
||||
|
||||
fetch = Fetch(*parser.parse_args())
|
||||
fetch.run()
|
||||
@ -26,11 +26,12 @@ define import.CONTRIB.defs
|
||||
##
|
||||
## target: fetch
|
||||
##
|
||||
$(1).FETCH.tar = $$(CONTRIB.download/)$$(notdir $$(firstword $$($(1).FETCH.url)))
|
||||
$(1).FETCH.url = FETCH_IS_UNDEFINED
|
||||
$(1).FETCH.target = $$($(1).FETCH.tar)
|
||||
$(1).FETCH.url = FETCH_URL_IS_UNDEFINED
|
||||
$(1).FETCH.basename = $$(notdir $$(firstword $$($(1).FETCH.url)))
|
||||
$(1).FETCH.distfile = $$(CONTRIB.download/)$$($(1).FETCH.basename)
|
||||
$(1).FETCH.target = $$($(1).FETCH.distfile)
|
||||
define $(1).FETCH
|
||||
$$(FETCH.exe) --config $(BUILD/)fetch.cfg --output-dir $$(dir $$@) $$(if $$($(1).FETCH.md5),--md5 $$($(1).FETCH.md5)) $$($(1).FETCH.url)
|
||||
$$(DF.FETCH.exe) --config $(BUILD/)distfile.cfg $$(if $$($(1).FETCH.md5),--md5 $$($(1).FETCH.md5)) --output $$@ $$($(1).FETCH.url)
|
||||
endef
|
||||
|
||||
##
|
||||
@ -38,19 +39,19 @@ define import.CONTRIB.defs
|
||||
##
|
||||
$(1).VERIFY.target = $$($(1).build/).stamp.verify
|
||||
define $(1).VERIFY
|
||||
$$(FETCH.exe) --config $(BUILD/)fetch.cfg --output-dir $$(dir $$($(1).FETCH.tar)) $$(if $$($(1).FETCH.md5),--md5 $$($(1).FETCH.md5)) $$($(1).FETCH.url)
|
||||
$$(DF.VERIFY.exe) --config $(BUILD/)distfile.cfg $$(if $$($(1).FETCH.md5),--md5 $$($(1).FETCH.md5)) $$($(1).FETCH.distfile)
|
||||
$$(TOUCH.exe) $$@
|
||||
endef
|
||||
|
||||
##
|
||||
## target: extract
|
||||
##
|
||||
$(1).EXTRACT.tarbase = $$(strip $$(foreach x,tar.bz2 tar.gz,$$(patsubst %.$$(x),%,$$(filter %.$$(x),$$(notdir $$($(1).FETCH.url))))))
|
||||
$(1).EXTRACT.tarbase = $$(strip $$(foreach x,tar.bz2 tar.gz,$$(patsubst %.$$(x),%,$$(filter %.$$(x),$$($(1).FETCH.basename)))))
|
||||
$(1).EXTRACT.dir/ = $$($(1).build/)$$($(1).EXTRACT.tarbase)/
|
||||
$(1).EXTRACT.target = $$($(1).build/).stamp.extract
|
||||
define $(1).EXTRACT
|
||||
$$(RM.exe) -fr $$($(1).EXTRACT.dir/)
|
||||
$$(TAR.exe) xfC $$($(1).FETCH.tar) $$($(1).build/)
|
||||
$$(TAR.exe) xfC $$($(1).FETCH.distfile) $$($(1).build/)
|
||||
$$(TOUCH.exe) $$@
|
||||
endef
|
||||
|
||||
@ -345,6 +346,8 @@ $($(1).name): $($(1).name).build
|
||||
##
|
||||
contrib.fetch: $($(1).name).fetch
|
||||
contrib.verify: $($(1).name).verify
|
||||
contrib.verify.touch: $($(1).name).verify.touch
|
||||
contrib.verify.untouch: $($(1).name).verify.untouch
|
||||
contrib.extract: $($(1).name).extract
|
||||
contrib.patch: $($(1).name).patch
|
||||
contrib.configure: $($(1).name).configure
|
||||
|
||||
@ -1,13 +1,14 @@
|
||||
AR.exe = ar
|
||||
CP.exe = cp
|
||||
FETCH.exe = $(SRC/)make/python_launcher $(SRC/)make/fetch.py
|
||||
M4.exe = m4
|
||||
MKDIR.exe = mkdir
|
||||
PATCH.exe = patch
|
||||
RM.exe = rm
|
||||
TAR.exe = tar
|
||||
TOUCH.exe = touch
|
||||
MV.exe = mv
|
||||
ZIP.exe = zip
|
||||
LN.exe = ln
|
||||
GIT.exe = git
|
||||
AR.exe = ar
|
||||
CP.exe = cp
|
||||
DF.FETCH.exe = $(SRC/)make/python_launcher $(SRC/)make/df-fetch.py
|
||||
DF.VERIFY.exe = $(SRC/)make/python_launcher $(SRC/)make/df-verify.py
|
||||
M4.exe = m4
|
||||
MKDIR.exe = mkdir
|
||||
PATCH.exe = patch
|
||||
RM.exe = rm
|
||||
TAR.exe = tar
|
||||
TOUCH.exe = touch
|
||||
MV.exe = mv
|
||||
ZIP.exe = zip
|
||||
LN.exe = ln
|
||||
GIT.exe = git
|
||||
|
||||
106
make/lib/hb_distfile.py
Normal file
106
make/lib/hb_distfile.py
Normal file
@ -0,0 +1,106 @@
|
||||
###############################################################################
|
||||
##
|
||||
## Coded for minimum version of Python 2.7 .
|
||||
##
|
||||
## Python3 is incompatible.
|
||||
##
|
||||
## Authors: konablend
|
||||
##
|
||||
###############################################################################
|
||||
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import string
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
from optparse import OptionParser
|
||||
|
||||
###############################################################################
|
||||
|
||||
class Tool(object):
|
||||
LOG_QUIET = 0
|
||||
LOG_INFO = 1
|
||||
LOG_VERBOSE = 2
|
||||
LOG_DEBUG = 3
|
||||
|
||||
def __init__(self):
|
||||
self.name = os.path.splitext(os.path.basename(sys.argv[0]))[0]
|
||||
self.parser = OptionParser()
|
||||
self.parser.add_option('-v', '--verbose', default=Tool.LOG_INFO, action='count', dest='verbosity', help='increase verbosity')
|
||||
self.parser.add_option('--config', default=None, action='callback', metavar='FILE', type='str', callback=self._load_config, help='specify configuration file')
|
||||
|
||||
def _parse(self):
|
||||
(self.options,self.args) = self.parser.parse_args()
|
||||
|
||||
## be sure not to use any methods referencing self.options as we are still parsing args
|
||||
def _load_config(self, option, opt, value, parser):
|
||||
with open(value, 'r') as file:
|
||||
data = json.load(file)
|
||||
parser.values.verbosity = data['verbosity']
|
||||
extend = getattr(self, '_load_config2', None)
|
||||
if extend:
|
||||
extend(parser, data)
|
||||
|
||||
## newline not required
|
||||
def errln(self, format, *args, **kwargs):
|
||||
s = (format % args)
|
||||
if re.match('^.*[!?:;.]$', s):
|
||||
if kwargs.get('exit', None) != None:
|
||||
sys.stderr.write('ERROR: %s stop.\n' % (s))
|
||||
sys.exit(1)
|
||||
sys.stderr.write('ERROR: %s continuing\n' % (s))
|
||||
else:
|
||||
if kwargs.get('exit', None) != None:
|
||||
sys.stderr.write('ERROR: %s; stop.\n' % (s))
|
||||
sys.exit(1)
|
||||
sys.stderr.write('ERROR: %s; continuing.\n' % (s))
|
||||
|
||||
## newline not required
|
||||
def warnln(self, format, *args):
|
||||
s = (format % args)
|
||||
if re.match( '^.*[!?:;.]$', s ):
|
||||
sys.stdout.write('WARNING: %s continuing.\n' % (s))
|
||||
else:
|
||||
sys.stdout.write('WARNING: %s; continuing.\n' % (s))
|
||||
|
||||
## newline required
|
||||
def infof(self, format, *args):
|
||||
if self.options.verbosity >= Tool.LOG_INFO:
|
||||
sys.stdout.write(format % args)
|
||||
|
||||
## newline required
|
||||
def verbosef(self, format, *args):
|
||||
if self.options.verbosity >= Tool.LOG_VERBOSE:
|
||||
sys.stdout.write(format % args)
|
||||
|
||||
## newline required
|
||||
def debugf(self, format, *args):
|
||||
if self.options.verbosity >= Tool.LOG_DEBUG:
|
||||
sys.stdout.write(format % args)
|
||||
|
||||
def debug_exception(self, xinfo=None):
|
||||
if self.options.verbosity >= Tool.LOG_DEBUG:
|
||||
if not xinfo:
|
||||
xinfo = sys.exc_info()
|
||||
traceback.print_exception(*xinfo)
|
||||
|
||||
## generate a temporary filename - not worried about race conditions
|
||||
def mktmpname(self, filename):
|
||||
return filename + '.tmp.' + ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(8))
|
||||
|
||||
###############################################################################
|
||||
|
||||
class ToolError(Exception):
|
||||
def __init__(self, op='unknown', text=None):
|
||||
self.op = op
|
||||
self.text = text
|
||||
|
||||
def __call__(self, text):
|
||||
self.text = text
|
||||
return self
|
||||
|
||||
def __str__(self):
|
||||
return self.text
|
||||
Loading…
x
Reference in New Issue
Block a user