* Lib/urllib.py: implemented new quoting rules; added splituser,

splitpasswd, splitattr, splitvalue; new ftp syntax (user:passwd,
cwd to each subdir, type={a,i,d})

Lib/urllib.py

@@ -14,6 +14,7 @@
 # used to query various info about the object, if available.
 # (rfc822.Message objects are queried with the getheader() method.)
 
+import string
 import socket
 import regex
 
@@ -147,8 +148,12 @@ class URLopener:
 		if not host: raise IOError, ('gopher error', 'no host given')
 		type, selector = splitgophertype(selector)
 		selector, query = splitquery(selector)
-		if query: fp = gopherlib.send_query(selector, query, host)
+		selector = unquote(selector)
-		else: fp = gopherlib.send_selector(selector, host)
+		if query:
+			query = unquote(query)
+			fp = gopherlib.send_query(selector, query, host)
+		else:
+			fp = gopherlib.send_selector(selector, host)
 		return addinfo(fp, noheaders())
 
 	# Use local file or FTP depending on form of URL
@@ -165,23 +170,41 @@ class URLopener:
 		host, port = splitport(host)
 		if not port and socket.gethostbyname(host) in (
 			  localhost(), thishost()):
+			file = unquote(file)
 			return addinfo(open(file, 'r'), noheaders())
 		raise IOError, ('local file error', 'not on local host')
 
 	# Use FTP protocol
 	def open_ftp(self, url):
-		host, file = splithost(url)
+		host, path = splithost(url)
 		if not host: raise IOError, ('ftp error', 'no host given')
 		host, port = splitport(host)
+		user, host = splituser(host)
+		if user: user, passwd = splitpasswd(user)
+		else: passwd = None
 		host = socket.gethostbyname(host)
 		if not port:
 			import ftplib
 			port = ftplib.FTP_PORT
-		key = (host, port)
+		path, attrs = splitattr(path)
+		dirs = string.splitfields(path, '/')
+		dirs, file = dirs[:-1], dirs[-1]
+		if dirs and not dirs[0]: dirs = dirs[1:]
+		key = (user, host, port, string.joinfields(dirs, '/'))
+		print 'key =', key
 		try:
 			if not self.ftpcache.has_key(key):
-				self.ftpcache[key] = ftpwrapper(host, port)
+				self.ftpcache[key] = \
-			return addinfo(self.ftpcache[key].retrfile(file),
+						   ftpwrapper(user, passwd,
+							      host, port, dirs)
+			if not file: type = 'D'
+			else: type = 'I'
+			for attr in attrs:
+				attr, value = splitvalue(attr)
+				if string.lower(attr) == 'type' and \
+				   value in ('a', 'A', 'i', 'I', 'd', 'D'):
+					type = string.upper(value)
+			return addinfo(self.ftpcache[key].retrfile(file, type),
 				  noheaders())
 		except ftperrors(), msg:
 			raise IOError, ('ftp error', msg)
@@ -232,24 +255,33 @@ def noheaders():
 
 # Class used by open_ftp() for cache of open FTP connections
 class ftpwrapper:
-	def __init__(self, host, port):
+	def __init__(self, user, passwd, host, port, dirs):
+		self.user = unquote(user or '')
+		self.passwd = unquote(passwd or '')
 		self.host = host
 		self.port = port
+		self.dirs = []
+		for dir in dirs:
+			self.dirs.append(unquote(dir))
 		self.init()
 	def init(self):
 		import ftplib
 		self.ftp = ftplib.FTP()
 		self.ftp.connect(self.host, self.port)
-		self.ftp.login()
+		self.ftp.login(self.user, self.passwd)
-	def retrfile(self, file):
+		for dir in self.dirs:
+			self.ftp.cwd(dir)
+	def retrfile(self, file, type):
 		import ftplib
+		if type in ('d', 'D'): cmd = 'TYPE A'; isdir = 1
+		else: cmd = 'TYPE ' + type; isdir = 0
 		try:
-			self.ftp.voidcmd('TYPE I')
+			self.ftp.voidcmd(cmd)
 		except ftplib.all_errors:
 			self.init()
-			self.ftp.voidcmd('TYPE I')
+			self.ftp.voidcmd(cmd)
 		conn = None
-		if file:
+		if file and not isdir:
 			try:
 				cmd = 'RETR ' + file
 				conn = self.ftp.transfercmd(cmd)
@@ -317,7 +349,6 @@ def basejoin(base, url):
 	basepath, basequery = splitquery(basepath)
 	type = basetype or 'file'
 	if path[:1] != '/':
-		import string
 		i = string.rfind(basepath, '/')
 		if i < 0: basepath = '/'
 		else: basepath = basepath[:i+1]
@@ -327,19 +358,23 @@ def basejoin(base, url):
 	else: return type + ':' + path
 
 
-# Utilities to parse URLs:
+# Utilities to parse URLs (most of these return None for missing parts):
 # unwrap('<URL:type//host/path>') --> 'type//host/path'
 # splittype('type:opaquestring') --> 'type', 'opaquestring'
 # splithost('//host[:port]/path') --> 'host[:port]', '/path'
+# splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'
+# splitpasswd('user:passwd') -> 'user', 'passwd'
 # splitport('host:port') --> 'host', 'port'
 # splitquery('/path?query') --> '/path', 'query'
 # splittag('/path#tag') --> '/path', 'tag'
+# splitattr('/path;attr1=value1;attr2=value2;...') ->
+#   '/path', ['attr1=value1', 'attr2=value2', ...]
+# splitvalue('attr=value') --> 'attr', 'value'
 # splitgophertype('/Xselector') --> 'X', 'selector'
 # unquote('abc%20def') -> 'abc def'
 # quote('abc def') -> 'abc%20def')
 
 def unwrap(url):
-	import string
 	url = string.strip(url)
 	if url[:1] == '<' and url[-1:] == '>':
 		url = string.strip(url[1:-1])
@@ -356,6 +391,16 @@ def splithost(url):
 	if _hostprog.match(url) >= 0: return _hostprog.group(1, 2)
 	return None, url
 
+_userprog = regex.compile('^\([^@]*\)@\(.*\)$')
+def splituser(host):
+	if _userprog.match(host) >= 0: return _userprog.group(1, 2)
+	return None, host
+
+_passwdprog = regex.compile('^\([^:]*\):\(.*\)$')
+def splitpasswd(user):
+	if _passwdprog.match(user) >= 0: return _passwdprog.group(1, 2)
+	return user, None
+
 _portprog = regex.compile('^\(.*\):\([0-9]+\)$')
 def splitport(host):
 	if _portprog.match(host) >= 0: return _portprog.group(1, 2)
@@ -371,6 +416,15 @@ def splittag(url):
 	if _tagprog.match(url) >= 0: return _tagprog.group(1, 2)
 	return url, None
 
+def splitattr(url):
+	words = string.splitfields(url, ';')
+	return words[0], words[1:]
+
+_valueprog = regex.compile('^\([^=]*\)=\(.*\)$')
+def splitvalue(attr):
+	if _valueprog.match(attr) >= 0: return _valueprog.group(1, 2)
+	return attr, None
+
 def splitgophertype(selector):
 	if selector[:1] == '/' and selector[1:2]:
 		return selector[1], selector[2:]
@@ -378,7 +432,6 @@ def splitgophertype(selector):
 
 _quoteprog = regex.compile('%[0-9a-fA-F][0-9a-fA-F]')
 def unquote(s):
-	import string
 	i = 0
 	n = len(s)
 	res = ''
@@ -391,13 +444,15 @@ def unquote(s):
 		i = j+3
 	return res
 
-_acceptable = \
+always_safe = string.letters + string.digits + '_,.+-'
-	  'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._@'
+def quote(s, safe = '/'):
-def quote(s):
+	safe = always_safe + safe
 	res = ''
 	for c in s:
-		if c in _acceptable: res = res + c
+		if c in safe:
-		else: res = res + '%%%02x' % ord(c)
+			res = res + c
+		else:
+			res = res + '%%%02x' % ord(c)
 	return res
 
 # Test and time quote() and unquote()
@@ -452,5 +507,5 @@ def test():
 
 # Run test program when run as a script
 if __name__ == '__main__':
-	test1()
+##	test1()
 	test()