Preliminary support for future nested scopes

compile.h: #define NESTED_SCOPES_DEFAULT 0 for Python 2.1
           __future__ feature name: "nested_scopes"

symtable.h: Add st_nested_scopes slot.  Define flags to track exec and
    import star.

Lib/test/test_scope.py: requires nested scopes

compile.c: Fiddle with error messages.

    Reverse the sense of ste_optimized flag on
    PySymtableEntryObjects.  If it is true, there is an optimization
    conflict.

    Modify get_ref_type to respect st_nested_scopes flags.

    Refactor symtable_load_symbols() into several smaller functions,
    which use struct symbol_info to share variables.  In new function
    symtable_update_flags(), raise an error or warning for import * or
    bare exec that conflicts with nested scopes.  Also, modify handle
    for free variables to respect st_nested_scopes flag.

    In symtable_init() assign st_nested_scopes flag to
    NESTED_SCOPES_DEFAULT (defined in compile.h).

    Add preliminary and often incorrect implementation of
    symtable_check_future().

    Add symtable_lookup() helper for future use.

Include/compile.h

@@ -7,6 +7,9 @@
 extern "C" {
 #endif
 
+#define NESTED_SCOPES_DEFAULT 0
+#define FUTURE_NESTED_SCOPES "nested_scopes"
+
 /* Bytecode object */
 typedef struct {
     PyObject_HEAD

Include/symtable.h

@@ -20,6 +20,7 @@ struct _symtable_entry;
 
 struct symtable {
 	int st_pass;             /* pass == 1 or 2 */
+	int st_nested_scopes;    /* true if nested scopes are enabled */
 	char *st_filename;       /* name of file being compiled */
 	struct _symtable_entry *st_cur; /* current symbol table entry */
 	PyObject *st_symbols;    /* dictionary of symbol table entries */
@@ -40,7 +41,7 @@ typedef struct _symtable_entry {
 	PyObject *ste_children;  /* list of child ids */
 	int ste_type;            /* module, class, or function */
 	int ste_lineno;          /* first line of scope */
-	int ste_optimized;       /* true if namespace is optimized */
+	int ste_optimized;       /* true if namespace can't be optimized */
 	int ste_nested;          /* true if scope is nested */
 	int ste_child_free;      /* true if a child scope has free variables,
 				    including free refs to globals */
@@ -84,6 +85,10 @@ DL_IMPORT(void) PySymtable_Free(struct symtable *);
 #define FREE 4
 #define CELL 5
 
+#define OPT_IMPORT_STAR 1
+#define OPT_EXEC 2
+#define OPT_BARE_EXEC 4
+
 #ifdef __cplusplus
 }
 #endif

Lib/test/test_scope.py

@@ -1,3 +1,5 @@
+from __future__ import nested_scopes
+
 from test.test_support import verify, TestFailed, check_syntax
 
 print "1. simple nesting"

Python/compile.c

@@ -55,6 +55,18 @@ int Py_OptimizeFlag = 0;
 #define ILLEGAL_DYNAMIC_SCOPE \
 "%.100s: exec or 'import *' makes names ambiguous in nested scope"
 
+#define UNDEFINED_FUTURE_FEATURE \
+"future feature %.100s is not defined"
+
+#define GLOBAL_AFTER_ASSIGN \
+"name '%.400s' is assigned to before global declaration"
+
+#define GLOBAL_AFTER_USE \
+"name '%.400s' is used prior to global declaration"
+
+#define LOCAL_GLOBAL \
+"name '%.400s' is a function paramter and declared global"
+
 #define MANGLE_LEN 256
 
 #define OFF(x) offsetof(PyCodeObject, x)
@@ -424,7 +436,6 @@ com_error(struct compiling *c, PyObject *exc, char *msg)
 	set_error_location(c->c_filename, c->c_lineno);
 }
 
-
 /* Interface to the block stack */
 
 static void
@@ -449,7 +460,6 @@ block_pop(struct compiling *c, int type)
 	}
 }
 
-
 /* Prototype forward declarations */
 
 static int com_init(struct compiling *, char *);
@@ -2106,6 +2116,9 @@ static int
 com_make_closure(struct compiling *c, PyCodeObject *co)
 {
 	int i, free = PyTuple_GET_SIZE(co->co_freevars);
+	/* If the code is compiled with st->st_nested_scopes == 0,
+	   then no variable will ever be added to co_freevars. 
+	*/
 	if (free == 0)
 		return 0;
 	for (i = 0; i < free; ++i) {
@@ -3939,6 +3952,7 @@ static int
 get_ref_type(struct compiling *c, char *name)
 {
 	PyObject *v;
+	if (c->c_symtable->st_nested_scopes) {
 		if (PyDict_GetItemString(c->c_cellvars, name) != NULL)
 			return CELL;
 		if (PyDict_GetItemString(c->c_locals, name) != NULL)
@@ -3953,6 +3967,18 @@ get_ref_type(struct compiling *c, char *name)
 				return GLOBAL_IMPLICIT;
 			}
 		}
+	} else {
+		if (PyDict_GetItemString(c->c_locals, name) != NULL)
+			return LOCAL;
+		v = PyDict_GetItemString(c->c_globals, name);
+		if (v) {
+			if (v == Py_None)
+				return GLOBAL_EXPLICIT;
+			else {
+				return GLOBAL_IMPLICIT;
+			}
+		}
+	}
 	{
 		char buf[350];
 		sprintf(buf, 
@@ -3983,15 +4009,146 @@ symtable_build(struct compiling *c, node *n)
 	return 0;
 }
 
+static int
+symtable_init_compiling_symbols(struct compiling *c)
+{
+	PyObject *varnames;
+
+	varnames = c->c_symtable->st_cur->ste_varnames;
+	if (varnames == NULL) {
+		varnames = PyList_New(0);
+		if (varnames == NULL)
+			return -1;
+		c->c_symtable->st_cur->ste_varnames = varnames;
+		Py_INCREF(varnames);
+	} else
+		Py_INCREF(varnames);
+	c->c_varnames = varnames;
+
+	c->c_globals = PyDict_New();
+	if (c->c_globals == NULL)
+		return -1;
+	c->c_freevars = PyDict_New();
+	if (c->c_freevars == NULL)
+		return -1;
+	c->c_cellvars = PyDict_New();
+	if (c->c_cellvars == NULL)
+		return -1;
+	return 0;
+}
+
+struct symbol_info {
+	int si_nlocals;
+	int si_ncells;
+	int si_nfrees;
+	int si_nimplicit;
+};
+
+static void
+symtable_init_info(struct symbol_info *si)
+{
+	si->si_nlocals = 0;
+	si->si_ncells = 0;
+	si->si_nfrees = 0;
+	si->si_nimplicit = 0;
+}
+
+static int
+symtable_resolve_free(struct compiling *c, PyObject *name, 
+		      struct symbol_info *si)
+{
+	PyObject *dict, *v;
+
+	/* Seperate logic for DEF_FREE.  If it occurs in a function,
+	   it indicates a local that we must allocate storage for (a
+	   cell var).  If it occurs in a class, then the class has a
+	   method and a free variable with the same name.
+	*/
+
+	if (c->c_symtable->st_cur->ste_type == TYPE_FUNCTION) {
+		v = PyInt_FromLong(si->si_ncells++);
+		dict = c->c_cellvars;
+	} else {
+		v = PyInt_FromLong(si->si_nfrees++);
+		dict = c->c_freevars;
+	}
+	if (v == NULL)
+		return -1;
+	if (PyDict_SetItem(dict, name, v) < 0) {
+		Py_DECREF(v);
+		return -1;
+	}
+	Py_DECREF(v);
+	return 0;
+}
+
+static int
+symtable_freevar_offsets(PyObject *freevars, int offset)
+{
+	PyObject *name, *v;
+	int pos;
+
+	/* The cell vars are the first elements of the closure,
+	   followed by the free vars.  Update the offsets in
+	   c_freevars to account for number of cellvars. */  
+	pos = 0;
+	while (PyDict_Next(freevars, &pos, &name, &v)) {
+		int i = PyInt_AS_LONG(v) + offset;
+		PyObject *o = PyInt_FromLong(i);
+		if (o == NULL)
+			return -1;
+		if (PyDict_SetItem(freevars, name, o) < 0) {
+			Py_DECREF(o);
+			return -1;
+		}
+		Py_DECREF(o);
+	}
+	return 0;
+}
+
+static int
+symtable_update_flags(struct compiling *c, PySymtableEntryObject *ste,
+		      struct symbol_info *si)
+{
+	if (ste->ste_type != TYPE_MODULE)
+		c->c_flags |= CO_NEWLOCALS;
+	if (ste->ste_type == TYPE_FUNCTION) {
+		c->c_nlocals = si->si_nlocals;
+		if (ste->ste_optimized == 0)
+			c->c_flags |= CO_OPTIMIZED;
+		else if (si->si_ncells || si->si_nfrees 
+			 || (ste->ste_nested && si->si_nimplicit)
+			 || ste->ste_child_free) {
+			if (c->c_symtable->st_nested_scopes) {
+				PyErr_Format(PyExc_SyntaxError,
+					     ILLEGAL_DYNAMIC_SCOPE, 
+				     PyString_AS_STRING(ste->ste_name));
+				set_error_location(c->c_symtable->st_filename,
+						   ste->ste_lineno);
+				return -1;
+			} else {
+				char buf[200];
+				sprintf(buf, ILLEGAL_DYNAMIC_SCOPE,
+					PyString_AS_STRING(ste->ste_name));
+				if (PyErr_Warn(PyExc_SyntaxWarning,
+					       buf) < 0) {
+					return -1;
+				}
+			}
+		}
+	}
+	return 0;
+}
+
 static int
 symtable_load_symbols(struct compiling *c)
 {
 	static PyObject *implicit = NULL;
-	PyObject *name, *varnames, *v;
-	int i, flags, pos;
-	int nlocals, nfrees, ncells, nimplicit;
 	struct symtable *st = c->c_symtable;
 	PySymtableEntryObject *ste = st->st_cur;
+	PyObject *name, *varnames, *v;
+	int i, flags, pos;
+	struct symbol_info si;
 
 	if (implicit == NULL) {
 		implicit = PyInt_FromLong(1);
@@ -4000,33 +4157,14 @@ symtable_load_symbols(struct compiling *c)
 	}
 	v = NULL;
 
+	if (symtable_init_compiling_symbols(c) < 0)
+		goto fail;
+	symtable_init_info(&si);
 	varnames = st->st_cur->ste_varnames;
-	if (varnames == NULL) {
+	si.si_nlocals = PyList_GET_SIZE(varnames);
-		varnames = PyList_New(0);
+	c->c_argcount = si.si_nlocals;
-		if (varnames == NULL)
-			return -1;
-		ste->ste_varnames = varnames;
-		Py_INCREF(varnames);
-	} else
-		Py_INCREF(varnames);
-	c->c_varnames = varnames;
 
-	c->c_globals = PyDict_New();
+	for (i = 0; i < si.si_nlocals; ++i) {
-	if (c->c_globals == NULL)
-		goto fail;
-	c->c_freevars = PyDict_New();
-	if (c->c_freevars == NULL)
-		goto fail;
-	c->c_cellvars = PyDict_New();
-	if (c->c_cellvars == NULL)
-		goto fail;
-
-	nlocals = PyList_GET_SIZE(varnames);
-	c->c_argcount = nlocals;
-	nfrees = 0;
-	ncells = 0;
-	nimplicit = 0;
-	for (i = 0; i < nlocals; ++i) {
 		v = PyInt_FromLong(i);
 		if (PyDict_SetItem(c->c_locals, 
 				   PyList_GET_ITEM(varnames, i), v) < 0)
@@ -4044,29 +4182,9 @@ symtable_load_symbols(struct compiling *c)
 			/* undo the original DEF_FREE */
 			flags &= ~(DEF_FREE | DEF_FREE_CLASS);
 
-		/* Seperate logic for DEF_FREE.  If it occurs in a
-		   function, it indicates a local that we must
-		   allocate storage for (a cell var).  If it occurs in
-		   a class, then the class has a method and a free
-		   variable with the same name.
-		*/
-
 		if ((flags & (DEF_FREE | DEF_FREE_CLASS))
-		    && (flags & (DEF_LOCAL | DEF_PARAM))) {
+		    && (flags & (DEF_LOCAL | DEF_PARAM)))
-			PyObject *dict;
+			symtable_resolve_free(c, name, &si);
-			if (ste->ste_type == TYPE_FUNCTION) {
-				v = PyInt_FromLong(ncells++);
-				dict = c->c_cellvars;
-			} else {
-				v = PyInt_FromLong(nfrees++);
-				dict = c->c_freevars;
-			}
-			if (v == NULL)
-				return -1;
-			if (PyDict_SetItem(dict, name, v) < 0)
-				goto fail;
-			Py_DECREF(v);
-		}
 
 		if (flags & DEF_STAR) {
 			c->c_argcount--;
@@ -4077,23 +4195,22 @@ symtable_load_symbols(struct compiling *c)
 		} else if (flags & DEF_INTUPLE) 
 			c->c_argcount--;
 		else if (flags & DEF_GLOBAL) {
-			if ((flags & DEF_PARAM) 
+			if (flags & DEF_PARAM) {
-			    && (PyString_AS_STRING(name)[0] != '.')){
+				PyErr_Format(PyExc_SyntaxError, LOCAL_GLOBAL,
-				PyErr_Format(PyExc_SyntaxError,
-				     "name '%.400s' is local and global",
 					     PyString_AS_STRING(name));
 				set_error_location(st->st_filename, 
 						   ste->ste_lineno);
+				st->st_errors++;
 				goto fail;
 			}
 			if (PyDict_SetItem(c->c_globals, name, Py_None) < 0)
 				goto fail;
 		} else if (flags & DEF_FREE_GLOBAL) {
-			nimplicit++;
+			si.si_nimplicit++;
 			if (PyDict_SetItem(c->c_globals, name, implicit) < 0)
 				goto fail;
 		} else if ((flags & DEF_LOCAL) && !(flags & DEF_PARAM)) {
-			v = PyInt_FromLong(nlocals++);
+			v = PyInt_FromLong(si.si_nlocals++);
 			if (v == NULL)
 				goto fail;
 			if (PyDict_SetItem(c->c_locals, name, v) < 0)
@@ -4103,15 +4220,15 @@ symtable_load_symbols(struct compiling *c)
 				if (PyList_Append(c->c_varnames, name) < 0)
 					goto fail;
 		} else if (is_free(flags)) {
-			if (ste->ste_nested) {
+			if (ste->ste_nested && st->st_nested_scopes) {
-				v = PyInt_FromLong(nfrees++);
+				v = PyInt_FromLong(si.si_nfrees++);
 				if (v == NULL)
 					goto fail;
 				if (PyDict_SetItem(c->c_freevars, name, v) < 0)
 					goto fail;
 				Py_DECREF(v);
 			} else {
-				nimplicit++;
+				si.si_nimplicit++;
 				if (PyDict_SetItem(c->c_globals, name,
 						   implicit) < 0)
 					goto fail;
@@ -4119,40 +4236,9 @@ symtable_load_symbols(struct compiling *c)
 		}
 	}
 
-	/* The cell vars are the first elements of the closure,
+	if (symtable_freevar_offsets(c->c_freevars, si.si_ncells) < 0)
-	   followed by the free vars.  Update the offsets in
-	   c_freevars to account for number of cellvars. */ 
-	pos = 0;
-	while (PyDict_Next(c->c_freevars, &pos, &name, &v)) {
-		int i = PyInt_AS_LONG(v) + ncells;
-		PyObject *o = PyInt_FromLong(i);
-		if (PyDict_SetItem(c->c_freevars, name, o) < 0) {
-			Py_DECREF(o);
 		return -1;
-		}
+	return symtable_update_flags(c, ste, &si);
-		Py_DECREF(o);
-	}
-
-	if (ste->ste_type == TYPE_FUNCTION)
-		c->c_nlocals = nlocals;
-
-	if (ste->ste_type != TYPE_MODULE)
-		c->c_flags |= CO_NEWLOCALS;
-	if (ste->ste_type == TYPE_FUNCTION) {
-		if (ste->ste_optimized)
-			c->c_flags |= CO_OPTIMIZED;
-		else if (ncells || nfrees 
-			 || (ste->ste_nested && nimplicit)
-			 || ste->ste_child_free) {
-			PyErr_Format(PyExc_SyntaxError, ILLEGAL_DYNAMIC_SCOPE,
-				     PyString_AS_STRING(ste->ste_name));
-			set_error_location(st->st_filename, ste->ste_lineno);
-			return -1;
-		}
-	}
-
-	return 0;
-
  fail:
 	/* is this always the right thing to do? */
 	Py_XDECREF(v);
@@ -4168,6 +4254,7 @@ symtable_init()
 	if (st == NULL)
 		return NULL;
 	st->st_pass = 1;
+	st->st_nested_scopes = NESTED_SCOPES_DEFAULT;
 	st->st_filename = NULL;
 	if ((st->st_stack = PyList_New(0)) == NULL)
 		goto fail;
@@ -4193,6 +4280,44 @@ PySymtable_Free(struct symtable *st)
 	PyMem_Free((void *)st);
 }
 
+/* XXX this code is a placeholder for correct code.
+   from __future__ import name set language options */
+
+static int
+symtable_check_future(struct symtable *st, node *n)
+{
+	int i;
+	node *name = CHILD(n, 1);
+
+	if (strcmp(STR(CHILD(name, 0)), "__future__") != 0)
+		return 0;
+	/* It is only legal to define __future__ features at the top
+	   of a module.  If the current scope is not the module level
+	   or if there are any symbols defined, it is too late. */
+	if (st->st_cur->ste_symbols != st->st_global
+	    || PyDict_Size(st->st_cur->ste_symbols) != 0) {
+		PyErr_SetString(PyExc_SyntaxError, 
+	"imports from __future__ are only legal at the beginning of a module");
+		return -1;
+	}
+	for (i = 3; i < NCH(n); ++i) {
+		char *feature = STR(CHILD(CHILD(n, i), 0));
+		/* Do a linear search through the defined features,
+		   assuming there aren't very many of them. */ 
+		if (strcmp(feature, FUTURE_NESTED_SCOPES) == 0) {
+			st->st_nested_scopes = 1;
+		} else {
+			PyErr_Format(PyExc_SyntaxError,
+				     UNDEFINED_FUTURE_FEATURE, feature);
+			set_error_location(st->st_filename,
+					   st->st_cur->ste_lineno);
+			st->st_errors++;
+			return -1;
+		}
+	}
+	return 1;
+}
+
 /* When the compiler exits a scope, it must should update the scope's
    free variable information with the list of free variables in its
    children.
@@ -4322,12 +4447,17 @@ symtable_undo_free(struct symtable *st, PyObject *id,
 	return 0;
 }
 
+/* symtable_enter_scope() gets a reference via PySymtableEntry_New().
+   This reference is released when the scope is exited, via the DECREF
+   in symtable_exit_scope().
+*/
+
 static int
 symtable_exit_scope(struct symtable *st)
 {
 	int end;
 
-	if (st->st_pass == 1)
+	if (st->st_pass == 1 && st->st_nested_scopes)
 		symtable_update_free_vars(st);
 	Py_DECREF(st->st_cur);
 	end = PyList_GET_SIZE(st->st_stack) - 1;
@@ -4363,6 +4493,27 @@ symtable_enter_scope(struct symtable *st, char *name, int type,
 	}
 }
 
+static int
+symtable_lookup(struct symtable *st, char *name)
+{
+	char buffer[MANGLE_LEN];
+	PyObject *v;
+	int flags;
+
+	if (mangle(st->st_private, name, buffer, sizeof(buffer)))
+		name = buffer;
+	v = PyDict_GetItemString(st->st_cur->ste_symbols, name);
+	if (v == NULL) {
+		if (PyErr_Occurred())
+			return -1;
+		else
+			return 0;
+	}
+
+	flags = PyInt_AS_LONG(v);
+	return flags;
+}
+
 static int
 symtable_add_def(struct symtable *st, char *name, int flag)
 {
@@ -4485,10 +4636,12 @@ symtable_node(struct symtable *st, node *n)
 		symtable_import(st, n);
 		break;
 	case exec_stmt: {
-		st->st_cur->ste_optimized = 0;
+		st->st_cur->ste_optimized |= OPT_EXEC;
 		symtable_node(st, CHILD(n, 1));
 		if (NCH(n) > 2)
 			symtable_node(st, CHILD(n, 3));
+		else
+			st->st_cur->ste_optimized |= OPT_BARE_EXEC;
 		if (NCH(n) > 4)
 			symtable_node(st, CHILD(n, 5));
 		break;
@@ -4698,8 +4851,10 @@ symtable_global(struct symtable *st, node *n)
 {
 	int i;
 
-	for (i = 1; i < NCH(n); i += 2)
+	for (i = 1; i < NCH(n); i += 2) {
-		symtable_add_def(st, STR(CHILD(n, i)), DEF_GLOBAL);
+		char *name = STR(CHILD(n, i));
+		symtable_add_def(st, name, DEF_GLOBAL);
+	}
 }
 
 static void
@@ -4725,10 +4880,10 @@ symtable_import(struct symtable *st, node *n)
                                 ('*' | import_as_name (',' import_as_name)*)
 	   import_as_name: NAME [NAME NAME]
 	*/
-
 	if (STR(CHILD(n, 0))[0] == 'f') {  /* from */
+		symtable_check_future(st, n);
 		if (TYPE(CHILD(n, 3)) == STAR) {
-			st->st_cur->ste_optimized = 0;
+			st->st_cur->ste_optimized |= OPT_IMPORT_STAR;
 		} else {
 			for (i = 3; i < NCH(n); i += 2) {
 				node *c = CHILD(n, i);

Python/symtable.c

@@ -43,7 +43,7 @@ PySymtableEntry_New(struct symtable *st, char *name, int type, int lineno)
 	    goto fail;
 	ste->ste_children = v;
 
-	ste->ste_optimized = 1;
+	ste->ste_optimized = 0;
 	ste->ste_lineno = lineno;
 	switch (type) {
 	case funcdef: