From 0539192a402c4caddcbd39fe01d3f96432fbd782 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 23 Jun 2017 14:06:41 +0200 Subject: [PATCH] bpo-26145: Implement PEP 511, code transformer * Add -o command line option: value stored into a new sys.implementation.optim_tag attribute * -o noopt now disables the peephole optimizer * Add get_code_transformers() and set_code_transformers() functions to the sys module * Add PyCF_TRANSFORMED_AST compiler flag * Add test_code_transformer unit tests * Change how importlib generated the .pyc filename: it now also uses the sys.implementation.optim_tag * Add _PySys_CodeTransformer structure --- Doc/library/sys.rst | 20 ++ Include/code.h | 2 + Include/compile.h | 1 + Include/pydebug.h | 1 + Include/pystate.h | 8 +- Include/sysmodule.h | 20 ++ Lib/distutils/tests/test_bdist_dumb.py | 4 +- Lib/distutils/tests/test_build_py.py | 13 +- Lib/distutils/tests/test_install.py | 4 +- Lib/importlib/_bootstrap_external.py | 35 ++- Lib/test/test_code_transformer.py | 137 +++++++++ Lib/test/test_compileall.py | 7 +- Lib/test/test_imp.py | 4 +- Lib/test/test_importlib/test_util.py | 44 +-- Lib/test/test_py_compile.py | 9 +- Lib/test/test_sys.py | 88 +++++- Modules/main.c | 15 +- Objects/object.c | 3 + Parser/asdl_c.py | 2 + Python/Python-ast.c | 2 + Python/bltinmodule.c | 5 +- Python/compile.c | 107 ++++++- Python/peephole.c | 157 ++++++++++ Python/pylifecycle.c | 1 + Python/pythonrun.c | 117 +++++++- Python/sysmodule.c | 391 ++++++++++++++++++++++++- 26 files changed, 1131 insertions(+), 66 deletions(-) create mode 100644 Lib/test/test_code_transformer.py diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 54b99e0fd999f1..aac8ef56b8107a 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -413,6 +413,13 @@ always available. .. versionadded:: 3.7 +.. function:: get_code_transformers() + + Return the list of code transformers.; see :func:`set_code_transformers`. + + .. versionadded:: 3.6 + + .. function:: getcheckinterval() Return the interpreter's "check interval"; see :func:`setcheckinterval`. @@ -719,6 +726,9 @@ always available. ``cache_tag`` is set to ``None``, it indicates that module caching should be disabled. + *optim_tag* is the optimized tag used by the import machinery in the + filenames of cached modules. + :data:`sys.implementation` may contain additional attributes specific to the Python implementation. These non-standard attributes must start with an underscore, and are not described here. Regardless of its contents, @@ -728,6 +738,9 @@ always available. .. versionadded:: 3.3 + .. versionchanged:: 3.6 + Added *optim_tag* attribute. + .. data:: int_info @@ -978,6 +991,13 @@ always available. implement a dynamic prompt. +.. function:: set_code_transformers(transformers) + + Set code transformers: *transformers* is a sequence of code transformers. + + .. versionadded:: 3.6 + + .. function:: setcheckinterval(interval) Set the interpreter's "check interval". This integer value determines how often diff --git a/Include/code.h b/Include/code.h index 385258f93ce491..6015b64233808d 100644 --- a/Include/code.h +++ b/Include/code.h @@ -149,6 +149,8 @@ PyAPI_FUNC(int) _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra); #endif +PyAPI_DATA(PyTypeObject) _PyPeepholeOptimizer_Type; + #ifdef __cplusplus } #endif diff --git a/Include/compile.h b/Include/compile.h index 3cc351d4098ebe..2218eed49d44bf 100644 --- a/Include/compile.h +++ b/Include/compile.h @@ -22,6 +22,7 @@ PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *); #define PyCF_DONT_IMPLY_DEDENT 0x0200 #define PyCF_ONLY_AST 0x0400 #define PyCF_IGNORE_COOKIE 0x0800 +#define PyCF_TRANSFORMED_AST 0x1000 #ifndef Py_LIMITED_API typedef struct { diff --git a/Include/pydebug.h b/Include/pydebug.h index 6e23a896c3d1b1..eef967a7bebfe8 100644 --- a/Include/pydebug.h +++ b/Include/pydebug.h @@ -23,6 +23,7 @@ PyAPI_DATA(int) Py_NoUserSiteDirectory; PyAPI_DATA(int) Py_UnbufferedStdioFlag; PyAPI_DATA(int) Py_HashRandomizationFlag; PyAPI_DATA(int) Py_IsolatedFlag; +PyAPI_DATA(PyObject*) _Py_OptimTag; #ifdef MS_WINDOWS PyAPI_DATA(int) Py_LegacyWindowsStdioFlag; diff --git a/Include/pystate.h b/Include/pystate.h index edfb08b15bc476..aeb45610864d0b 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -52,11 +52,11 @@ typedef struct _is { int64_t id; - PyObject *modules; + PyObject *sysdict; /* sys.__dict__ */ + PyObject *modules; /* sys.modules list */ PyObject *modules_by_index; - PyObject *sysdict; - PyObject *builtins; - PyObject *importlib; + PyObject *builtins; /* dictionary of the builtins module */ + PyObject *importlib; /* importlib module */ PyObject *codec_search_path; PyObject *codec_search_cache; diff --git a/Include/sysmodule.h b/Include/sysmodule.h index c5547ff6742e06..218d916122fca8 100644 --- a/Include/sysmodule.h +++ b/Include/sysmodule.h @@ -37,6 +37,26 @@ PyAPI_FUNC(PyObject *) PySys_GetXOptions(void); PyAPI_FUNC(size_t) _PySys_GetSizeOf(PyObject *); #endif +#ifndef Py_LIMITED_API +typedef struct { + /* The code transformer object */ + PyObject *transformer; + + /* name of the transformer */ + PyObject *name; + + /* ast_transformer() method of transformer, or NULL */ + PyObject *ast_transformer; + + /* code_transformer() method of transformer, or NULL */ + PyObject *code_transformer; +} _PySys_CodeTransformer; + +PyAPI_FUNC(int) _PySys_GetCodeTransformers( + _PySys_CodeTransformer **transformers, + Py_ssize_t *ntransformer); +#endif + #ifdef __cplusplus } #endif diff --git a/Lib/distutils/tests/test_bdist_dumb.py b/Lib/distutils/tests/test_bdist_dumb.py index c8ccdc2383dead..76074000e17fef 100644 --- a/Lib/distutils/tests/test_bdist_dumb.py +++ b/Lib/distutils/tests/test_bdist_dumb.py @@ -87,7 +87,9 @@ def test_simple_built(self): contents = sorted(os.path.basename(fn) for fn in contents) wanted = ['foo-0.1-py%s.%s.egg-info' % sys.version_info[:2], 'foo.py'] if not sys.dont_write_bytecode: - wanted.append('foo.%s.pyc' % sys.implementation.cache_tag) + fn = 'foo.%s.%s-0.pyc' % (sys.implementation.cache_tag, + sys.implementation.optim_tag) + wanted.append(fn) self.assertEqual(contents, sorted(wanted)) def test_suite(): diff --git a/Lib/distutils/tests/test_build_py.py b/Lib/distutils/tests/test_build_py.py index 0712e92c6aba4c..da7039308b3572 100644 --- a/Lib/distutils/tests/test_build_py.py +++ b/Lib/distutils/tests/test_build_py.py @@ -62,8 +62,9 @@ def test_package_data(self): self.assertFalse(os.path.exists(pycache_dir)) else: pyc_files = os.listdir(pycache_dir) - self.assertIn("__init__.%s.pyc" % sys.implementation.cache_tag, - pyc_files) + fn = '__init__.%s.%s-0.pyc' % (sys.implementation.cache_tag, + sys.implementation.optim_tag) + self.assertIn(fn, pyc_files) def test_empty_package_dir(self): # See bugs #1668596/#1720897 @@ -102,8 +103,10 @@ def test_byte_compile(self): found = os.listdir(cmd.build_lib) self.assertEqual(sorted(found), ['__pycache__', 'boiledeggs.py']) found = os.listdir(os.path.join(cmd.build_lib, '__pycache__')) + fn = 'boiledeggs.%s.%s-0.pyc' % (sys.implementation.cache_tag, + sys.implementation.optim_tag) self.assertEqual(found, - ['boiledeggs.%s.pyc' % sys.implementation.cache_tag]) + [fn]) @unittest.skipIf(sys.dont_write_bytecode, 'byte-compile disabled') def test_byte_compile_optimized(self): @@ -120,7 +123,9 @@ def test_byte_compile_optimized(self): found = os.listdir(cmd.build_lib) self.assertEqual(sorted(found), ['__pycache__', 'boiledeggs.py']) found = os.listdir(os.path.join(cmd.build_lib, '__pycache__')) - expect = 'boiledeggs.{}.opt-1.pyc'.format(sys.implementation.cache_tag) + expect = ('boiledeggs.%s.%s-1.pyc' + % (sys.implementation.cache_tag, + sys.implementation.optim_tag)) self.assertEqual(sorted(found), [expect]) def test_dir_in_package_data(self): diff --git a/Lib/distutils/tests/test_install.py b/Lib/distutils/tests/test_install.py index 287ab1989e4083..f5c54cf776c3ff 100644 --- a/Lib/distutils/tests/test_install.py +++ b/Lib/distutils/tests/test_install.py @@ -191,7 +191,9 @@ def test_record(self): f.close() found = [os.path.basename(line) for line in content.splitlines()] - expected = ['hello.py', 'hello.%s.pyc' % sys.implementation.cache_tag, + fn_pyc = 'hello.%s.%s-0.pyc' % (sys.implementation.cache_tag, + sys.implementation.optim_tag) + expected = ['hello.py', fn_pyc, 'sayhi', 'UNKNOWN-0.0.0-py%s.%s.egg-info' % sys.version_info[:2]] self.assertEqual(found, expected) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 32354042f6f233..7269f777107b72 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -253,7 +253,6 @@ def _write_atomic(path, data, mode=0o666): _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c _PYCACHE = '__pycache__' -_OPT = 'opt-' SOURCE_SUFFIXES = ['.py'] # _setup() adds .pyw as needed. @@ -294,15 +293,15 @@ def cache_from_source(path, debug_override=None, *, optimization=None): raise NotImplementedError('sys.implementation.cache_tag is None') almost_filename = ''.join([(base if base else rest), sep, tag]) if optimization is None: - if sys.flags.optimize == 0: - optimization = '' - else: - optimization = sys.flags.optimize + optimization = sys.flags.optimize + elif optimization == '': + optimization = 0 optimization = str(optimization) if optimization != '': if not optimization.isalnum(): raise ValueError('{!r} is not alphanumeric'.format(optimization)) - almost_filename = '{}.{}{}'.format(almost_filename, _OPT, optimization) + part = sys.implementation.optim_tag + '-' + almost_filename = '{}.{}{}'.format(almost_filename, part, optimization) return _path_join(head, _PYCACHE, almost_filename + BYTECODE_SUFFIXES[0]) @@ -329,10 +328,12 @@ def source_from_cache(path): '{!r}'.format(pycache_filename)) elif dot_count == 3: optimization = pycache_filename.rsplit('.', 2)[-2] - if not optimization.startswith(_OPT): + optim_tag = sys.implementation.optim_tag + '-' + if optimization.startswith(optim_tag): + opt_level = optimization[len(optim_tag):] + else: raise ValueError("optimization portion of filename does not start " - "with {!r}".format(_OPT)) - opt_level = optimization[len(_OPT):] + "with {!r}".format(optim_tag)) if not opt_level.isalnum(): raise ValueError("optimization level {!r} is not an alphanumeric " "value".format(optimization)) @@ -683,6 +684,13 @@ def load_module(self, fullname): return _bootstrap._load_module_shim(self, fullname) +def _transformers_tag(): + transformers = sys.get_code_transformers() + if not transformers: + return 'noopt' + return '-'.join(transformer.name for transformer in transformers) + + class SourceLoader(_LoaderBasics): def path_mtime(self, path): @@ -778,6 +786,15 @@ def get_code(self, fullname): return _compile_bytecode(bytes_data, name=fullname, bytecode_path=bytecode_path, source_path=source_path) + + optim_tag= sys.implementation.optim_tag + transformers_tag = _transformers_tag() + if optim_tag != transformers_tag: + # AST transformers are missing, the code cannot be compiled + raise ImportError("missing AST transformers for %r: " + "optim_tag=%r, transformers tag=%r" + % (source_path, optim_tag, transformers_tag)) + source_bytes = self.get_data(source_path) code_object = self.source_to_code(source_bytes, source_path) _bootstrap._verbose_message('code object from {}', source_path) diff --git a/Lib/test/test_code_transformer.py b/Lib/test/test_code_transformer.py new file mode 100644 index 00000000000000..69f6e3c7fdfeba --- /dev/null +++ b/Lib/test/test_code_transformer.py @@ -0,0 +1,137 @@ +""" +Test the implemenentation of the PEP 511: code transformers. +""" +import unittest +import ast +import os.path +import sys +import types +from test.support.script_helper import assert_python_ok, assert_python_failure + + +class BytecodeTransformer: + name = "bytecode" + + def __init__(self): + self.call = None + + def code_transformer(self, code, context): + self.call = (code, context) + consts = ['Ni! Ni! Ni!' if isinstance(const, str) else const + for const in code.co_consts] + return types.CodeType(code.co_argcount, + code.co_kwonlyargcount, + code.co_nlocals, + code.co_stacksize, + code.co_flags, + code.co_code, + tuple(consts), + code.co_names, + code.co_varnames, + code.co_filename, + code.co_name, + code.co_firstlineno, + code.co_lnotab, + code.co_freevars, + code.co_cellvars) + + +class KnightsWhoSayNi(ast.NodeTransformer): + def visit_Str(self, node): + node.s = 'Ni! Ni! Ni!' + return node + + +class ASTTransformer: + name = "ast" + + def __init__(self): + self.transformer = KnightsWhoSayNi() + self.call = None + + def ast_transformer(self, tree, context): + self.call = (tree, context) + self.transformer.visit(tree) + return tree + + +class CodeTransformerTests(unittest.TestCase): + def setUp(self): + transformers = sys.get_code_transformers() + self.addCleanup(sys.set_code_transformers, transformers) + + sys.set_code_transformers([]) + + def test_bytecode(self): + sys.set_code_transformers([BytecodeTransformer()]) + code = compile('print("Hello World")', '', 'exec') + self.assertEqual(code.co_consts, ('Ni! Ni! Ni!', None)) + + def test_bytecode_call(self): + expr = 'x + 1' + filename = "test.py" + expected = compile(expr, filename, "exec") + + transformer = BytecodeTransformer() + sys.set_code_transformers([transformer]) + code = compile(expr, filename, "exec") + + code, context = transformer.call + self.assertEqual(code, expected) + self.assertEqual(context.filename, filename) + + def test_ast(self): + expected = ast.parse('print("Ni! Ni! Ni!")') + + sys.set_code_transformers([ASTTransformer()]) + tree = compile('print("Hello World")', 'string', 'exec', + flags=ast.PyCF_TRANSFORMED_AST) + self.assertEqual(ast.dump(tree), ast.dump(expected)) + + def test_ast_call(self): + code = '1 + 1' + expected_ast = ast.dump(ast.parse(code)) + + filename = "test.py" + transformer = ASTTransformer() + sys.set_code_transformers([transformer]) + compile(code, filename, "exec") + + tree, context = transformer.call + self.assertEqual(ast.dump(tree), expected_ast) + self.assertEqual(context.filename, filename) + + +class DefaultTransformerTests(unittest.TestCase): + def check_default(self, expected, *args): + code = ('import sys; ', + 'transformers = list(map(type, sys.get_code_transformers()))', + 'print(transformers, sys.implementation.optim_tag)') + res = assert_python_ok(*args, '-c', '\n'.join(code)) + self.assertEqual(res.out.rstrip(), expected) + + def test_default(self): + self.check_default(b"[] opt") + + def test_opt(self): + self.check_default(b"[] opt", '-o', 'opt') + + def test_noopt(self): + self.check_default(b"[] noopt", '-o', 'noopt') + + +class OptimTagTests(unittest.TestCase): + """Test -o command line option.""" + + def test_invalid_optim_tags(self): + invalid_tags = ['a.b', 'a%sb' % os.path.sep] + if os.path.altsep: + invalid_tags.append('a%sb' % os.path.altsep) + for optim_tag in invalid_tags: + code = 'import sys; print(sys.implementation.optim_tag)' + res = assert_python_failure('-o', optim_tag, '-c', code) + self.assertIn(b'invalid optimization tag', res.err.rstrip()) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 2356efcaec78be..51c5790f7d6c69 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -312,10 +312,11 @@ def test_no_args_respects_quiet_flag(self): # Ensure that the default behavior of compileall's CLI is to create # PEP 3147/PEP 488 pyc files. + optim_tag = sys.implementation.optim_tag for name, ext, switch in [ - ('normal', 'pyc', []), - ('optimize', 'opt-1.pyc', ['-O']), - ('doubleoptimize', 'opt-2.pyc', ['-OO']), + ('normal', '%s-0.pyc' % optim_tag, ['-o', optim_tag]), + ('optimize', '%s-1.pyc' % optim_tag, ['-o', optim_tag, '-O']), + ('doubleoptimize', '%s-2.pyc' % optim_tag, ['-o', optim_tag, '-OO']), ]: def f(self, ext=ext, switch=switch): script_helper.assert_python_ok(*(switch + diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index 4ece3654108e49..af0b271fc4b6e0 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -364,8 +364,8 @@ def test_cache_from_source(self): # Given the path to a .py file, return the path to its PEP 3147 # defined .pyc file (i.e. under __pycache__). path = os.path.join('foo', 'bar', 'baz', 'qux.py') - expect = os.path.join('foo', 'bar', 'baz', '__pycache__', - 'qux.{}.pyc'.format(self.tag)) + pyc = 'qux.{}.{}-0.pyc'.format(self.tag, sys.implementation.optim_tag) + expect = os.path.join('foo', 'bar', 'baz', '__pycache__', pyc) self.assertEqual(imp.cache_from_source(path, True), expect) @unittest.skipUnless(sys.implementation.cache_tag is not None, diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index 56a0b0e7a518cd..b44f2b073213d6 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -556,6 +556,12 @@ class PEP3147Tests: """Tests of PEP 3147-related functions: cache_from_source and source_from_cache.""" tag = sys.implementation.cache_tag + optim_tag = sys.implementation.optim_tag + + def pyc_filename(self, name, optim_level=0, name_dot=True): + if name_dot: + name += '.' + return f'{name}{self.tag}.{self.optim_tag}-{optim_level}.pyc' @unittest.skipUnless(sys.implementation.cache_tag is not None, 'requires sys.implementation.cache_tag not be None') @@ -563,8 +569,9 @@ def test_cache_from_source(self): # Given the path to a .py file, return the path to its PEP 3147 # defined .pyc file (i.e. under __pycache__). path = os.path.join('foo', 'bar', 'baz', 'qux.py') + filename = self.pyc_filename('qux') expect = os.path.join('foo', 'bar', 'baz', '__pycache__', - 'qux.{}.pyc'.format(self.tag)) + filename) self.assertEqual(self.util.cache_from_source(path, optimization=''), expect) @@ -577,8 +584,8 @@ def test_cache_from_source_no_cache_tag(self): def test_cache_from_source_no_dot(self): # Directory with a dot, filename without dot. path = os.path.join('foo.bar', 'file') - expect = os.path.join('foo.bar', '__pycache__', - 'file{}.pyc'.format(self.tag)) + filename = self.pyc_filename('file', name_dot=False) + expect = os.path.join('foo.bar', '__pycache__', filename) self.assertEqual(self.util.cache_from_source(path, optimization=''), expect) @@ -601,7 +608,8 @@ def test_cache_from_source_debug_override(self): def test_cache_from_source_cwd(self): path = 'foo.py' - expect = os.path.join('__pycache__', 'foo.{}.pyc'.format(self.tag)) + filename = self.pyc_filename('foo') + expect = os.path.join('__pycache__', filename) self.assertEqual(self.util.cache_from_source(path, optimization=''), expect) @@ -626,7 +634,8 @@ def __bool__(self): raise RuntimeError def test_cache_from_source_optimization_empty_string(self): # Setting 'optimization' to '' leads to no optimization tag (PEP 488). path = 'foo.py' - expect = os.path.join('__pycache__', 'foo.{}.pyc'.format(self.tag)) + filename = self.pyc_filename('foo') + expect = os.path.join('__pycache__', filename) self.assertEqual(self.util.cache_from_source(path, optimization=''), expect) @@ -635,30 +644,31 @@ def test_cache_from_source_optimization_None(self): # (PEP 488) path = 'foo.py' optimization_level = sys.flags.optimize - almost_expect = os.path.join('__pycache__', 'foo.{}'.format(self.tag)) - if optimization_level == 0: - expect = almost_expect + '.pyc' - elif optimization_level <= 2: - expect = almost_expect + '.opt-{}.pyc'.format(optimization_level) + if optimization_level <= 2: + filename = self.pyc_filename('foo', optimization_level) else: msg = '{!r} is a non-standard optimization level'.format(optimization_level) self.skipTest(msg) self.assertEqual(self.util.cache_from_source(path, optimization=None), - expect) + os.path.join('__pycache__', filename)) def test_cache_from_source_optimization_set(self): # The 'optimization' parameter accepts anything that has a string repr # that passes str.alnum(). path = 'foo.py' valid_characters = string.ascii_letters + string.digits - almost_expect = os.path.join('__pycache__', 'foo.{}'.format(self.tag)) + got = self.util.cache_from_source(path, optimization=valid_characters) # Test all valid characters are accepted. + filename = self.pyc_filename('foo', valid_characters) self.assertEqual(got, - almost_expect + '.opt-{}.pyc'.format(valid_characters)) + os.path.join('__pycache__', filename)) + # str() should be called on argument. + filename = self.pyc_filename('foo', 42) self.assertEqual(self.util.cache_from_source(path, optimization=42), - almost_expect + '.opt-42.pyc') + os.path.join('__pycache__', filename)) + # Invalid characters raise ValueError. with self.assertRaises(ValueError): self.util.cache_from_source(path, optimization='path/is/bad') @@ -724,7 +734,7 @@ def test_source_from_cache_too_few_dots(self): def test_source_from_cache_too_many_dots(self): with self.assertRaises(ValueError): self.util.source_from_cache( - '__pycache__/foo.cpython-32.opt-1.foo.pyc') + '__pycache__/foo.cpython-32.{}-1.foo.pyc'.format(self.optim_tag)) def test_source_from_cache_not_opt(self): # Non-`opt-` path component -> ValueError @@ -740,12 +750,12 @@ def test_source_from_cache_no__pycache__(self): def test_source_from_cache_optimized_bytecode(self): # Optimized bytecode is not an issue. - path = os.path.join('__pycache__', 'foo.{}.opt-1.pyc'.format(self.tag)) + path = os.path.join('__pycache__', self.pyc_filename('foo', 1)) self.assertEqual(self.util.source_from_cache(path), 'foo.py') def test_source_from_cache_missing_optimization(self): # An empty optimization level is a no-no. - path = os.path.join('__pycache__', 'foo.{}.opt-.pyc'.format(self.tag)) + path = os.path.join('__pycache__', self.pyc_filename('foo', '')) with self.assertRaises(ValueError): self.util.source_from_cache(path) diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py index 4a6caa571875d7..305948f770c09b 100644 --- a/Lib/test/test_py_compile.py +++ b/Lib/test/test_py_compile.py @@ -107,11 +107,11 @@ def test_double_dot_no_clobber(self): pyc_path = weird_path + 'c' head, tail = os.path.split(cache_path) penultimate_tail = os.path.basename(head) + pyc = 'foo.bar.{}.{}-0.pyc'.format(sys.implementation.cache_tag, + sys.implementation.optim_tag) self.assertEqual( os.path.join(penultimate_tail, tail), - os.path.join( - '__pycache__', - 'foo.bar.{}.pyc'.format(sys.implementation.cache_tag))) + os.path.join( '__pycache__', pyc)) with open(weird_path, 'w') as file: file.write('x = 123\n') py_compile.compile(weird_path) @@ -120,7 +120,8 @@ def test_double_dot_no_clobber(self): def test_optimization_path(self): # Specifying optimized bytecode should lead to a path reflecting that. - self.assertIn('opt-2', py_compile.compile(self.source_path, optimize=2)) + expected = '{}-2'.format(sys.implementation.optim_tag) + self.assertIn(expected, py_compile.compile(self.source_path, optimize=2)) if __name__ == "__main__": diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 3844812ba28873..fe9c5a8299ea95 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1231,8 +1231,94 @@ def test_asyncgen_hooks(self): self.assertIsNone(cur.finalizer) +class MyCodeTransformer: + name = 'mytransformer' + + def code_transformer(self, bytecode): + # no-op + return bytecode + + def ast_transformers(self, tree, context): + # no-op + return tree + + +class CodeTransformersTests(unittest.TestCase): + def setUp(self): + transformers = sys.get_code_transformers() + self.addCleanup(sys.set_code_transformers, transformers) + + def test_sys_get_transformers(self): + transformers = sys.get_code_transformers() + self.assertIsInstance(transformers, list) + for transformer in transformers: + self.assertIsInstance(transformer.name, str) + + def test_sys_set_code_transformers(self): + transformer = MyCodeTransformer() + sys.set_code_transformers([transformer]) + + transformers = sys.get_code_transformers() + self.assertEqual(len(transformers), 1) + self.assertIs(transformers[0], transformer) + + def test_optim_tag(self): + transformer1 = MyCodeTransformer() + transformer1.name = "a" + sys.set_code_transformers([transformer1]) + self.assertEqual(sys.implementation.optim_tag, "a") + + transformer2 = MyCodeTransformer() + transformer2.name = "b" + sys.set_code_transformers([transformer1, transformer2]) + self.assertEqual(sys.implementation.optim_tag, "a-b") + + sys.set_code_transformers([]) + self.assertEqual(sys.implementation.optim_tag, "noopt") + + def test_invalid_type(self): + with self.assertRaises(AttributeError): + sys.set_code_transformers([123]) + + def test_invalid_name(self): + transformer = MyCodeTransformer() + + # None name + with self.assertRaises(TypeError): + transformer.name = None + sys.set_code_transformers([transformer]) + + # empty name + with self.assertRaises(ValueError): + transformer.name = '' + sys.set_code_transformers([transformer]) + + # invalid characters + invalid_chars = '.-' + os.path.sep + if os.path.altsep: + invalid_chars += os.path.altsep + for invalid_char in invalid_chars: + with self.assertRaises(ValueError): + transformer.name = 'a%sb' % invalid_char + sys.set_code_transformers([transformer]) + + def test_no_transformation(self): + class InvalidTransformer: + name = "invalid" + + transformer = InvalidTransformer() + + # a code transformer must have a code_transformer() + # and/or a ast_transformer() method + with self.assertRaises(ValueError): + sys.set_code_transformers([transformer]) + + def test_main(): - test.support.run_unittest(SysModuleTest, SizeofTest) + test.support.run_unittest(SysModuleTest, + SizeofTest, + CodeTransformersTests) + if __name__ == "__main__": test_main() diff --git a/Modules/main.c b/Modules/main.c index 08b22760de1125..f1a13714f98db5 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -40,7 +40,7 @@ static wchar_t **orig_argv; static int orig_argc; /* command line options */ -#define BASE_OPTS L"bBc:dEhiIJm:OqRsStuvVW:xX:?" +#define BASE_OPTS L"bBc:dEhiIJm:Oo:qRsStuvVW:xX:?" #define PROGRAM_OPTS BASE_OPTS @@ -66,6 +66,7 @@ static const char usage_2[] = "\ -m mod : run library module as a script (terminates option list)\n\ -O : optimize generated bytecode slightly; also PYTHONOPTIMIZE=x\n\ -OO : remove doc-strings in addition to the -O optimizations\n\ +-o opt : optimisation tag\n\ -q : don't print version and copyright messages on interactive startup\n\ -s : don't add user site directory to sys.path; also PYTHONNOUSERSITE\n\ -S : don't imply 'import site' on initialization\n\ @@ -506,6 +507,18 @@ read_command_line(int argc, wchar_t **argv, _Py_CommandLineDetails *cmdline) /* Ignored */ break; + case 'o': + if (wcslen(_PyOS_optarg) == 0 + || wcschr(_PyOS_optarg, L'.') + || wcschr(_PyOS_optarg, SEP) +#ifdef ALTSEP + || wcschr(_PyOS_optarg, ALTSEP) +#endif + ) + Py_FatalError("invalid optimization tag"); + _Py_OptimTag = PyUnicode_FromWideChar(_PyOS_optarg, -1); + break; + /* This space reserved for other options */ default: diff --git a/Objects/object.c b/Objects/object.c index 2ba6e572ea61ff..5c08b1f7c630f8 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1820,6 +1820,9 @@ _Py_ReadyTypes(void) if (PyType_Ready(&_PyCoroWrapper_Type) < 0) Py_FatalError("Can't initialize coroutine wrapper type"); + + if (PyType_Ready(&_PyPeepholeOptimizer_Type) < 0) + Py_FatalError("Can't initialize peephole optimizer type"); } diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 096f5f8c7494f7..085e3ea628ae3c 100644 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1039,6 +1039,8 @@ def visitModule(self, mod): self.emit('if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return NULL;', 1) self.emit('if (PyModule_AddIntMacro(m, PyCF_ONLY_AST) < 0)', 1) self.emit("return NULL;", 2) + self.emit('if (PyModule_AddIntMacro(m, PyCF_TRANSFORMED_AST) < 0)', 1) + self.emit("return NULL;", 2) for dfn in mod.dfns: self.visit(dfn) self.emit("return m;", 1) diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 2759b2fe9c4b33..1bfd49d11b17d0 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -7920,6 +7920,8 @@ PyInit__ast(void) if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return NULL; if (PyModule_AddIntMacro(m, PyCF_ONLY_AST) < 0) return NULL; + if (PyModule_AddIntMacro(m, PyCF_TRANSFORMED_AST) < 0) + return NULL; if (PyDict_SetItemString(d, "mod", (PyObject*)mod_type) < 0) return NULL; if (PyDict_SetItemString(d, "Module", (PyObject*)Module_type) < 0) return NULL; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index d16b1b4407ac05..4aa38f2f4cf4fd 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -697,7 +697,8 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, cf.cf_flags = flags | PyCF_SOURCE_IS_UTF8; if (flags & - ~(PyCF_MASK | PyCF_MASK_OBSOLETE | PyCF_DONT_IMPLY_DEDENT | PyCF_ONLY_AST)) + ~(PyCF_MASK | PyCF_MASK_OBSOLETE | PyCF_DONT_IMPLY_DEDENT + | PyCF_ONLY_AST | PyCF_TRANSFORMED_AST)) { PyErr_SetString(PyExc_ValueError, "compile(): unrecognised flags"); @@ -731,7 +732,7 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, if (is_ast == -1) goto error; if (is_ast) { - if (flags & PyCF_ONLY_AST) { + if (flags & (PyCF_ONLY_AST | PyCF_TRANSFORMED_AST)) { Py_INCREF(source); result = source; } diff --git a/Python/compile.c b/Python/compile.c index 280ddc39e317b8..b7e39bf5494727 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -5281,6 +5281,96 @@ compute_code_flags(struct compiler *c) return flags; } +static PyObject* +code_transformers_context(struct compiler *c) +{ + PyObject *dict, *namespace; + + dict = PyDict_New(); + if (dict == NULL) + return NULL; + + if (PyDict_SetItemString(dict, "filename", c->c_filename) < 0) { + Py_DECREF(dict); + return NULL; + } + + namespace = _PyNamespace_New(dict); + Py_DECREF(dict); + return namespace; +} + +static PyObject* +code_transformers(struct compiler *c, PyObject *code_orig) +{ + PyObject *code = NULL; + PyObject *context = NULL; + _PySys_CodeTransformer *transformers; + Py_ssize_t i, ntransformer, nbytecodetransformer; + + if (_PySys_GetCodeTransformers(&transformers, &ntransformer) < 0) { + PyErr_SetString(PyExc_RuntimeError, + "failed to get code transformers"); + return NULL; + } + + nbytecodetransformer = 0; + for (i=0; i < ntransformer; i++) { + if (transformers[i].code_transformer) + nbytecodetransformer++; + } + if (nbytecodetransformer == 0) { + /* No AST transformer registered: nothing to do. + * Optimization to avoid calling the expensive functions + * PyAST_mod2obj(), PyAST_obj2mod(), PyAST_Validate(). */ + Py_INCREF(code_orig); + return code_orig; + } + + /* FIXME: the bytecode transformer can technically call + * sys.set_code_transformers(), we need to keep a strong reference on each + * bytecode transformer */ + + context = code_transformers_context(c); + + Py_INCREF(code_orig); + code = code_orig; + + for (i=0; i < ntransformer; i++) { + PyObject *transformer, *new_code; + + transformer = transformers[i].code_transformer; + if (transformer == NULL) + continue; + + new_code = PyObject_CallFunction(transformer, "OO", code, context); + if (new_code == NULL) { + /* FIXME: raise a better exception and chain the previous one */ + goto error; + } + + if (!PyCode_Check(new_code)) { + PyErr_Format(PyExc_ValueError, + "bytecode transformer must return " + "a code object, got %s", + Py_TYPE(new_code)->tp_name); + Py_DECREF(new_code); + return NULL; + } + + Py_SETREF(code, new_code); + } + + Py_DECREF(context); + return code; + +error: + /* FIXME: release reference to code transformers */ + Py_XDECREF(context); + Py_XDECREF(code); + return NULL; +} + static PyCodeObject * makecode(struct compiler *c, struct assembler *a) { @@ -5293,6 +5383,7 @@ makecode(struct compiler *c, struct assembler *a) PyObject *freevars = NULL; PyObject *cellvars = NULL; PyObject *bytecode = NULL; + PyObject *lnotab = NULL; Py_ssize_t nlocals; int nlocals_int; int flags; @@ -5324,9 +5415,10 @@ makecode(struct compiler *c, struct assembler *a) if (flags < 0) goto error; - bytecode = PyCode_Optimize(a->a_bytecode, consts, names, a->a_lnotab); - if (!bytecode) - goto error; + bytecode = a->a_bytecode; + lnotab = a->a_lnotab; + a->a_bytecode = NULL; + a->a_lnotab = NULL; tmp = PyList_AsTuple(consts); /* PyCode_New requires a tuple */ if (!tmp) @@ -5342,7 +5434,14 @@ makecode(struct compiler *c, struct assembler *a) freevars, cellvars, c->c_filename, c->u->u_name, c->u->u_firstlineno, - a->a_lnotab); + lnotab); + if (co == NULL) + goto error; + + tmp = code_transformers(c, (PyObject *)co); + Py_DECREF(co); + co = (PyCodeObject *)tmp; + error: Py_XDECREF(consts); Py_XDECREF(names); diff --git a/Python/peephole.c b/Python/peephole.c index 1bee7102d9cc8a..aeba42315fd23a 100644 --- a/Python/peephole.c +++ b/Python/peephole.c @@ -783,3 +783,160 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, PyMem_Free(codestr); return code; } + + +/* PeepholeOptimizer */ + +static PyObject* +optimizer_code_transformer(PyObject *self, PyObject *args) +{ + PyCodeObject *code, *new_code; + PyObject *context; + PyObject *lnotab = NULL, *consts = NULL, *bytecode = NULL; + PyObject *tmp; + Py_ssize_t i, len; + + if (!PyArg_ParseTuple(args, "O!O", + &PyCode_Type, &code, &context)) + return NULL; + + + len = PyBytes_GET_SIZE(code->co_lnotab); + if (len != 0) { + lnotab = PyBytes_FromStringAndSize(NULL, len); + if (lnotab == NULL) + goto error; + + assert(Py_REFCNT(lnotab) == 1); + memcpy(PyBytes_AS_STRING(lnotab), PyBytes_AS_STRING(code->co_lnotab), + len); + Py_SIZE(lnotab) = len; + } + else { + Py_INCREF(code->co_lnotab); + lnotab = code->co_lnotab; + } + + len = PyTuple_GET_SIZE(code->co_consts); + consts = PyList_New(len); + if (consts == NULL) + goto error; + for (i=0; i < len; i++) { + PyObject *item = PyTuple_GET_ITEM(code->co_consts, i); + Py_INCREF(item); + PyList_SET_ITEM(consts, i, item); + } + + bytecode = PyCode_Optimize(code->co_code, + consts, + code->co_names, + lnotab); + if (bytecode == NULL) + goto error; + + tmp = PyList_AsTuple(consts); + if (tmp == NULL) + goto error; + Py_SETREF(consts, tmp); + + new_code = PyCode_New(code->co_argcount, + code->co_kwonlyargcount, + code->co_nlocals, + code->co_stacksize, + code->co_flags, + bytecode, + consts, + code->co_names, + code->co_varnames, + code->co_freevars, + code->co_cellvars, + code->co_filename, + code->co_name, + code->co_firstlineno, + lnotab); + Py_DECREF(lnotab); + Py_DECREF(consts); + Py_DECREF(bytecode); + + return (PyObject *)new_code; + +error: + Py_XDECREF(lnotab); + Py_XDECREF(consts); + Py_XDECREF(bytecode); + return NULL; +} + +PyDoc_STRVAR(code_transformer_doc, +"code_transformer(code, consts, names, lnotab, context) -> tuple\n\ +\n\ +Run the peephole optimizer.\n" +"Return optimized code as (code, lnotab) tuple."); + +/* FIXME: remove useless constructor? */ +static PyObject * +optimizer_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + assert(type != NULL && type->tp_alloc != NULL); + return type->tp_alloc(type, 0); +} + +static PyObject * +optimizer_get_name(PyFunctionObject *op) +{ + return PyUnicode_FromString("opt"); +} + +static PyMethodDef optimizer_methods[] = { + {"code_transformer", optimizer_code_transformer, METH_VARARGS, + code_transformer_doc}, + {NULL, NULL} /* sentinel */ +}; + +static PyGetSetDef optimizer_getsetlist[] = { + {"name", (getter)optimizer_get_name, NULL}, + {NULL} /* Sentinel */ +}; + + +PyTypeObject _PyPeepholeOptimizer_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "PeepholeOptimizer", + 0, + 0, + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + optimizer_methods, /* tp_methods */ + 0, /* tp_members */ + optimizer_getsetlist, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + optimizer_new, /* tp_new */ + 0, /* tp_free */ +}; diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 953bc90a456bdb..17acd696a949b1 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -98,6 +98,7 @@ int Py_IsolatedFlag = 0; /* for -I, isolate from user's env */ int Py_LegacyWindowsFSEncodingFlag = 0; /* Uses mbcs instead of utf-8 */ int Py_LegacyWindowsStdioFlag = 0; /* Uses FileIO instead of WindowsConsoleIO */ #endif +PyObject* _Py_OptimTag = NULL; /* optimization tag for importlib */ PyThreadState *_Py_Finalizing = NULL; diff --git a/Python/pythonrun.c b/Python/pythonrun.c index f31b3ee5a5dcd1..e97960a12b8208 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1036,7 +1036,7 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start, PyArena_Free(arena); return NULL; } - if (flags && (flags->cf_flags & PyCF_ONLY_AST)) { + if (flags && (flags->cf_flags & (PyCF_ONLY_AST | PyCF_TRANSFORMED_AST))) { PyObject *result = PyAST_mod2obj(mod); PyArena_Free(arena); return result; @@ -1104,6 +1104,111 @@ Py_SymtableString(const char *str, const char *filename_str, int start) return st; } +static PyObject* +ast_transformers_context(PyObject *filename) +{ + PyObject *dict, *namespace; + + dict = PyDict_New(); + if (dict == NULL) + return NULL; + + if (PyDict_SetItemString(dict, "filename", filename) < 0) { + Py_DECREF(dict); + return NULL; + } + + namespace = _PyNamespace_New(dict); + Py_DECREF(dict); + return namespace; +} + +static mod_ty +ast_transformers(mod_ty mod, PyArena *arena, int cf_flags, + PyObject *filename, int start) +{ + PyObject *ast, *context = NULL; + int mode; + _PySys_CodeTransformer *transformers; + Py_ssize_t i, ntransformer, nasttransformer; + + assert(filename != NULL); + + if (cf_flags & PyCF_ONLY_AST) + return mod; + + if (_PySys_GetCodeTransformers(&transformers, &ntransformer) < 0) { + PyErr_SetString(PyExc_RuntimeError, + "failed to get code transformers"); + return NULL; + } + + nasttransformer = 0; + for (i=0; i < ntransformer; i++) { + if (transformers[i].ast_transformer) + nasttransformer++; + } + if (nasttransformer == 0) { + /* No AST transformer registered: nothing to do. + * Optimization to avoid calling the expensive functions + * PyAST_mod2obj(), PyAST_obj2mod(), PyAST_Validate(). */ + return mod; + } + + /* FIXME: the AST transformer can technically call + * sys.set_code_transformers(), we need to keep a strong reference on each + * AST transformer */ + + context = ast_transformers_context(filename); + + if (start == Py_file_input) + mode = 0; /* exec */ + else if (start == Py_eval_input) + mode = 1; /* eval */ + else { + assert(start == Py_single_input); + mode = 2; /* single */ + } + + ast = PyAST_mod2obj(mod); + if (ast == NULL) + goto error; + + for (i=0; i < ntransformer; i++) { + PyObject *ast_transformer, *new_ast; + + ast_transformer = transformers[i].ast_transformer; + if (ast_transformer == NULL) + continue; + + new_ast = PyObject_CallFunction(ast_transformer, "OO", ast, context); + Py_DECREF(ast); + + if (new_ast == NULL) { + /* FIXME: raise a better exception and chain the previous one */ + goto error; + } + + ast = new_ast; + } + + mod = PyAST_obj2mod(ast, arena, mode); + Py_DECREF(ast); + if (mod == NULL) + goto error; + + if (!PyAST_Validate(mod)) + goto error; + + Py_DECREF(context); + return mod; + +error: + /* FIXME: release reference to AST transformers */ + Py_XDECREF(context); + return NULL; +} + /* Preferred access to parser is through AST. */ mod_ty PyParser_ASTFromStringObject(const char *s, PyObject *filename, int start, @@ -1131,7 +1236,10 @@ PyParser_ASTFromStringObject(const char *s, PyObject *filename, int start, mod = NULL; } err_free(&err); - return mod; + if (mod == NULL) + return NULL; + + return ast_transformers(mod, arena, flags->cf_flags, filename, start); } mod_ty @@ -1178,7 +1286,10 @@ PyParser_ASTFromFileObject(FILE *fp, PyObject *filename, const char* enc, mod = NULL; } err_free(&err); - return mod; + if (mod == NULL) + return NULL; + + return ast_transformers(mod, arena, flags->cf_flags, filename, start); } mod_ty diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 424a88f7086b3e..5b42a5d6d19f84 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -38,11 +38,26 @@ _Py_IDENTIFIER(__sizeof__); _Py_IDENTIFIER(buffer); _Py_IDENTIFIER(builtins); _Py_IDENTIFIER(encoding); +_Py_IDENTIFIER(name); _Py_IDENTIFIER(path); _Py_IDENTIFIER(stdout); _Py_IDENTIFIER(stderr); _Py_IDENTIFIER(write); +/* State of the sys module, per PEP 3121. */ +typedef struct { + Py_ssize_t ntransformer; + /* FIXME: free memory at exit */ + _PySys_CodeTransformer *transformers; +} PySysState; + +/* Given a module object, get its per-module state. */ +static PySysState * +_PySys_GetState(PyObject *module) +{ + return (PySysState *)PyModule_GetState(module); +} + PyObject * _PySys_GetObjectId(_Py_Identifier *key) { @@ -785,6 +800,263 @@ Return a namedtuple of installed asynchronous generators hooks \ ); +static void +code_transformers_dealloc(Py_ssize_t ntransformer, + _PySys_CodeTransformer *transformers) +{ + Py_ssize_t i; + + for (i=0; i < ntransformer; i++) { + Py_DECREF(transformers[i].transformer); + Py_DECREF(transformers[i].name); + Py_XDECREF(transformers[i].ast_transformer); + Py_XDECREF(transformers[i].code_transformer); + } + PyMem_Free(transformers); +} + +static int +sys_check_code_transformer_name(PyObject *name, Py_ssize_t len, Py_UCS4 ch) +{ + Py_ssize_t pos; + + pos = PyUnicode_FindChar(name, ch, 0, len, 1); + if (pos == -2) + return -1; + if (pos != -1) { + PyErr_SetString(PyExc_ValueError, + "invalid transformer name"); + return -1; + } + + return 0; +} + +static int +sys_init_code_transformer(_PySys_CodeTransformer *self, PyObject *transformer) +{ + PyObject *name = NULL, *ast_transformer = NULL, *code_transformer = NULL; + Py_ssize_t len; + int res = -1; + + name = _PyObject_GetAttrId(transformer, &PyId_name); + if (name == NULL) + goto exit; + + if (!PyUnicode_Check(name)) { + PyErr_Format(PyExc_TypeError, + "transformer name type must be str, got %s", + Py_TYPE(name)->tp_name); + goto exit; + } + + if (PyUnicode_READY(name) == -1) + goto exit; + len = PyUnicode_GET_LENGTH(name); + + if (len == 0) { + PyErr_SetString(PyExc_ValueError, + "transformer name must be non-empty"); + goto exit; + } + + if (sys_check_code_transformer_name(name, len, '-') < 0) + goto exit; + if (sys_check_code_transformer_name(name, len, '.') < 0) + goto exit; + if (sys_check_code_transformer_name(name, len, '\0') < 0) + goto exit; + if (sys_check_code_transformer_name(name, len, SEP) < 0) + goto exit; +#ifdef ALTSEP + if (sys_check_code_transformer_name(name, len, ALTSEP) < 0) + goto exit; +#endif + + ast_transformer = PyObject_GetAttrString(transformer, "ast_transformer"); + if (ast_transformer == NULL) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) + goto exit; + PyErr_Clear(); + } + + code_transformer = PyObject_GetAttrString(transformer, "code_transformer"); + if (code_transformer == NULL) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) + goto exit; + PyErr_Clear(); + } + + if (ast_transformer == NULL && code_transformer == NULL) { + PyErr_SetString(PyExc_ValueError, + "a code transformer must have a code_transformer() " + "and/or a ast_transformer() method"); + goto exit; + } + + Py_INCREF(transformer); + self->transformer = transformer; + Py_INCREF(name); + self->name = name; + Py_XINCREF(ast_transformer); + self->ast_transformer = ast_transformer; + Py_XINCREF(code_transformer); + self->code_transformer = code_transformer; + res = 0; + +exit: + Py_XDECREF(name); + Py_XDECREF(code_transformer); + Py_XDECREF(ast_transformer); + return res; +} + +static PyObject * +sys_set_code_transformers(PyObject *self, PyObject *transformers_seq) +{ + PySysState *st = _PySys_GetState(self); + PyObject *seq = NULL; + PyObject *names = NULL, *optim_tag, *implementation; + _PySys_CodeTransformer *transformers = NULL; + Py_ssize_t i, ntransformer, ninitialized = 0, size; + + implementation = PySys_GetObject("implementation"); + if (implementation == NULL) { + PyErr_SetString(PyExc_RuntimeError, "lost sys.implementation"); + goto error; + } + + seq = PySequence_Fast(transformers_seq, + "transformers must be a sequence " + "of code transformers"); + if (seq == NULL) + goto error; + + ntransformer = PySequence_Fast_GET_SIZE(seq); + + if (ntransformer > PY_SSIZE_T_MAX / (Py_ssize_t)sizeof(transformers[0])) { + PyErr_NoMemory(); + goto error; + } + size = ntransformer * sizeof(transformers[0]); + transformers = PyMem_Malloc(size); + if (transformers == NULL) { + PyErr_NoMemory(); + goto error; + } + + names = PyList_New(0); + if (names == NULL) + goto error; + + for (i=0; i < ntransformer; i++) { + PyObject *transformer = PySequence_Fast_GET_ITEM(seq, i); + if (sys_init_code_transformer(&transformers[i], transformer) < 0) + goto error; + ninitialized++; + + if (PyList_Append(names, transformers[i].name) < 0) + goto error; + } + + if (PyList_GET_SIZE(names) > 0) { + PyObject *dash = PyUnicode_FromString("-"); + if (dash == NULL) + goto error; + + optim_tag = PyUnicode_Join(dash, names); + Py_DECREF(dash); + } + else { + optim_tag = PyUnicode_FromString("noopt"); + if (optim_tag == NULL) + goto error; + } + + if (PyObject_SetAttrString(implementation, "optim_tag", optim_tag) < 0) { + Py_DECREF(optim_tag); + goto error; + } + Py_DECREF(optim_tag); + + /* release memory */ + Py_DECREF(names); + Py_DECREF(seq); + + code_transformers_dealloc(st->ntransformer, st->transformers); + st->ntransformer = ntransformer; + st->transformers = transformers; + + Py_RETURN_NONE; + +error: + Py_XDECREF(seq); + code_transformers_dealloc(ninitialized, transformers); + Py_XDECREF(names); + return NULL; +} + +PyDoc_STRVAR(set_code_transformers_doc, +"set_code_transformers(transformers)\n" +"\n" +"Set code transformers."); + +static PyObject* +sys_get_module(void) +{ + PyThreadState *tstate; + tstate = PyThreadState_GET(); + return PyDict_GetItemString(tstate->interp->modules, "sys"); +} + +int +_PySys_GetCodeTransformers(_PySys_CodeTransformer **transformers, + Py_ssize_t *ntransformer) +{ + PyObject *sysmod; + PySysState *st; + + sysmod = sys_get_module(); + if (sysmod == NULL) { + return -1; + } + + st = _PySys_GetState(sysmod); + if (st == NULL) { + return -1; + } + + *ntransformer = st->ntransformer; + *transformers = st->transformers; + return 0; +} + +static PyObject * +sys_get_code_transformers(PyObject *self, PyObject *args) +{ + PySysState *st = _PySys_GetState(self); + PyObject *list; + Py_ssize_t i; + + list = PyList_New(st->ntransformer); + if (list == NULL) + return NULL; + + for (i=0; i < st->ntransformer; i++) { + PyObject *transformer = st->transformers[i].transformer; + Py_INCREF(transformer); + PyList_SET_ITEM(list, i, transformer); + } + + return list; +} + +PyDoc_STRVAR(get_code_transformers_doc, +"get_code_transformers() -> list\n" +"\n" +"Return the list of code transformers."); + + static PyTypeObject Hash_InfoType; PyDoc_STRVAR(hash_info_doc, @@ -1451,6 +1723,10 @@ static PyMethodDef sys_methods[] = { {"getandroidapilevel", (PyCFunction)sys_getandroidapilevel, METH_NOARGS, getandroidapilevel_doc}, #endif + {"set_code_transformers", sys_set_code_transformers, METH_O, + set_code_transformers_doc}, + {"get_code_transformers", sys_get_code_transformers, METH_NOARGS, + get_code_transformers_doc}, {NULL, NULL} /* sentinel */ }; @@ -1855,6 +2131,20 @@ make_impl_info(PyObject *version_info) if (res < 0) goto error; + if (_Py_OptimTag) { + Py_INCREF(_Py_OptimTag); + value = _Py_OptimTag; + } + else { + value = PyUnicode_FromString("opt"); + if (value == NULL) + goto error; + } + res = PyDict_SetItemString(impl_info, "optim_tag", value); + Py_DECREF(value); + if (res < 0) + goto error; + res = PyDict_SetItemString(impl_info, "version", version_info); if (res < 0) goto error; @@ -1888,18 +2178,85 @@ make_impl_info(PyObject *version_info) return NULL; } +static int +sys_traverse(PyObject *self, visitproc visit, void *arg) +{ + PySysState *st = _PySys_GetState(self); + Py_ssize_t i; + + for (i=0; i < st->ntransformer; i++) { + Py_VISIT(st->transformers[i].transformer); + } + return 0; +} + +static int +sys_clear(PyObject *self) +{ + PySysState *st = _PySys_GetState(self); + + code_transformers_dealloc(st->ntransformer, st->transformers); + + st->ntransformer = 0; + st->transformers = NULL; + return 0; +} + +static void +sys_free(void *self) +{ + sys_clear((PyObject *)self); +} + static struct PyModuleDef sysmodule = { PyModuleDef_HEAD_INIT, "sys", sys_doc, - -1, /* multiple "initialization" just copies the module dict. */ + sizeof(PySysState), /* multiple "initialization" just copies the module dict. */ sys_methods, NULL, - NULL, - NULL, - NULL + sys_traverse, + sys_clear, + sys_free }; +static int +set_peephole_optimizer(void) +{ + PyObject *sysmod; + PySysState *st; + PyObject *optimizer; + + sysmod = sys_get_module(); + if (sysmod == NULL) { + return -1; + } + st = _PySys_GetState(sysmod); + + assert(st->ntransformer == 0); + assert(st->transformers == NULL); + + optimizer = PyObject_CallFunction((PyObject *)&_PyPeepholeOptimizer_Type, NULL); + if (optimizer == NULL) + return -1; + + st->transformers = PyMem_Malloc(sizeof(st->transformers[0])); + if (st->transformers == NULL) { + PyErr_NoMemory(); + Py_DECREF(optimizer); + return -1; + } + + if (sys_init_code_transformer(&st->transformers[0], optimizer) < 0) { + Py_DECREF(optimizer); + return -1; + } + Py_DECREF(optimizer); + + st->ntransformer = 1; + return 0; +} + /* Updating the sys namespace, returning NULL pointer on error */ #define SET_SYS_FROM_STRING_BORROW(key, value) \ do { \ @@ -1926,9 +2283,13 @@ static struct PyModuleDef sysmodule = { PyObject * _PySys_BeginInit(void) { - PyObject *m, *sysdict, *version_info; + PyObject *m, *sysdict; int res; + /* Py_NewInterpreter() calls _PyImport_FindBuiltin("sys") which requires + * an init method because the sys module has a state. */ + sysmodule.m_base.m_init = _PySys_BeginInit; + m = PyModule_Create(&sysmodule); if (m == NULL) return NULL; @@ -2013,8 +2374,6 @@ _PySys_BeginInit(void) &version_info_desc) < 0) return NULL; } - version_info = make_version_info(); - SET_SYS_FROM_STRING("version_info", version_info); /* prevent user from creating new instances */ VersionInfoType.tp_init = NULL; VersionInfoType.tp_new = NULL; @@ -2022,9 +2381,6 @@ _PySys_BeginInit(void) if (res < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) PyErr_Clear(); - /* implementation */ - SET_SYS_FROM_STRING("implementation", make_impl_info(version_info)); - /* flags */ if (FlagsType.tp_name == 0) { if (PyStructSequence_InitType2(&FlagsType, &flags_desc) < 0) @@ -2143,6 +2499,21 @@ _PySys_EndInit(PyObject *sysdict) SET_SYS_FROM_STRING_BORROW_INT_RESULT("_xoptions", get_xoptions()); + /* FIXME: move version_info init back to _PySys_BeginInit, + get it from the dict */ + PyObject *version_info = make_version_info(); + SET_SYS_FROM_STRING_INT_RESULT("version_info", version_info); + + /* implementation */ + SET_SYS_FROM_STRING_INT_RESULT("implementation", make_impl_info(version_info)); + + if (!_Py_OptimTag + || PyUnicode_CompareWithASCIIString(_Py_OptimTag, "opt") == 0) { + if (set_peephole_optimizer() < 0) { + return -1; + } + } + if (PyErr_Occurred()) return -1; return 0;