Commit 3c1786d6 authored by Richard Plangger's avatar Richard Plangger

catchup default

parents 3c2558b9 a4460bf8
......@@ -22,6 +22,7 @@ syntax: regexp
^pypy/module/cpyext/test/.+\.obj$
^pypy/module/cpyext/test/.+\.manifest$
^pypy/module/test_lib_pypy/ctypes_tests/.+\.o$
^pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test\.o$
^pypy/module/cppyy/src/.+\.o$
^pypy/module/cppyy/bench/.+\.so$
^pypy/module/cppyy/bench/.+\.root$
......@@ -35,7 +36,6 @@ syntax: regexp
^pypy/module/test_lib_pypy/cffi_tests/__pycache__.+$
^pypy/doc/.+\.html$
^pypy/doc/config/.+\.rst$
^pypy/doc/basicblock\.asc$
^pypy/doc/.+\.svninfo$
^rpython/translator/c/src/libffi_msvc/.+\.obj$
^rpython/translator/c/src/libffi_msvc/.+\.dll$
......@@ -45,53 +45,33 @@ syntax: regexp
^rpython/translator/c/src/cjkcodecs/.+\.obj$
^rpython/translator/c/src/stacklet/.+\.o$
^rpython/translator/c/src/.+\.o$
^rpython/translator/jvm/\.project$
^rpython/translator/jvm/\.classpath$
^rpython/translator/jvm/eclipse-bin$
^rpython/translator/jvm/src/pypy/.+\.class$
^rpython/translator/benchmark/docutils$
^rpython/translator/benchmark/templess$
^rpython/translator/benchmark/gadfly$
^rpython/translator/benchmark/mako$
^rpython/translator/benchmark/bench-custom\.benchmark_result$
^rpython/translator/benchmark/shootout_benchmarks$
^rpython/translator/llvm/.+\.so$
^rpython/translator/goal/target.+-c$
^rpython/translator/goal/.+\.exe$
^rpython/translator/goal/.+\.dll$
^pypy/goal/pypy-translation-snapshot$
^pypy/goal/pypy-c
^pypy/goal/pypy-jvm
^pypy/goal/pypy-jvm.jar
^pypy/goal/.+\.exe$
^pypy/goal/.+\.dll$
^pypy/goal/.+\.lib$
^pypy/_cache$
^pypy/doc/statistic/.+\.html$
^pypy/doc/statistic/.+\.eps$
^pypy/doc/statistic/.+\.pdf$
^rpython/translator/cli/src/pypylib\.dll$
^rpython/translator/cli/src/query\.exe$
^rpython/translator/cli/src/main\.exe$
^lib-python/2.7/lib2to3/.+\.pickle$
^lib_pypy/__pycache__$
^lib_pypy/ctypes_config_cache/_.+_cache\.py$
^lib_pypy/ctypes_config_cache/_.+_.+_\.py$
^lib_pypy/_libmpdec/.+.o$
^rpython/translator/cli/query-descriptions$
^pypy/doc/discussion/.+\.html$
^include/.+\.h$
^include/.+\.inl$
^pypy/doc/_build/.*$
^pypy/doc/config/.+\.html$
^pypy/doc/config/style\.css$
^pypy/doc/jit/.+\.html$
^pypy/doc/jit/style\.css$
^pypy/doc/image/lattice1\.png$
^pypy/doc/image/lattice2\.png$
^pypy/doc/image/lattice3\.png$
^pypy/doc/image/stackless_informal\.png$
^pypy/doc/image/parsing_example.+\.png$
^rpython/doc/_build/.*$
^pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test\.o$
^compiled
^.git/
^.hypothesis/
......
......@@ -41,29 +41,29 @@ copyrighted by one or more of the following people and organizations:
Amaury Forgeot d'Arc
Antonio Cuni
Samuele Pedroni
Matti Picus
Alex Gaynor
Brian Kearns
Matti Picus
Philip Jenvey
Michael Hudson
Ronan Lamy
David Schneider
Manuel Jacob
Holger Krekel
Christian Tismer
Hakan Ardo
Manuel Jacob
Ronan Lamy
Benjamin Peterson
Richard Plangger
Anders Chrigstrom
Eric van Riet Paap
Wim Lavrijsen
Richard Plangger
Richard Emslie
Alexander Schremmer
Dan Villiom Podlaski Christiansen
Remi Meier
Lukas Diekmann
Sven Hager
Anders Lehmann
Remi Meier
Aurelien Campeas
Niklaus Haldimann
Camillo Bruni
......@@ -72,8 +72,8 @@ copyrighted by one or more of the following people and organizations:
Romain Guillebert
Leonardo Santagada
Seo Sanghyeon
Justin Peel
Ronny Pfannschmidt
Justin Peel
David Edelsohn
Anders Hammarquist
Jakub Gustak
......@@ -95,6 +95,7 @@ copyrighted by one or more of the following people and organizations:
Tyler Wade
Michael Foord
Stephan Diehl
Vincent Legoll
Stefan Schwarzer
Valentino Volonghi
Tomek Meka
......@@ -105,9 +106,9 @@ copyrighted by one or more of the following people and organizations:
Jean-Paul Calderone
Timo Paulssen
Squeaky
Marius Gedminas
Alexandre Fayolle
Simon Burton
Marius Gedminas
Martin Matusiak
Konstantin Lopuhin
Wenzhu Man
......@@ -116,16 +117,20 @@ copyrighted by one or more of the following people and organizations:
Ivan Sichmann Freitas
Greg Price
Dario Bertini
Stefano Rivera
Mark Pearse
Simon Cross
Andreas Stührk
Stefano Rivera
Edd Barrett
Jean-Philippe St. Pierre
Guido van Rossum
Pavel Vinogradov
Jeremy Thurgood
Paweł Piotr Przeradowski
Spenser Bauman
Paul deGrandis
Ilya Osadchiy
marky1991
Tobias Oberstein
Adrian Kuhn
Boris Feigin
......@@ -134,14 +139,12 @@ copyrighted by one or more of the following people and organizations:
Georg Brandl
Bert Freudenberg
Stian Andreassen
Edd Barrett
Tobias Pape
Wanja Saatkamp
Gerald Klix
Mike Blume
Tobias Pape
Oscar Nierstrasz
Stefan H. Muller
Jeremy Thurgood
Rami Chowdhury
Eugene Oden
Henry Mason
......@@ -153,6 +156,8 @@ copyrighted by one or more of the following people and organizations:
Lukas Renggli
Guenter Jantzen
Ned Batchelder
Tim Felgentreff
Anton Gulenko
Amit Regmi
Ben Young
Nicolas Chauvat
......@@ -162,12 +167,12 @@ copyrighted by one or more of the following people and organizations:
Nicholas Riley
Jason Chu
Igor Trindade Oliveira
Tim Felgentreff
Yichao Yu
Rocco Moretti
Gintautas Miliauskas
Michael Twomey
Lucian Branescu Mihaila
Yichao Yu
Devin Jeanpierre
Gabriel Lavoie
Olivier Dormond
Jared Grubb
......@@ -191,33 +196,33 @@ copyrighted by one or more of the following people and organizations:
Stanislaw Halik
Mikael Schönenberg
Berkin Ilbeyi
Elmo M?ntynen
Elmo Mäntynen
Faye Zhao
Jonathan David Riehl
Anders Qvist
Corbin Simpson
Chirag Jadwani
Beatrice During
Alex Perry
Vincent Legoll
Vaibhav Sood
Alan McIntyre
Spenser Bauman
William Leslie
Alexander Sedov
Attila Gobi
Jasper.Schulz
Christopher Pope
Devin Jeanpierre
Vaibhav Sood
Christian Tismer
Marc Abramowitz
Dan Stromberg
Arjun Naik
Valentina Mukhamedzhanova
Stefano Parmesan
Mark Young
Alexis Daboville
Jens-Uwe Mager
Carl Meyer
Karl Ramm
Pieter Zieschang
Anton Gulenko
Gabriel
Lukas Vacek
Andrew Dalke
......@@ -225,6 +230,7 @@ copyrighted by one or more of the following people and organizations:
Jakub Stasiak
Nathan Taylor
Vladimir Kryachko
Omer Katz
Jacek Generowicz
Alejandro J. Cura
Jacob Oscarson
......@@ -239,6 +245,7 @@ copyrighted by one or more of the following people and organizations:
Lars Wassermann
Philipp Rustemeuer
Henrik Vendelbo
Richard Lancaster
Dan Buch
Miguel de Val Borro
Artur Lisiecki
......@@ -250,18 +257,18 @@ copyrighted by one or more of the following people and organizations:
Tomo Cocoa
Kim Jin Su
Toni Mattis
Amber Brown
Lucas Stadler
Julian Berman
Markus Holtermann
roberto@goyle
Yury V. Zaytsev
Anna Katrina Dominguez
William Leslie
Bobby Impollonia
Faye Zhao
timo@eistee.fritz.box
Andrew Thompson
Yusei Tahara
Aaron Tubbs
Ben Darnell
Roberto De Ioris
Juan Francisco Cantero Hurtado
......@@ -273,6 +280,7 @@ copyrighted by one or more of the following people and organizations:
Christopher Armstrong
Michael Hudson-Doyle
Anders Sigfridsson
Nikolay Zinov
Yasir Suhail
Jason Michalski
rafalgalczynski@gmail.com
......@@ -282,6 +290,7 @@ copyrighted by one or more of the following people and organizations:
Gustavo Niemeyer
Stephan Busemann
Rafał Gałczyński
Matt Bogosian
Christian Muirhead
Berker Peksag
James Lan
......@@ -316,9 +325,9 @@ copyrighted by one or more of the following people and organizations:
Stefan Marr
jiaaro
Mads Kiilerich
Richard Lancaster
opassembler.py
Antony Lee
Jason Madden
Yaroslav Fedevych
Jim Hunziker
Markus Unterwaditzer
......@@ -327,6 +336,7 @@ copyrighted by one or more of the following people and organizations:
squeaky
Zearin
soareschen
Jonas Pfannschmidt
Kurt Griffiths
Mike Bayer
Matthew Miller
......
......@@ -7,6 +7,7 @@ def get_hashed_dir(cfile):
content = fid.read()
# from cffi's Verifier()
key = '\x00'.join([sys.version[:3], content])
key += 'cpyext-gc-support-2' # this branch requires recompilation!
if sys.version_info >= (3,):
key = key.encode('utf-8')
k1 = hex(binascii.crc32(key[0::2]) & 0xffffffff)
......@@ -62,7 +63,7 @@ def compile_shared(csource, modulename, output_dir=None):
if sys.platform == 'win32':
# XXX pyconfig.h uses a pragma to link to the import library,
# which is currently python27.lib
library = os.path.join(thisdir, '..', 'include', 'python27')
library = os.path.join(thisdir, '..', 'libs', 'python27')
if not os.path.exists(library + '.lib'):
# For a local translation or nightly build
library = os.path.join(thisdir, '..', 'pypy', 'goal', 'python27')
......
......@@ -550,6 +550,7 @@ class FFI(object):
lst.append(value)
#
if '__pypy__' in sys.builtin_module_names:
import os
if sys.platform == "win32":
# we need 'libpypy-c.lib'. Current distributions of
# pypy (>= 4.1) contain it as 'libs/python27.lib'.
......@@ -558,11 +559,15 @@ class FFI(object):
ensure('library_dirs', os.path.join(sys.prefix, 'libs'))
else:
# we need 'libpypy-c.{so,dylib}', which should be by
# default located in 'sys.prefix/bin'
# default located in 'sys.prefix/bin' for installed
# systems.
pythonlib = "pypy-c"
if hasattr(sys, 'prefix'):
import os
ensure('library_dirs', os.path.join(sys.prefix, 'bin'))
# On uninstalled pypy's, the libpypy-c is typically found in
# .../pypy/goal/.
if hasattr(sys, 'prefix'):
ensure('library_dirs', os.path.join(sys.prefix, 'pypy', 'goal'))
else:
if sys.platform == "win32":
template = "python%d%d"
......
......@@ -36,11 +36,16 @@ working_modules.update([
"cStringIO", "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
"binascii", "_multiprocessing", '_warnings', "_collections",
"_multibytecodec", "micronumpy", "_continuation", "_cffi_backend",
"_csv", "cppyy", "_pypyjson", "_vmprof",
"_csv", "cppyy", "_pypyjson",
])
if os.uname()[4] == 's390x':
working_modules.remove("_vmprof")
from rpython.jit.backend import detect_cpu
try:
if detect_cpu.autodetect().startswith('x86'):
working_modules.add('_vmprof')
except detect_cpu.ProcessorAutodetectError:
pass
translation_modules = default_modules.copy()
translation_modules.update([
......@@ -165,12 +170,8 @@ pypy_optiondescription = OptionDescription("objspace", "Object Space Options", [
cmdline="--translationmodules",
suggests=[("objspace.allworkingmodules", False)]),
BoolOption("usepycfiles", "Write and read pyc files when importing",
default=True),
BoolOption("lonepycfiles", "Import pyc files with no matching py file",
default=False,
requires=[("objspace.usepycfiles", True)]),
default=False),
StrOption("soabi",
"Tag to differentiate extension modules built for different Python interpreters",
......
======================
Rawrefcount and the GC
======================
GC Interface
------------
"PyObject" is a raw structure with at least two fields, ob_refcnt and
ob_pypy_link. The ob_refcnt is the reference counter as used on
CPython. If the PyObject structure is linked to a live PyPy object,
its current address is stored in ob_pypy_link and ob_refcnt is bumped
by either the constant REFCNT_FROM_PYPY, or the constant
REFCNT_FROM_PYPY_LIGHT (== REFCNT_FROM_PYPY + SOME_HUGE_VALUE)
(to mean "light finalizer").
Most PyPy objects exist outside cpyext, and conversely in cpyext it is
possible that a lot of PyObjects exist without being seen by the rest
of PyPy. At the interface, however, we can "link" a PyPy object and a
PyObject. There are two kinds of link:
rawrefcount.create_link_pypy(p, ob)
Makes a link between an exising object gcref 'p' and a newly
allocated PyObject structure 'ob'. ob->ob_refcnt must be
initialized to either REFCNT_FROM_PYPY, or
REFCNT_FROM_PYPY_LIGHT. (The second case is an optimization:
when the GC finds the PyPy object and PyObject no longer
referenced, it can just free() the PyObject.)
rawrefcount.create_link_pyobj(p, ob)
Makes a link from an existing PyObject structure 'ob' to a newly
allocated W_CPyExtPlaceHolderObject 'p'. You must also add
REFCNT_FROM_PYPY to ob->ob_refcnt. For cases where the PyObject
contains all the data, and the PyPy object is just a proxy. The
W_CPyExtPlaceHolderObject should have only a field that contains
the address of the PyObject, but that's outside the scope of the
GC.
rawrefcount.from_obj(p)
If there is a link from object 'p' made with create_link_pypy(),
returns the corresponding 'ob'. Otherwise, returns NULL.
rawrefcount.to_obj(Class, ob)
Returns ob->ob_pypy_link, cast to an instance of 'Class'.
Collection logic
----------------
Objects existing purely on the C side have ob->ob_pypy_link == 0;
these are purely reference counted. On the other hand, if
ob->ob_pypy_link != 0, then ob->ob_refcnt is at least REFCNT_FROM_PYPY
and the object is part of a "link".
The idea is that links whose 'p' is not reachable from other PyPy
objects *and* whose 'ob->ob_refcnt' is REFCNT_FROM_PYPY or
REFCNT_FROM_PYPY_LIGHT are the ones who die. But it is more messy
because PyObjects still (usually) need to have a tp_dealloc called,
and this cannot occur immediately (and can do random things like
accessing other references this object points to, or resurrecting the
object).
Let P = list of links created with rawrefcount.create_link_pypy()
and O = list of links created with rawrefcount.create_link_pyobj().
The PyPy objects in the list O are all W_CPyExtPlaceHolderObject: all
the data is in the PyObjects, and all outsite references (if any) are
in C, as "PyObject *" fields.
So, during the collection we do this about P links:
for (p, ob) in P:
if ob->ob_refcnt != REFCNT_FROM_PYPY
and ob->ob_refcnt != REFCNT_FROM_PYPY_LIGHT:
mark 'p' as surviving, as well as all its dependencies
At the end of the collection, the P and O links are both handled like
this:
for (p, ob) in P + O:
if p is not surviving: # even if 'ob' might be surviving
unlink p and ob
if ob->ob_refcnt == REFCNT_FROM_PYPY_LIGHT:
free(ob)
elif ob->ob_refcnt > REFCNT_FROM_PYPY_LIGHT:
ob->ob_refcnt -= REFCNT_FROM_PYPY_LIGHT
else:
ob->ob_refcnt -= REFCNT_FROM_PYPY
if ob->ob_refcnt == 0:
invoke _Py_Dealloc(ob) later, outside the GC
GC Implementation
-----------------
We need two copies of both the P list and O list, for young or old
objects. All four lists can be regular AddressLists of 'ob' objects.
We also need an AddressDict mapping 'p' to 'ob' for all links in the P
list, and update it when PyPy objects move.
Further notes
-------------
XXX
XXX the rest is the ideal world, but as a first step, we'll look
XXX for the minimal tweaks needed to adapt the existing cpyext
XXX
For objects that are opaque in CPython, like <dict>, we always create
a PyPy object, and then when needed we make an empty PyObject and
attach it with create_link_pypy()/REFCNT_FROM_PYPY_LIGHT.
For <int> and <float> objects, the corresponding PyObjects contain a
"long" or "double" field too. We link them with create_link_pypy()
and we can use REFCNT_FROM_PYPY_LIGHT too: 'tp_dealloc' doesn't
need to be called, and instead just calling free() is fine.
For <type> objects, we need both a PyPy and a PyObject side. These
are made with create_link_pypy()/REFCNT_FROM_PYPY.
For custom PyXxxObjects allocated from the C extension module, we
need create_link_pyobj().
For <str> or <unicode> objects coming from PyPy, we use
create_link_pypy()/REFCNT_FROM_PYPY_LIGHT with a PyObject
preallocated with the size of the string. We copy the string
lazily into that area if PyString_AS_STRING() is called.
For <str>, <unicode>, <tuple> or <list> objects in the C extension
module, we first allocate it as only a PyObject, which supports
mutation of the data from C, like CPython. When it is exported to
PyPy we could make a W_CPyExtPlaceHolderObject with
create_link_pyobj().
For <tuple> objects coming from PyPy, if they are not specialized,
then the PyPy side holds a regular reference to the items. Then we
can allocate a PyTupleObject and store in it borrowed PyObject
pointers to the items. Such a case is created with
create_link_pypy()/REFCNT_FROM_PYPY_LIGHT. If it is specialized,
then it doesn't work because the items are created just-in-time on the
PyPy side. In this case, the PyTupleObject needs to hold real
references to the PyObject items, and we use create_link_pypy()/
REFCNT_FROM_PYPY. In all cases, we have a C array of PyObjects
that we can directly return from PySequence_Fast_ITEMS, PyTuple_ITEMS,
PyTuple_GetItem, and so on.
For <list> objects coming from PyPy, we can use a cpyext list
strategy. The list turns into a PyListObject, as if it had been
allocated from C in the first place. The special strategy can hold
(only) a direct reference to the PyListObject, and we can use either
create_link_pyobj() or create_link_pypy() (to be decided).
PySequence_Fast_ITEMS then works for lists too, and PyList_GetItem
can return a borrowed reference, and so on.
Making a PyPy Release
=====================
The PyPy Release Process
========================
Release Policy
++++++++++++++
We try to create a stable release a few times a year. These are released on
a branch named like release-2.x or release-4.x, and each release is tagged,
for instance release-4.0.1.
After release, inevitably there are bug fixes. It is the responsibility of
the commiter who fixes a bug to make sure this fix is on the release branch,
so that we can then create a tagged bug-fix release, which will hopefully
happen more often than stable releases.
How to Create a PyPy Release
++++++++++++++++++++++++++++
Overview
--------
......
......@@ -72,6 +72,7 @@ alias = {
'Anton Gulenko':['anton gulenko', 'anton_gulenko'],
'Richard Lancaster':['richardlancaster'],
'William Leslie':['William ML Leslie'],
'Spenser Bauman':['Spenser Andrew Bauman'],
}
alias_map = {}
......
......@@ -153,6 +153,37 @@ Refactor vmprof to work cross-operating-system.
Seperate structmember.h from Python.h Also enhance creating api functions
to specify which header file they appear in (previously only pypy_decl.h)
.. branch: llimpl
Refactor register_external(), remove running_on_llinterp mechanism and
apply sandbox transform on externals at the end of annotation.
.. branch: cffi-embedding-win32
.. branch: windows-vmprof-support
vmprof should work on Windows.
.. branch: reorder-map-attributes
When creating instances and adding attributes in several different orders
depending on some condition, the JIT would create too much code. This is now
fixed.
.. branch: cpyext-gc-support-2
Improve CPython C API support, which means lxml now runs unmodified
(after removing pypy hacks, pending pull request)
.. branch: look-inside-tuple-hash
Look inside tuple hash, improving mdp benchmark
.. branch: vlen-resume
Compress resume data, saving 10-20% of memory consumed by the JIT
.. branch: memop-simplify3
Further simplifying the backend operations malloc_cond_varsize and zero_array.
......
......@@ -277,7 +277,6 @@ class PyPyTarget(object):
if config.translation.sandbox:
config.objspace.lonepycfiles = False
config.objspace.usepycfiles = False
config.translating = True
......
......@@ -27,7 +27,7 @@ unpackiterable_driver = jit.JitDriver(name='unpackiterable',
class W_Root(object):
"""This is the abstract root class of all wrapped objects that live
in a 'normal' object space like StdObjSpace."""
__slots__ = ()
__slots__ = ('__weakref__',)
user_overridden_class = False
def getdict(self, space):
......
......@@ -91,6 +91,7 @@ def generate_tokens(lines, flags):
strstart = (0, 0, "")
for line in lines:
lnum = lnum + 1
line = universal_newline(line)
pos, max = 0, len(line)
if contstr:
......@@ -259,3 +260,14 @@ def generate_tokens(lines, flags):
token_list.append((tokens.ENDMARKER, '', lnum, pos, line))
return token_list
def universal_newline(line):
# show annotator that indexes below are non-negative
line_len_m2 = len(line) - 2
if line_len_m2 >= 0 and line[-2] == '\r' and line[-1] == '\n':
return line[:line_len_m2] + '\n'
line_len_m1 = len(line) - 1
if line_len_m1 >= 0 and line[-1] == '\r':
return line[:line_len_m1] + '\n'
return line
......@@ -158,3 +158,10 @@ pass"""
def test_print_function(self):
self.parse("from __future__ import print_function\nx = print\n")
def test_universal_newlines(self):
fmt = 'stuff = """hello%sworld"""'
expected_tree = self.parse(fmt % '\n')
for linefeed in ["\r\n","\r"]:
tree = self.parse(fmt % linefeed)
assert expected_tree == tree
......@@ -156,20 +156,6 @@ def get_unique_interplevel_subclass(config, cls, hasdict, wants_slots,
get_unique_interplevel_subclass._annspecialcase_ = "specialize:memo"
_subclass_cache = {}
def enum_interplevel_subclasses(config, cls):
"""Return a list of all the extra interp-level subclasses of 'cls' that
can be built by get_unique_interplevel_subclass()."""
result = []
for flag1 in (False, True):
for flag2 in (False, True):
for flag3 in (False, True):
for flag4 in (False, True):
result.append(get_unique_interplevel_subclass(
config, cls, flag1, flag2, flag3, flag4))
result = dict.fromkeys(result)
assert len(result) <= 6
return result.keys()
def _getusercls(config, cls, wants_dict, wants_slots, wants_del, weakrefable):
typedef = cls.typedef
if wants_dict and typedef.hasdict:
......@@ -262,7 +248,7 @@ def _builduserclswithfeature(config, supercls, *features):
def user_setup(self, space, w_subtype):
self.space = space
self.w__class__ = w_subtype
self.user_setup_slots(w_subtype.nslots)
self.user_setup_slots(w_subtype.layout.nslots)
def user_setup_slots(self, nslots):
assert nslots == 0
......
......@@ -57,7 +57,7 @@ def patch_sys(space):
# pypy_init_embedded_cffi_module().
if not glob.patched_sys:
space.appexec([], """():
import os
import os, sys
sys.stdin = sys.__stdin__ = os.fdopen(0, 'rb', 0)
sys.stdout = sys.__stdout__ = os.fdopen(1, 'wb', 0)
sys.stderr = sys.__stderr__ = os.fdopen(2, 'wb', 0)
......