Commit ddbe5e00 authored by mattip's avatar mattip

merge nditer-external_loop which implements numpy's nditer external_loop argument

parents d3bd39b5 b4d29788
......@@ -39,3 +39,7 @@ interested in background information.
.. branch: kill-multimethod
Kill multimethod machinery, all multimethods were removed earlier.
.. branch nditer-external_loop
Implement `external_loop` arguement to numpy's nditer
......@@ -449,7 +449,7 @@ class SliceArray(BaseConcreteArray):
strides.reverse()
backstrides.reverse()
new_shape.reverse()
return SliceArray(self.start, strides, backstrides, new_shape,
return self.__class__(self.start, strides, backstrides, new_shape,
self, orig_array)
new_strides = calc_new_strides(new_shape, self.get_shape(),
self.get_strides(),
......@@ -460,10 +460,16 @@ class SliceArray(BaseConcreteArray):
new_backstrides = [0] * len(new_shape)
for nd in range(len(new_shape)):
new_backstrides[nd] = (new_shape[nd] - 1) * new_strides[nd]
return SliceArray(self.start, new_strides, new_backstrides, new_shape,
return self.__class__(self.start, new_strides, new_backstrides, new_shape,
self, orig_array)
class NonWritableSliceArray(SliceArray):
def descr_setitem(self, space, orig_array, w_index, w_value):
raise OperationError(space.w_ValueError, space.wrap(
"assignment destination is read-only"))
class VoidBoxStorage(BaseConcreteArray):
def __init__(self, size, dtype):
self.storage = alloc_raw_storage(size)
......
......@@ -8,8 +8,8 @@ Given an array x: x.shape == [5,6], where each element occupies one byte
At which byte in x.data does the item x[3,4] begin?
if x.strides==[1,5]:
pData = x.pData + (x.start + 3*1 + 4*5)*sizeof(x.pData[0])
pData = x.pData + (x.start + 24) * sizeof(x.pData[0])
so the offset of the element is 24 elements after the first
pData = x.pData + (x.start + 23) * sizeof(x.pData[0])
so the offset of the element is 23 elements after the first
What is the next element in x after coordinates [3,4]?
if x.order =='C':
......@@ -33,7 +33,7 @@ shape dimension
which is x.strides[1] * (x.shape[1] - 1) + x.strides[0]
so if we precalculate the overflow backstride as
[x.strides[i] * (x.shape[i] - 1) for i in range(len(x.shape))]
we can go faster.
we can do only addition while iterating
All the calculations happen in next()
"""
from rpython.rlib import jit
......@@ -41,6 +41,16 @@ from pypy.module.micronumpy import support, constants as NPY
from pypy.module.micronumpy.base import W_NDimArray
from pypy.module.micronumpy.flagsobj import _update_contiguous_flags
class OpFlag(object):
def __init__(self):
self.rw = ''
self.broadcast = True
self.force_contig = False
self.force_align = False
self.native_byte_order = False
self.tmp_copy = ''
self.allocate = False
class PureShapeIter(object):
def __init__(self, shape, idx_w):
......@@ -89,11 +99,13 @@ class IterState(object):
class ArrayIter(object):
_immutable_fields_ = ['contiguous', 'array', 'size', 'ndim_m1', 'shape_m1[*]',
'strides[*]', 'backstrides[*]', 'factors[*]',
'track_index']
'slice_shape', 'slice_stride', 'slice_backstride',
'track_index', 'operand_type', 'slice_operand_type']
track_index = True
def __init__(self, array, size, shape, strides, backstrides):
def __init__(self, array, size, shape, strides, backstrides, op_flags=OpFlag()):
from pypy.module.micronumpy import concrete
assert len(shape) == len(strides) == len(backstrides)
_update_contiguous_flags(array)
self.contiguous = (array.flags & NPY.ARRAY_C_CONTIGUOUS and
......@@ -105,6 +117,12 @@ class ArrayIter(object):
self.shape_m1 = [s - 1 for s in shape]
self.strides = strides
self.backstrides = backstrides
self.slice_shape = 1
self.slice_stride = -1
if strides:
self.slice_stride = strides[-1]
self.slice_backstride = 1
self.slice_operand_type = concrete.SliceArray
ndim = len(shape)
factors = [0] * ndim
......@@ -114,6 +132,10 @@ class ArrayIter(object):
else:
factors[ndim-i-1] = factors[ndim-i] * shape[ndim-i]
self.factors = factors
if op_flags.rw == 'r':
self.operand_type = concrete.ConcreteNonWritableArrayWithBase
else:
self.operand_type = concrete.ConcreteArrayWithBase
@jit.unroll_safe
def reset(self, state=None):
......@@ -193,6 +215,12 @@ class ArrayIter(object):
assert state.iterator is self
self.array.setitem(state.offset, elem)
def getoperand(self, st, base):
impl = self.operand_type
res = impl([], self.array.dtype, self.array.order, [], [],
self.array.storage, base)
res.start = st.offset
return res
def AxisIter(array, shape, axis, cumulative):
strides = array.get_strides()
......@@ -216,3 +244,42 @@ def AllButAxisIter(array, axis):
size /= shape[axis]
shape[axis] = backstrides[axis] = 0
return ArrayIter(array, size, shape, array.strides, backstrides)
class SliceIter(ArrayIter):
'''
used with external loops, getitem and setitem return a SliceArray
view into the original array
'''
_immutable_fields_ = ['base', 'slice_shape[*]', 'slice_stride[*]', 'slice_backstride[*]']
def __init__(self, array, size, shape, strides, backstrides, slice_shape,
slice_stride, slice_backstride, op_flags, base):
from pypy.module.micronumpy import concrete
ArrayIter.__init__(self, array, size, shape, strides, backstrides, op_flags)
self.slice_shape = slice_shape
self.slice_stride = slice_stride
self.slice_backstride = slice_backstride
self.base = base
if op_flags.rw == 'r':
self.slice_operand_type = concrete.NonWritableSliceArray
else:
self.slice_operand_type = concrete.SliceArray
def getitem(self, state):
# XXX cannot be called - must return a boxed value
assert False
def getitem_bool(self, state):
# XXX cannot be called - must return a boxed value
assert False
def setitem(self, state, elem):
# XXX cannot be called - must return a boxed value
assert False
def getoperand(self, state, base):
assert state.iterator is self
impl = self.slice_operand_type
arr = impl(state.offset, [self.slice_stride], [self.slice_backstride],
[self.slice_shape], self.array, self.base)
return arr
......@@ -83,8 +83,12 @@ class __extend__(W_NDimArray):
raise OperationError(space.w_AttributeError, space.wrap(
"Cannot delete array dtype"))
def ndims(self):
return len(self.get_shape())
ndims._always_inline_ = True
def descr_get_ndim(self, space):
return space.wrap(len(self.get_shape()))
return space.wrap(self.ndims())
def descr_get_itemsize(self, space):
return space.wrap(self.get_dtype().elsize)
......@@ -103,14 +107,14 @@ class __extend__(W_NDimArray):
return space.wrap(loop.tostring(space, self))
def getitem_filter(self, space, arr):
if len(arr.get_shape()) > 1 and arr.get_shape() != self.get_shape():
if arr.ndims() > 1 and arr.get_shape() != self.get_shape():
raise OperationError(space.w_ValueError, space.wrap(
"boolean index array should have 1 dimension"))
if arr.get_size() > self.get_size():
raise OperationError(space.w_ValueError, space.wrap(
"index out of range for array"))
size = loop.count_all_true(arr)
if len(arr.get_shape()) == 1:
if arr.ndims() == 1:
res_shape = [size] + self.get_shape()[1:]
else:
res_shape = [size]
......@@ -119,7 +123,7 @@ class __extend__(W_NDimArray):
return loop.getitem_filter(w_res, self, arr)
def setitem_filter(self, space, idx, val):
if len(idx.get_shape()) > 1 and idx.get_shape() != self.get_shape():
if idx.ndims() > 1 and idx.get_shape() != self.get_shape():
raise OperationError(space.w_ValueError, space.wrap(
"boolean index array should have 1 dimension"))
if idx.get_size() > self.get_size():
......@@ -210,7 +214,7 @@ class __extend__(W_NDimArray):
if space.is_w(w_idx, space.w_Ellipsis):
return self
elif isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool() \
and len(w_idx.get_shape()) > 0:
and w_idx.ndims() > 0:
return self.getitem_filter(space, w_idx)
try:
return self.implementation.descr_getitem(space, self, w_idx)
......@@ -228,7 +232,7 @@ class __extend__(W_NDimArray):
self.implementation.setslice(space, convert_to_array(space, w_value))
return
elif isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool() \
and len(w_idx.get_shape()) > 0:
and w_idx.ndims() > 0:
self.setitem_filter(space, w_idx, convert_to_array(space, w_value))
return
try:
......@@ -289,7 +293,7 @@ class __extend__(W_NDimArray):
shape=shape, backward_broadcast=backward_broadcast)
def is_scalar(self):
return len(self.get_shape()) == 0
return self.ndims() == 0
def set_scalar_value(self, w_val):
return self.implementation.setitem(self.implementation.start, w_val)
......@@ -408,7 +412,7 @@ class __extend__(W_NDimArray):
"""
if axis1 == axis2:
return self
n = len(self.get_shape())
n = self.ndims()
if n <= 1:
return self
if axis1 < 0:
......@@ -426,7 +430,7 @@ class __extend__(W_NDimArray):
return self.implementation.nonzero(space, index_type)
def descr_tolist(self, space):
if len(self.get_shape()) == 0:
if self.ndims() == 0:
return self.get_scalar_value().item(space)
l_w = []
for i in range(self.get_shape()[0]):
......@@ -514,7 +518,7 @@ class __extend__(W_NDimArray):
if len(args_w) == 0:
raise OperationError(space.w_ValueError, space.wrap(
"itemset must have at least one argument"))
if len(args_w) != len(self.get_shape()) + 1:
if len(args_w) != self.ndims() + 1:
raise OperationError(space.w_ValueError, space.wrap(
"incorrect number of indices for array"))
self.descr_setitem(space, space.newtuple(args_w[:-1]), args_w[-1])
......@@ -647,14 +651,14 @@ class __extend__(W_NDimArray):
@unwrap_spec(offset=int, axis1=int, axis2=int)
def descr_diagonal(self, space, offset=0, axis1=0, axis2=1):
if len(self.get_shape()) < 2:
if self.ndims() < 2:
raise OperationError(space.w_ValueError, space.wrap(
"need at least 2 dimensions for diagonal"))
if (axis1 < 0 or axis2 < 0 or axis1 >= len(self.get_shape()) or
axis2 >= len(self.get_shape())):
if (axis1 < 0 or axis2 < 0 or axis1 >= self.ndims() or
axis2 >= self.ndims()):
raise oefmt(space.w_ValueError,
"axis1(=%d) and axis2(=%d) must be withing range "
"(ndim=%d)", axis1, axis2, len(self.get_shape()))
"(ndim=%d)", axis1, axis2, self.ndims())
if axis1 == axis2:
raise OperationError(space.w_ValueError, space.wrap(
"axis1 and axis2 cannot be the same"))
......@@ -733,7 +737,7 @@ class __extend__(W_NDimArray):
raise OperationError(space.w_NotImplementedError, space.wrap(
'sorter not supported in searchsort'))
side = searchside_converter(space, w_side)
if len(self.get_shape()) != 1:
if self.ndims() != 1:
raise oefmt(space.w_ValueError, "a must be a 1-d array")
v = convert_to_array(space, w_v)
ret = W_NDimArray.from_shape(
......@@ -972,7 +976,7 @@ class __extend__(W_NDimArray):
if other.is_scalar():
#Note: w_out is not modified, this is numpy compliant.
return self.descr_mul(space, other)
elif len(self.get_shape()) < 2 and len(other.get_shape()) < 2:
elif self.ndims() < 2 and other.ndims() < 2:
w_res = self.descr_mul(space, other)
assert isinstance(w_res, W_NDimArray)
return w_res.descr_sum(space, space.wrap(-1), out)
......@@ -989,7 +993,7 @@ class __extend__(W_NDimArray):
matches = False
elif not out.implementation.order == "C":
matches = False
elif len(out.get_shape()) != len(out_shape):
elif out.ndims() != len(out_shape):
matches = False
else:
for i in range(len(out_shape)):
......
This diff is collapsed.
......@@ -63,9 +63,6 @@ class AppTestNDIter(BaseNumpyAppTest):
from numpy import arange, nditer, array
a = arange(24).reshape(2, 3, 4)
import sys
if '__pypy__' in sys.builtin_module_names:
raises(NotImplementedError, nditer, a, flags=['external_loop'])
skip('nditer external_loop not implmented')
r = []
n = 0
for x in nditer(a, flags=['external_loop']):
......@@ -79,7 +76,9 @@ class AppTestNDIter(BaseNumpyAppTest):
r.append(x)
n += 1
assert n == 12
assert (array(r) == [[ 0, 12], [ 4, 16], [ 8, 20], [ 1, 13], [ 5, 17], [ 9, 21], [ 2, 14], [ 6, 18], [10, 22], [ 3, 15], [ 7, 19], [11, 23]]).all()
assert (array(r) == [[ 0, 12], [ 4, 16], [ 8, 20], [ 1, 13], [ 5, 17], [ 9, 21],
[ 2, 14], [ 6, 18], [10, 22], [ 3, 15], [ 7, 19], [11, 23],
]).all()
e = raises(ValueError, 'r[0][0] = 0')
assert str(e.value) == 'assignment destination is read-only'
r = []
......@@ -222,9 +221,6 @@ class AppTestNDIter(BaseNumpyAppTest):
def test_outarg(self):
from numpy import nditer, zeros, arange
import sys
if '__pypy__' in sys.builtin_module_names:
raises(NotImplementedError, nditer, [1, 2], flags=['external_loop'])
skip('nditer external_loop not implmented')
def square1(a):
it = nditer([a, None])
......@@ -233,6 +229,9 @@ class AppTestNDIter(BaseNumpyAppTest):
return it.operands[1]
assert (square1([1, 2, 3]) == [1, 4, 9]).all()
if '__pypy__' in sys.builtin_module_names:
raises(NotImplementedError, nditer, [1, 2], flags=['buffered'])
skip('nditer buffered not implmented')
def square2(a, out=None):
it = nditer([a, out], flags=['external_loop', 'buffered'],
op_flags=[['readonly'],
......@@ -252,10 +251,11 @@ class AppTestNDIter(BaseNumpyAppTest):
from numpy import nditer, arange
a = arange(3)
import sys
if '__pypy__' in sys.builtin_module_names:
raises(NotImplementedError, nditer, a, flags=['external_loop'])
skip('nditer external_loop not implmented')
b = arange(8).reshape(2,4)
if '__pypy__' in sys.builtin_module_names:
raises(NotImplementedError, nditer, [a, b, None], flags=['external_loop'],
op_axes=[[0, -1, -1], [-1, 0, 1], None])
skip('nditer op_axes not implemented yet')
it = nditer([a, b, None], flags=['external_loop'],
op_axes=[[0, -1, -1], [-1, 0, 1], None])
for x, y, z in it:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment