001/*-
002 *******************************************************************************
003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
004 * All rights reserved. This program and the accompanying materials
005 * are made available under the terms of the Eclipse Public License v1.0
006 * which accompanies this distribution, and is available at
007 * http://www.eclipse.org/legal/epl-v10.html
008 *
009 * Contributors:
010 *    Peter Chang - initial API and implementation and/or initial documentation
011 *******************************************************************************/
012
013package org.eclipse.january.dataset;
014
015import java.io.Serializable;
016import java.lang.reflect.Array;
017import java.text.Format;
018import java.util.Arrays;
019import java.util.List;
020
021import org.eclipse.january.DatasetException;
022import org.eclipse.january.IMonitor;
023import org.eclipse.january.MetadataException;
024import org.eclipse.january.metadata.ErrorMetadata;
025import org.eclipse.january.metadata.MetadataFactory;
026import org.eclipse.january.metadata.StatisticsMetadata;
027import org.eclipse.january.metadata.internal.ErrorMetadataImpl;
028import org.eclipse.january.metadata.internal.StatisticsMetadataImpl;
029import org.slf4j.Logger;
030import org.slf4j.LoggerFactory;
031
032/**
033 * Generic container class for data 
034 * <p>
035 * Each subclass has an array of primitive types, elements of this array are grouped or
036 * compounded to make items 
037 * <p>
038 * Data items can be boolean, integer, float, complex float, vector float, etc
039 */
040public abstract class AbstractDataset extends LazyDatasetBase implements Dataset {
041        // pin UID to base class
042        private static final long serialVersionUID = Dataset.serialVersionUID;
043
044        private static final Logger logger = LoggerFactory.getLogger(AbstractDataset.class);
045
046        protected int size; // number of items
047
048        transient protected AbstractDataset base; // is null when not a view
049        protected int[] stride; // can be null for row-major, contiguous datasets
050        protected int offset;
051
052        /**
053         * The data itself, held in a 1D array, but the object will wrap it to appear as possessing as many dimensions as
054         * wanted
055         */
056        protected Serializable odata = null;
057
058        /**
059         * Set aliased data as base data
060         */
061        abstract protected void setData();
062
063        /**
064         * Constructor required for serialisation.
065         */
066        public AbstractDataset() {
067        }
068
069        @Override
070        public synchronized Dataset synchronizedCopy() {
071                return clone();
072        }
073
074        @Override
075        public Class<?> getElementClass() {
076                return InterfaceUtils.getElementClass(getClass());
077        }
078
079        @Override
080        public int getDType() {
081                return DTypeUtils.getDType(getClass());
082        }
083
084        @Override
085        public int hashCode() {
086                return getStats().getHash(shape);
087        }
088
089        @Override
090        abstract public AbstractDataset clone();
091
092        protected Format stringFormat = null;
093
094        @Override
095        public void setStringFormat(Format format) {
096                stringFormat = format;
097        }
098
099        @Override
100        public Dataset copy(final int dtype) {
101                return copy(DTypeUtils.getInterface(dtype));
102        }
103
104        @Override
105        public <T extends Dataset> T copy(Class<T> clazz) {
106                return DatasetUtils.copy(clazz, this);
107        }
108
109        @Override
110        public Dataset cast(final int dtype) {
111                return cast(DTypeUtils.getInterface(dtype));
112        }
113
114        @SuppressWarnings("unchecked")
115        @Override
116        public <T extends Dataset> T cast(Class<T> clazz) {
117                if (clazz.isInstance(this)) {
118                        return (T) this;
119                }
120                return DatasetUtils.cast(clazz, this);
121        }
122
123        @SuppressWarnings("unchecked")
124        @Override
125        public <T extends Dataset> T cast(final int isize, Class<T> clazz, final boolean repeat) {
126                if (clazz.isInstance(this) && getElementsPerItem() == isize) {
127                        return (T) this;
128                }
129                return DatasetUtils.cast(isize, clazz, this, repeat);
130        }
131
132        @Override
133        public Dataset cast(final boolean repeat, final int dtype, final int isize) {
134                return cast(isize, DTypeUtils.getInterface(dtype), repeat);
135        }
136
137        @Override
138        abstract public AbstractDataset getView(boolean deepCopyMetadata);
139
140        /**
141         * Copy fields from original to view
142         * @param orig original
143         * @param view destination
144         * @param clone if true, then clone everything but bulk data
145         * @param cloneMetadata if true, clone metadata
146         */
147        protected static void copyToView(Dataset orig, AbstractDataset view, boolean clone, boolean cloneMetadata) {
148                view.name = orig.getName();
149                view.size = orig.getSize();
150                view.odata = orig.getBuffer();
151                view.offset = orig.getOffset();
152                AbstractDataset a = orig instanceof AbstractDataset ? (AbstractDataset) orig : null;
153                view.base = a == null ? null : a.base;
154
155                int[] s = a == null ? null : a.stride;
156                if (clone) {
157                        view.shape = orig.getShapeRef() == null ? null : orig.getShape();
158                        view.stride = s == null ? null : s.clone();
159                } else {
160                        view.shape = orig.getShapeRef();
161                        view.stride = s;
162                }
163
164                view.metadata = getMetadataMap(orig, cloneMetadata);
165                Class<? extends Dataset> oc = InterfaceUtils.findSubInterface(orig.getClass());
166                Class<? extends Dataset> vc = InterfaceUtils.findSubInterface(view.getClass());
167                if (!oc.equals(vc)) {
168                        view.setDirty();
169                }
170        }
171
172        @Override
173        public IntegerDataset getIndices() {
174                final IntegerDataset ret = DatasetUtils.indices(shape);
175                if (getName() != null) {
176                        ret.setName("Indices of " + getName());
177                }
178                return ret;
179        }
180
181        @Override
182        public Dataset getTransposedView(int... axes) {
183                axes = checkPermutatedAxes(shape, axes);
184
185                AbstractDataset t = getView(true);
186                if (axes == null || getRank() == 1)
187                        return t;
188
189                int rank = shape.length;
190                int[] tstride = new int[rank];
191                int[] toffset = new int[1];
192                int[] nshape = createStrides(new SliceND(shape), this, tstride, toffset);
193                int[] nstride = new int[rank];
194                for (int i = 0; i < rank; i++) {
195                        final int ax = axes[i];
196                        nstride[i] = tstride[ax];
197                        nshape[i] = shape[ax];
198                }
199                t.shape = nshape;
200                t.stride = nstride;
201                t.offset = toffset[0];
202                t.base = this;
203                t.setDirty();
204                t.transposeMetadata(axes);
205                return t;
206        }
207
208        @Override
209        public Dataset transpose(int... axes) {
210                Dataset t = getTransposedView(axes);
211                return t == null ? clone() : t.clone();
212        }
213
214        @Override
215        public Dataset swapAxes(int axis1, int axis2) {
216                int rank = shape.length;
217                axis1 = ShapeUtils.checkAxis(rank, axis1);
218                axis2 = ShapeUtils.checkAxis(rank, axis2);
219
220                if (rank == 1 || axis1 == axis2) {
221                        return this;
222                }
223
224                int[] axes = new int[rank];
225                for (int i = 0; i < rank; i++) {
226                        axes[i] = i;
227                }               
228
229                axes[axis1] = axis2;
230                axes[axis2] = axis1;
231                return getTransposedView(axes);
232        }
233
234        boolean isContiguous() {
235                if (stride == null)
236                        return true;
237
238                if (offset != 0)
239                        return false;
240
241                int s = getElementsPerItem();
242                for (int j = getRank() - 1; j >= 0; j--) {
243                        if (stride[j] != s) {
244                                return false;
245                        }
246                        s *= shape[j];
247                }
248
249                return true;
250        }
251
252        @Override
253        public Dataset flatten() {
254                if (!isContiguous()) { // need to make a copy if not contiguous
255                        return clone().flatten();
256                }
257                return reshape(size);
258        }
259
260        /**
261         * Fill dataset from object at depth dimension
262         * @param obj fill value
263         * @param depth dimension
264         * @param pos position
265         */
266        protected void fillData(Object obj, final int depth, final int[] pos) {
267                if (obj == null) {
268                        Class<?> c = InterfaceUtils.getElementClass(getClass());
269                        if (Float.class.equals(c)) {
270                                set(Float.NaN, pos);
271                        } else if (Float.class.equals(c)) {
272                                set(Double.NaN, pos);
273                        }
274                        return;
275                }
276
277                Class<?> clazz = obj.getClass();
278                if (obj instanceof List<?>) {
279                        List<?> jl = (List<?>) obj;
280                        int l = jl.size();
281                        for (int i = 0; i < l; i++) {
282                                Object lo = jl.get(i);
283                                fillData(lo, depth + 1, pos);
284                                pos[depth]++;
285                        }
286                        pos[depth] = 0;
287                } else if (clazz.isArray()) {
288                        int l = Array.getLength(obj);
289                        if (clazz.equals(odata.getClass())) {
290                                System.arraycopy(obj, 0, odata, get1DIndex(pos), l);
291                        } else if (clazz.getComponentType().isPrimitive()) {
292                                for (int i = 0; i < l; i++) {
293                                        set(Array.get(obj, i), pos);
294                                        pos[depth]++;
295                                }
296                                pos[depth] = 0;
297                        } else {
298                                for (int i = 0; i < l; i++) {
299                                        fillData(Array.get(obj, i), depth + 1, pos);
300                                        pos[depth]++;
301                                }
302                                pos[depth] = 0;
303                        }
304                } else if (obj instanceof IDataset) {
305                        boolean[] a = new boolean[shape.length];
306                        Arrays.fill(a, depth, a.length, true);
307                        setSlice(obj, getSliceIteratorFromAxes(pos, a));
308                } else {
309                        set(obj, pos);
310                }
311        }
312
313        @Override
314        public IndexIterator getIterator(final boolean withPosition) {
315                if (stride != null) {
316                        return base.getSize() == 1  ? (withPosition ? new PositionIterator(offset, shape) :
317                                new SingleItemIterator(offset, size)) : new StrideIterator(shape, stride, offset);
318                }
319                if (shape == null) {
320                        return new NullIterator();
321                }
322
323                return withPosition ? new ContiguousIteratorWithPosition(shape, size) : new ContiguousIterator(size);
324        }
325
326        @Override
327        public IndexIterator getIterator() {
328                return getIterator(false);
329        }
330
331        @Override
332        public PositionIterator getPositionIterator(final int... axes) {
333                return new PositionIterator(shape, axes);
334        }
335
336        @Override
337        public IndexIterator getSliceIterator(final int[] start, final int[] stop, final int[] step) {
338                return internalGetSliceIterator(new SliceND(shape, start, stop, step));
339        }
340
341        /**
342         * @param slice an n-D slice
343         * @return an slice iterator that operates like an IndexIterator
344         */
345        public IndexIterator getSliceIterator(SliceND slice) {
346                if (slice != null) {
347                        checkSliceND(slice);
348                }
349                return internalGetSliceIterator(slice);
350        }
351
352        /**
353         * @param slice an n-D slice
354         * @return an slice iterator that operates like an IndexIterator
355         */
356        protected IndexIterator internalGetSliceIterator(SliceND slice) {
357                if (ShapeUtils.calcLongSize(slice.getShape()) == 0) {
358                        return new NullIterator(shape, slice.getShape());
359                }
360                if (stride != null) {
361                        return new StrideIterator(getElementsPerItem(), shape, stride, offset, slice);
362                }
363                return new SliceIterator(shape, size, slice);
364        }
365
366        @Override
367        public SliceIterator getSliceIteratorFromAxes(final int[] pos, boolean[] axes) {
368                int rank = shape.length;
369                int[] start;
370                int[] stop = new int[rank];
371                int[] step = new int[rank];
372
373                if (pos == null) {
374                        start = new int[rank];
375                } else if (pos.length == rank) {
376                        start = pos.clone();
377                } else {
378                        throw new IllegalArgumentException("pos array length is not equal to rank of dataset");
379                }
380                if (axes == null) {
381                        axes = new boolean[rank];
382                        Arrays.fill(axes, true);
383                } else if (axes.length != rank) {
384                        throw new IllegalArgumentException("axes array length is not equal to rank of dataset");
385                }
386
387                for (int i = 0; i < rank; i++) {
388                        if (axes[i]) {
389                                stop[i] = shape[i];
390                        } else {
391                                stop[i] = start[i] + 1;
392                        }
393                        step[i] = 1;
394                }
395                return (SliceIterator) getSliceIterator(start, stop, step);
396        }
397
398        @Override
399        public BooleanIterator getBooleanIterator(Dataset choice) {
400                return getBooleanIterator(choice, true);
401        }
402
403        @Override
404        public BooleanIterator getBooleanIterator(Dataset choice, boolean value) {
405                return BooleanIterator.createIterator(value, this, choice, this);
406        }
407
408        @Override
409        public Dataset getByBoolean(Dataset selection) {
410                checkCompatibility(selection);
411
412                final int length = ((Number) selection.sum()).intValue();
413                final int is = getElementsPerItem();
414                Dataset r = DatasetFactory.zeros(is, getClass(), length);
415                BooleanIterator biter = getBooleanIterator(selection);
416
417                int i = 0;
418                while (biter.hasNext()) {
419                        r.setObjectAbs(i, getObjectAbs(biter.index));
420                        i += is;
421                }
422                return r;
423        }
424
425        @Override
426        public Dataset getBy1DIndex(IntegerDataset index) {
427                final int is = getElementsPerItem();
428                final Dataset r = DatasetFactory.zeros(is, getClass(), index.getShapeRef());
429                final IntegerIterator iter = new IntegerIterator(index, size, is);
430
431                int i = 0;
432                while (iter.hasNext()) {
433                        r.setObjectAbs(i, getObjectAbs(iter.index));
434                        i += is;
435                }
436                return r;
437        }
438
439        @Override
440        public Dataset getByIndexes(final Object... indexes) {
441                final IntegersIterator iter = new IntegersIterator(shape, indexes);
442                final int is = getElementsPerItem();
443                final Dataset r = DatasetFactory.zeros(is, getClass(), iter.getShape());
444
445                final int[] pos = iter.getPos();
446                int i = 0;
447                while (iter.hasNext()) {
448                        r.setObjectAbs(i, getObject(pos));
449                        i += is;
450                }
451                return r;
452        }
453
454        @Override
455        public boolean hasFloatingPointElements() {
456                Class<?> cls = getElementClass();
457                return cls == Float.class || cls == Double.class;
458        }
459
460        @Override
461        public int getElementsPerItem() {
462                return InterfaceUtils.getElementsPerItem(getClass());
463        }
464
465        @Override
466        public int getItemBytes() {
467                return InterfaceUtils.getItemBytes(getElementsPerItem(), getClass());
468        }
469
470        @Override
471        public int getSize() {
472                return size;
473        }
474
475        @Override
476        public int[] getShape() {
477                // make a copy of the dimensions data, and put that out
478                if (shape == null) {
479                        logger.warn("Shape is null!!!");
480                        return new int[] {};
481                }
482                return shape.clone();
483        }
484
485        @Override
486        public int getRank() {
487                return shape == null ? 0 : shape.length;
488        }
489
490        @Override
491        public int getNbytes() {
492                return getSize() * getItemBytes();
493        }
494
495        /**
496         * Check for -1 placeholder in shape and replace if necessary
497         * @param shape to use
498         * @param size expected size
499         */
500        private void checkShape(int[] shape, int size) {
501                if (shape == null) {
502                        if (size == 0) {
503                                return;
504                        }
505                        logger.error("New shape must not be null for nonzero-sized dataset");
506                        throw new IllegalArgumentException("New shape must not be null for nonzero-sized dataset");
507                }
508                int rank = shape.length;
509                int found = -1;
510                int nsize = 1;
511                for (int i = 0; i < rank; i++) {
512                        int d = shape[i];
513                        if (d == -1) {
514                                if (found == -1) {
515                                        found = i;
516                                } else {
517                                        logger.error("Can only have one -1 placeholder in shape");
518                                        throw new IllegalArgumentException("Can only have one -1 placeholder in shape");
519                                }
520                        } else {
521                                nsize *= d;
522                        }
523                }
524                if (found >= 0) {
525                        shape[found] = size/nsize;
526                } else if (nsize != size && !(rank == 0 && size == 0)) {
527                        logger.error("New shape is not same size as old shape");
528                        throw new IllegalArgumentException("New size is not same as the old size. Old size is "+size+" new size is "+nsize+" and shape is "+Arrays.toString(shape));
529                }
530        }
531
532        @Override
533        public void setShape(final int... shape) {
534                int[] nshape = shape == null ? null : shape.clone();
535                checkShape(nshape, size);
536                if (Arrays.equals(this.shape, nshape)) {
537                        return;
538                }
539
540                if (stride != null) {
541                        // the only compatible shapes are ones where new dimensions are factors of old dimensions
542                        // or are combined adjacent old dimensions 
543                        int[] oshape = this.shape;
544                        int orank = oshape.length;
545                        int nrank = nshape.length;
546                        int diff = nrank - orank;
547                        int[] nstride = new int[nrank];
548                        boolean ones = true;
549                        // work forwards for broadcasting cases
550                        for (int i = 0, j = 0; i < orank || j < nrank;) {
551                                if (j >= diff && i < orank && j < nrank && oshape[i] == nshape[j]) {
552                                        nstride[j++] = stride[i++];
553                                } else if (j < nrank && nshape[j] == 1) {
554                                        nstride[j++] = 0;
555                                } else if (i < orank && oshape[i] == 1) {
556                                        i++;
557                                } else {
558                                        if (j < nrank)
559                                                j++;
560                                        if (i < orank)
561                                                i++;
562                                        ones = false;
563                                }
564                        }
565                        if (!ones) { // not just ones differ in shapes
566                                int[] ostride = stride;
567                                int ob = 0;
568                                int oe = 1;
569                                int nb = 0;
570                                int ne = 1;
571                                while (ob < orank && nb < nrank) {
572                                        int ol = oshape[ob];
573                                        int nl = nshape[nb];
574                                        
575                                        if (nl < ol) { // find group of shape dimensions that form common size
576                                                do { // case where new shape spreads single dimension over several dimensions
577                                                        if (ne == nrank) {
578                                                                break;
579                                                        }
580                                                        nl *= nshape[ne++];
581                                                } while (nl < ol);
582                                                if (nl != ol) {
583                                                        logger.error("Subshape is incompatible with single dimension");
584                                                        throw new IllegalArgumentException("Subshape is incompatible with single dimension");
585                                                }
586                                                int on = ne - 1;
587                                                while (nshape[on] == 1) {
588                                                        on--;
589                                                }
590
591                                                nstride[on] = ostride[ob];
592                                                for (int n = on - 1; n >= nb; n--) {
593                                                        if (nshape[n] == 1)
594                                                                continue;
595
596                                                        nstride[n] = nshape[on] * nstride[on];
597                                                        on = n;
598                                                }
599                                        } else if (ol < nl) {
600                                                do { // case where new shape combines several dimensions into one dimension
601                                                        if (oe == orank) {
602                                                                break;
603                                                        }
604                                                        ol *= oshape[oe++];
605                                                } while (ol < nl);
606                                                if (nl != ol) {
607                                                        logger.error("Single dimension is incompatible with subshape");
608                                                        throw new IllegalArgumentException("Single dimension is incompatible with subshape");
609                                                }
610
611                                                int oo = oe - 1;
612                                                while (oshape[oo] == 1) {
613                                                        oo--;
614                                                }
615                                                int os = ostride[oo];
616                                                for (int o = oo - 1; o >= ob; o--) {
617                                                        if (oshape[o] == 1)
618                                                                continue;
619                                                        if (ostride[o] != oshape[oo] * ostride[oo]) {
620                                                                logger.error("Subshape cannot be a non-contiguous view");
621                                                                throw new IllegalArgumentException("Subshape cannot be a non-contiguous view");
622                                                        }
623                                                        oo = o;
624                                                }
625                                                nstride[nb] = os;
626                                        } else {
627                                                nstride[nb] = ostride[ob];
628                                        }
629
630                                        ob = oe++;
631                                        nb = ne++;
632                                }
633                        }
634        
635                        stride = nstride;
636                }
637
638                setDirty();
639                if (this.shape != null && metadata != null) {
640                        reshapeMetadata(this.shape, nshape);
641                }
642                this.shape = nshape;
643        }
644
645        @Override
646        public int[] getShapeRef() {
647                return shape;
648        }
649
650        @Override
651        public int getOffset() {
652                return offset;
653        }
654
655        @Override
656        public int[] getStrides() {
657                return stride;
658        }
659
660        @Override
661        public Serializable getBuffer() {
662                return odata;
663        }
664
665        @Override
666        public void overrideInternal(Serializable buffer, int... shape) {
667                if (buffer != null) {
668                        odata = buffer;
669                        setData();
670                        setDirty();
671                }
672        
673                if (shape != null) {
674                        this.shape = shape.clone();
675                        size = ShapeUtils.calcSize(this.shape);
676                }
677        }
678
679        /**
680         * Create a stride array from dataset
681         * @param a dataset
682         * @param offset output offset
683         * @return new strides
684         */
685        public static int[] createStrides(Dataset a, final int[] offset) {
686                return createStrides(a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), offset);
687        }
688
689        /**
690         * Create a stride array from dataset
691         * @param isize item size
692         * @param shape to use
693         * @param oStride original stride
694         * @param oOffset original offset (only used if there is an original stride)
695         * @param offset output offset
696         * @return new strides
697         */
698        public static int[] createStrides(final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] offset) {
699                int rank = shape.length;
700                final int[] stride;
701                if (oStride == null) {
702                        offset[0] = 0;
703                        stride = new int[rank];
704                        int s = isize;
705                        for (int j = rank - 1; j >= 0; j--) {
706                                stride[j] = s;
707                                s *= shape[j];
708                        }
709                } else {
710                        offset[0] = oOffset;
711                        stride = oStride.clone();
712                }
713                return stride;
714        }
715
716        /**
717         * Create a stride array from slice information and a dataset
718         * @param slice an n-D slice
719         * @param a dataset
720         * @param stride output stride
721         * @param offset output offset
722         * @return new shape
723         */
724        public static int[] createStrides(final SliceND slice, final Dataset a, final int[] stride, final int[] offset) {
725                return createStrides(slice, a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), stride, offset);
726        }
727
728        /**
729         * Create a stride array from slice and dataset information
730         * @param slice an n-D slice
731         * @param isize item size
732         * @param shape to use
733         * @param oStride original stride
734         * @param oOffset original offset (only used if there is an original stride)
735         * @param stride output stride
736         * @param offset output offset
737         * @return new shape
738         */
739        public static int[] createStrides(final SliceND slice, final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] stride, final int[] offset) {
740                int[] lstart = slice.getStart();
741                int[] lstep = slice.getStep();
742                int[] newShape = slice.getShape();
743                int rank = shape.length;
744
745                if (oStride == null) {
746                        int s = isize;
747                        offset[0] = 0;
748                        for (int j = rank - 1; j >= 0; j--) {
749                                stride[j] = s * lstep[j];
750                                offset[0] += s * lstart[j];
751                                s *= shape[j];
752                        }
753                } else {
754                        offset[0] = oOffset;
755                        for (int j = 0; j < rank; j++) {
756                                int s = oStride[j];
757                                stride[j] = lstep[j] * s;
758                                offset[0] += lstart[j] * s;
759                        }
760                }
761
762                return newShape;
763        }
764
765        @Override
766        public Dataset getBroadcastView(int... broadcastShape) {
767                AbstractDataset view = getView(true);
768                
769                if (!Arrays.equals(shape, broadcastShape)) {
770                        List<int[]> nShapes = BroadcastUtils.broadcastShapesToMax(broadcastShape, shape);
771                        view.setShape(nShapes.get(0));
772                        view.stride = BroadcastUtils.createBroadcastStrides(view, broadcastShape);
773                        view.base = this;
774                        view.shape = broadcastShape.clone();
775                        view.size = ShapeUtils.calcSize(broadcastShape);
776                        if (view.name == null || view.name.isEmpty()) {
777                                view.name = "Broadcast from " + Arrays.toString(shape);
778                        } else {
779                                view.name = "Broadcast of " + view.name + " from " + Arrays.toString(shape);
780                        }
781                }
782                return view;
783        }
784
785        @Override
786        public Dataset getSliceView(final int[] start, final int[] stop, final int[] step) {
787                return internalGetSliceView(new SliceND(shape, start, stop, step));
788        }
789
790        @Override
791        public Dataset getSliceView(Slice... slice) {
792                if (slice == null || slice.length == 0) {
793                        return getView(true);
794                }
795
796                return internalGetSliceView(new SliceND(shape, slice));
797        }
798
799        /**
800         * Get a slice of the dataset. The returned dataset is a view on a selection of items
801         * @param slice an n-D slice
802         * @return slice view
803         */
804        @Override
805        public Dataset getSliceView(SliceND slice) {
806                if (slice != null) {
807                        checkSliceND(slice);
808                }
809                return internalGetSliceView(slice);
810        }
811
812        private Dataset internalGetSliceView(SliceND slice) {
813                if (slice.isAll()) {
814                        return getView(true);
815                }
816
817                final int rank = shape.length;
818                int[] sStride = new int[rank];
819                int[] sOffset = new int[1];
820
821                int[] sShape = createStrides(slice, this, sStride, sOffset);
822        
823                AbstractDataset s = getView(false);
824                s.shape = sShape;
825                s.size = ShapeUtils.calcSize(sShape);
826                s.stride = sStride;
827                s.offset = sOffset[0];
828                s.base = this;
829
830                s.metadata = copyMetadata();
831                s.sliceMetadata(true, slice);
832
833                s.setDirty();
834                s.setName(name + BLOCK_OPEN + slice + BLOCK_CLOSE);
835
836                return s;
837        }
838
839        /**
840         * Get flattened view index of given position 
841         * @param pos
842         *            the integer array specifying the n-D position
843         * @return the index on the flattened dataset
844         */
845        private int getFlat1DIndex(final int[] pos) {
846                final int imax = pos.length;
847                if (imax == 0) {
848                        return 0;
849                }
850
851                return get1DIndexFromShape(pos);
852        }
853
854        /**
855         * @return index of first element
856         * @since 2.0
857         */
858        protected int getFirst1DIndex() {
859                if (shape == null) {
860                        throw new IllegalArgumentException("Cannot find an index from a null shape");
861                }
862                return stride == null ? 0 : offset;
863        }
864
865        @Override
866        public int get1DIndex(final int... n) {
867                if (n.length == 0 && shape.length == 0)
868                        return offset;
869
870                return stride == null ? get1DIndexFromShape(n) : get1DIndexFromStrides(n);
871        }
872
873        private static void throwAIOOBException(int i, int s, int d) {
874                throw new ArrayIndexOutOfBoundsException("Index (" + i + ") out of range [-" + s + "," + s
875                                + "] in dimension " + d);
876        }
877
878        /**
879         * @param i position in first dimension
880         * @return the index on the data array corresponding to that location
881         */
882        protected int get1DIndex(int i) {
883                if (shape == null) {
884                        throw new IllegalArgumentException("Cannot find an index from a null shape");
885                }
886                if (shape.length > 1) {
887                        logger.error("This dataset is not 1D but was addressed as such");
888                        throw new UnsupportedOperationException("This dataset is not 1D but was addressed as such");
889                }
890                if (i < 0) {
891                        i += shape[0];
892                }
893                if (i < 0 || i >= shape[0]) {
894                        throwAIOOBException(i, shape[0], 0);
895                }
896                return stride == null ? i : i*stride[0] + offset;
897        }
898
899        /**
900         * @param i position in first dimension
901         * @param j position in second dimension
902         * @return the index on the data array corresponding to that location
903         */
904        protected int get1DIndex(int i, int j) {
905                if (shape == null) {
906                        throw new IllegalArgumentException("Cannot find an index from a null shape");
907                }
908                if (shape.length != 2) {
909                        logger.error("This dataset is not 2D but was addressed as such");
910                        throw new UnsupportedOperationException("This dataset is not 2D but was addressed as such");
911                }
912                if (i < 0) {
913                        i += shape[0];
914                }
915                if (i < 0 || i >= shape[0]) {
916                        throwAIOOBException(i, shape[0], 0);
917                }
918                if (j < 0) {
919                        j += shape[1];
920                }
921                if (j < 0 || j >= shape[1]) {
922                        throwAIOOBException(i, shape[1], 1);
923                }
924                return stride == null ? i*shape[1] + j : i*stride[0] + j*stride[1] + offset;
925        }
926
927        protected int get1DIndexFromShape(final int[] n) {
928                return get1DIndexFromShape(shape, n);
929        }
930
931        protected static int get1DIndexFromShape(final int[] shape, final int[] n) {
932                if (shape == null) {
933                        throw new IllegalArgumentException("Cannot find an index from a null shape");
934                }
935                final int rank = shape.length;
936                if (rank != n.length) {
937                        String errMsg = String.format("Number of position values must be equal to rank of %d", rank);
938                        logger.error(errMsg);
939                        throw new IllegalArgumentException(errMsg);
940                }
941                int index = 0;
942                for (int i = 0; i < rank; i++) {
943                        final int si = shape[i];
944                        int ni = n[i];
945                        if (ni < 0) {
946                                ni += si;
947                        }
948                        if (ni < 0 || ni >= si) {
949                                throwAIOOBException(ni, si, i);
950                        }
951                        index = index * si + ni;
952                }
953
954                return index;
955        }
956
957        private int get1DIndexFromStrides(final int[] n) {
958                return get1DIndexFromStrides(shape, stride, offset, n);
959        }
960
961        private static int get1DIndexFromStrides(final int[] shape, final int[] stride, final int offset, final int[] n) {
962                if (shape == null) {
963                        throw new IllegalArgumentException("Cannot find an index from a null shape");
964                }
965                final int rank = shape.length;
966                if (rank != n.length) {
967                        String errMsg = String.format("Number of position values must be equal to rank of %d", rank);
968                        logger.error(errMsg);
969                        throw new IllegalArgumentException(errMsg);
970                }
971                int index = offset;
972                for (int i = 0; i < rank; i++) {
973                        final int st = stride[i];
974                        if (st != 0) { // not broadcasted
975                                final int si = shape[i];
976                                int ni = n[i];
977                                if (ni < 0) {
978                                        ni += si;
979                                }
980                                if (ni < 0 || ni >= si) {
981                                        throwAIOOBException(ni, si, i);
982                                }
983                                index += st * ni;
984                        }
985                }
986                return index;
987        }
988
989        @Override
990        public int[] getNDPosition(final int n) {
991                if (isIndexInRange(n)) {
992                        throw new IllegalArgumentException("Index provided " + n
993                                        + "is larger then the size of the containing array");
994                }
995
996                return stride == null ? ShapeUtils.getNDPositionFromShape(n, shape) : getNDPositionFromStrides(n);
997        }
998
999        private boolean isIndexInRange(final int n) {
1000                if (stride == null) {
1001                        return n >= size;
1002                }
1003                return n >= getBufferLength();
1004        }
1005
1006        /**
1007         * @return entire buffer length
1008         */
1009        abstract protected int getBufferLength();
1010
1011        private int[] getNDPositionFromStrides(int n) {
1012                n -= offset;
1013                int rank = shape.length;
1014                if (rank == 1) {
1015                        return new int[] { n / stride[0] };
1016                }
1017
1018                int[] output = new int[rank];
1019                int i = 0;
1020                while (i != n) { // TODO find more efficient way than this exhaustive search
1021                        int j = rank - 1;
1022                        for (; j >= 0; j--) {
1023                                output[j]++;
1024                                i += stride[j];
1025                                if (output[j] >= shape[j]) {
1026                                        output[j] = 0;
1027                                        i -= shape[j] * stride[j];
1028                                } else {
1029                                        break;
1030                                }
1031                        }
1032                        if (j == -1) {
1033                                logger.error("Index was not found in this strided dataset");
1034                                throw new IllegalArgumentException("Index was not found in this strided dataset");
1035                        }
1036                }
1037
1038                return output;
1039        }
1040
1041        @Override
1042        public int checkAxis(int axis) {
1043                return ShapeUtils.checkAxis(shape.length, axis);
1044        }
1045
1046        @Deprecated
1047        protected static int checkAxis(int rank, int axis) {
1048                return ShapeUtils.checkAxis(rank, axis);
1049        }
1050
1051        protected static final char BLOCK_OPEN = '[';
1052        protected static final char BLOCK_CLOSE = ']';
1053        protected final static String ELLIPSIS = "...";
1054
1055        @Override
1056        public String toString() {
1057                final int rank = shape == null ? 0 : shape.length;
1058                final StringBuilder out = new StringBuilder();
1059
1060                if (InterfaceUtils.isElemental(getClass())) {
1061                        out.append("Dataset ");
1062                } else {
1063                        out.append("Compound dataset (");
1064                        out.append(getElementsPerItem());
1065                        out.append(") ");
1066                }
1067
1068                if (name != null && name.length() > 0) {
1069                        out.append("'");
1070                        out.append(name);
1071                        out.append("' has shape ");
1072                } else {
1073                        out.append("shape is ");
1074                }
1075
1076                out.append(BLOCK_OPEN);
1077                if (rank > 0) {
1078                        out.append(shape[0]);
1079                }
1080                for (int i = 1; i < rank; i++) {
1081                        out.append(", " + shape[i]);
1082                }
1083                out.append(BLOCK_CLOSE);
1084                return out.toString();
1085        }
1086
1087        @Override
1088        public String toString(boolean showData) {
1089                if (!showData) {
1090                        return toString();
1091                }
1092
1093                if (size == 0) {
1094                        return "[]";
1095                }
1096
1097                final int rank = shape == null ? 0 : shape.length;
1098                final StringBuilder out = new StringBuilder();
1099
1100                if (rank > 0) {
1101                        int[] pos = new int[rank];
1102                        final StringBuilder lead = new StringBuilder();
1103                        printBlocks(out, lead, 0, pos);
1104                } else {
1105                        out.append(getString());
1106                }
1107                return out.toString();
1108        }
1109
1110        /**
1111         * Limit to strings output via the toString() method
1112         */
1113        private static int maxStringLength = 120;
1114
1115        /**
1116         * Set maximum line length for toString() method
1117         * @param maxLineLength limit on length of line
1118         */
1119        public static void setMaxLineLength(int maxLineLength) {
1120                maxStringLength = maxLineLength;
1121        }
1122
1123        /**
1124         * @return maximum line length for toString() method
1125         */
1126        public static int getMaxLineLength() {
1127                return maxStringLength;
1128        }
1129
1130        /**
1131         * Limit to number of sub-blocks output via the toString() method
1132         */
1133        private static final int MAX_SUBBLOCKS = 6;
1134
1135        private final static String SEPARATOR = ",";
1136        private final static String SPACE = " ";
1137        private final static String NEWLINE = "\n";
1138
1139        /**
1140         * Make a line of output for last dimension of dataset
1141         * @param end
1142         * @param start
1143         * @return line
1144         */
1145        private StringBuilder makeLine(final int end, final int[] start) {
1146                StringBuilder line = new StringBuilder();
1147                final int[] pos;
1148                if (end >= start.length) {
1149                        pos = Arrays.copyOf(start, end + 1);
1150                } else {
1151                        pos = start;
1152                }
1153                pos[end] = 0;
1154                line.append(BLOCK_OPEN);
1155                line.append(getString(pos));
1156
1157                final int length = shape[end];
1158
1159                // trim elements printed if length exceed estimate of maximum elements
1160                int excess = length - maxStringLength / 3; // space + number + separator
1161                int midIndex = -1;
1162                if (excess > 0) {
1163                        int index = (length - excess) / 2;
1164                        for (int y = 1; y < index; y++) {
1165                                line.append(SEPARATOR + SPACE);
1166                                pos[end] = y;
1167                                line.append(getString(pos));
1168                        }
1169                        midIndex = line.length() + 2;
1170                        index = (length + excess) / 2;
1171                        for (int y = index; y < length; y++) {
1172                                line.append(SEPARATOR + SPACE);
1173                                pos[end] = y;
1174                                line.append(getString(pos));
1175                        }
1176                } else {
1177                        for (int y = 1; y < length; y++) {
1178                                line.append(SEPARATOR + SPACE);
1179                                pos[end] = y;
1180                                line.append(getString(pos));
1181                        }
1182                }
1183                line.append(BLOCK_CLOSE);
1184
1185                // trim string down to limit
1186                int lineLength = line.length();
1187                excess = lineLength - maxStringLength - ELLIPSIS.length() - 1;
1188                if (excess > 0) {
1189                        int index = (lineLength - excess) / 2;
1190                        if (midIndex > 0 && index > midIndex) {
1191                                index = midIndex;
1192                        } else {
1193                                index = line.lastIndexOf(SEPARATOR, index) + 2;
1194                        }
1195                        StringBuilder out = new StringBuilder(line.subSequence(0, index));
1196                        out.append(ELLIPSIS + SEPARATOR);
1197                        index = (lineLength + excess) / 2;
1198                        if (midIndex > 0 && index <= midIndex) {
1199                                index = midIndex - 1;
1200                        } else {
1201                                index = line.indexOf(SEPARATOR, index) + 1;
1202                        }
1203                        out.append(line.subSequence(index, lineLength));
1204                        return out;
1205                } else if (midIndex > 0) { // add ellipsis
1206                        StringBuilder out = new StringBuilder(line.subSequence(0, midIndex));
1207                        out.append(ELLIPSIS + SEPARATOR + SPACE);
1208                        out.append(line.subSequence(midIndex, lineLength));
1209                        return out;
1210                }
1211
1212                return line;
1213        }
1214
1215        /**
1216         * recursive method to print blocks
1217         */
1218        private void printBlocks(final StringBuilder out, final StringBuilder lead, final int level, final int[] pos) {
1219                if (out.length() > 0) {
1220                        char last = out.charAt(out.length() - 1);
1221                        if (last != BLOCK_OPEN) {
1222                                out.append(lead);
1223                        }
1224                }
1225                final int end = getRank() - 1;
1226                if (level != end) {
1227                        out.append(BLOCK_OPEN);
1228                        int length = shape[level];
1229
1230                        // first sub-block
1231                        pos[level] = 0;
1232                        StringBuilder newlead = new StringBuilder(lead);
1233                        newlead.append(SPACE);
1234                        printBlocks(out, newlead, level + 1, pos);
1235                        if (length < 2) { // escape
1236                                out.append(BLOCK_CLOSE);
1237                                return;
1238                        }
1239
1240                        out.append(SEPARATOR + NEWLINE);
1241                        for (int i = level + 1; i < end; i++) {
1242                                out.append(NEWLINE);
1243                        }
1244
1245                        // middle sub-blocks
1246                        if (length < MAX_SUBBLOCKS) {
1247                                for (int x = 1; x < length - 1; x++) {
1248                                        pos[level] = x;
1249                                        printBlocks(out, newlead, level + 1, pos);
1250                                        if (end <= level + 1) {
1251                                                out.append(SEPARATOR + NEWLINE);
1252                                        } else {
1253                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
1254                                        }
1255                                }
1256                        } else {
1257                                final int excess = length - MAX_SUBBLOCKS;
1258                                int xmax = (length - excess) / 2;
1259                                for (int x = 1; x < xmax; x++) {
1260                                        pos[level] = x;
1261                                        printBlocks(out, newlead, level + 1, pos);
1262                                        if (end <= level + 1) {
1263                                                out.append(SEPARATOR + NEWLINE);
1264                                        } else {
1265                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
1266                                        }
1267                                }
1268                                out.append(newlead);
1269                                out.append(ELLIPSIS + SEPARATOR + NEWLINE);
1270                                xmax = (length + excess) / 2;
1271                                for (int x = xmax; x < length - 1; x++) {
1272                                        pos[level] = x;
1273                                        printBlocks(out, newlead, level + 1, pos);
1274                                        if (end <= level + 1) {
1275                                                out.append(SEPARATOR + NEWLINE);
1276                                        } else {
1277                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
1278                                        }
1279                                }
1280                        }
1281
1282                        // last sub-block
1283                        pos[level] = length - 1;
1284                        printBlocks(out, newlead, level + 1, pos);
1285                        out.append(BLOCK_CLOSE);
1286                } else {
1287                        out.append(makeLine(end, pos));
1288                }
1289        }
1290
1291        @Override
1292        public Dataset squeezeEnds() {
1293                return squeeze(true);
1294        }
1295
1296        @Override
1297        public Dataset squeeze() {
1298                return squeeze(false);
1299        }
1300
1301        @Override
1302        public Dataset squeeze(boolean onlyFromEnds) {
1303                final int[] tshape = ShapeUtils.squeezeShape(shape, onlyFromEnds);
1304                final int[] oshape = shape;
1305                if (stride == null) {
1306                        shape = tshape;
1307                } else {
1308                        int rank = shape.length;
1309                        int trank = tshape.length;
1310                        if (trank < rank) {
1311                                int[] tstride = new int[tshape.length];
1312                                if (onlyFromEnds) {
1313                                        for (int i = 0; i < rank; i++) {
1314                                                if (shape[i] != 1) {
1315                                                        for (int k = 0; k < trank; k++) {
1316                                                                tstride[k] = stride[i++];
1317                                                        }
1318                                                        break;
1319                                                }
1320                                        }
1321                                } else {
1322                                        int t = 0;
1323                                        for (int i = 0; i < rank; i++) {
1324                                                if (shape[i] != 1) {
1325                                                        tstride[t++] = stride[i];
1326                                                }
1327                                        }
1328                                }
1329                                shape = tshape;
1330                                stride = tstride;
1331                        }
1332                }
1333
1334                setDirty();
1335                reshapeMetadata(oshape, shape);
1336                return this;
1337        }
1338
1339        @Override
1340        public boolean isCompatibleWith(final ILazyDataset g) {
1341                return ShapeUtils.areShapesCompatible(shape, g.getShape());
1342        }
1343
1344        @Override
1345        public void checkCompatibility(final ILazyDataset g) throws IllegalArgumentException {
1346                ShapeUtils.checkCompatibility(this, g);
1347        }
1348
1349        @Override
1350        public Dataset reshape(final int... shape) {
1351                Dataset a;
1352                try {
1353                        a = getView(true);
1354                        a.setShape(shape);
1355                } catch (IllegalArgumentException e) {
1356                        a = clone();
1357                        a.setShape(shape);
1358                }
1359                return a;
1360        }
1361
1362        /**
1363         * @param start begin
1364         * @param stop exclusive end
1365         * @param step number to skip
1366         * @return number of steps to take
1367         */
1368        protected static int calcSteps(final double start, final double stop, final double step) {
1369                return Math.max(0, (int) Math.ceil((stop - start) / step));
1370        }
1371
1372        @Override
1373        public boolean isComplex() {
1374                return false;
1375        }
1376
1377        @Override
1378        public Dataset getRealPart() {
1379                return this;
1380        }
1381
1382        @Override
1383        public Dataset getRealView() {
1384                return getView(true);
1385        }
1386
1387        @Override
1388        public Dataset getSlice(final int[] start, final int[] stop, final int[] step) {
1389                return internalGetSlice(new SliceND(shape, start, stop, step));
1390        }
1391
1392        @Override
1393        public Dataset getSlice(Slice... slice) {
1394                return internalGetSlice(new SliceND(shape, slice));
1395        }
1396
1397        @Override
1398        public Dataset getSlice(IMonitor monitor, Slice... slice) {
1399                return getSlice(slice);
1400        }
1401
1402        @Override
1403        public Dataset getSlice(IMonitor monitor, SliceND slice) {
1404                return getSlice(slice);
1405        }
1406
1407        @Override
1408        public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) {
1409                return getSlice(start, stop, step);
1410        }
1411
1412        /**
1413         * Get a slice of the dataset. The returned dataset is a copied selection of items
1414         * @param slice an n-D slice
1415         * @return The dataset of the sliced data
1416         */
1417        @Override
1418        public Dataset getSlice(final SliceND slice) {
1419                if (slice != null) {
1420                        checkSliceND(slice);
1421                }
1422                return internalGetSlice(slice);
1423        }
1424
1425        private Dataset internalGetSlice(final SliceND slice) {
1426                SliceIterator it = (SliceIterator) internalGetSliceIterator(slice);
1427                AbstractDataset s = getSlice(it);
1428                s.metadata = copyMetadata();
1429                s.setDirty();
1430                s.sliceMetadata(true, slice);
1431                return s;
1432        }
1433
1434        /**
1435         * Get a slice of the dataset. The returned dataset is a copied selection of items
1436         * 
1437         * @param iterator Slice iterator
1438         * @return The dataset of the sliced data
1439         */
1440        abstract public AbstractDataset getSlice(final SliceIterator iterator);
1441
1442        @Override
1443        public Dataset setSlice(final Object obj, final SliceND slice) {
1444                if (slice != null) {
1445                        checkSliceND(slice);
1446                }
1447                return internalSetSlice(obj, slice);
1448        }
1449
1450        private Dataset internalSetSlice(final Object obj, final SliceND slice) {
1451                Dataset ds;
1452                if (obj instanceof Dataset) {
1453                        ds = (Dataset) obj;
1454                } else if (obj instanceof IDataset) {
1455                        ds = DatasetUtils.convertToDataset((IDataset) obj);
1456                } else {
1457                        Class<? extends Dataset> dClass = getClass();
1458                        if (!BooleanDataset.class.equals(dClass)) {
1459                                dClass = InterfaceUtils.getLargestInterface(this);
1460                        }
1461                        ds = DatasetFactory.createFromObject(getElementsPerItem(), dClass, obj);
1462                }
1463
1464                return setSlicedView(getSliceView(slice), ds);
1465        }
1466
1467        @Override
1468        public Dataset setSlice(final Object obj, final int[] start, final int[] stop, final int[] step) {
1469                return internalSetSlice(obj, new SliceND(shape, start, stop, step));
1470        }
1471
1472        /**
1473         * Set a view of current dataset to given dataset with broadcasting
1474         * @param view destination
1475         * @param d source of data
1476         * @return this dataset
1477         */
1478        abstract Dataset setSlicedView(Dataset view, Dataset d);
1479
1480        @Override
1481        public Dataset setSlice(Object obj, Slice... slice) {
1482                if (slice == null || slice.length == 0) {
1483                        return internalSetSlice(obj, new SliceND(shape));
1484                }
1485                return internalSetSlice(obj, new SliceND(shape, slice));
1486        }
1487
1488        @Override
1489        public boolean all() {
1490                return Comparisons.allTrue(this);
1491        }
1492
1493        @Override
1494        public BooleanDataset all(final int axis) {
1495                return Comparisons.allTrue(this, axis);
1496        }
1497
1498        @Override
1499        public boolean any() {
1500                return Comparisons.anyTrue(this);
1501        }
1502
1503        @Override
1504        public BooleanDataset any(final int axis) {
1505                return Comparisons.anyTrue(this, axis);
1506        }
1507
1508        @Override
1509        public Dataset ifloorDivide(final Object o) {
1510                return idivide(o).ifloor();
1511        }
1512
1513        @Override
1514        public double residual(final Object o) {
1515                return residual(o, null, false);
1516        }
1517
1518        @Override
1519        public double residual(final Object o, boolean ignoreNaNs) {
1520                return residual(o, null, ignoreNaNs);
1521        }
1522
1523        /**
1524         * @return statistics
1525         * @since 2.0
1526         */
1527        @SuppressWarnings("unchecked")
1528        protected StatisticsMetadata<Number> getStats() {
1529                StatisticsMetadata<Number> md = getFirstMetadata(StatisticsMetadata.class);
1530                if (md == null || md.isDirty(this)) {
1531                        md = new StatisticsMetadataImpl<Number>();
1532                        md.initialize(this);
1533                        setMetadata(md);
1534                }
1535                return md;
1536        }
1537
1538        /**
1539         * @return statistics
1540         * @since 2.0
1541         */
1542        @SuppressWarnings("unchecked")
1543        protected StatisticsMetadata<String> getStringStats() {
1544                StatisticsMetadata<String> md = getFirstMetadata(StatisticsMetadata.class);
1545                if (md == null || md.isDirty(this)) {
1546                        md = new StatisticsMetadataImpl<String>();
1547                        md.initialize(this);
1548                        setMetadata(md);
1549                }
1550                return md;
1551        }
1552
1553        @Override
1554        public Number max(boolean... ignoreInvalids) {
1555                return getStats().getMaximum(ignoreInvalids);
1556        }
1557
1558        @Override
1559        public Dataset max(int axis, boolean... ignoreInvalids) {
1560                return getStats().getMaximum(axis, ignoreInvalids);
1561        }
1562
1563        @Override
1564        public Dataset max(int[] axes, boolean... ignoreInvalids) {
1565                return getStats().getMaximum(axes, ignoreInvalids);
1566        }
1567
1568        @Override
1569        public Number min(boolean... ignoreInvalids) {
1570                return getStats().getMinimum(ignoreInvalids);
1571        }
1572
1573        @Override
1574        public Dataset min(int axis, boolean... ignoreInvalids) {
1575                return getStats().getMinimum(axis, ignoreInvalids);
1576        }
1577
1578        @Override
1579        public Dataset min(int[] axes, boolean... ignoreInvalids) {
1580                return getStats().getMinimum(axes, ignoreInvalids);
1581        }
1582
1583        @Override
1584        public int argMax(boolean... ignoreInvalids) {
1585                return getFlat1DIndex(maxPos(ignoreInvalids));
1586        }
1587
1588        /**
1589         * @since 2.0
1590         */
1591        @Override
1592        public IntegerDataset argMax(int axis, boolean... ignoreInvalids) {
1593                return (IntegerDataset) getStats().getArgMaximum(axis, ignoreInvalids);
1594        }
1595
1596        @Override
1597        public int argMin(boolean... ignoreInvalids) {
1598                return getFlat1DIndex(minPos(ignoreInvalids));
1599        }
1600
1601        /**
1602         * @since 2.0
1603         */
1604        @Override
1605        public IntegerDataset argMin(int axis, boolean... ignoreInvalids) {
1606                return (IntegerDataset) getStats().getArgMinimum(axis, ignoreInvalids);
1607        }
1608
1609        @Override
1610        public Number peakToPeak(boolean... ignoreInvalids) {
1611                return InterfaceUtils.fromDoubleToBiggestNumber(getClass(), max(ignoreInvalids).doubleValue() - min(ignoreInvalids).doubleValue());
1612        }
1613
1614        @Override
1615        public Dataset peakToPeak(int axis,  boolean... ignoreInvalids) {
1616                return Maths.subtract(max(axis, ignoreInvalids), min(axis, ignoreInvalids));
1617        }
1618
1619        @Override
1620        public Dataset peakToPeak(int[] axes,  boolean... ignoreInvalids) {
1621                return Maths.subtract(max(axes, ignoreInvalids), min(axes, ignoreInvalids));
1622        }
1623
1624        @Override
1625        public long count(boolean... ignoreInvalids) {
1626                return getStats().getCount(ignoreInvalids);
1627        }
1628
1629        @Override
1630        public Dataset count(int axis, boolean... ignoreInvalids) {
1631                return getStats().getCount(axis, ignoreInvalids);
1632        }
1633
1634        @Override
1635        public Dataset count(int[] axes, boolean... ignoreInvalids) {
1636                return getStats().getCount(axes, ignoreInvalids);
1637        }
1638
1639        @Override
1640        public Object sum(boolean... ignoreInvalids) {
1641                return InterfaceUtils.toBiggestNumber(getClass(), getStats().getSum(ignoreInvalids));
1642        }
1643
1644        @Override
1645        public Dataset sum(int axis, boolean... ignoreInvalids) {
1646                return getStats().getSum(axis, ignoreInvalids);
1647        }
1648
1649        @Override
1650        public Dataset sum(int[] axes, boolean... ignoreInvalids) {
1651                return getStats().getSum(axes, ignoreInvalids);
1652        }
1653
1654        @Override
1655        public Object product(boolean... ignoreInvalids) {
1656                return Stats.product(this, ignoreInvalids);
1657        }
1658
1659        @Override
1660        public Dataset product(int axis, boolean... ignoreInvalids) {
1661                return Stats.product(this, axis, ignoreInvalids);
1662        }
1663
1664        @Override
1665        public Dataset product(int[] axes, boolean... ignoreInvalids) {
1666                return Stats.product(this, axes, ignoreInvalids);
1667        }
1668
1669        @Override
1670        public Object mean(boolean... ignoreInvalids) {
1671                return getStats().getMean(ignoreInvalids);
1672        }
1673
1674        @Override
1675        public Dataset mean(int axis, boolean... ignoreInvalids) {
1676                return getStats().getMean(axis, ignoreInvalids);
1677        }
1678
1679        @Override
1680        public Dataset mean(int[] axes, boolean... ignoreInvalids) {
1681                return getStats().getMean(axes, ignoreInvalids);
1682        }
1683
1684        @Override
1685        public double variance() {
1686                return variance(false);
1687        }
1688
1689        @Override
1690        public double variance(boolean isWholePopulation, boolean... ignoreInvalids) {
1691                return getStats().getVariance(isWholePopulation, ignoreInvalids);
1692        }
1693
1694        @Override
1695        public Dataset variance(int axis) {
1696                return getStats().getVariance(axis, false);
1697        }
1698
1699        @Override
1700        public Dataset variance(int[] axes) {
1701                return getStats().getVariance(axes, false);
1702        }
1703
1704        @Override
1705        public Dataset variance(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
1706                return getStats().getVariance(axis, isWholePopulation, ignoreInvalids);
1707        }
1708
1709        @Override
1710        public Dataset variance(int[] axes, boolean isWholePopulation, boolean... ignoreInvalids) {
1711                return getStats().getVariance(axes, isWholePopulation, ignoreInvalids);
1712        }
1713
1714        @Override
1715        public double stdDeviation() {
1716                return Math.sqrt(variance());
1717        }
1718
1719        @Override
1720        public double stdDeviation(boolean isWholePopulation, boolean... ignoreInvalids) {
1721                return Math.sqrt(variance(isWholePopulation, ignoreInvalids));
1722        }
1723
1724        @Override
1725        public Dataset stdDeviation(int axis) {
1726                return Maths.sqrt(variance(axis, false));
1727        }
1728
1729        @Override
1730        public Dataset stdDeviation(int[] axes) {
1731                return Maths.sqrt(variance(axes, false));
1732        }
1733
1734        @Override
1735        public Dataset stdDeviation(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
1736                return Maths.sqrt(variance(axis, isWholePopulation, ignoreInvalids));
1737        }
1738
1739        @Override
1740        public Dataset stdDeviation(int[] axes, boolean isWholePopulation, boolean... ignoreInvalids) {
1741                return Maths.sqrt(variance(axes, isWholePopulation, ignoreInvalids));
1742        }
1743
1744        @Override
1745        public double rootMeanSquare(boolean... ignoreInvalids) {
1746                StatisticsMetadata<Number> stats = getStats();
1747                final double mean = stats.getMean(ignoreInvalids).doubleValue();
1748                final double var = stats.getVariance(true, ignoreInvalids);
1749                return Math.sqrt(var + mean * mean);
1750        }
1751
1752        @Override
1753        public Dataset rootMeanSquare(int axis, boolean... ignoreInvalids) {
1754                StatisticsMetadata<Number> stats = getStats();
1755                Dataset v = stats.getVariance(axis, true, ignoreInvalids);
1756                Dataset m = stats.getMean(axis, ignoreInvalids);
1757                Dataset result = Maths.multiply(m, m);
1758                return Maths.sqrt(result.iadd(v));
1759        }
1760
1761        @Override
1762        public Dataset rootMeanSquare(int[] axes, boolean... ignoreInvalids) {
1763                StatisticsMetadata<Number> stats = getStats();
1764                Dataset v = stats.getVariance(axes, true, ignoreInvalids);
1765                Dataset m = stats.getMean(axes, ignoreInvalids);
1766                Dataset result = Maths.multiply(m, m);
1767                return Maths.sqrt(result.iadd(v));
1768        }
1769
1770        /**
1771         * Set item from compatible dataset in a direct and speedy way. Remember to setDirty afterwards.
1772         * 
1773         * @param dindex destination index
1774         * @param sindex source index
1775         * @param src
1776         *            is the source data buffer
1777         */
1778        protected abstract void setItemDirect(final int dindex, final int sindex, final Object src);
1779
1780        /**
1781         * @return error broadcasted to current shape
1782         */
1783        private Dataset getBroadcastedInternalError() {
1784                if (shape == null) {
1785                        throw new IllegalArgumentException("Cannot get error for null dataset");
1786                }
1787                ILazyDataset led = super.getErrors();
1788                if (led == null)
1789                        return null;
1790
1791                Dataset ed = null;
1792                try {
1793                        ed = DatasetUtils.sliceAndConvertLazyDataset(led);
1794                } catch (DatasetException e) {
1795                        logger.error("Could not get data from lazy dataset", e);
1796                }
1797                if (led != ed) {
1798                        setErrors(ed); // set back
1799                }
1800
1801                return ed.getBroadcastView(shape);
1802        }
1803
1804        @Override
1805        public Dataset getErrors() {
1806                Dataset ed = getBroadcastedInternalError();
1807                if (ed == null)
1808                        return null;
1809
1810                return ed;
1811        }
1812
1813        @Override
1814        public double getError() {
1815                Dataset ed = getBroadcastedInternalError();
1816                if (ed == null)
1817                        return 0;
1818
1819                return ed.getDouble();
1820        }
1821
1822        @Override
1823        public double getError(final int i) {
1824                Dataset ed = getBroadcastedInternalError();
1825                if (ed == null)
1826                        return 0;
1827
1828                return ed.getDouble(i);
1829        }
1830
1831        @Override
1832        public double getError(final int i, final int j) {
1833                Dataset ed = getBroadcastedInternalError();
1834                if (ed == null)
1835                        return 0;
1836
1837                return ed.getDouble(i, j);
1838        }
1839
1840        @Override
1841        public double getError(int... pos) {
1842                Dataset ed = getBroadcastedInternalError();
1843                if (ed == null)
1844                        return 0;
1845
1846                return ed.getDouble(pos);
1847        }
1848
1849        @Override
1850        public double[] getErrorArray(final int i) {
1851                Dataset ed = getBroadcastedInternalError();
1852                if (ed == null)
1853                        return null;
1854
1855                return new double[] {getError(i)};
1856        }
1857
1858        @Override
1859        public double[] getErrorArray(final int i, final int j) {
1860                Dataset ed = getBroadcastedInternalError();
1861                if (ed == null)
1862                        return null;
1863
1864                return new double[] {getError(i, j)};
1865        }
1866
1867        @Override
1868        public double[] getErrorArray(int... pos) {
1869                Dataset ed = getBroadcastedInternalError();
1870                if (ed == null)
1871                        return null;
1872
1873                return new double[] {getError(pos)};
1874        }
1875
1876        protected Dataset getInternalSquaredError() {
1877                Dataset sed = getErrorBuffer().getBroadcastView(shape);
1878                return sed;
1879        }
1880
1881        @Override
1882        public Dataset getErrorBuffer() {
1883                ErrorMetadata emd = getErrorMetadata();
1884                if (emd == null)
1885                        return null;
1886
1887                if (!(emd instanceof ErrorMetadataImpl)) {
1888                        ILazyDataset led = emd.getError();
1889                        Dataset ed;
1890                        try {
1891                                ed = DatasetUtils.sliceAndConvertLazyDataset(led);
1892                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
1893                                setMetadata(emd);
1894                                emd.setError(ed);
1895                        } catch (MetadataException me) {
1896                                logger.error("Could not create metadata", me);
1897                        } catch (DatasetException e) {
1898                                logger.error("Could not get data from lazy dataset", e);
1899                        }
1900                }
1901
1902                return ((ErrorMetadataImpl) emd).getSquaredError();
1903        }
1904
1905        /**
1906         * Set a copy of the buffer that backs the (squared) error data
1907         * @param buffer can be null, anything that can be used to create a DoubleDataset or CompoundDoubleDataset
1908         */
1909        @Override
1910        public void setErrorBuffer(Serializable buffer) {
1911                if (shape == null) {
1912                        throw new IllegalArgumentException("Cannot set error buffer for null dataset");
1913                }
1914                if (buffer == null) {
1915                        clearMetadata(ErrorMetadata.class);
1916                        return;
1917                }
1918
1919                IDataset d = (IDataset) createFromSerializable(buffer, false);
1920                ErrorMetadata emd = getErrorMetadata();
1921                if (!(emd instanceof ErrorMetadataImpl)) {
1922                        try {
1923                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
1924                                setMetadata(emd);
1925                        } catch (MetadataException me) {
1926                                logger.error("Could not create metadata", me);
1927                        }
1928                }
1929                ((ErrorMetadataImpl) emd).setSquaredError(d);
1930        }
1931}