001/*-
002 *******************************************************************************
003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
004 * All rights reserved. This program and the accompanying materials
005 * are made available under the terms of the Eclipse Public License v1.0
006 * which accompanies this distribution, and is available at
007 * http://www.eclipse.org/legal/epl-v10.html
008 *
009 * Contributors:
010 *    Peter Chang - initial API and implementation and/or initial documentation
011 *******************************************************************************/
012
013package org.eclipse.january.dataset;
014
015import java.io.Serializable;
016import java.lang.annotation.Annotation;
017import java.lang.reflect.Array;
018import java.lang.reflect.Field;
019import java.util.ArrayList;
020import java.util.Arrays;
021import java.util.Collection;
022import java.util.concurrent.ConcurrentMap;
023import java.util.concurrent.ConcurrentHashMap;
024import java.util.List;
025import java.util.Map;
026
027import org.eclipse.january.DatasetException;
028import org.eclipse.january.MetadataException;
029import org.eclipse.january.metadata.Dirtiable;
030import org.eclipse.january.metadata.ErrorMetadata;
031import org.eclipse.january.metadata.IMetadata;
032import org.eclipse.january.metadata.MetadataFactory;
033import org.eclipse.january.metadata.MetadataType;
034import org.eclipse.january.metadata.Reshapeable;
035import org.eclipse.january.metadata.Sliceable;
036import org.eclipse.january.metadata.Transposable;
037import org.slf4j.Logger;
038import org.slf4j.LoggerFactory;
039
040/**
041 * Common base for both lazy and normal dataset implementations
042 */
043public abstract class LazyDatasetBase implements ILazyDataset, Serializable {
044
045        private static final long serialVersionUID = 767926846438976050L;
046
047        protected static final Logger logger = LoggerFactory.getLogger(LazyDatasetBase.class);
048
049        protected static boolean catchExceptions;
050
051        static {
052                /**
053                 * Boolean to set to true if running jython scripts that utilise ScisoftPy in IDE
054                 */
055                try {
056                        catchExceptions = Boolean.getBoolean("run.in.eclipse");
057                } catch (SecurityException e) {
058                        // set a default for when the security manager does not allow access to the requested key
059                        catchExceptions = false;
060                }
061        }
062
063        transient private boolean dirty = true; // indicate dirty state of metadata
064        protected String name = "";
065
066        /**
067         * The shape or dimensions of the dataset
068         */
069        protected int[] shape;
070
071        protected ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> metadata = null;
072
073        /**
074         * @return type of dataset item
075         */
076        abstract public int getDType();
077
078        @Override
079        public Class<?> getElementClass() {
080                return DTypeUtils.getElementClass(getDType());
081        }
082
083        @Override
084        public LazyDatasetBase clone() {
085                return null;
086        }
087
088        @Override
089        public boolean equals(Object obj) {
090                if (this == obj) {
091                        return true;
092                }
093                if (obj == null) {
094                        return false;
095                }
096                if (!getClass().equals(obj.getClass())) {
097                        return false;
098                }
099        
100                LazyDatasetBase other = (LazyDatasetBase) obj;
101                if (getDType() != other.getDType()) {
102                        return false;
103                }
104                if (getElementsPerItem() != other.getElementsPerItem()) {
105                        return false;
106                }
107                if (!Arrays.equals(shape, other.shape)) {
108                        return false;
109                }
110                return true;
111        }
112
113        @Override
114        public int hashCode() {
115                int hash = getDType() * 17 + getElementsPerItem();
116                int rank = shape.length;
117                for (int i = 0; i < rank; i++) {
118                        hash = hash*17 + shape[i];
119                }
120                return hash;
121        }
122
123        @Override
124        public String getName() {
125                return name;
126        }
127
128        @Override
129        public void setName(String name) {
130                this.name = name;
131        }
132
133        @Override
134        public int[] getShape() {
135                return shape.clone();
136        }
137
138        @Override
139        public int getRank() {
140                return shape.length;
141        }
142
143        /**
144         * This method allows anything that dirties the dataset to clear various metadata values
145         * so that the other methods can work correctly.
146         * @since 2.1
147         */
148        public void setDirty() {
149                dirty = true;
150        }
151
152        /**
153         * Find first sub-interface of (or class that directly implements) MetadataType
154         * @param clazz
155         * @return sub-interface
156         * @exception IllegalArgumentException when given class is {@link MetadataType} or an anonymous sub-class of it
157         */
158        @SuppressWarnings("unchecked")
159        public static Class<? extends MetadataType> findMetadataTypeSubInterfaces(Class<? extends MetadataType> clazz) {
160                if (clazz.equals(MetadataType.class)) {
161                        throw new IllegalArgumentException("Cannot accept MetadataType");
162                }
163
164                if (clazz.isInterface()) {
165                        return clazz;
166                }
167
168                if (clazz.isAnonymousClass()) { // special case
169                        Class<?> s = clazz.getSuperclass();
170                        if (!s.equals(Object.class)) {
171                                // only use super class if it is not an anonymous class of an interface
172                                clazz = (Class<? extends MetadataType>) s;
173                        }
174                }
175
176                for (Class<?> c : clazz.getInterfaces()) {
177                        if (c.equals(MetadataType.class)) {
178                                if (clazz.isAnonymousClass()) {
179                                        throw new IllegalArgumentException("Cannot accept anonymous subclasses of MetadataType");
180                                }
181                                return clazz;
182                        }
183                        if (MetadataType.class.isAssignableFrom(c)) {
184                                return (Class<? extends MetadataType>) c;
185                        }
186                }
187
188                Class<?> c = clazz.getSuperclass(); // Naughty: someone has sub-classed a metadata class
189                if (c != null) {
190                        return findMetadataTypeSubInterfaces((Class<? extends MetadataType>) c);
191                }
192
193                logger.error("Somehow the search for metadata type interface ended in a bad place");
194                assert false; // should not be able to get here!!!
195                return null;
196        }
197
198        @Override
199        public void setMetadata(MetadataType metadata) {
200                addMetadata(metadata, true);
201        }
202
203        @Override
204        public void addMetadata(MetadataType metadata) {
205                addMetadata(metadata, false);
206        }
207
208        private synchronized void addMetadata(MetadataType metadata, boolean clear) {
209                if (metadata == null)
210                        return;
211
212                if (this.metadata == null) {
213                        this.metadata = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
214                }
215
216                Class<? extends MetadataType> clazz = findMetadataTypeSubInterfaces(metadata.getClass());
217                if (!this.metadata.containsKey(clazz)) {
218                        this.metadata.put(clazz, new ArrayList<MetadataType>());
219                } else if (clear) {
220                        this.metadata.get(clazz).clear();
221                }
222                this.metadata.get(clazz).add(metadata);
223
224                // add for special case of sub-interfaces of IMetadata
225                if (!IMetadata.class.equals(clazz) && IMetadata.class.isAssignableFrom(clazz)) {
226                        clazz = IMetadata.class;
227                        if (!this.metadata.containsKey(clazz)) {
228                                this.metadata.put(clazz, new ArrayList<MetadataType>());
229                        } else if (clear) {
230                                this.metadata.get(clazz).clear();
231                        }
232                        this.metadata.get(clazz).add(metadata);
233                }
234        }
235
236        @Override
237        @Deprecated
238        public synchronized IMetadata getMetadata() {
239                return getFirstMetadata(IMetadata.class);
240        }
241
242        @SuppressWarnings("unchecked")
243        @Override
244        public synchronized <S extends MetadataType, T extends S> List<S> getMetadata(Class<T> clazz) throws MetadataException {
245                if (metadata == null) {
246                        dirty = false;
247                        return null;
248                }
249
250                if (dirty) {
251                        dirtyMetadata();
252                        dirty = false;
253                }
254
255                if (clazz == null) {
256                        List<S> all = new ArrayList<S>();
257                        for (Class<? extends MetadataType> c : metadata.keySet()) {
258                                all.addAll((Collection<S>) metadata.get(c));
259                        }
260                        return all;
261                }
262
263                return (List<S>) metadata.get(findMetadataTypeSubInterfaces(clazz));
264        }
265
266        @Override
267        public synchronized <S extends MetadataType, T extends S> S getFirstMetadata(Class<T> clazz) {
268                try {
269                        List<S> ml = getMetadata(clazz);
270                        if (ml == null) {
271                                return null;
272                        }
273                        for (S t : ml) {
274                                if (clazz.isInstance(t)) {
275                                        return t;
276                                }
277                        }
278                } catch (Exception e) {
279                        logger.error("Get metadata failed!",e);
280                }
281
282                return null;
283        }
284
285        @Override
286        public synchronized void clearMetadata(Class<? extends MetadataType> clazz) {
287                if (metadata == null)
288                        return;
289
290                if (clazz == null) {
291                        metadata.clear();
292                        return;
293                }
294
295                List<MetadataType> list = metadata.get(findMetadataTypeSubInterfaces(clazz));
296                if( list != null) {
297                        list.clear();
298                }
299        }
300
301        /**
302         * @since 2.0
303         */
304        protected synchronized ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata() {
305                return copyMetadata(metadata, null);
306        }
307
308        /**
309         * @since 2.0
310         */
311        protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> metadata) {
312                return copyMetadata(metadata, null);
313        }
314
315        /**
316         * Copy metadata. If oMetadata is not null, then copy from that when it has the corresponding items
317         * @since 2.1
318         * @param metadata
319         * @param oMetadata can be null
320         * @return
321         */
322        protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> metadata, Map<Class<? extends MetadataType>, List<MetadataType>> oMetadata) {
323                if (metadata == null)
324                        return null;
325
326                ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
327
328                for (Class<? extends MetadataType> c : metadata.keySet()) {
329                        List<MetadataType> l = metadata.get(c);
330                        if (!l.isEmpty() && oMetadata != null && oMetadata.containsKey(c)) { // only override when not empty
331                                l = oMetadata.get(c); 
332                        }
333                        List<MetadataType> nl = new ArrayList<MetadataType>(l.size());
334                        map.put(c, nl);
335                        for (MetadataType m : l) {
336                                nl.add(m.clone());
337                        }
338                }
339                return map;
340        }
341
342        interface MetadatasetAnnotationOperation {
343                /**
344                 * Process value of given field
345                 * <p>
346                 * When the field is not a container then the returned value
347                 * may replace the old value
348                 * @param f given field
349                 * @param o value of field
350                 * @return transformed field
351                 */
352                Object processField(Field f, Object o);
353
354                /**
355                 * @return annotated class
356                 */
357                Class<? extends Annotation> getAnnClass();
358
359                /**
360                 * @param axis
361                 * @return number of dimensions to insert or remove
362                 */
363                int change(int axis);
364
365                /**
366                 * 
367                 * @return rank or -1 to match
368                 */
369                int getNewRank();
370
371                /**
372                 * Run on given lazy dataset
373                 * @param lz
374                 * @return 
375                 */
376                ILazyDataset run(ILazyDataset lz);
377        }
378
379        class MdsSlice implements MetadatasetAnnotationOperation {
380                private boolean asView;
381                private SliceND slice;
382                private int[] oShape;
383                private long oSize;
384
385                public MdsSlice(boolean asView, SliceND slice) {
386                        this.asView = asView;
387                        this.slice = slice;
388                        oShape = slice.getSourceShape();
389                        oSize = ShapeUtils.calcLongSize(oShape);
390                }
391
392                @Override
393                public Object processField(Field field, Object o) {
394                        return o;
395                }
396
397                @Override
398                public Class<? extends Annotation> getAnnClass() {
399                        return Sliceable.class;
400                }
401
402                @Override
403                public int change(int axis) {
404                        return 0;
405                }
406
407                @Override
408                public int getNewRank() {
409                        return -1;
410                }
411
412                @Override
413                public ILazyDataset run(ILazyDataset lz) {
414                        int rank = lz.getRank();
415                        if (slice.getStart().length != rank) {
416                                throw new IllegalArgumentException("Slice rank does not match dataset!");
417                        }
418
419                        int[] shape = lz.getShape();
420                        SliceND nslice;
421                        if (!ShapeUtils.areShapesBroadcastCompatible(oShape, shape)) {
422                                nslice = new SliceND(shape);
423                                for (int i = 0; i < rank; i++) {
424                                        int s = shape[i];
425                                        int os = oShape[i];
426                                        if (s >= os) {
427                                                nslice.setSlice(i, 0, os, 1);
428                                        } else if (s == 1) {
429                                                nslice.setSlice(i, 0, 1, 1);
430                                        } else {
431                                                throw new IllegalArgumentException("Sliceable dataset has non-unit dimension less than host!");
432                                        }
433                                }
434                                lz = lz.getSliceView(nslice);
435                        }
436                        if (lz.getSize() == oSize) {
437                                nslice = slice;
438                        } else {
439                                nslice = slice.clone();
440                        }
441                        for (int i = 0; i < rank; i++) {
442                                int s = shape[i];
443                                if (s >= oShape[i]) {
444                                        continue;
445                                } else if (s == 1) {
446                                        nslice.setSlice(i, 0, 1, 1);
447                                } else {
448                                        throw new IllegalArgumentException("Sliceable dataset has non-unit dimension less than host!");
449                                }
450                        }
451
452                        if (asView || (lz instanceof IDataset)) {
453                                return lz.getSliceView(nslice);
454                        }
455                        try {
456                                return lz.getSlice(nslice);
457                        } catch (DatasetException e) {
458                                logger.error("Could not slice dataset in metadata", e);
459                                return null;
460                        }
461                }
462        }
463
464        class MdsReshape implements MetadatasetAnnotationOperation {
465                private boolean matchRank;
466                private int[] oldShape;
467                private int[] newShape;
468                boolean onesOnly;
469                int[] differences;
470
471                /*
472                 * if only ones then record differences (insertions and deletions)
473                 * 
474                 * if shape changing, find broadcasted dimensions and disallow
475                 * merging that include those dimensions
476                 */
477                public MdsReshape(final int[] oldShape, final int[] newShape) {
478                        this.oldShape = oldShape;
479                        this.newShape = newShape;
480                        differences = null;
481                }
482
483                @Override
484                public Object processField(Field field, Object o) {
485                        Annotation a = field.getAnnotation(Reshapeable.class);
486                        if (a != null) { // cannot be null
487                                matchRank = ((Reshapeable) a).matchRank();
488                        }
489                        return o;
490                }
491
492                @Override
493                public Class<? extends Annotation> getAnnClass() {
494                        return Reshapeable.class;
495                }
496
497                @Override
498                public int change(int axis) {
499                        if (matchRank) {
500                                if (differences == null)
501                                        init();
502
503                                if (onesOnly) {
504                                        return differences[axis];
505                                }
506                                throw new UnsupportedOperationException("TODO support other shape operations");
507                        }
508                        return 0;
509                }
510
511                @Override
512                public int getNewRank() {
513                        return matchRank ? newShape.length : -1;
514                }
515
516                private void init() {
517                        int or = oldShape.length - 1;
518                        int nr = newShape.length - 1;
519                        if (or < 0 || nr < 0) { // zero-rank shapes
520                                onesOnly = true;
521                                differences = new int[1];
522                                differences[0] = or < 0 ? nr + 1 : or + 1;
523                                return;
524                        }
525                        int ob = 0;
526                        int nb = 0;
527                        onesOnly = true;
528                        do {
529                                while (oldShape[ob] == 1 && ob < or) {
530                                        ob++; // next non-unit dimension
531                                }
532                                while (newShape[nb] == 1 && nb < nr) {
533                                        nb++;
534                                }
535                                if (oldShape[ob++] != newShape[nb++]) {
536                                        onesOnly = false;
537                                        break;
538                                }
539                        } while (ob <= or && nb <= nr);
540
541                        ob = 0;
542                        nb = 0;
543                        differences = new int[or + 2];
544                        if (onesOnly) {
545                                // work out unit dimensions removed from or add to old
546                                int j = 0;
547                                do {
548                                        if (oldShape[ob] != 1 && newShape[nb] != 1) {
549                                                ob++;
550                                                nb++;
551                                        } else {
552                                                while (oldShape[ob] == 1 && ob < or) {
553                                                        ob++;
554                                                        differences[j]--;
555                                                }
556                                                while (newShape[nb] == 1 && nb < nr) {
557                                                        nb++;
558                                                        differences[j]++;
559                                                }
560                                        }
561                                        j++;
562                                } while (ob <= or && nb <= nr && j <= or);
563                                while (ob <= or && oldShape[ob] == 1) {
564                                        ob++;
565                                        differences[j]--;
566                                }
567                                while (nb <= nr && newShape[nb] == 1) {
568                                        nb++;
569                                        differences[j]++;
570                                }
571                        } else {
572                                if (matchRank) {
573                                        logger.error("Combining dimensions is currently not supported");
574                                        throw new IllegalArgumentException("Combining dimensions is currently not supported");
575                                }
576                                // work out mapping: contiguous dimensions can be grouped or split
577                                while (ob <= or && nb <= nr) {
578                                        int ol = oldShape[ob];
579                                        while (ol == 1 && ol <= or) {
580                                                ob++;
581                                                ol = oldShape[ob];
582                                        }
583                                        int oe = ob + 1;
584                                        int nl = newShape[nb];
585                                        while (nl == 1 && nl <= nr) {
586                                                nb++;
587                                                nl = newShape[nb];
588                                        }
589                                        int ne = nb + 1;
590                                        if (ol < nl) {
591                                                differences[ob] = 1;
592                                                do { // case where new shape combines several dimensions into one dimension
593                                                        if (oe == (or + 1)) {
594                                                                break;
595                                                        }
596                                                        differences[oe] = 1;
597                                                        ol *= oldShape[oe++];
598                                                } while (ol < nl);
599                                                differences[oe - 1] = oe - ob; // signal end with difference
600                                                if (nl != ol) {
601                                                        logger.error("Single dimension is incompatible with subshape");
602                                                        throw new IllegalArgumentException("Single dimension is incompatible with subshape");
603                                                }
604                                        } else if (ol > nl) {
605                                                do { // case where new shape spreads single dimension over several dimensions
606                                                        if (ne == (nr + 1)) {
607                                                                break;
608                                                        }
609                                                        nl *= newShape[ne++];
610                                                } while (nl < ol);
611                                                if (nl != ol) {
612                                                        logger.error("Subshape is incompatible with single dimension");
613                                                        throw new IllegalArgumentException("Subshape is incompatible with single dimension");
614                                                }
615
616                                        }
617
618                                        ob = oe;
619                                        nb = ne;
620                                }
621
622                        }
623                }
624
625                @Override
626                public ILazyDataset run(ILazyDataset lz) {
627                        if (differences == null)
628                                init();
629
630                        int[] lshape = lz.getShape();
631                        if (Arrays.equals(newShape, lshape)) {
632                                return lz;
633                        }
634                        int or = lz.getRank();
635                        int nr = newShape.length;
636                        int[] nshape = new int[nr];
637                        Arrays.fill(nshape, 1);
638                        if (onesOnly) {
639                                // ignore omit removed dimensions
640                                for (int i = 0, si = 0, di = 0; i < (or+1) && si <= or && di < nr; i++) {
641                                        int c = differences[i];
642                                        if (c == 0) {
643                                                nshape[di++] = lshape[si++];
644                                        } else if (c > 0) {
645                                                while (c-- > 0 && di < nr) {
646                                                        di++;
647                                                }
648                                        } else if (c < 0) {
649                                                si -= c; // remove dimensions by skipping forward in source array
650                                        }
651                                }
652                        } else {
653                                boolean[] broadcast = new boolean[or];
654                                for (int ob = 0; ob < or; ob++) {
655                                        broadcast[ob] = oldShape[ob] != 1 && lshape[ob] == 1;
656                                }
657                                int osize = lz.getSize();
658
659                                // cannot do 3x5x... to 15x... if metadata is broadcasting (i.e. 1x5x...)
660                                int ob = 0;
661                                int nsize = 1;
662                                for (int i = 0; i < nr; i++) {
663                                        if (ob < or && broadcast[ob]) {
664                                                if (differences[ob] != 0) {
665                                                        logger.error("Metadata contains a broadcast axis which cannot be reshaped");
666                                                        throw new IllegalArgumentException("Metadata contains a broadcast axis which cannot be reshaped");
667                                                }
668                                        } else {
669                                                nshape[i] = nsize < osize ? newShape[i] : 1;
670                                        }
671                                        nsize *= nshape[i];
672                                        ob++;
673                                }
674                        }
675
676                        ILazyDataset nlz = lz.getSliceView();
677                        if (lz instanceof Dataset) {
678                                nlz = ((Dataset) lz).reshape(nshape);
679                        } else {
680                                nlz = lz.getSliceView();
681                                nlz.setShape(nshape);
682                        }
683                        return nlz;
684                }
685        }
686
687        class MdsTranspose implements MetadatasetAnnotationOperation {
688                int[] map;
689
690                public MdsTranspose(final int[] axesMap) {
691                        map = axesMap;
692                }
693
694                @SuppressWarnings({ "rawtypes", "unchecked" })
695                @Override
696                public Object processField(Field f, Object o) {
697                        // reorder arrays and lists according the axes map
698                        if (o.getClass().isArray()) {
699                                int l = Array.getLength(o);
700                                if (l == map.length) {
701                                        Object narray = Array.newInstance(o.getClass().getComponentType(), l);
702                                        for (int i = 0; i < l; i++) {
703                                                Array.set(narray, i, Array.get(o, map[i]));
704                                        }
705                                        for (int i = 0; i < l; i++) {
706                                                Array.set(o, i, Array.get(narray, i));
707                                        }
708                                }
709                        } else if (o instanceof List<?>) {
710                                List list = (List) o;
711                                int l = list.size();
712                                if (l == map.length) {
713                                        Object narray = Array.newInstance(o.getClass().getComponentType(), l);
714                                        for (int i = 0; i < l; i++) {
715                                                Array.set(narray, i, list.get(map[i]));
716                                        }
717                                        list.clear();
718                                        for (int i = 0; i < l; i++) {
719                                                list.add(Array.get(narray, i));
720                                        }
721                                }
722                        }
723                        return o;
724                }
725
726                @Override
727                public Class<? extends Annotation> getAnnClass() {
728                        return Transposable.class;
729                }
730
731                @Override
732                public int change(int axis) {
733                        return 0;
734                }
735
736                @Override
737                public int getNewRank() {
738                        return -1;
739                }
740
741                @Override
742                public ILazyDataset run(ILazyDataset lz) {
743                        return lz.getTransposedView(map);
744                }
745        }
746
747        class MdsDirty implements MetadatasetAnnotationOperation {
748
749                @Override
750                public Object processField(Field f, Object o) {
751                        // throw exception if not boolean???
752                        Class<?> t = f.getType();
753                        if (t.equals(boolean.class) || t.equals(Boolean.class)) {
754                                if (o.equals(false)) {
755                                        o = true;
756                                }
757                        }
758                        return o;
759                }
760
761                @Override
762                public Class<? extends Annotation> getAnnClass() {
763                        return Dirtiable.class;
764                }
765
766                @Override
767                public int change(int axis) {
768                        return 0;
769                }
770
771                @Override
772                public int getNewRank() {
773                        return -1;
774                }
775
776                @Override
777                public ILazyDataset run(ILazyDataset lz) {
778                        return lz;
779                }
780        }
781
782        /**
783         * Slice all datasets in metadata that are annotated by @Sliceable. Call this on the new sliced
784         * dataset after cloning the metadata
785         * @param asView if true then just a view
786         * @param slice
787         */
788        protected void sliceMetadata(boolean asView, final SliceND slice) {
789                processAnnotatedMetadata(new MdsSlice(asView, slice), true);
790        }
791
792        /**
793         * Reshape all datasets in metadata that are annotated by @Reshapeable. Call this when squeezing
794         * or setting the shape
795         * 
796         * @param newShape
797         */
798        protected void reshapeMetadata(final int[] oldShape, final int[] newShape) {
799                processAnnotatedMetadata(new MdsReshape(oldShape, newShape), true);
800        }
801
802        /**
803         * Transpose all datasets in metadata that are annotated by @Transposable. Call this on the transposed
804         * dataset after cloning the metadata
805         * @param axesMap
806         */
807        protected void transposeMetadata(final int[] axesMap) {
808                processAnnotatedMetadata(new MdsTranspose(axesMap), true);
809        }
810
811        /**
812         * Dirty metadata that are annotated by @Dirtiable. Call this when the dataset has been modified
813         * @since 2.0
814         */
815        protected void dirtyMetadata() {
816                processAnnotatedMetadata(new MdsDirty(), true);
817        }
818
819        @SuppressWarnings("unchecked")
820        private void processAnnotatedMetadata(MetadatasetAnnotationOperation op, boolean throwException) {
821                if (metadata == null)
822                        return;
823
824                for (Class<? extends MetadataType> c : metadata.keySet()) {
825                        for (MetadataType m : metadata.get(c)) {
826                                if (m == null)
827                                        continue;
828
829                                Class<? extends MetadataType> mc = m.getClass();
830                                do { // iterate over super-classes
831                                        processClass(op, m, mc, throwException);
832                                        Class<?> sclazz = mc.getSuperclass();
833                                        if (!MetadataType.class.isAssignableFrom(sclazz))
834                                                break;
835                                        mc = (Class<? extends MetadataType>) sclazz;
836                                } while (true);
837                        }
838                }
839        }
840
841        @SuppressWarnings({ "unchecked", "rawtypes" })
842        private static void processClass(MetadatasetAnnotationOperation op, MetadataType m, Class<? extends MetadataType> mc, boolean throwException) {
843                for (Field f : mc.getDeclaredFields()) {
844                        if (!f.isAnnotationPresent(op.getAnnClass()))
845                                continue;
846
847                        try {
848                                f.setAccessible(true);
849                                Object o = f.get(m);
850                                if (o == null)
851                                        continue;
852
853                                Object no = op.processField(f, o);
854                                if (no != o) {
855                                        f.set(m, no);
856                                        continue;
857                                }
858                                Object r = null;
859                                if (o instanceof ILazyDataset) {
860                                        try {
861                                                f.set(m, op.run((ILazyDataset) o));
862                                        } catch (Exception e) {
863                                                logger.error("Problem processing " + o, e);
864                                                if (!catchExceptions)
865                                                        throw e;
866                                        }
867                                } else if (o.getClass().isArray()) {
868                                        int l = Array.getLength(o);
869                                        if (l <= 0)
870                                                continue;
871
872                                        for (int i = 0; r == null && i < l; i++) {
873                                                r = Array.get(o, i);
874                                        }
875                                        int n = op.getNewRank();
876                                        if (r == null) {
877                                                if (n < 0 || n != l) { // all nulls be need to match rank as necessary
878                                                        f.set(m, Array.newInstance(o.getClass().getComponentType(), n < 0 ? l : n));
879                                                }
880                                                continue;
881                                        }
882                                        if (n < 0)
883                                                n = l;
884                                        Object narray = Array.newInstance(r.getClass(), n);
885                                        for (int i = 0, si = 0, di = 0; di < n && si < l; i++) {
886                                                int c = op.change(i);
887                                                if (c == 0) {
888                                                        Array.set(narray, di++, processObject(op, Array.get(o, si++)));
889                                                } else if (c > 0) {
890                                                        di += c; // add nulls by skipping forward in destination array
891                                                } else if (c < 0) {
892                                                        si -= c; // remove dimensions by skipping forward in source array
893                                                }
894                                        }
895                                        if (n == l) {
896                                                for (int i = 0; i < l; i++) {
897                                                        Array.set(o, i, Array.get(narray, i));
898                                                }
899                                        } else {
900                                                f.set(m, narray);
901                                        }
902                                } else if (o instanceof List<?>) {
903                                        List list = (List) o;
904                                        int l = list.size();
905                                        if (l <= 0)
906                                                continue;
907
908                                        for (int i = 0; r == null && i < l; i++) {
909                                                r = list.get(i);
910                                        }
911                                        int n = op.getNewRank();
912                                        if (r == null) {
913                                                if (n < 0 || n != l) { // all nulls be need to match rank as necessary
914                                                        list.clear();
915                                                        for (int i = 0, imax = n < 0 ? l : n; i < imax; i++) {
916                                                                list.add(null);
917                                                        }
918                                                }
919                                                continue;
920                                        }
921
922                                        if (n < 0)
923                                                n = l;
924                                        Object narray = Array.newInstance(r.getClass(), n);
925                                        for (int i = 0, si = 0, di = 0; i < l && si < l; i++) {
926                                                int c = op.change(i);
927                                                if (c == 0) {
928                                                        Array.set(narray, di++, processObject(op, list.get(si++)));
929                                                } else if (c > 0) {
930                                                        di += c; // add nulls by skipping forward in destination array
931                                                } else if (c < 0) {
932                                                        si -= c; // remove dimensions by skipping forward in source array
933                                                }
934                                        }
935                                        list.clear();
936                                        for (int i = 0; i < n; i++) {
937                                                list.add(Array.get(narray, i));
938                                        }
939                                } else if (o instanceof Map<?,?>) {
940                                        Map map = (Map) o;
941                                        for (Object k : map.keySet()) {
942                                                map.put(k, processObject(op, map.get(k)));
943                                        }
944                                }
945                        } catch (Exception e) {
946                                logger.error("Problem occurred when processing metadata of class {}: {}", mc.getCanonicalName(), e);
947                                if (throwException)
948                                        throw new RuntimeException(e);
949                        }
950                }
951        }
952
953        @SuppressWarnings({ "unchecked", "rawtypes" })
954        private static Object processObject(MetadatasetAnnotationOperation op, Object o) throws Exception {
955                if (o == null)
956                        return o;
957
958                if (o instanceof ILazyDataset) {
959                        try {
960                                return op.run((ILazyDataset) o);
961                        } catch (Exception e) {
962                                logger.error("Problem processing " + o, e);
963                                if (!catchExceptions)
964                                        throw e;
965                        }
966                } else if (o.getClass().isArray()) {
967                        int l = Array.getLength(o);
968                        for (int i = 0; i < l; i++) {
969                                Array.set(o, i, processObject(op, Array.get(o, i)));
970                        }
971                } else if (o instanceof List<?>) {
972                        List list = (List) o;
973                        for (int i = 0, imax = list.size(); i < imax; i++) {
974                                list.set(i, processObject(op, list.get(i)));
975                        }
976                } else if (o instanceof Map<?,?>) {
977                        Map map = (Map) o;
978                        for (Object k : map.keySet()) {
979                                map.put(k, processObject(op, map.get(k)));
980                        }
981                }
982                return o;
983        }
984
985        protected void restoreMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> oldMetadata) {
986                for (Class<? extends MetadataType> mc : oldMetadata.keySet()) {
987                        metadata.put(mc, oldMetadata.get(mc));
988                }
989        }
990
991        protected ILazyDataset createFromSerializable(Serializable blob, boolean keepLazy) {
992                ILazyDataset d = null;
993                if (blob instanceof ILazyDataset) {
994                        d = (ILazyDataset) blob;
995                        if (d instanceof IDataset) {
996                                Dataset ed = DatasetUtils.convertToDataset((IDataset) d);
997                                int is = ed.getElementsPerItem();
998                                if (is != 1 && is != getElementsPerItem()) {
999                                        throw new IllegalArgumentException("Dataset has incompatible number of elements with this dataset");
1000                                }
1001                                d = ed.cast(is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64);
1002                        } else if (!keepLazy) {
1003                                final int is = getElementsPerItem();
1004                                try {
1005                                        d = DatasetUtils.cast(d.getSlice(), is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64);
1006                                } catch (DatasetException e) {
1007                                        logger.error("Could not get data from lazy dataset", e);
1008                                        return null;
1009                                }
1010                        }
1011                } else {
1012                        final int is = getElementsPerItem();
1013                        if (is == 1) {
1014                                d = DatasetFactory.createFromObject(DoubleDataset.class, blob);
1015                        } else {
1016                                try {
1017                                        d = DatasetFactory.createFromObject(is, CompoundDoubleDataset.class, blob);
1018                                } catch (IllegalArgumentException e) { // if only single value supplied try again
1019                                        d = DatasetFactory.createFromObject(DoubleDataset.class, blob);
1020                                }
1021                        }
1022                        if (d.getSize() == getSize() && !Arrays.equals(d.getShape(), shape)) {
1023                                d.setShape(shape.clone());
1024                        }
1025                }
1026                List<int[]> s = BroadcastUtils.broadcastShapesToMax(shape, d.getShape());
1027                d.setShape(s.get(0));
1028
1029                return d;
1030        }
1031
1032        @Override
1033        public void setErrors(Serializable errors) {
1034                if (shape == null) {
1035                        throw new IllegalArgumentException("Cannot set errors for null dataset");
1036                }
1037                if (errors == null) {
1038                        clearMetadata(ErrorMetadata.class);
1039                        return;
1040                }
1041                if (errors == this) {
1042                        logger.warn("Ignoring setting error to itself as this will lead to infinite recursion");
1043                        return;
1044                }
1045
1046                ILazyDataset errorData = createFromSerializable(errors, true);
1047
1048                ErrorMetadata emd = getErrorMetadata();
1049                if (emd == null) {
1050                        try {
1051                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
1052                                setMetadata(emd);
1053                        } catch (MetadataException me) {
1054                                logger.error("Could not create metadata", me);
1055                        }
1056                }
1057                emd.setError(errorData);
1058        }
1059
1060        protected ErrorMetadata getErrorMetadata() {
1061                try {
1062                        List<ErrorMetadata> el = getMetadata(ErrorMetadata.class);
1063                        if (el != null && !el.isEmpty()) {
1064                                 return el.get(0);
1065                        }
1066                } catch (Exception e) {
1067                }
1068                return null;
1069        }
1070
1071        @Override
1072        public ILazyDataset getErrors() {
1073                ErrorMetadata emd = getErrorMetadata();
1074                return emd == null ? null : emd.getError();
1075        }
1076
1077        @Override
1078        public boolean hasErrors() {
1079                return LazyDatasetBase.this.getErrors() != null;
1080        }
1081
1082        /**
1083         * Check permutation axes
1084         * @param shape
1085         * @param axes
1086         * @return cleaned up axes or null if trivial
1087         */
1088        public static int[] checkPermutatedAxes(int[] shape, int... axes) {
1089                int rank = shape == null ? 0 : shape.length;
1090
1091                if (axes == null || axes.length == 0) {
1092                        axes = new int[rank];
1093                        for (int i = 0; i < rank; i++) {
1094                                axes[i] = rank - 1 - i;
1095                        }
1096                }
1097
1098                if (axes.length != rank) {
1099                        logger.error("axis permutation has length {} that does not match dataset's rank {}", axes.length, rank);
1100                        throw new IllegalArgumentException("axis permutation does not match shape of dataset");
1101                }
1102        
1103                // check all permutation values are within bounds
1104                for (int i = 0; i < rank; i++) {
1105                        axes[i] = ShapeUtils.checkAxis(rank, axes[i]);
1106                }
1107        
1108                // check for a valid permutation (is this an unnecessary restriction?)
1109                int[] perm = axes.clone();
1110                Arrays.sort(perm);
1111
1112                for (int i = 0; i < rank; i++) {
1113                        if (perm[i] != i) {
1114                                logger.error("axis permutation is not valid: it does not contain complete set of axes");
1115                                throw new IllegalArgumentException("axis permutation does not contain complete set of axes");
1116                        }
1117                }
1118
1119                if (Arrays.equals(axes, perm))
1120                        return null; // signal identity or trivial permutation
1121
1122                return axes;
1123        }
1124}