001    /*
002     * Copyright (c) 1998-2014 ChemAxon Ltd. All Rights Reserved.
003     *
004     * This software is the confidential and proprietary information of
005     * ChemAxon. You shall not disclose such Confidential Information
006     * and shall use it only in accordance with the terms of the agreements
007     * you entered into with ChemAxon.
008     *
009     */
010    package com.chemaxon.overlap.io;
011    
012    import chemaxon.formats.MolFormatException;
013    import chemaxon.struc.Molecule;
014    import com.chemaxon.calculations.common.SubProgressObserver;
015    import com.google.common.base.Optional;
016    import java.io.InputStream;
017    import java.io.Serializable;
018    import java.lang.reflect.Array;
019    import java.util.ArrayList;
020    import java.util.Iterator;
021    import java.util.LinkedList;
022    import java.util.List;
023    import java.util.Queue;
024    import java.util.concurrent.Callable;
025    import java.util.concurrent.CancellationException;
026    import java.util.concurrent.ExecutionException;
027    import java.util.concurrent.ExecutorService;
028    import java.util.concurrent.Future;
029    
030    /**
031     * Helper method for managing {@link MasterStorages} instances.
032     *
033     * @author Gabor Imre
034     */
035    public final class MasterStorages {
036    
037        /**
038         * Page size to use on import.
039         */
040        public static final int DEFAULT_IMPORT_PAGESIZE = 50;
041    
042        /**
043         * Max queue size for import.
044         */
045        public static final int MAX_QUEUE_SIZE = 300;
046    
047        /**
048         * No constructor exposed.
049         */
050        private MasterStorages() {
051        }
052    
053        /**
054         * Builder class for a master storage.
055         *
056         * @param <T> Type of stored properties
057         */
058        public interface MasterStorageBuilder<T> {
059    
060            /**
061             * Add next represented data.
062             *
063             * @param masterIndex Next index
064             * @param data Next data
065             * @throws IllegalArgumentException when passed masterIndex differs from expected
066             */
067            void add(int masterIndex, T data);
068    
069            /**
070             * Add missing data.
071             *
072             * @param masterIndex Next index
073             * @throws IllegalArgumentException when passed masterIndex differs from expected
074             */
075            void addMissing(int masterIndex);
076    
077            /**
078             * Build immutable, serializable storage from the current state.
079             *
080             * @return Built storage
081             */
082            MasterStorage<T> build();
083        }
084    
085        /**
086         * Implementation for String storage builder.
087         *
088         * @param <T> Type of stored properties
089         */
090        private static class MasterStorageBuilderImpl<T extends Serializable> implements MasterStorageBuilder<T> {
091    
092            /**
093             * Number of initial skips.
094             */
095            private final int skips;
096    
097            /**
098             * Type of elements.
099             */
100            private final Class<T> elements;
101            /**
102             * Stored properties.
103             */
104            private final List<T> storage;
105    
106            /**
107             * Next exepcted index (readno).
108             */
109            private int nextExpectedIndex;
110    
111            /**
112             * Construct new buidler.
113             *
114             * @param skips Initial skips.
115             */
116            MasterStorageBuilderImpl(int skips, Class<T> elements) {
117                if (skips < 0) {
118                    throw new IllegalArgumentException("Skips can not be negative " + skips);
119                }
120                this.elements = elements;
121                this.skips = skips;
122                this.storage = new ArrayList<T>();
123                this.nextExpectedIndex = skips;
124            }
125    
126            /**
127             * Check index for correct order.
128             *
129             * @param index Passed readno
130             * @throws IllegalArgumentException when readno does not match
131             */
132            private void checkIndex(int index) {
133                if (this.nextExpectedIndex != index) {
134                    throw new IllegalArgumentException("Expected " + this.nextExpectedIndex + ", got " + index);
135                }
136                this.nextExpectedIndex++;
137            }
138    
139            @Override
140            public void add(int masterIndex, T data) {
141                checkIndex(masterIndex);
142                this.storage.add(data);
143            }
144    
145            @Override
146            public void addMissing(int masterIndex) {
147                checkIndex(masterIndex);
148                this.storage.add(null);
149            }
150    
151            @Override
152            public MasterStorage<T> build() {
153                return new ArrayStorage<T>(this.storage, this.skips, this.elements);
154            }
155    
156        }
157    
158        /**
159         * Create a String storage builder.
160         *
161         * @param skip Initial skip count
162         * @return A String storage builder
163         */
164        public static MasterStorageBuilder<String> createStringStoreBuilder(int skip) {
165            return new MasterStorageBuilderImpl<String>(skip, String.class);
166        }
167    
168        /**
169         * Create a String storage builder.
170         *
171         * @return A String storage builder
172         */
173        public static MasterStorageBuilder<String> createStringStoreBuilder() {
174            return createStringStoreBuilder(0);
175        }
176    
177        /**
178         * Create storage from a strcuture file.
179         *
180         * <p>
181         * This method blocks until completion. Passed stream and progress observer callback is accessed only from the
182         * calling thread.</p>
183         *
184         * @param is InputStream to read from. Will not be closed upon completion/abort.
185         * @param opts Options to pass to underlying importer. Use empty string when no import option is used.
186         * @param count Max number of structures to input
187         * @param standardizer Standardizer to apply on imported structures
188         * @param ex ExecutorService to run parsing/converting
189         * @param po Observer to track progress. Will be closed by invoking {@link SubProgressObserver#done()} upon
190         * completion/abortion
191         * @param mcb Callbacks to also report molecules or null
192         * @return Created master storage
193         */
194        public static MasterMoleculeStorage masterMoleculeStorageFromFile(
195                InputStream is,
196                String opts,
197                int count,
198                StandardizerWrapper standardizer,
199                ExecutorService ex,
200                SubProgressObserver po,
201                List<? extends MoleculeCallback> mcb) {
202    
203            try {
204                final Iterator<StructureRecord> it = new StructureRecordIterator(is, opts);
205                final Queue<Future<ImportTask>> queue = new LinkedList<Future<ImportTask>>();
206    
207                // final ImmutableList.Builder<String> ret = new ImmutableList.Builder<String>();
208                final CompactStringStorage.Builder ret = new CompactStringStorage.Builder(100000, 0);
209    
210                int readct = 0;
211    
212                while ((it.hasNext() && readct < count) || !queue.isEmpty()) {
213                    if ((it.hasNext() && readct < count)) {
214                        final int firstId = readct;
215                        final List<StructureRecord> in = new ArrayList<StructureRecord>(DEFAULT_IMPORT_PAGESIZE);
216                        for (int i = 0; i < DEFAULT_IMPORT_PAGESIZE && it.hasNext() && readct < count; i++) {
217                            in.add(it.next());
218                            readct++;
219                        }
220                        queue.add(ex.submit(new ImportTask(standardizer, in, firstId)));
221                    }
222    
223                    while (!queue.isEmpty() && (queue.element().isDone() || queue.size() >= MAX_QUEUE_SIZE)) {
224                        final ImportTask task = queue.remove().get();
225                        final List<String> res = task.getRes();
226                        if (mcb != null) {
227                            task.applyCallbacks(mcb);
228                        }
229    
230                        int i = task.firstID;
231                        for (String s : res) {
232                            ret.add(i++, s);
233                        }
234    
235                        po.worked(res.size());
236                        if (po.isCancelled()) {
237                            throw new CancellationException();
238                        }
239                    }
240    
241                }
242    
243                return new MasterMoleculeStorage(ret.build());
244            } catch (InterruptedException e) {
245                throw new CancellationException();
246            } catch (ExecutionException e) {
247                // not expected to die
248                throw new IllegalStateException(e);
249            } finally {
250                po.done();
251            }
252        }
253    
254        /**
255         * Molecule callback based builder.
256         */
257        public static class MoleculeCallbackBuilder implements MoleculeCallback {
258    
259            /**
260             * Underlying immutable list builder.
261             */
262            //private final ImmutableList.Builder<String> list;
263            private final CompactStringStorage.Builder storage;
264    
265            /**
266             * Next exepcted index (readno).
267             */
268            private int nextExpectedReadno;
269    
270            /**
271             * Construct new empty builder.
272             */
273            public MoleculeCallbackBuilder() {
274                // this.list = new ImmutableList.Builder<String>();
275                this.storage = new CompactStringStorage.Builder(100000, 0);
276                this.nextExpectedReadno = 0;
277            }
278    
279            /**
280             * Check index for correct order.
281             *
282             * @param readno Passed readno
283             * @throws IllegalArgumentException when readno does not match
284             */
285            private void checkReadno(int readno) {
286                if (this.nextExpectedReadno != readno) {
287                    throw new IllegalArgumentException("Expected " + this.nextExpectedReadno + ", got " + readno);
288                }
289                this.nextExpectedReadno++;
290            }
291    
292            @Override
293            public void notifyMolecule(int readno, Optional<String> molString, Molecule m, int index) {
294                checkReadno(readno);
295                this.storage.add(readno, MasterMoleculeStorage.bestEffortConvert(m));
296    
297            }
298    
299            @Override
300            public void notifyProcessingError(int readno, Optional<String> molString, Molecule m, Throwable t) {
301                checkReadno(readno);
302                this.storage.add(readno, MasterMoleculeStorage.bestEffortConvert(m));
303            }
304    
305            @Override
306            public void notifyParseError(int readno, Optional<String> molString, Throwable t) {
307                checkReadno(readno);
308                // We have no molecule for this index
309                this.storage.addMissing(readno);
310                // this.list.add(MasterMoleculeStorage.MISSING_STRUCTURE);
311            }
312    
313            /**
314             * Create immutable representation.
315             *
316             * @return Immutable representation
317             */
318            public MasterMoleculeStorage build() {
319                return new MasterMoleculeStorage(this.storage.build());
320            }
321    
322        }
323    
324        /**
325         * Import task.
326         */
327        private static class ImportTask implements Callable<ImportTask> {
328    
329            /**
330             * Standardizer to use.
331             */
332            private final StandardizerWrapper standardizer;
333    
334            /**
335             * Structure records to process.
336             */
337            private final List<StructureRecord> structures;
338    
339            /**
340             * Parsed molecules or null.
341             */
342            private final List<Molecule> molecules;
343    
344            /**
345             * Errors or null.
346             */
347            private final List<Throwable> errors;
348    
349            /**
350             * Resulting Strings.
351             */
352            private final List<String> res;
353    
354            /**
355             * ID for first.
356             */
357            private final int firstID;
358    
359            /**
360             * Construct task.
361             *
362             * @param standardizer Underlying standardizer
363             * @param structures Structures to process
364             */
365            public ImportTask(StandardizerWrapper standardizer, List<StructureRecord> structures, int firstId) {
366                this.standardizer = standardizer;
367                this.structures = structures;
368                this.res = new ArrayList<String>(this.structures.size());
369                this.molecules = new ArrayList<Molecule>(this.structures.size());
370                this.errors = new ArrayList<Throwable>(this.structures.size());
371                this.firstID = firstId;
372            }
373    
374            @Override
375            public ImportTask call() throws Exception {
376                for (StructureRecord record : this.structures) {
377                    try {
378                        final Molecule m = record.parseMolecule();
379                        this.res.add(MasterMoleculeStorage.bestEffortConvert(m));
380                        this.molecules.add(m);
381                        this.errors.add(null);
382                    } catch (MolFormatException e) {
383                        //this.res.add(MasterMoleculeStorage.MISSING_STRUCTURE);
384                        this.res.add("");
385                        this.molecules.add(null);
386                        this.errors.add(e);
387                    }
388                }
389    
390                return this;
391            }
392            /**
393             * Get resulting String representations.
394             *
395             * @return Results of String representations
396             */
397            List<String> getRes() {
398                if (this.res.size() != this.structures.size()) {
399                    throw new IllegalStateException("Results size is " + this.res.size()
400                            + ", expected " + this.structures.size());
401                }
402                return this.res;
403            }
404    
405            /**
406             * Invoke all callbacks.
407             *
408             * @param callbacks Callbacks to invoke
409             */
410            public void applyCallbacks(List<? extends MoleculeCallback> callbacks) {
411                if (this.molecules.size() != this.structures.size()) {
412                    throw new IllegalStateException("Results size is " + this.molecules.size()
413                            + ", expected " + this.structures.size());
414                }
415                if (callbacks == null || callbacks.isEmpty()) {
416                    return;
417                }
418                int id = this.firstID;
419                for (int i = 0; i < this.molecules.size(); i++) {
420    
421                    for (MoleculeCallback cb : callbacks) {
422                        if (this.molecules.get(i) != null) {
423                            cb.notifyMolecule(id, Optional.<String>absent(), this.molecules.get(i), id);
424                        } else {
425                            cb.notifyParseError(id, Optional.<String>absent(), this.errors.get(i));
426                        }
427                    }
428    
429                    id++;
430                }
431    
432            }
433    
434        }
435    
436        /**
437         * Simple array backed storage.
438         *
439         */
440        private static final class ArrayStorage<T extends Serializable> implements MasterStorage<T>, Serializable {
441    
442            /**
443             * Serial version.
444             */
445            private static final long serialVersionUID = 0;
446    
447            /**
448             * Backing array; <code>null</code> for missings.
449             */
450            private final T[] propArray;
451    
452            /**
453             * Absent count.
454             */
455            private final int absents;
456    
457            /**
458             * Number of initial skips.
459             */
460            private final int skips;
461    
462            /**
463             * Construct from a List of Strings.
464             *
465             * @param props Props to represent
466             * @param skips Number of initial skips
467             * @param elements Type of elements
468             */
469            ArrayStorage(List<T> props, int skips, Class<T> elements) {
470                if (skips < 0) {
471    
472                    throw new IllegalArgumentException("Negative skips " + skips);
473                }
474                this.skips = skips;
475                final T[] ar = (T[]) Array.newInstance(elements, 0);
476                this.propArray = props.toArray(ar);
477                int a = 0;
478                for (T s : this.propArray) {
479                    if (s == null) {
480                        a++;
481                    }
482                }
483                this.absents = a;
484            }
485    
486            @Override
487            public int size() {
488                return this.propArray.length + this.skips;
489            }
490    
491            @Override
492            public Optional<T> get(int index) {
493                if (index < this.skips || this.propArray[index - this.skips] == null) {
494                    return Optional.<T>absent();
495                } else {
496                    return Optional.of(this.propArray[index - this.skips]);
497                }
498            }
499    
500            @Override
501            public int getAbsentCount() {
502                return this.absents + this.skips;
503            }
504    
505            @Override
506            public String toString() {
507                return "String master storage; size: "
508                        + size() + ", absents: " + getAbsentCount() + ", skips: " + this.skips;
509            }
510    
511        }
512    
513    
514    }