redberry-redberry-pipe-7323a27293f8/.hg_archival.txt0000644000000000000000000000020113353377423020264 0ustar 00000000000000repo: 7914145f3a4e71eb7e190f8da8c4f56eeaa1e194 node: 7323a27293f8a9fee5f4eda2c06c42d6b32fae9f branch: default tag: v1.0.0-alpha0 redberry-redberry-pipe-7323a27293f8/.flow0000644000000000000000000000017313353377423016156 0ustar 00000000000000[branchname] master = default develop = develop feature = feature/ release = release/ hotfix = hotfix/ support = support/ redberry-redberry-pipe-7323a27293f8/.hgflow0000644000000000000000000000017413353377423016476 0ustar 00000000000000[branchname] master = default develop = develop feature = feature/ release = release/ hotfix = hotfix/ support = support/ redberry-redberry-pipe-7323a27293f8/.hgignore0000644000000000000000000000014613353377423017011 0ustar 00000000000000target nbactions.*\.xml$ nb-configuration.xml$ \.log$ \.log\. \.iml$ \.ipr$ \.iws$ \.idea \.DS_Store$ redberry-redberry-pipe-7323a27293f8/.hgtags0000644000000000000000000000057613353377423016473 0ustar 000000000000006888a683c930f5ddaf0c6ccd97f2709ebedda7e9 v0.9 fd180a2531ee9268d7b8f3b0724db52acb9a7fb5 v0.9.1 0fc6a342c9c6f0f516353eb8248120e2416bae0c v0.9.2 32ac825eca9c81e9fca2018e5f707c3c1e646a8a v0.9.3 7f330fab9bed5b039021e4a2661721a57d906c19 v0.9.4 7a6fa9fc0b45f54c99ad028766cd12130ec64c8b v0.9.5 5117244cf5e36a1f0fa536724b912d7262e2209e v0.9.6 0c85a16ffc6b0d43b501c48e523e4dcb94c6e095 v0.9.7 redberry-redberry-pipe-7323a27293f8/pom.xml0000644000000000000000000001522213353377423016524 0ustar 00000000000000 4.0.0 cc.redberry pipe 1.0.0-alpha0 org.sonatype.oss oss-parent 9 pipe Java library for implementation of concurrent pipelines. GNU General Public License, version 3 http://www.gnu.org/licenses/gpl-3.0.html repo dbolotin bolotin.dmitriy@gmail.com Bolotin Dmitriy IBCH RAS http://www.ibch.ru/ architect developer tester documentation UTC+04:00 https://bitbucket.org/dbolotin mikesh mikhail.shugay@gmail.com Mikhail Shugay IBCH RAS http://www.ibch.ru/ documentation UTC+04:00 https://bitbucket.org/mikesh scm:hg:http://bitbucket.org/redberry/redberry-pipe https://bitbucket.org/redberry/redberry-pipe/src/ Bitbucket https://bitbucket.org/redberry/redberry-pipe/issues junit junit 4.8.2 test org.apache.commons commons-math 2.2 test UTF-8 release org.apache.maven.plugins maven-source-plugin 3.0.1 attach-sources jar-no-fork org.apache.maven.plugins maven-javadoc-plugin 3.0.1 attach-javadocs jar none org.apache.maven.plugins maven-gpg-plugin 1.6 sign-artifacts verify sign org.apache.maven.plugins maven-compiler-plugin 3.8.0 1.8 1.8 org.sonatype.plugins nexus-staging-maven-plugin 1.6.8 true ossrh https://oss.sonatype.org/ redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/CUtils.java0000644000000000000000000004252413353377423025056 0ustar 00000000000000/* * cc.redberry.pipe: java library for implementation of concurrent pipelines * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ package cc.redberry.pipe; import cc.redberry.pipe.blocks.Merger; import cc.redberry.pipe.util.Chunk; import cc.redberry.pipe.util.SimpleProcessorWrapper; import cc.redberry.pipe.util.SimpleProcessorWrapperSynchronized; import cc.redberry.pipe.util.SimpleVoidProcessorExecutor; import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; /** * Utility methods. */ public final class CUtils { public static final OutputPort EMPTY_OUTPUT_PORT = new OutputPort() { @Override public Object take() { return null; } }; public static final OutputPortCloseable EMPTY_OUTPUT_PORT_CLOSEABLE = new OutputPortCloseable() { @Override public void close() { } @Override public Object take() { return null; } }; /** * Wraps processor and input port into single output port. * * @param inputPort output port to be wrapped * @param processor processor to wrap * @param type of input objects * @param type of output objects * @return wrapped output port */ public static OutputPort wrap(OutputPort inputPort, Processor processor) { return new SimpleProcessorWrapper<>(inputPort, processor); } /** * Wraps processor and input port into single output port. Wrapper will introduce additional synchronization. * * @param inputPort output port to be wrapped * @param processor processor to wrap * @param type of input objects * @param type of output objects * @return wrapped output port */ public static OutputPort wrapSynchronized(OutputPort inputPort, Processor processor) { return new SimpleProcessorWrapperSynchronized<>(inputPort, processor); } /** * Returns void processor wrapper of an input port. * * @param inputPort input port to wrap * @param type of objects * @return wrapped input port */ public static VoidProcessor asVoidProcessor(final InputPort inputPort) { return new VoidProcessor() { @Override public void process(T input) { inputPort.put(input); } }; } /** * Drains all content of output port into input port.
This method will close a to port by putting null into * it. * * @param from port to take elements from * @param to port to put elements to * @param type of elements * @throws InterruptedException */ public static void drain(OutputPort from, InputPort to) throws InterruptedException { T element; while ((element = from.take()) != null) to.put(element); to.put(null); //Closing } /** * Drains all content of output port into input port. * * @param from port to take elements from * @param to port to put elements to * @param type of elements * @throws InterruptedException */ public static void drainWithoutClose(OutputPort from, InputPort to) throws InterruptedException { T element; while ((element = from.take()) != null) to.put(element); } /** * Converts VoidProcessor to Processor * * @param processor VoidProcessor to convert * @return Processor wrapper of VoidProcessor */ public static Processor asProcessor(final VoidProcessor processor) { return new Processor() { @Override public T process(T input) { processor.process(input); return input; } }; } /** * Process no more than {@code maxCount} elements from input by provided processor in current thread. * * @param input input object stream * @param processor void processor * @param maxCount maximum number of objects to process * @param type of processed objects * @return true if some objects left in input, false if not * @throws InterruptedException */ public static boolean processPart(OutputPort input, VoidProcessor processor, int maxCount) throws InterruptedException { if (maxCount <= 0) throw new IllegalArgumentException("maxCount should be greater than 0"); T object = null; while ((--maxCount) >= 0 && (object = input.take()) != null) processor.process(object); return object != null; } /** * Process all elements from output port in current thread. * * @param input input object stream * @param processor void processor * @param type of processed objects * @throws InterruptedException */ public static void processAll(OutputPort input, VoidProcessor processor) throws InterruptedException { T object; while ((object = input.take()) != null) processor.process(object); } /** * Process all elements from output port in current thread. * * @param input input object stream * @param processor void processor * @param threadCount number of parallel threads * @param type of processed objects * @throws InterruptedException */ public static void processAllInParallel(OutputPort input, VoidProcessor processor, int threadCount) throws InterruptedException { --threadCount; Thread[] threads = new Thread[threadCount]; for (int i = 0; i < threadCount; ++i) { threads[i] = new Thread(new SimpleVoidProcessorExecutor<>(input, processor)); threads[i].start(); } T object; while ((object = input.take()) != null) processor.process(object); for (int i = 0; i < threadCount; ++i) threads[i].join(); } /** * Process all elements from output port in current thread. * * @param input input object stream * @param processor void processor * @param threadCount number of parallel threads * @param executorService executor service to create threads * @param type of processed objects * @throws InterruptedException */ public static void processAllInParallel(OutputPort input, VoidProcessor processor, int threadCount, ExecutorService executorService) throws InterruptedException { --threadCount; Future[] futures = new Future[threadCount]; for (int i = 0; i < threadCount; ++i) futures[i] = executorService.submit(new SimpleVoidProcessorExecutor<>(input, processor)); T object; while ((object = input.take()) != null) processor.process(object); try { for (int i = 0; i < threadCount; ++i) futures[i].get(); } catch (ExecutionException ex) { throw (RuntimeException) ex.getCause(); } } /** * Process all elements from output port in current thread. * * @param input input object stream * @param processorsFactory void processor factory * @param threadCount number of parallel threads * @param type of processed objects * @throws InterruptedException */ public static void processAllInParallel(OutputPort input, VoidProcessorFactory processorsFactory, int threadCount) throws InterruptedException { --threadCount; Thread[] threads = new Thread[threadCount]; for (int i = 0; i < threadCount; ++i) { threads[i] = new Thread(new SimpleVoidProcessorExecutor(input, processorsFactory.create())); threads[i].start(); } VoidProcessor processor = processorsFactory.create(); T object; while ((object = input.take()) != null) processor.process(object); for (int i = 0; i < threadCount; ++i) threads[i].join(); } /** * Processes elements in N different threads. For each thread a separate processor is created by a provided * factory.
In this implementation threads are created by an ExecutorsService. * * @param input elements to process * @param processorsFactory factory to createInputStream processors * @param threadCount count of parallel processing threads * @param executorService service to execute this task (thread pool) * @param type of processed objects */ public static void processAllInParallel(OutputPort input, VoidProcessorFactory processorsFactory, int threadCount, ExecutorService executorService) throws InterruptedException { --threadCount; Future[] futures = new Future[threadCount]; for (int i = 0; i < threadCount; ++i) futures[i] = executorService.submit(new SimpleVoidProcessorExecutor<>(input, processorsFactory.create())); VoidProcessor processor = processorsFactory.create(); T object; while ((object = input.take()) != null) processor.process(object); try { for (int i = 0; i < threadCount; ++i) futures[i].get(); } catch (ExecutionException ex) { throw (RuntimeException) ex.getCause(); } } public static OutputPort asOutputPort(Iterable iterable) { final Iterator iterator = iterable.iterator(); return new OutputPort() { @Override public synchronized T take() { if (iterator.hasNext()) return iterator.next(); else return null; } }; } public static OutputPort asUnsafeOutputPort(Iterable iterable) { final Iterator iterator = iterable.iterator(); return new OutputPort() { @Override public T take() { if (iterator.hasNext()) return iterator.next(); else return null; } }; } public static OutputPort> chunked(final OutputPortCloseable input, final int chunkSize) { return new OutputPortCloseable>() { @Override public void close() { input.close(); } @Override public Chunk take() { return Chunk.readChunk(input, chunkSize); } }; } public static OutputPort> chunked(final OutputPort input, final int chunkSize) { if (input instanceof OutputPortCloseable) return chunked((OutputPortCloseable) input, chunkSize); else return new OutputPort>() { @Override public Chunk take() { return Chunk.readChunk(input, chunkSize); } }; } public static OutputPortCloseable unchunked(OutputPort> chunkedPort) { return new UnChunkingPort<>(chunkedPort); } public static Processor, Chunk> chunked(final Processor processor) { return new Processor, Chunk>() { @Override public Chunk process(Chunk input) { Object[] buffer = new Object[input.size()]; for (int i = 0; i < input.size(); ++i) buffer[i] = processor.process(input.get(i)); return new Chunk<>(buffer); } }; } public static VoidProcessor> chunked(final VoidProcessor processor) { return new VoidProcessor>() { @Override public void process(Chunk input) { for (I i : input) processor.process(i); } }; } public static Merger buffered(OutputPort input, int bufferSize) { Merger merger = new Merger<>(bufferSize); merger.merge(input); merger.start(); return merger; } public static Merger buffered(OutputPort input, int bufferSize, ExecutorService executorService) { Merger merger = new Merger<>(bufferSize); merger.merge(input, executorService); merger.start(); return merger; } public static OutputPort asOutputPort(T... array) { return asOutputPort(Arrays.asList(array)); } public static OutputPort asOutputPort(final List list) { return new OutputPort() { final AtomicInteger counter = new AtomicInteger(); @Override public T take() { int index = counter.getAndIncrement(); if (index >= list.size()) return null; return list.get(index); } }; } public static Iterable it(final OutputPort input) { return new Iterable() { @Override public Iterator iterator() { return new OPIterator<>(input); } }; } public static class OPIterator implements Iterator { protected final OutputPort op; T next = null; public OPIterator(OutputPort op) { this.op = op; } @Override public boolean hasNext() { if (next != null) return true; else return (next = op.take()) != null; } @Override public T next() { T n = next; next = null; return n; } @Override public void remove() { throw new IllegalStateException("Not supported."); } } private static final class UnChunkingPort implements OutputPortCloseable { final OutputPort> innerPort; final AtomicReference> currentChunk = new AtomicReference(); private UnChunkingPort(OutputPort> innerPort) { this.innerPort = innerPort; } @Override public void close() { if (innerPort instanceof OutputPortCloseable) ((OutputPortCloseable) innerPort).close(); } @Override public T take() { ChunkHolder holder = currentChunk.get(); T element; while (holder == null || (element = holder.take()) == null) { // Current holder is empty // Checking if this holder is final in the sequence if (holder != null && holder.isFinal()) // End of sequence return null; // Creating new holder ChunkHolder newHolder = new ChunkHolder<>(); // Trying to acquire reference to holder if (currentChunk.compareAndSet(holder, newHolder)) { // Reference to holder successfully acquired // Initializing holder (this call also supports null as input) newHolder.set(innerPort.take()); holder = newHolder; } else // Somebody already acquired reference to holder // Getting this reference holder = currentChunk.get(); } return element; } } private static final class ChunkHolder implements OutputPort { private volatile OutputPort chunkPort; private OutputPort getPort() { // Spin-loop while (chunkPort == null) ; return chunkPort; } @Override public T take() { return getPort().take(); } public boolean isFinal() { return getPort() == EMPTY_OUTPUT_PORT; } public void set(Chunk value) { if (value == null) chunkPort = EMPTY_OUTPUT_PORT; else chunkPort = value.outputPort(); } } } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/InputPort.java0000644000000000000000000000264213353377423025614 0ustar 00000000000000/* * cc.redberry.pipe: java library for implementation of concurrent pipelines * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ package cc.redberry.pipe; /** * Base interface for passive object consumer. * * @param type of consumed objects * @author Bolotin Dmitriy (bolotin.dmitriy@gmail.com) */ public interface InputPort { /** * Put an element into this input port. * *

Putting of {@code null} is allowed only once and this indicates that the port should be closed. No * invocations of this method allowed for closed input port.

* * @param object object to put into this input port, or {@code null} if port should be closed * @throws InterruptedException */ void put(T object); } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/OutputPort.java0000644000000000000000000000237613353377423026021 0ustar 00000000000000/* * cc.redberry.pipe: java library for implementation of concurrent pipelines * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ package cc.redberry.pipe; /** * Base interface for passive object producer. * * @param type of produced objects * @author Bolotin Dmitriy (bolotin.dmitriy@gmail.com) * @see cc.redberry.pipe.blocks.AbstractOutputPort */ public interface OutputPort { /** * Returns next element from this output port, or {@code null} if port is closed. * * @return next element from this output port, or {@code null} if port is closed */ T take(); } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/OutputPortCloseable.java0000644000000000000000000000272313353377423027627 0ustar 00000000000000package cc.redberry.pipe; /** * This interface should be implemented along with the {@link OutputPort} interface to indicate that current output port * should be closed (by invoking {@link #close()} method) after downstream consumer (e.g. {@link * cc.redberry.pipe.blocks.Merger}) is closed. All standard blocks will automatically and thread-safe (the latter will * be applied only if output port is not marked as thread-safe using {@link ThreadSafe}) close current output port after * their own job is finished. * * Don't use this method to indicate that {@link InputPort} should be closed, because all input ports are by default * closable, and will be closed by putting null into them. */ public interface OutputPortCloseable extends AutoCloseable, OutputPort { /** * Close current output port and frees all resources used by it (threads, file streams, etc...). * *

Contract for implementation:

    * *
  • The implementation of this method must not throw an exceptions if port is already closed.
    • * *

* *

If the implementation of this method does not fulfill this contract, than methods {@link * cc.redberry.pipe.blocks.Merger#close()}, {@link cc.redberry.pipe.blocks.ParallelProcessor#close()}, etc... will * have an unpredictable behavior.

* *

Use {@link cc.redberry.pipe.blocks.AbstractOutputPort} for proper synchronization.

*/ @Override void close(); } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/PInterruptedException.java0000644000000000000000000000062613353377423030154 0ustar 00000000000000package cc.redberry.pipe; public class PInterruptedException extends RuntimeException { public PInterruptedException(Throwable cause) { super(cause); } public PInterruptedException(String message, Throwable cause) { super(message, cause); } public PInterruptedException(String message) { super(message); } public PInterruptedException() { } } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/Processor.java0000644000000000000000000000221213353377423025620 0ustar 00000000000000/* * cc.redberry.pipe: java library for implementation of concurrent pipelines * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ package cc.redberry.pipe; /** * Interface for processor which produces a single output object from a single input object. * * @param type of consumed object * @param type of produced object * @author Bolotin Dmitriy (bolotin.dmitriy@gmail.com) */ public interface Processor { OutputT process(InputT input); } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/ProcessorFactory.java0000644000000000000000000000220413353377423027151 0ustar 00000000000000/* * cc.redberry.pipe: java library for implementation of concurrent pipelines * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ package cc.redberry.pipe; /** * {@link Processor} factory. * * @param type of consumed object of processor * @param type of produced object of processor * @author Bolotin Dmitriy (bolotin.dmitriy@gmail.com) * @see Processor */ public interface ProcessorFactory { Processor create(); } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/VoidProcessor.java0000644000000000000000000000201313353377423026441 0ustar 00000000000000/* * cc.redberry.pipe: java library for implementation of concurrent pipelines * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ package cc.redberry.pipe; /** * Interface for processor without output. * * @param * @author Bolotin Dmitriy (bolotin.dmitriy@gmail.com) */ public interface VoidProcessor { void process(InputT input); } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/VoidProcessorFactory.java0000644000000000000000000000177613353377423030010 0ustar 00000000000000/* * cc.redberry.pipe: java library for implementation of concurrent pipelines * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ package cc.redberry.pipe; /** * Factory for {@link VoidProcessor}. * * @author Bolotin Dmitriy (bolotin.dmitriy@gmail.com) */ public interface VoidProcessorFactory { VoidProcessor create(); } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/blocks/Buffer.java0000644000000000000000000002120213353377423026327 0ustar 00000000000000/* * cc.redberry.pipe: java library for implementation of concurrent pipelines * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ package cc.redberry.pipe.blocks; import cc.redberry.pipe.InputPort; import cc.redberry.pipe.OutputPort; import cc.redberry.pipe.PInterruptedException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; /** * Fixed size FIFO buffer that concatenates data streams from several created (by {@link #createInputPort()} method) * input ports and acting as a single output port. * *

Code is adapted from {@link java.util.concurrent.ArrayBlockingQueue}.

* * @param type of objects to route * @author Bolotin Dmitriy * @author Mikhail Shugay */ public final class Buffer implements OutputPort, BufferStatusProvider { public static final int DEFAULT_SIZE = 512; /** * The queued items */ final Object[] items; /** * Collection to store created input ports */ final List inputPorts = new ArrayList<>(); /** * Main lock guarding all access */ final ReentrantLock lock; /** * Condition for waiting - take */ private final Condition notEmpty; /** * Condition for waiting - put */ private final Condition notFull; /** * Condition for waitClosed */ private final Condition closedCondition; /** * Buffer state counters */ private long putCount, takeCount; /* * Concurrency control uses the classic double-condition algorithm * found in any textbook. */ /** * Is this port closed */ boolean closed; /** * Index of item for next take, pull, peek or remove */ int takeIndex; /** * Index of item for next put, offer, or add */ int putIndex; /** * Number of elements in the queue */ int count; /** * Creates a {@code Buffer} with default size. */ public Buffer() { this(DEFAULT_SIZE); } /** * Creates a {@code Buffer} with the given (fixed) size. * * @param size the size of this queue * @throws IllegalArgumentException if {@code size < 1} */ public Buffer(int size) { this(size, false); } /** * Creates a {@code Buffer} with the given (fixed) size and the specified access policy. * * @param size the size of this queue * @param fair if {@code true} then queue accesses for threads blocked on insertion or removal are processed in FIFO * order; the access order is unspecified otherwise * @throws IllegalArgumentException if {@code size < 1} */ public Buffer(int size, boolean fair) { if (size <= 0) throw new IllegalArgumentException(); this.items = new Object[size]; lock = new ReentrantLock(fair); notEmpty = lock.newCondition(); notFull = lock.newCondition(); closedCondition = lock.newCondition(); } // Internal helper methods @SuppressWarnings("unchecked") static E cast(Object item) { return (E) item; } /** * Inserts the specified element at the tail of this queue, waiting for space to become available if the queue is * full. Invoked only through {@link Buffer.BufferInputPort} instances. */ private void put(E e) { final ReentrantLock lock = this.lock; try { lock.lockInterruptibly(); try { while (count == items.length && !closed) notFull.await(); if (closed) return; //If buffer is closed drop input object insert(e); } finally { lock.unlock(); } } catch (InterruptedException ie) { throw new PInterruptedException(ie); } } /** * Take first object that was put through some of input ports owned by this buffer. * * @return {@inheritDoc} * @throws InterruptedException */ @Override public E take() { if (inputPorts.isEmpty()) throw new IllegalStateException("No input ports created for this Buffer."); final ReentrantLock lock = this.lock; try { lock.lockInterruptibly(); try { while (count == 0 && !closed) notEmpty.await(); if (closed && count == 0) return null; return extract(); } finally { lock.unlock(); } } catch (InterruptedException ie) { throw new PInterruptedException(ie); } } /** * Closes current buffer. * *

Any objects putted to this buffer through {@link InputPort#put(Object)} method of owned input ports will be * dropped and will never reach the output.

*/ public void close() { lock.lock(); try { closed = true; notFull.signalAll(); notEmpty.signalAll(); closedCondition.signalAll(); } finally { lock.unlock(); } } /** * Creates new input port to put elements to this buffer. Buffer will be closed after all created input ports are * closed. * * @return new InputPort for this buffer */ public InputPort createInputPort() { BufferInputPort port = new BufferInputPort(); inputPorts.add(port); return port; } /** * This method will return after this buffer is closed. * * @throws InterruptedException */ public void waitClosed() throws InterruptedException { lock.lock(); try { while (!closed) closedCondition.await(); } finally { lock.unlock(); } } @Override public BufferStatus getStatus() { lock.lock(); try { return new BufferStatus(closed, items.length, putCount, takeCount); } finally { lock.unlock(); } } //Helper methods /** * Invoked on each input port closing event. Tests whether each input port is closed, and if so closes the buffer. */ private void testClosed() { lock.lock(); try { if (closed) return; for (BufferInputPort port : inputPorts) if (!port.currentPortClosed.get()) return; closed = true; notEmpty.signalAll(); closedCondition.signalAll(); } finally { lock.unlock(); } } /** * Circularly increment i. */ final int inc(int i) { return (++i == items.length) ? 0 : i; } /** * Inserts element at current put position, advances, and signals. Call only when holding lock. */ private void insert(E x) { items[putIndex] = x; putIndex = inc(putIndex); ++count; notEmpty.signal(); ++putCount; } /** * Extracts element at current take position, advances, and signals. Call only when holding lock. */ private E extract() { final Object[] items = this.items; E x = this.cast(items[takeIndex]); items[takeIndex] = null; takeIndex = inc(takeIndex); --count; notFull.signal(); ++takeCount; return x; } /** * Implementation of InputPort for buffer. */ private class BufferInputPort implements InputPort { AtomicBoolean currentPortClosed = new AtomicBoolean(false); public void put(E object) { if (object == null) { if (!currentPortClosed.compareAndSet(false, true)) throw new IllegalArgumentException("Worker is already closed."); testClosed(); return; } Buffer.this.put(object); } } } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/blocks/BufferStatus.java0000644000000000000000000000572313353377423027545 0ustar 00000000000000package cc.redberry.pipe.blocks; import java.util.Objects; /** * Buffer status snapshot. */ public final class BufferStatus { /** * Is buffer closed (no more elements will be added by upstream process, it may still contain some elements to be * consumed by downstream process) */ public final boolean closed; /** * Current buffer capacity (maximal number of objects it can hold) */ public final int capacity; /** * Number of objects added to the buffer by upstream process(es) */ public final long putCount; /** * Number of objects taken from the buffer bu downstream process(es) */ public final long takeCount; /** * Value of System.currentTimeMillis() for the moment of object creation */ public final long timestamp; public BufferStatus(boolean closed, int capacity, long putCount, long takeCount) { this(closed, capacity, putCount, takeCount, System.currentTimeMillis()); } public BufferStatus(boolean closed, int capacity, long putCount, long takeCount, long timestamp) { if (putCount < takeCount) throw new IllegalArgumentException(); if (putCount - takeCount > capacity) throw new IllegalArgumentException(); this.closed = closed; this.capacity = capacity; this.putCount = putCount; this.takeCount = takeCount; this.timestamp = timestamp; } /** * Returns buffer size (current number of elements in the buffer). * * size = putCount - takeCount * * @return buffer size */ public int size() { return (int) (putCount - takeCount); } /** * Returns buffer fullness: double value from 1.0 to 0.0, 1.0 - for completely full buffer, 0.0 for empty buffer. * * fullness = size() / capacity * * @return buffer fullness */ public double fullness() { return 1.0 * size() / capacity; } /** * Returns true if this buffer is closed and contain no elements */ public boolean isClosedAndEmpty() { return size() == 0 && closed; } @Override public boolean equals(Object o) { if (this == o) return true; if (!(o instanceof BufferStatus)) return false; BufferStatus that = (BufferStatus) o; return closed == that.closed && capacity == that.capacity && putCount == that.putCount && takeCount == that.takeCount && timestamp == that.timestamp; } @Override public int hashCode() { return Objects.hash(closed, capacity, putCount, takeCount, timestamp); } @Override public String toString() { return "BufferStatus{" + "closed=" + closed + ", capacity=" + capacity + ", putCount=" + putCount + ", takeCount=" + takeCount + ", timestamp=" + timestamp + '}'; } } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/blocks/BufferStatusProvider.java0000644000000000000000000000044613353377423031255 0ustar 00000000000000package cc.redberry.pipe.blocks; /** * Interface for the object that can report buffer status (fullness and total number of objects being transferred) */ public interface BufferStatusProvider { /** * Returns snapshot of current buffer status */ BufferStatus getStatus(); } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/blocks/CCUtils.java0000644000000000000000000000243413353377423026432 0ustar 00000000000000/* * cc.redberry.pipe: java library for implementation of concurrent pipelines * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ package cc.redberry.pipe.blocks; import cc.redberry.pipe.Processor; import cc.redberry.pipe.ProcessorFactory; /** * Inner utility methods. */ final class CCUtils { private CCUtils() { } static Processor[] createArray(ProcessorFactory factory, int size) { Processor[] result = new Processor[size]; for (int i = 0; i < size; ++i) result[i] = factory.create(); return result; } } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/blocks/FilteringPort.java0000644000000000000000000000611013353377423027707 0ustar 00000000000000/* * cc.redberry.pipe: java library for implementation of concurrent pipelines * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ package cc.redberry.pipe.blocks; import cc.redberry.pipe.InputPort; import cc.redberry.pipe.OutputPort; import cc.redberry.pipe.OutputPortCloseable; import cc.redberry.primitives.Filter; import java.util.concurrent.atomic.AtomicInteger; /** * Filtering OutputPort wrapper. Propagates only elements accepted by filter. * * @param type of filtered objects * @author Bolotin Dmitriy (bolotin.dmitriy@gmail.com) */ public final class FilteringPort implements OutputPortCloseable { private final OutputPort port; private final Filter filter; private final AtomicInteger accepted = new AtomicInteger(0); private final AtomicInteger rejected = new AtomicInteger(0); private InputPort discardedPort; /** * Creates filtering port. * * @param port input object stream * @param filter filter */ public FilteringPort(OutputPort port, Filter filter) { this(port, filter, null); } /** * Creates filtering port. * * @param port input object stream * @param filter filter * @param discardedPort input port to put rejected objects of {@code null} */ public FilteringPort(OutputPort port, Filter filter, InputPort discardedPort) { if (port == null || filter == null) throw new NullPointerException(); this.port = port; this.filter = filter; this.discardedPort = discardedPort; } @Override public T take() { T o; while ((o = port.take()) != null && !filter.accept(o)) { if (o != null && discardedPort != null) discardedPort.put(o); rejected.incrementAndGet(); } if (o != null) accepted.incrementAndGet(); return o; } public void attachDiscardPort(InputPort discardedPort) { this.discardedPort = discardedPort; } public int getAcceptedCount() { return accepted.get(); } public int getRejectedCount() { return rejected.get(); } public int getTotalCount() { return accepted.get() + rejected.get(); } @Override public void close() { if (port instanceof OutputPortCloseable) ((OutputPortCloseable) port).close(); } } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/blocks/InputOutputPair.java0000644000000000000000000000070613353377423030260 0ustar 00000000000000package cc.redberry.pipe.blocks; /** * Tuple storing input value passed to {@link cc.redberry.pipe.Processor} and result it has produced. * * @param input type * @param output type */ public final class InputOutputPair { public final InputT input; public final OutputT result; public InputOutputPair(InputT input, OutputT result) { this.input = input; this.result = result; } } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/blocks/Merger.java0000644000000000000000000001602013353377423026341 0ustar 00000000000000/* * cc.redberry.pipe: java library for implementation of concurrent pipelines * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ package cc.redberry.pipe.blocks; import cc.redberry.pipe.InputPort; import cc.redberry.pipe.OutputPort; import cc.redberry.pipe.OutputPortCloseable; import cc.redberry.pipe.util.ExceptionHandler; import java.util.ArrayList; import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicBoolean; /** * Merges data streams form several output ports. * *

It starts a new thread for each attached output port to pull data from it and place it into output buffer.

* * @param type of merged stream objects * @author Bolotin Dmitriy * @author Mikhail Shugay */ // TODO better implementation possible, with only one thread public final class Merger implements OutputPortCloseable { final Buffer buffer; final List> inputPorts = new ArrayList<>(); final List> futures = new ArrayList<>(); final List threads = new ArrayList<>(); final List transmitters = new ArrayList<>(); private final ExceptionHandler exceptionsHandler = new ExceptionHandler() { @Override public void handle(RuntimeException exception, O2ITransmitter source) { if (thrown == null) thrown = exception; close(); } }; private final AtomicBoolean closed = new AtomicBoolean(false); List targetExecutors = new ArrayList<>(); private boolean failSafe = false; private volatile RuntimeException thrown = null; /** * Creates merger with default output buffer size. */ public Merger() { this(Buffer.DEFAULT_SIZE); } /** * Creates merger with defined output buffer size. * * @param bufferSize size of output buffer */ public Merger(int bufferSize) { this.buffer = new Buffer<>(bufferSize); } public void setFailSafe(boolean failSafe) { this.failSafe = failSafe; } /** * Attache output port to merger. * *

Each port will be processed in individual thread, so no concurrent calls to {@link * cc.redberry.pipe.OutputPort#take()} will be performed.

* * @param port port to be served * @param executorService executor service that should be used to create new thread to serve provided output port */ public void merge(OutputPort port, ExecutorService executorService) { if (targetExecutors == null) throw new RuntimeException("This merger is already started."); inputPorts.add(port); targetExecutors.add(executorService); } /** * Attache output port to merger. * *

Each port will be processed in individual thread, so no concurrent calls to {@link * cc.redberry.pipe.OutputPort#take()} will be performed.

* * @param port port to be served */ public void merge(OutputPort port) { if (targetExecutors == null) throw new RuntimeException("This merger is already started."); inputPorts.add(port); targetExecutors.add(null); } /** * Starts threads for current merger */ public synchronized void start() { if (targetExecutors == null) throw new RuntimeException("This merger is already started."); ExecutorService service; OutputPort port; O2ITransmitter task; int i; InputPort[] iPorts = new InputPort[inputPorts.size()]; for (i = 0; i < inputPorts.size(); ++i) iPorts[i] = buffer.createInputPort(); for (i = 0; i < inputPorts.size(); ++i) { service = targetExecutors.get(i); port = inputPorts.get(i); transmitters.add(task = new O2ITransmitter<>(port, iPorts[i], exceptionsHandler)); if (service == null) { Thread thread = new Thread(task); threads.add(thread); thread.start(); } else futures.add(service.submit(task)); } targetExecutors = null; } /** * Join all threads owned by this buffer. * * @throws RuntimeException if some of {@link cc.redberry.pipe.OutputPort#take()} thrown som exception * @throws InterruptedException */ public void join() throws InterruptedException { try { for (Future future : futures) future.get(); } catch (ExecutionException ee) { throw new RuntimeException("This expression should not be thrown.", ee); } for (Thread thread : threads) thread.join(); if (Thread.interrupted()) throw new InterruptedException(); } /** * Take object from output buffer * * @return {@inheritDoc} * @throws InterruptedException */ @Override public E take() { E result = buffer.take(); // Checking whether some processor has thrown an exception if buffer is closed if (result == null) { if (thrown != null) synchronized (this) { if (thrown != null) { RuntimeException e = thrown; thrown = null; throw e; } } return null; } return result; } /** * Stops current merger. * *

This method propagates close signal to upstream (input) output port.

*/ @Override public void close() { if (!closed.compareAndSet(false, true)) return; for (OutputPort input : inputPorts) if (input instanceof OutputPortCloseable) ((OutputPortCloseable) input).close(); for (O2ITransmitter tasks : transmitters) tasks.stop(); if (!failSafe) buffer.close(); } /** * Returns output buffer status provider to monitor its fullness and other characteristics. */ public BufferStatusProvider getBufferStatusProvider() { return buffer; } @Override public String toString() { return buffer.toString(); } } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/blocks/O2ITransmitter.java0000644000000000000000000000452513353377423027755 0ustar 00000000000000/* * cc.redberry.pipe: java library for implementation of concurrent pipelines * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ package cc.redberry.pipe.blocks; import cc.redberry.pipe.InputPort; import cc.redberry.pipe.OutputPort; import cc.redberry.pipe.PInterruptedException; import cc.redberry.pipe.util.ExceptionHandler; /** * Output to input transmitter. * * @param type of transmitted objects * @author Bolotin Dmitriy * @author Mikhail Shugay */ public final class O2ITransmitter implements Runnable { public final ExceptionHandler handler; public final OutputPort oPort; public final InputPort iPort; public volatile boolean stopped = false; public O2ITransmitter(OutputPort output, InputPort input, ExceptionHandler exceptionHandler) { this.handler = exceptionHandler; this.oPort = output; this.iPort = input; } public O2ITransmitter(OutputPort output, InputPort input) { this(output, input, null); } public OutputPort getInput() { return oPort; } public InputPort getOutput() { return iPort; } public void stop() { stopped = true; } @Override public void run() { try { T element; while (!stopped && (element = oPort.take()) != null) iPort.put(element); } catch (PInterruptedException ex) { } catch (RuntimeException re) { if (handler != null) handler.handle(re, this); } finally { iPort.put(null); //close upstream input port any way } } } ././@LongLink0000000000000000000000000000014600000000000011216 Lustar 00000000000000redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/blocks/OutputPortSynchronizer.javaredberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/blocks/OutputPortSynchronizer.jav0000644000000000000000000000031213353377423031537 0ustar 00000000000000package cc.redberry.pipe.blocks; import cc.redberry.pipe.OutputPort; public class OutputPortSynchronizer implements OutputPort { @Override public T take() { return null; } } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/blocks/ParallelProcessor.java0000644000000000000000000003134113353377423030557 0ustar 00000000000000/* * cc.redberry.pipe: java library for implementation of concurrent pipelines * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ package cc.redberry.pipe.blocks; import cc.redberry.pipe.*; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicLong; /** * This class helps to process data from one OutputPort in N parallel threads, collecting results in one output buffer. * *

It could be used either to access one processor concurrently or N processors from their corresponding threads. See * constructor descriptions for details.

* * @param input type * @param output type * @author Bolotin Dmitriy * @author Mikhail Shugay */ public final class ParallelProcessor implements OutputPortCloseable { private static AtomicLong processorIdCounter = new AtomicLong(0); private final Buffer outputBuffer; private final Worker[] workers; private final Thread[] threads; private final Future[] futures; private final OutputPort input; private volatile boolean closed = false, failSafe = false; private volatile RuntimeException thrown = null; /** * Creates ParallelProcessor working in one thread per processor mode with default buffer size. * * @param input Worker to take elements from. * @param factory Factory to createInputStream processors. * @param threads Number of parallel threads for processing. Also it is the number of processors to * createInputStream. * @param executorService Executor service to create threads with. */ public ParallelProcessor(OutputPort input, ProcessorFactory factory, int threads, ExecutorService executorService) { this(input, CCUtils.createArray(factory, threads), Buffer.DEFAULT_SIZE, executorService); } /** * Creates ParallelProcessor working in one thread per processor mode. * * @param input Worker to take elements from. * @param factory Factory to createInputStream processors. * @param bufferSize Buffer size. * @param threads Number of parallel threads for processing. Also it is the number of processors to * createInputStream. * @param executorService Executor service to create threads with. */ public ParallelProcessor(OutputPort input, ProcessorFactory factory, int bufferSize, int threads, ExecutorService executorService) { this(input, CCUtils.createArray(factory, threads), bufferSize, executorService); } /** * Creates ParallelProcessor working in one thread per processor mode. * * @param input Worker to take elements from. * @param factory Factory to createInputStream processors. * @param bufferSize Buffer size. * @param threads Number of parallel threads of processing. Also it is the number of processors to * createInputStream. */ public ParallelProcessor(OutputPort input, ProcessorFactory factory, int bufferSize, int threads) { this(input, CCUtils.createArray(factory, threads), bufferSize); } /** * Creates ParallelProcessor working in one thread per processor mode with default buffer size. * * @param input Worker to take elements from. * @param factory Factory to createInputStream processors. * @param threads Number of parallel threads of processing. Also it is the number of processors to * createInputStream. */ public ParallelProcessor(OutputPort input, ProcessorFactory factory, int threads) { this(input, CCUtils.createArray(factory, threads), Buffer.DEFAULT_SIZE); } /** * Creates ParallelProcessor working in concurrent processor access mode. * * @param input Worker to take elements from. * @param processor Processor. It will be accessed concurrently. * @param bufferSize Buffer size. * @param threads Number of parallel threads of processing. */ public ParallelProcessor(OutputPort input, Processor processor, int bufferSize, int threads) { this(input, toArray(processor, threads), bufferSize); } /** * Creates ParallelProcessor working in concurrent processor access mode with default buffer size. * * @param input Worker to take elements from. * @param processor Processor. It will be accessed concurrently. * @param threads Number of parallel threads of processing. */ public ParallelProcessor(OutputPort input, Processor processor, int threads) { this(input, toArray(processor, threads), Buffer.DEFAULT_SIZE); } /** * Creates ParallelProcessor working in concurrent processor access mode. * * @param input Worker to take elements from. * @param processor Processor. It will be accessed concurrently. * @param bufferSize Buffer size. * @param threads Number of parallel threads of processing. * @param executorService Executor service to create threads with. */ public ParallelProcessor(OutputPort input, Processor processor, int bufferSize, int threads, ExecutorService executorService) { this(input, toArray(processor, threads), bufferSize, executorService); } /** * Creates ParallelProcessor working in one thread per processor mode. * * @param input Worker to take elements from. * @param processors Processors. Each processor will be accessed only by one thread. * @param bufferSize Buffer size. */ public ParallelProcessor(OutputPort input, Processor[] processors, int bufferSize) { this.input = input; this.outputBuffer = new Buffer<>(bufferSize); this.workers = new ParallelProcessor.Worker[processors.length]; this.threads = new Thread[processors.length]; this.futures = null; int i; InputPort[] iPorts = new InputPort[processors.length]; for (i = 0; i < processors.length; ++i) iPorts[i] = this.outputBuffer.createInputPort(); long id = processorIdCounter.incrementAndGet(); for (i = 0; i < processors.length; ++i) { this.workers[i] = new Worker(iPorts[i], processors[i]); this.threads[i] = new Thread(this.workers[i], "ParallelProcessor-" + id + "-" + i); this.threads[i].start(); } } /** * Creates ParallelProcessor working in one thread per processor mode. * * @param input Worker to take elements from. * @param processors Processors. Each processor will be accessed only by one thread. * @param bufferSize Buffer size. * @param executorService Executor service to create threads with. */ public ParallelProcessor(OutputPort input, Processor[] processors, int bufferSize, ExecutorService executorService) { this.input = input; this.outputBuffer = new Buffer<>(bufferSize); this.workers = new ParallelProcessor.Worker[processors.length]; this.threads = null; this.futures = new Future[processors.length]; int i; InputPort[] iPorts = new InputPort[processors.length]; for (i = 0; i < processors.length; ++i) iPorts[i] = this.outputBuffer.createInputPort(); for (i = 0; i < processors.length; ++i) { this.workers[i] = new Worker(iPorts[i], processors[i]); this.futures[i] = executorService.submit(this.workers[i]); } } private static Processor[] toArray(Processor processor, int size) { //if (size > 1 // && !(processor instanceof ThreadSafe)) // throw new IllegalArgumentException("Processor not marked as thread-safe. Use ThreadSafe interface to mark " + // "thread-safe Processor implementations."); Processor[] processors = new Processor[size]; for (int i = 0; i < size; ++i) processors[i] = processor; return processors; } public ParallelProcessor setFailSafe(boolean failSafe) { this.failSafe = failSafe; return this; } /** * Join all threads owned by this parallel processor. * * @throws InterruptedException if this thread was interrupted */ public void join() throws InterruptedException { if (futures == null) for (Thread t : threads) t.join(); else try { for (Future future : futures) future.get(); } catch (ExecutionException ee) { throw new RuntimeException("This exception should not be thrown.", ee); } if (thrown != null) throw new RuntimeException(thrown); if (Thread.interrupted()) throw new InterruptedException(); } /** * Take first object that was processed so far from output buffer. * * @return {@inheritDoc} */ public OutputT take() { OutputT result = outputBuffer.take(); // Checking whether some processor has thrown an exception if buffer is closed if (result == null) { if (thrown != null) synchronized (this) { if (thrown != null) { RuntimeException re = thrown; thrown = null; throw re; } } return null; } return result; } /** * Returns output buffer status provider to monitor its fullness and other characteristics. */ public BufferStatusProvider getOutputBufferStatusProvider() { return outputBuffer; } @Override public String toString() { return outputBuffer.toString(); } /** * Stops current processor. * *

Some elements taken from input may be dropped, and may never reach the output if {@code failSafe} flag is set * to false.

* *

This method propagates close signal to upstream (input) output port.

*/ @Override public void close() { closed = true; if (input instanceof OutputPortCloseable) ((OutputPortCloseable) input).close(); if (!failSafe) outputBuffer.close(); } private class Worker implements Runnable { private final InputPort toBuffer; private final Processor processor; private Worker(InputPort toBuffer, Processor processor) { this.toBuffer = toBuffer; this.processor = processor; } @Override public void run() { //For performance final OutputPort input = ParallelProcessor.this.input; try { InputT inputValue; OutputT outputValue; while (!closed && (inputValue = input.take()) != null) { outputValue = processor.process(inputValue); toBuffer.put(outputValue); if (Thread.interrupted()) //Just in case return; } } catch (PInterruptedException in) { } catch (RuntimeException re) { if (thrown == null) synchronized (ParallelProcessor.this) { if (thrown == null) thrown = re; } close(); //Close all! } finally { toBuffer.put(null); //close upstream input port in any case } } } } ././@LongLink0000000000000000000000000000015000000000000011211 Lustar 00000000000000redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/blocks/ResultAttachingProcessor.javaredberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/blocks/ResultAttachingProcessor.j0000644000000000000000000000152513353377423031435 0ustar 00000000000000package cc.redberry.pipe.blocks; import cc.redberry.pipe.Processor; /** * {@link cc.redberry.pipe.Processor} wrapper used to save input value passed to the wrapped processor and attache it to * the output. * * @author Bolotin Dmitriy * @see InputOutputPair */ public class ResultAttachingProcessor implements Processor> { private final Processor processor; /** * Use factory method. * * @param processor */ public ResultAttachingProcessor(Processor processor) { this.processor = processor; } @Override public final InputOutputPair process(InputT input) { OutputT output = processor.process(input); return new InputOutputPair(input, output); } } ././@LongLink0000000000000000000000000000015700000000000011220 Lustar 00000000000000redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/blocks/ResultAttachingProcessorFactory.javaredberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/blocks/ResultAttachingProcessorFa0000644000000000000000000000136113353377423031452 0ustar 00000000000000package cc.redberry.pipe.blocks; import cc.redberry.pipe.Processor; import cc.redberry.pipe.ProcessorFactory; /** * {@link cc.redberry.pipe.ProcessorFactory} wrapper to produce {@link ResultAttachingProcessor}. */ public final class ResultAttachingProcessorFactory implements ProcessorFactory> { private final ProcessorFactory processorFactory; public ResultAttachingProcessorFactory(ProcessorFactory processorFactory) { this.processorFactory = processorFactory; } @Override public Processor> create() { return new ResultAttachingProcessor<>(processorFactory.create()); } } ././@LongLink0000000000000000000000000000015300000000000011214 Lustar 00000000000000redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/blocks/ThreadSafeOutputPortWrapper.javaredberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/blocks/ThreadSafeOutputPortWrappe0000644000000000000000000000034113353377423031452 0ustar 00000000000000package cc.redberry.pipe.blocks; /** * Created with IntelliJ IDEA. User: dbolotin Date: 24.09.13 Time: 19:37 To change this template use File | Settings | * File Templates. */ public class ThreadSafeOutputPortWrapper { } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/blocks/package-info.java0000644000000000000000000000160113353377423027443 0ustar 00000000000000/* * cc.redberry.pipe: java library for implementation of concurrent pipelines * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ /** * Core building blocks for pipelines construction */ package cc.redberry.pipe.blocks;redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/package-info.java0000644000000000000000000000153213353377423026171 0ustar 00000000000000/* * cc.redberry.pipe: java library for implementation of concurrent pipelines * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ /** * Core interfaces */ package cc.redberry.pipe;redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/util/Chunk.java0000644000000000000000000000516713353377423025702 0ustar 00000000000000package cc.redberry.pipe.util; import cc.redberry.pipe.OutputPort; import java.util.Arrays; import java.util.Iterator; import java.util.concurrent.atomic.AtomicInteger; public final class Chunk implements Iterable { final Object[] data; final int size; public Chunk(Object[] data, int size) { this.data = data; this.size = size; } public Chunk(Object[] data) { this.data = data; this.size = data.length; } public Object[] toArray() { if (data.length == size) return data; else return Arrays.copyOf(data, size); } @SuppressWarnings("unchecked") public T get(int index) { return (T) data[index]; } public int size() { return size; } @Override public Iterator iterator() { return new It(); } public OutputPort outputPort() { return new Op(); } /** * Reads a chunk from input port. * *

Normally returns chunk with {@code size == chunkSize}. Or chunk with {@code size < chunkSize} if there are no * elements left in port after this operation or {@code null} if there are no elements in port.

* * @param port output port to read from * @param chunkSize desired chunk size * @return returns chunk with {@code size == chunkSize}, or chunk with {@code size < chunkSize} if there are no * elements left in port after this operation, or {@code null} if there are no elements in port */ public static Chunk readChunk(OutputPort port, int chunkSize) { Object[] buffer = new Object[chunkSize]; int size = 0; for (; size < chunkSize; ++size) { Object element = port.take(); if (element == null) break; buffer[size] = element; } if (size == 0) return null; return new Chunk<>(buffer, size); } private final class It implements Iterator { int pointer = 0; @Override public boolean hasNext() { return pointer < size; } @Override public T next() { return get(pointer++); } @Override public void remove() { throw new UnsupportedOperationException(); } } private final class Op implements OutputPort { final AtomicInteger pointer = new AtomicInteger(); @Override public T take() { int index = pointer.getAndIncrement(); if (index >= size) return null; return get(index); } } } ././@LongLink0000000000000000000000000000015500000000000011216 Lustar 00000000000000redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/util/ClosePreventingInputPortWrapper.javaredberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/util/ClosePreventingInputPortWrap0000644000000000000000000000324113353377423031527 0ustar 00000000000000/* * cc.redberry.pipe: java library for implementation of concurrent pipelines * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ package cc.redberry.pipe.util; import cc.redberry.pipe.InputPort; /** * This class prevents putting null to the nested input port. * *

To close nested port use {@code close()} method.

* * @param type of port */ public class ClosePreventingInputPortWrapper implements InputPort { private final InputPort innerInputPort; public ClosePreventingInputPortWrapper(InputPort innerInputPort) { if (innerInputPort == null) throw new NullPointerException(); this.innerInputPort = innerInputPort; } @Override public void put(T object) { if (object != null) innerInputPort.put(object); } /** * Use this method to close (put null) inner port. * * @throws InterruptedException */ public void close() throws InterruptedException { innerInputPort.put(null); } } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/util/CountLimitingOutputPort.java0000644000000000000000000000467313353377423031506 0ustar 00000000000000/* * cc.redberry.pipe: java library for implementation of concurrent pipelines * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ package cc.redberry.pipe.util; import cc.redberry.pipe.OutputPort; import cc.redberry.pipe.OutputPortCloseable; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; public class CountLimitingOutputPort implements OutputPortCloseable { private final OutputPort innerPort; private final long limit; private final AtomicLong count; private final AtomicBoolean finished = new AtomicBoolean(false); public CountLimitingOutputPort(OutputPort innerPort, long limit) { if (innerPort == null) throw new NullPointerException(); this.innerPort = innerPort; this.limit = limit; this.count = new AtomicLong(-limit); } public long getLimit() { return limit; } public long getElementsLeft() { long left = count.get(); if (left > 0) return 0; return -left; } public boolean isClosed() { return finished.get(); } @Override public T take() { if (finished.get()) return null; long index = count.incrementAndGet(); if (index > 0) { if (finished.compareAndSet(false, true)) if (innerPort instanceof OutputPortCloseable) ((OutputPortCloseable) innerPort).close(); return null; } T value = innerPort.take(); if (value == null) finished.set(true); return value; } @Override public void close() { finished.set(true); if (innerPort instanceof OutputPortCloseable) ((OutputPortCloseable) innerPort).close(); } } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/util/CountingInputPort.java0000644000000000000000000000261413353377423030277 0ustar 00000000000000/* * cc.redberry.pipe: java library for implementation of concurrent pipelines * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ package cc.redberry.pipe.util; import cc.redberry.pipe.InputPort; import java.util.concurrent.atomic.AtomicInteger; public final class CountingInputPort implements InputPort { private final AtomicInteger counter = new AtomicInteger(); private final InputPort inherited; public CountingInputPort() { this.inherited = null; } public CountingInputPort(InputPort inherited) { this.inherited = inherited; } @Override public void put(T object) { counter.incrementAndGet(); if (inherited != null) inherited.put(object); } } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/util/CountingOutputPort.java0000644000000000000000000000430213353377423030474 0ustar 00000000000000/* * cc.redberry.pipe: java library for implementation of concurrent pipelines * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ package cc.redberry.pipe.util; import cc.redberry.pipe.OutputPort; import cc.redberry.pipe.OutputPortCloseable; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; /** * Counting wrapper for {@link OutputPort}. * * @author Bolotin Dmitriy */ public class CountingOutputPort implements OutputPortCloseable { private final OutputPort innerOutputPort; private final AtomicBoolean closed = new AtomicBoolean(false); private final AtomicLong counter = new AtomicLong(); public CountingOutputPort(OutputPort innerOutputPort) { this.innerOutputPort = innerOutputPort; } @Override public T take() { if (closed.get()) return null; T value = innerOutputPort.take(); if (value != null) counter.incrementAndGet(); else closed.set(true); return value; } /** * Returns the count of object passed through this port. */ public long getCount() { return counter.get(); } public boolean isClosed() { return closed.get(); } @Override public String toString() { return Integer.toString(counter.intValue()); } @Override public void close() { if (innerOutputPort instanceof OutputPortCloseable) ((OutputPortCloseable) innerOutputPort).close(); } } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/util/DummyInputPort.java0000644000000000000000000000210713353377423027601 0ustar 00000000000000/* * cc.redberry.pipe: java library for implementation of concurrent pipelines * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ package cc.redberry.pipe.util; import cc.redberry.pipe.InputPort; public class DummyInputPort implements InputPort { public final static DummyInputPort INSTANCE = new DummyInputPort(); private DummyInputPort() { } @Override public void put(T object) { } } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/util/ExceptionHandler.java0000644000000000000000000000244113353377423030056 0ustar 00000000000000/* * cc.redberry.pipe: java library for implementation of concurrent pipelines * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ package cc.redberry.pipe.util; /** * Interface to handle exceptions thrown by {@link cc.redberry.pipe.OutputPort#take()}, {@link * cc.redberry.pipe.Processor#process(Object)} and {@link cc.redberry.pipe.VoidProcessor#process(Object)} methods in * threads created by framework infrastructure.. */ public interface ExceptionHandler { /** * Executed from execution threads. * * @param exception thrown exception */ public void handle(RuntimeException exception, S source); } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/util/FlatteningOutputPort.java0000644000000000000000000000215513353377423031005 0ustar 00000000000000package cc.redberry.pipe.util; import cc.redberry.pipe.OutputPortCloseable; import cc.redberry.pipe.OutputPort; //TODO much better implementation is possible public class FlatteningOutputPort implements OutputPortCloseable { private OutputPort> opop; private OutputPort op; public FlatteningOutputPort(OutputPort> opop) { this.opop = opop; } @Override public synchronized T take() { T result; //First invocation if (op == null) op = opop.take(); do { if (op == null) return null; result = op.take(); if (result == null) op = opop.take(); } while (result == null); return result; } @Override public synchronized void close() { if (opop instanceof OutputPortCloseable) ((OutputPortCloseable) opop).close(); if (op != null && op instanceof OutputPortCloseable) ((OutputPortCloseable) op).close(); opop = null; } } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/util/Indexer.java0000644000000000000000000000013513353377423026216 0ustar 00000000000000package cc.redberry.pipe.util; public interface Indexer { long getIndex(T object); } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/util/InputPortBuffer.java0000644000000000000000000000374113353377423027724 0ustar 00000000000000/* * cc.redberry.pipe: java library for implementation of concurrent pipelines * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ package cc.redberry.pipe.util; import cc.redberry.pipe.InputPort; import cc.redberry.pipe.OutputPort; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicInteger; /** * Concurrent unlimited objects buffer based on ConcurrentLinkedQueue.
It is a simple wrapper for * ConcurrentLinkedQueue.
For normal functioning all objects must be putted before first object can be taken. * * @param * @author Bolotin Dmitriy (bolotin.dmitriy@gmail.com) */ public final class InputPortBuffer implements InputPort, OutputPort { private final AtomicInteger elements = new AtomicInteger(); private final Queue queue; public InputPortBuffer() { queue = new ConcurrentLinkedQueue<>(); } @Override public void put(T object) { if (object == null) //Do nothing on close return; elements.incrementAndGet(); queue.add(object); } @Override public T take() { T element; if ((element = queue.poll()) != null) elements.decrementAndGet(); return element; } public int size() { return elements.get(); } } ././@LongLink0000000000000000000000000000014700000000000011217 Lustar 00000000000000redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/util/IteratorOutputPortAdapter.javaredberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/util/IteratorOutputPortAdapter.ja0000644000000000000000000000115213353377423031451 0ustar 00000000000000package cc.redberry.pipe.util; import cc.redberry.pipe.OutputPort; import java.util.Iterator; public class IteratorOutputPortAdapter implements OutputPort { private Iterator iterator; public IteratorOutputPortAdapter(Iterator iterator) { this.iterator = iterator; } public IteratorOutputPortAdapter(Iterable iterable) { this.iterator = iterable.iterator(); } @Override public synchronized T take() { if (iterator == null) return null; if (!iterator.hasNext()) return null; return iterator.next(); } } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/util/OrderedOutputPort.java0000644000000000000000000000335613353377423030302 0ustar 00000000000000package cc.redberry.pipe.util; import cc.redberry.pipe.OutputPort; import java.util.Comparator; import java.util.TreeSet; public final class OrderedOutputPort implements OutputPort { private final OutputPort innerOutputPort; private final Indexer indexer; private final TreeSet sortedSet; private long nextIndex; public OrderedOutputPort(OutputPort innerOutputPort, Indexer indexer) { this(innerOutputPort, indexer, 0); } public OrderedOutputPort(OutputPort innerOutputPort, Indexer indexer, long startsFrom) { this.innerOutputPort = innerOutputPort; this.indexer = indexer; this.sortedSet = new TreeSet<>(new IndexComparator<>(indexer)); this.nextIndex = startsFrom; } @Override public T take() { if (!sortedSet.isEmpty() && indexer.getIndex(sortedSet.first()) == nextIndex) { ++nextIndex; return sortedSet.pollFirst(); } T obj; while ((obj = innerOutputPort.take()) != null) { if (indexer.getIndex(obj) == nextIndex) { ++nextIndex; return obj; } sortedSet.add(obj); } if (!sortedSet.isEmpty()) throw new IllegalStateException("Some elements left in buffer."); return null; } private final static class IndexComparator implements Comparator { private final Indexer indexer; private IndexComparator(Indexer indexer) { this.indexer = indexer; } @Override public int compare(T o1, T o2) { return Long.compare(indexer.getIndex(o1), indexer.getIndex(o2)); } } } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/util/SimpleProcessorWrapper.java0000644000000000000000000000374113353377423031320 0ustar 00000000000000/* * cc.redberry.pipe: java library for implementation of concurrent pipelines * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ package cc.redberry.pipe.util; import cc.redberry.pipe.OutputPort; import cc.redberry.pipe.OutputPortCloseable; import cc.redberry.pipe.Processor; /** * This class is used by {@link cc.redberry.pipe.CUtils#wrap(cc.redberry.pipe.OutputPort, cc.redberry.pipe.Processor)} * method. * * @param type of consumed object * @param type of produced object * @author Bolotin Dmitriy (bolotin.dmitriy@gmail.com) */ public class SimpleProcessorWrapper implements OutputPortCloseable { private final OutputPort innerPort; private final Processor processor; public SimpleProcessorWrapper(OutputPort innerPort, Processor processor) { this.innerPort = innerPort; this.processor = processor; } @Override public OutputT take() { InputT input = innerPort.take(); if (input == null) return null; return processor.process(input); } @Override public void close() { if (innerPort instanceof OutputPortCloseable) ((OutputPortCloseable) innerPort).close(); } } ././@LongLink0000000000000000000000000000016000000000000011212 Lustar 00000000000000redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/util/SimpleProcessorWrapperSynchronized.javaredberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/util/SimpleProcessorWrapperSynchr0000644000000000000000000000413413353377423031564 0ustar 00000000000000/* * cc.redberry.pipe: java library for implementation of concurrent pipelines * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ package cc.redberry.pipe.util; import cc.redberry.pipe.OutputPort; import cc.redberry.pipe.OutputPortCloseable; import cc.redberry.pipe.Processor; /** * This class is used by {@link cc.redberry.pipe.CUtils#wrap(cc.redberry.pipe.OutputPort, cc.redberry.pipe.Processor)} * method. * * @param type of consumed object * @param type of produced object * @author Bolotin Dmitriy (bolotin.dmitriy@gmail.com) */ public class SimpleProcessorWrapperSynchronized implements OutputPortCloseable { private final OutputPort innerPort; private final Processor processor; public SimpleProcessorWrapperSynchronized(OutputPort innerPort, Processor processor) { this.innerPort = innerPort; this.processor = processor; } @Override public OutputT take() { InputT input = innerPort.take(); if (input == null) return null; OutputT result; synchronized (processor) { result = processor.process(input); } return result; } @Override public void close() { if (innerPort instanceof OutputPortCloseable) ((OutputPortCloseable) innerPort).close(); } } ././@LongLink0000000000000000000000000000015100000000000011212 Lustar 00000000000000redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/util/SimpleVoidProcessorExecutor.javaredberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/util/SimpleVoidProcessorExecutor.0000644000000000000000000000267613353377423031464 0ustar 00000000000000/* * cc.redberry.pipe: java library for implementation of concurrent pipelines * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ package cc.redberry.pipe.util; import cc.redberry.pipe.OutputPort; import cc.redberry.pipe.VoidProcessor; /** * @param * @author Bolotin Dmitriy (bolotin.dmitriy@gmail.com) */ public class SimpleVoidProcessorExecutor implements Runnable { private final OutputPort input; private final VoidProcessor processor; public SimpleVoidProcessorExecutor(OutputPort input, VoidProcessor processor) { this.input = input; this.processor = processor; } @Override public void run() { T object; while ((object = input.take()) != null) processor.process(object); } } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/util/StatusReporter.java0000644000000000000000000001246013353377423027632 0ustar 00000000000000package cc.redberry.pipe.util; import cc.redberry.pipe.blocks.BufferStatus; import cc.redberry.pipe.blocks.BufferStatusProvider; import java.io.IOException; import java.io.OutputStream; import java.text.DecimalFormat; import java.util.ArrayList; import java.util.function.Supplier; import java.util.stream.Collectors; /** * Universal periodic reporter */ public class StatusReporter extends Thread { private static int FULLNESS_SLOTS = 20; private static final DecimalFormat decFormat = new DecimalFormat("##.#"); final int reportPeriod; final OutputStream outputStream; final ArrayList buffers = new ArrayList<>(); public StatusReporter() { this(5000); } public StatusReporter(int reportPeriod) { this(reportPeriod, System.err); } /** * @param reportPeriod report period in milliseconds * @param outputStream stream to print report */ public StatusReporter(int reportPeriod, OutputStream outputStream) { this.reportPeriod = reportPeriod; this.outputStream = outputStream; this.setDaemon(true); } public synchronized void addBuffer(String name, BufferStatusProvider buffer) { buffers.add(new BufferHolder(name, buffer)); } public synchronized void addCustomProvider(StatusProvider provider) { buffers.add(provider); } public synchronized void addCustomProviderFromLambda(Supplier statusSupplier) { buffers.add(new StatusProvider() { volatile Status status; @Override public void updateStatus() { status = statusSupplier.get(); } @Override public boolean isFinished() { return status.isFinished; } @Override public String getStatus() { return status.status; } }); } @Override public void run() { try { while (true) { synchronized (this) { // Updating status buffers.forEach(StatusProvider::updateStatus); // Removing all closed and empty buffers buffers.removeIf(StatusProvider::isFinished); // Stopping reporting thread if there are no mre buffers if (buffers.isEmpty()) break; // Putting report to the stream // In single transaction, hoping this will be printed as single chunk of information outputStream.write( ("\n" + buffers.stream() .map(StatusProvider::getStatus) .collect(Collectors.joining("\n")) + "\n") .getBytes()); // Sleep until next cycle Thread.sleep(reportPeriod); } } } catch (IOException | InterruptedException e) { throw new RuntimeException(e); } } private final static class BufferHolder implements StatusProvider { final String bufferName; final BufferStatusProvider statusProvider; BufferStatus currentStatus, previousStatus; public BufferHolder(String bufferName, BufferStatusProvider statusProvider) { this.bufferName = bufferName; this.statusProvider = statusProvider; } @Override public void updateStatus() { previousStatus = currentStatus; currentStatus = statusProvider.getStatus(); } @Override public boolean isFinished() { return currentStatus.isClosedAndEmpty(); } @Override public String getStatus() { if (currentStatus == null) return "status unknown"; int fullnessBlocks = (int) (currentStatus.fullness() * FULLNESS_SLOTS); String mainPart = bufferName + ": -" + (currentStatus.closed ? "X" : ">") + " [" + new String(new char[fullnessBlocks]).replace("\0", "=") + new String(new char[FULLNESS_SLOTS - fullnessBlocks]).replace("\0", " ") + "] (" + currentStatus.size() + "/" + currentStatus.capacity + ") -> "; if (previousStatus == null) return mainPart; return mainPart + decFormat.format((currentStatus.takeCount - previousStatus.takeCount) * 1000.0 / (currentStatus.timestamp - previousStatus.timestamp)) + " ops/s"; } } public interface StatusProvider { /** * This method should get the information from underlying status source, and save it into internal state. */ void updateStatus(); /** * No status reporting required for this item. * * If true returned it will be removed from the reporters list. */ boolean isFinished(); /** * Return string status. */ String getStatus(); } public static final class Status { final String status; final boolean isFinished; public Status(String status, boolean isFinished) { this.status = status; this.isFinished = isFinished; } } } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/util/TBranchOutputPort.java0000644000000000000000000000451313353377423030233 0ustar 00000000000000/* * cc.redberry.pipe: java library for implementation of concurrent pipelines * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ package cc.redberry.pipe.util; import cc.redberry.pipe.OutputPortCloseable; import cc.redberry.pipe.InputPort; import cc.redberry.pipe.OutputPort; import java.util.concurrent.atomic.AtomicBoolean; /** * T-branch like element putting each taken from OutputPort object into InputPort and propagating it forward by it's * take method. * * @author Bolotin Dmitriy (bolotin.dmitriy@gmail.com) */ public class TBranchOutputPort implements OutputPortCloseable { private final InputPort inputPort; private final OutputPort outputPort; private final AtomicBoolean closed = new AtomicBoolean(false); public TBranchOutputPort(InputPort inputPort, OutputPort outputPort) { this.inputPort = inputPort; this.outputPort = outputPort; } public static TBranchOutputPort wrap(InputPort inputPort, OutputPort outputPort) { return new TBranchOutputPort(inputPort, outputPort); } @Override public T take() { if (closed.get()) return null; T object = outputPort.take(); if (object == null) { if (closed.compareAndSet(false, true)) inputPort.put(null); } else inputPort.put(object); return object; } @Override public void close() { if (outputPort instanceof OutputPortCloseable) ((OutputPortCloseable) outputPort).close(); if (closed.compareAndSet(false, true)) inputPort.put(null); } } ././@LongLink0000000000000000000000000000014600000000000011216 Lustar 00000000000000redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/util/UncaughtExceptionHandler.javaredberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/util/UncaughtExceptionHandler.jav0000644000000000000000000000245113353377423031415 0ustar 00000000000000/* * cc.redberry.pipe: java library for implementation of concurrent pipelines * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ package cc.redberry.pipe.util; /** * Interface to handle exceptions thrown by {@link cc.redberry.pipe.OutputPort#take()}, {@link * cc.redberry.pipe.Processor#process(Object)} and {@link cc.redberry.pipe.VoidProcessor#process(Object)} * methods in threads created by framework infrastructure.. */ public interface UncaughtExceptionHandler { /** * Executed from execution threads. * * @param exception thrown exception */ public void handle(RuntimeException exception, S source); } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/pipe/util/package-info.java0000644000000000000000000000153713353377423027153 0ustar 00000000000000/* * cc.redberry.pipe: java library for implementation of concurrent pipelines * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ /** * Utility classes */ package cc.redberry.pipe.util;redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/primitives/Filter.java0000644000000000000000000000160313353377423026327 0ustar 00000000000000/* * cc.redberry.pipe: java library for implementation of concurrent pipelines * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ package cc.redberry.primitives; public interface Filter { boolean accept(T object); } redberry-redberry-pipe-7323a27293f8/src/main/java/cc/redberry/primitives/FilterUtil.java0000644000000000000000000000553013353377423027170 0ustar 00000000000000/* * cc.redberry.pipe: java library for implementation of concurrent pipelines * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ package cc.redberry.primitives; public class FilterUtil { public static final Filter ACCEPT_ALL = new Filter() { @Override public boolean accept(Object object) { return true; } }; public static final Filter REJECT_ALL = new Filter() { @Override public boolean accept(Object object) { return true; } }; private FilterUtil() { } public static Filter getTrueFilter() { return new Filter() { @Override public boolean accept(Object object) { return true; } }; } public static Filter or(final Filter... filters) { return new Filter() { @Override public boolean accept(T object) { for (Filter f : filters) if (f.accept(object)) return true; return false; } }; } public static Filter and(final Filter... filters) { return new Filter() { @Override public boolean accept(T object) { for (Filter f : filters) if (!f.accept(object)) return false; return true; } }; } public static Filter not(final Filter filter) { return new Filter() { @Override public boolean accept(T object) { return !filter.accept(object); } }; } public static Filter count(final int min, final int max, final Filter... filters) { return new Filter() { @Override public boolean accept(T object) { int count = 0; for (Filter f : filters) if (f.accept(object)) if (++count > max) return false; return count >= min; } }; } } redberry-redberry-pipe-7323a27293f8/src/test/java/cc/redberry/pipe/CUtilsTest.java0000644000000000000000000001174613353377423025753 0ustar 00000000000000package cc.redberry.pipe; import cc.redberry.pipe.blocks.Merger; import cc.redberry.pipe.blocks.ParallelProcessor; import cc.redberry.pipe.util.Chunk; import org.apache.commons.math.random.RandomGenerator; import org.apache.commons.math.random.Well19937c; import org.junit.Assert; import org.junit.Test; import java.util.Arrays; import java.util.HashSet; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import static cc.redberry.pipe.CUtils.*; public class CUtilsTest { @Test public void testAsOutputPort1() throws Exception { OutputPort op = asOutputPort(Arrays.asList(1)); Assert.assertEquals((Integer) 1, op.take()); Assert.assertNull(op.take()); } @Test public void testAsOutputPort2() throws Exception { OutputPort op = asOutputPort(Arrays.asList(1, 3)); Assert.assertEquals((Integer) 1, op.take()); Assert.assertEquals((Integer) 3, op.take()); Assert.assertNull(op.take()); } @Test public void testChunks1() throws Exception { int count = 100000; RandomGenerator rd = new Well19937c(); Set values = new HashSet<>(); for (int i = 0; i < count; ++i) values.add(rd.nextInt()); Set valuesCopy = new HashSet<>(values); Merger> buffered = buffered(chunked(asUnsafeOutputPort(values), 128), 16); Processor proc = new Processor() { @Override public Integer process(Integer input) { return input + 1; } }; ParallelProcessor, Chunk> cParallelProcessor = new ParallelProcessor<>(buffered, chunked(proc), 16, 20); for (Integer integer : it(unchunked(cParallelProcessor))) Assert.assertTrue(valuesCopy.remove(integer - 1)); Assert.assertTrue(valuesCopy.isEmpty()); } @Test public void testChunks2() throws Exception { Assert.assertNull(unchunked(EMPTY_OUTPUT_PORT).take()); Assert.assertNull(unchunked(EMPTY_OUTPUT_PORT_CLOSEABLE).take()); } @Test public void testChunks3() throws Exception { int count = 100000; RandomGenerator rd = new Well19937c(); Set values = new HashSet<>(); for (int i = 0; i < count; ++i) values.add(rd.nextInt()); final ConcurrentHashMap valuesCopy = new ConcurrentHashMap<>(values.size()); for (Integer value : values) valuesCopy.put(value, value); Merger> buffered = buffered(chunked(asUnsafeOutputPort(values), 128), 16); Processor proc = new Processor() { @Override public Integer process(Integer input) { return input + 1; } }; ParallelProcessor, Chunk> cParallelProcessor = new ParallelProcessor<>(buffered, chunked(proc), 16, 20); processAllInParallel(unchunked(cParallelProcessor), new VoidProcessor() { @Override public void process(Integer input) { Assert.assertTrue(valuesCopy.remove(input - 1) != null); } }, 20); Assert.assertTrue(valuesCopy.isEmpty()); } @Test public void testChunks4() throws Exception { int count = 100000; RandomGenerator rd = new Well19937c(); Set values = new HashSet<>(); for (int i = 0; i < count; ++i) values.add(rd.nextInt()); final ConcurrentHashMap valuesCopy = new ConcurrentHashMap<>(values.size()); for (Integer value : values) valuesCopy.put(value, value); Merger> buffered = buffered(chunked(asUnsafeOutputPort(values), 128), 16); processAllInParallel(unchunked(buffered), new VoidProcessor() { @Override public void process(Integer input) { Assert.assertTrue(valuesCopy.remove(input) != null); } }, 20); Assert.assertTrue(valuesCopy.isEmpty()); } @Test public void testChunks5() throws Exception { int count = 100000; RandomGenerator rd = new Well19937c(); Set values = new HashSet<>(); for (int i = 0; i < count; ++i) values.add(rd.nextInt()); final ConcurrentHashMap valuesCopy = new ConcurrentHashMap<>(values.size()); for (Integer value : values) valuesCopy.put(value, value); OutputPort> chunked = chunked(asUnsafeOutputPort(values), 128); processAllInParallel(unchunked(chunked), new VoidProcessor() { @Override public void process(Integer input) { Assert.assertTrue(valuesCopy.remove(input) != null); } }, 20); Assert.assertTrue(valuesCopy.isEmpty()); } }redberry-redberry-pipe-7323a27293f8/src/test/java/cc/redberry/pipe/blocks/MergerTest.java0000644000000000000000000002117513353377423027243 0ustar 00000000000000/* * cc.redberry.concurrent: high-level Java concurrent library. * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ package cc.redberry.pipe.blocks; import cc.redberry.pipe.OutputPort; import org.junit.Assert; import org.junit.Test; import java.util.Random; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; public class MergerTest { public static final int THREADS = 20; @Test public void simpleTestThreads() throws InterruptedException { RandomGenerator[] rgs = new RandomGenerator[THREADS]; for (int i = 0; i < THREADS; ++i) rgs[i] = new RandomGenerator(1000); Merger merger = new Merger<>(1000); for (int i = 0; i < THREADS; ++i) merger.merge(rgs[i]); merger.start(); long sum = 0; Integer value; while ((value = merger.take()) != null) { sum += value; } Assert.assertTrue(sum > 0); for (int i = 0; i < THREADS; ++i) sum -= rgs[i].sum.longValue(); Assert.assertEquals(sum, 0L); } @Test public void simpleTestExecutorService() throws InterruptedException { ExecutorService es = Executors.newCachedThreadPool(); try { RandomGenerator[] rgs = new RandomGenerator[THREADS]; for (int i = 0; i < THREADS; ++i) rgs[i] = new RandomGenerator(1000); Merger merger = new Merger<>(1000); for (int i = 0; i < THREADS; ++i) merger.merge(rgs[i], es); merger.start(); long sum = 0; Integer value; while ((value = merger.take()) != null) { sum += value; } Assert.assertTrue(sum > 0); for (int i = 0; i < THREADS; ++i) sum -= rgs[i].sum.longValue(); Assert.assertEquals(sum, 0L); } finally { es.shutdownNow(); } } @Test public void closeTest0() throws InterruptedException { ExecutorService es = Executors.newCachedThreadPool(); try { RandomGenerator[] rgs = new RandomGenerator[THREADS]; for (int i = 0; i < THREADS; ++i) rgs[i] = new RandomGenerator(Integer.MAX_VALUE); final Merger merger = new Merger<>(1000); for (int i = 0; i < THREADS; ++i) merger.merge(rgs[i], es); merger.start(); new Thread(new Runnable() { @Override public void run() { try { Thread.sleep(100); merger.close(); } catch (InterruptedException e) { } } }).start(); long sum = 0; Integer value; while ((value = merger.take()) != null) { sum += value; } Assert.assertTrue(sum > 0); //Something was consumed for (int i = 0; i < THREADS; ++i) { sum -= rgs[i].sum.longValue(); } Assert.assertTrue(sum <= 0L); //Some elements between producers and consumer were lost (elements that were in buffer when close() method was called) } finally { Assert.assertEquals(es.shutdownNow().size(), 0); //All threads stopped. } } @Test public void closeTest1() throws InterruptedException { ExecutorService es = Executors.newCachedThreadPool(); try { RandomGenerator[] rgs = new RandomGenerator[THREADS]; for (int i = 0; i < THREADS; ++i) rgs[i] = new RandomGenerator(Integer.MAX_VALUE); final Merger merger = new Merger<>(1000); merger.setFailSafe(true); for (int i = 0; i < THREADS; ++i) merger.merge(rgs[i], es); merger.start(); new Thread(new Runnable() { @Override public void run() { try { Thread.sleep(100); merger.close(); } catch (InterruptedException e) { } } }).start(); long sum = 0; Integer value; while ((value = merger.take()) != null) { sum += value; } Assert.assertTrue(sum > 0); //Something was consumed for (int i = 0; i < THREADS; ++i) { sum -= rgs[i].sum.longValue(); } Assert.assertEquals(sum, 0L); } finally { Assert.assertEquals(es.shutdownNow().size(), 0); //All threads stopped. } } @Test public void exceptionTest1() throws InterruptedException { ExecutorService es = Executors.newCachedThreadPool(); try { ExceptionRandomGenerator[] rgs = new ExceptionRandomGenerator[THREADS]; for (int i = 0; i < THREADS; ++i) rgs[i] = new ExceptionRandomGenerator(); final Merger merger = new Merger<>(1000); merger.setFailSafe(true); for (int i = 0; i < THREADS; ++i) merger.merge(rgs[i], es); merger.start(); long sum = 0; boolean ex = false; try { Integer value; while ((value = merger.take()) != null) { sum += value; } } catch (RuntimeException e) { ex = true; } Assert.assertTrue(ex); Assert.assertTrue(sum > 0); //Something was consumed for (int i = 0; i < THREADS; ++i) { sum -= rgs[i].sum.longValue(); } Assert.assertEquals(sum, 0L); } finally { Assert.assertEquals(es.shutdownNow().size(), 0); //All threads stopped. } } @Test public void exceptionTest2() throws InterruptedException { ExceptionRandomGenerator[] rgs = new ExceptionRandomGenerator[THREADS]; for (int i = 0; i < THREADS; ++i) rgs[i] = new ExceptionRandomGenerator(); final Merger merger = new Merger<>(1000); merger.setFailSafe(true); for (int i = 0; i < THREADS; ++i) merger.merge(rgs[i]); merger.start(); long sum = 0; boolean ex = false; try { Integer value; while ((value = merger.take()) != null) { sum += value; } } catch (RuntimeException e) { ex = true; } Assert.assertTrue(ex); Assert.assertTrue(sum > 0); //Something was consumed for (int i = 0; i < THREADS; ++i) { sum -= rgs[i].sum.longValue(); } Assert.assertEquals(sum, 0L); } private static class RandomGenerator implements OutputPort { final Random random = new Random(); final AtomicLong sum = new AtomicLong(); final AtomicInteger limit; private RandomGenerator(int limit) { this.limit = new AtomicInteger(limit); } @Override public synchronized Integer take() { if (limit.decrementAndGet() <= 0) return null; int next = random.nextInt(100); sum.addAndGet(next); return next; } } private static class ExceptionRandomGenerator implements OutputPort { final Random random = new Random(); final AtomicLong sum = new AtomicLong(); @Override public synchronized Integer take() { int next = random.nextInt(100); if (next == 42) throw new TestException(); sum.addAndGet(next); return next; } } private static class TestException extends RuntimeException { } } redberry-redberry-pipe-7323a27293f8/src/test/java/cc/redberry/pipe/blocks/ParallelProcessorTest.java0000644000000000000000000002140713353377423031454 0ustar 00000000000000/* * cc.redberry.concurrent: high-level Java concurrent library. * Copyright (c) 2010-2012 * Bolotin Dmitriy * * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, either version 3 of the License, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ package cc.redberry.pipe.blocks; import cc.redberry.pipe.OutputPort; import cc.redberry.pipe.PInterruptedException; import cc.redberry.pipe.Processor; import org.junit.Assert; import org.junit.Test; import java.util.Random; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.logging.Level; import java.util.logging.Logger; public class ParallelProcessorTest { private static final int count = 100000; @Test public void testSimpleExecutorService() { ExecutorService es = Executors.newCachedThreadPool(); try { TProcessor p = new TProcessor(); ParallelProcessor cp = new ParallelProcessor(new Generator(), p, 100, 40, es); int i = 0; while (true) { IntPair pair = cp.take(); if (pair == null) { Assert.assertTrue(i == count); return; } ++i; Assert.assertTrue(pair.result == pair.value + pair.value * pair.value / 3); } } finally { es.shutdownNow(); } } @Test public void testSimpleThreads() { TProcessor p = new TProcessor(); ParallelProcessor cp = new ParallelProcessor(new Generator(), p, 100, 40); int i = 0; try { while (true) { IntPair pair = cp.take(); if (pair == null) { Assert.assertTrue(i == count); return; } ++i; Assert.assertTrue(pair.result == pair.value + pair.value * pair.value / 3); } } catch (PInterruptedException ex) { Logger.getLogger(ParallelProcessorTest.class.getName()).log(Level.SEVERE, null, ex); } } @Test public void testCloseExecutorService0() { ExecutorService es = Executors.newCachedThreadPool(); try { TProcessor p = new TProcessor(); final Generator generator = new Generator(Integer.MAX_VALUE); final ParallelProcessor cp = new ParallelProcessor(generator, p, 100, 40, es); //cp.setFailSafe(true); int i = 0; new Thread(new Runnable() { @Override public void run() { try { Thread.sleep(100); cp.close(); } catch (InterruptedException e) { } } }).start(); while (true) { IntPair pair = cp.take(); if (pair == null) { Assert.assertTrue(i <= generator.i); return; } ++i; Assert.assertTrue(pair.result == pair.value + pair.value * pair.value / 3); } } finally { Assert.assertEquals(es.shutdownNow().size(), 0); //All threads stopped. } } @Test public void testCloseExecutorService1() { ExecutorService es = Executors.newCachedThreadPool(); try { TProcessor p = new TProcessor(); final Generator generator = new Generator(Integer.MAX_VALUE); final ParallelProcessor cp = new ParallelProcessor(generator, p, 100, 40, es) .setFailSafe(true); int i = 0; new Thread(new Runnable() { @Override public void run() { try { Thread.sleep(100); cp.close(); } catch (InterruptedException e) { } } }).start(); while (true) { IntPair pair = cp.take(); if (pair == null) { Assert.assertEquals(i, generator.i); return; } ++i; Assert.assertTrue(pair.result == pair.value + pair.value * pair.value / 3); } } finally { Assert.assertEquals(es.shutdownNow().size(), 0); //All threads stopped. } } @Test public void testCloseThread1() { TProcessor p = new TProcessor(); final Generator generator = new Generator(Integer.MAX_VALUE); final ParallelProcessor cp = new ParallelProcessor(generator, p, 100, 40) .setFailSafe(true); int i = 0; new Thread(new Runnable() { @Override public void run() { try { Thread.sleep(100); cp.close(); } catch (InterruptedException e) { } } }).start(); while (true) { IntPair pair = cp.take(); if (pair == null) { Assert.assertEquals(i, generator.i); return; } ++i; Assert.assertTrue(pair.result == pair.value + pair.value * pair.value / 3); } } @Test(expected = RuntimeException.class) public void testExceptionExecutorService() { ExecutorService es = Executors.newCachedThreadPool(); try { TProcessorException p = new TProcessorException(); ParallelProcessor cp = new ParallelProcessor(new Generator(Integer.MAX_VALUE), p, 100, 40, es); int i = 0; while (true) { IntPair pair = cp.take(); if (pair == null) { break; } ++i; Assert.assertTrue(pair.result == pair.value + pair.value * pair.value / 3); } } finally { es.shutdownNow(); } } @Test(expected = RuntimeException.class) public void testExceptionThread() { try { TProcessorException p = new TProcessorException(); ParallelProcessor cp = new ParallelProcessor(new Generator(Integer.MAX_VALUE), p, 100, 40); int i = 0; while (true) { IntPair pair = cp.take(); if (pair == null) { break; } ++i; Assert.assertTrue(pair.result == pair.value + pair.value * pair.value / 3); } cp.join(); } catch (InterruptedException ex) { throw new RuntimeException(ex); } } private static class IntPair { public int value, result; public IntPair(int value) { this.value = value; } } private static class Generator implements OutputPort { private final int count; private Random random = new Random(); private int i = 0; private Generator() { this.count = ParallelProcessorTest.count; } private Generator(int count) { this.count = count; } public synchronized IntPair take() { int val = random.nextInt(100); if (i++ > count - 1) return null; return new IntPair(val); } } private static class TProcessor implements Processor { public IntPair process(IntPair input) { input.result = input.value + input.value * input.value / 3; return input; } } private static class TProcessorException implements Processor { public IntPair process(IntPair input) { if (input.value == 42) throw new TestException(); input.result = input.value + input.value * input.value / 3; return input; } } private static class TestException extends RuntimeException { } }././@LongLink0000000000000000000000000000015400000000000011215 Lustar 00000000000000redberry-redberry-pipe-7323a27293f8/src/test/java/cc/redberry/pipe/blocks/ResultAttachingProcessorTest.javaredberry-redberry-pipe-7323a27293f8/src/test/java/cc/redberry/pipe/blocks/ResultAttachingProcessorTe0000644000000000000000000000252213353377423031527 0ustar 00000000000000package cc.redberry.pipe.blocks; import cc.redberry.pipe.OutputPort; import cc.redberry.pipe.Processor; import org.junit.Assert; import org.junit.Test; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicInteger; public class ResultAttachingProcessorTest { @Test public void test0() throws InterruptedException { Generator gen = new Generator(); Proc proc = new Proc(); OutputPort> values = new ParallelProcessor<>(gen, new ResultAttachingProcessor(proc), 1000, 10); InputOutputPair i; while ((i = values.take()) != null) { Assert.assertEquals((long) (i.input * 4 - 1), (long) i.result); } } private static class Proc implements Processor { @Override public Integer process(Integer input) { return input * 4 - 1; } } private static class Generator implements OutputPort { private final AtomicInteger counter = new AtomicInteger(1000); private Generator() { } public synchronized Integer take() { if (counter.decrementAndGet() < 0) return null; return ThreadLocalRandom.current().nextInt(10000); } } } ././@LongLink0000000000000000000000000000014600000000000011216 Lustar 00000000000000redberry-redberry-pipe-7323a27293f8/src/test/java/cc/redberry/pipe/util/FlatteningOutputPortTest.javaredberry-redberry-pipe-7323a27293f8/src/test/java/cc/redberry/pipe/util/FlatteningOutputPortTest.jav0000644000000000000000000000236513353377423031542 0ustar 00000000000000package cc.redberry.pipe.util; import cc.redberry.pipe.OutputPort; import org.junit.Test; import java.util.Arrays; import java.util.List; import static junit.framework.Assert.assertEquals; public class FlatteningOutputPortTest { @Test public void test1() throws Exception { List strings0 = Arrays.asList(new String[]{"k1", "k2", "k3", "k4"}); List strings1 = Arrays.asList(new String[]{"a1", "a2", "a3"}); List strings2 = Arrays.asList(new String[]{"a1s", "as2", "sk3", "sk4", "a3s"}); List> ssList = (List) Arrays.asList(new OutputPort[]{ new IteratorOutputPortAdapter(strings0), new IteratorOutputPortAdapter(strings1), new IteratorOutputPortAdapter(strings2)}); OutputPort> opop = new IteratorOutputPortAdapter<>(ssList); OutputPort sPort = new FlatteningOutputPort(opop); for (String s : strings0) assertEquals(s, sPort.take()); for (String s : strings1) assertEquals(s, sPort.take()); for (String s : strings2) assertEquals(s, sPort.take()); assertEquals(null, sPort.take()); } } ././@LongLink0000000000000000000000000000015300000000000011214 Lustar 00000000000000redberry-redberry-pipe-7323a27293f8/src/test/java/cc/redberry/pipe/util/IteratorOutputPortAdapterTest.javaredberry-redberry-pipe-7323a27293f8/src/test/java/cc/redberry/pipe/util/IteratorOutputPortAdapterTes0000644000000000000000000000111013353377423031561 0ustar 00000000000000package cc.redberry.pipe.util; import cc.redberry.pipe.OutputPort; import org.junit.Test; import java.util.Arrays; import java.util.List; import static junit.framework.Assert.assertEquals; public class IteratorOutputPortAdapterTest { @Test public void test1() throws Exception { List strings = Arrays.asList(new String[]{"k1", "k2", "k3", "k4"}); OutputPort sPort = new IteratorOutputPortAdapter(strings); for (String s : strings) assertEquals(s, sPort.take()); assertEquals(null, sPort.take()); } } redberry-redberry-pipe-7323a27293f8/src/test/java/cc/redberry/pipe/util/OrderedOutputPortTest.java0000644000000000000000000000332513353377423031171 0ustar 00000000000000package cc.redberry.pipe.util; import cc.redberry.pipe.InputPort; import cc.redberry.pipe.OutputPort; import cc.redberry.pipe.blocks.Buffer; import org.apache.commons.math.random.RandomDataImpl; import org.apache.commons.math.random.Well19937c; import org.junit.Assert; import org.junit.Test; import java.util.concurrent.atomic.AtomicInteger; public class OrderedOutputPortTest { @Test public void test1() throws Exception { RandomDataImpl rdi = new RandomDataImpl(new Well19937c()); final int[] randomPermutation = rdi.nextPermutation(500, 500); int threadCount = 30; final AtomicInteger index = new AtomicInteger(randomPermutation.length - 1); Thread[] threads = new Thread[threadCount]; final Buffer buffer = new Buffer<>(); for (int i = 0; i < threadCount; ++i) { final InputPort ip = buffer.createInputPort(); threads[i] = new Thread() { @Override public void run() { int i; while ((i = index.getAndDecrement()) >= 0) { ip.put(randomPermutation[i]); } ip.put(null); } }; } for (int i = 0; i < threadCount; ++i) { threads[i].start(); } OutputPort ordered = new OrderedOutputPort<>(buffer, new Indexer() { @Override public long getIndex(Integer object) { return object; } }); Integer i; int assertI = 0; while ((i = ordered.take()) != null) { Assert.assertEquals(assertI++, i.intValue()); } } }