Read arrays by chunks using threads

Sometimes you need to process very large arrays, in this case multithreading can be helpful

Read an array and process it by chunks using threads. Very useful when processing big amounts of data. The number of threads is calculated each time from available processor:

 

/**
* Reads an array by chunks using multithreads
*
* @author freeeelta http://freedelta.free.fr 2012-02-21
*/
package testthreads;

import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

public class TestThreads {

    /**
     * @param args
     */
    public static void main(String[] args) {

        int numberOfLines=121792;
        
        int[] numbers=new int[numberOfLines];
        
        for (int i = 0; i < numbers.length; i++)
        {
            numbers[i]=i+1;
        }
        
        int chunks = Runtime.getRuntime().availableProcessors();
        int[] offsets = new int[chunks];
        
        int start=0;
        int end=0;
        
        int windowSize=numberOfLines/chunks;
        
        // Number of lines too small to make chunks
        if (numberOfLines < (chunks*chunks) )
        {
            chunks=1;
            windowSize=1;
        }
        
        // Offset: pages sizes
        for (int i = 1; i < chunks; i++)
        {
            offsets[i] = i*windowSize;
        }
        
        System.out.println("Number of lines: "+numberOfLines+" chunks: "+chunks+" window size: "+windowSize);
        
        
        // Process each chunk using a thread for each one
        ExecutorService service = Executors.newFixedThreadPool(chunks);
        for (int i = 0; i < chunks; i++)
        {
            if (i!=0)
            {
                start=(offsets[i])+1;    
            }
            
            end = i < chunks - 1 ? offsets[i + 1] : (numberOfLines-1);
        
            // Call service to process data by chunks
            service.execute(new ReadChunk(start, end));
        }
        service.shutdown();

    }
    
    static class ReadChunk implements Runnable {
           
        private final int start;
        private final int end;

        public ReadChunk(int start, int end)
        {
           
            this.start = start;
            this.end = end;
        }

        public void run()
        {
            // Reading chunks, you can process them here
            System.out.println("start: "+this.start+" end: "+this.end);
     
        }
    }
}

 

The output for this program is:

Number of lines: 121792 chunks: 4 window size: 30448
start: 0 end: 30448
start: 30449 end: 60896
start: 60897 end: 91344
start: 91345 end: 121791