inanzzz | A simple batch job processing application in Golang

In this example we are going to split multiple jobs into set of batch operations where each batch will contain maximum of 3 jobs. Next batch will wait for the previous batch to finish.

Example

package main

import "fmt"

var (
	// batchSize defines the batch size where each job batch will contain maximum certain number of jobs.
	batchSize = 3
	// jobs contains the resources we are going to process.
	jobs = []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}
)

func main() {
	fmt.Println("BEGIN")

	for i := 0; i < len(jobs); i += batchSize {
		j := i + batchSize
		if j > len(jobs) {
			j = len(jobs)
		}

		fmt.Printf("batch %d: ", batchNo(i))
		processor(jobs[i:j])
	}

	fmt.Println("END")
}

func processor(jobs []string) {
	fmt.Println(jobs)
}

func batchNo(i int) int  {
	if i == 0 {
		i = 1
	} else if i == batchSize {
		i = 2
	} else if i == batchSize * 2 {
		i = 3
	} else {
		i = (i / batchSize) + 1
	}

	return i
}

Output

BEGIN
batch 1: [a b c]
batch 2: [d e f]
batch 3: [g h i]
batch 4: [j]
END

Other examples

You can also use examples below.

func batcher() {
	batch := make(map[int][]string)

	j := 0
	for i := 0; i < len(jobs); i++ {
		if i != 0 && i%batchSize == 0 {
			j++
		}

		batch[j] = append(batch[j], jobs[i])
	}

	fmt.Println(batch)
}

func batcher() {
	j := 1
	for i := 0; i < len(jobs); i++ {
		if j == batchSize {
			fmt.Println(jobs[i])
			j = 1
			continue
		}

		fmt.Printf(jobs[i])
		j++
	}
}