forever-files/partitioner/partitioner.go

package partitioner

import (
	"fmt"
	"forever-files/db"
	"forever-files/types"
	"github.com/dustin/go-humanize"
	"sort"
)

func CalculatePartitions(store db.DB, targetSize int64) (partitions [][]types.FileMetadata, err error) {
	totalSize, err := store.GetTotalSize()
	if err != nil {
		return nil, fmt.Errorf("error getting total size: %w", err)
	}
	if targetSize <= 0 {
		targetSize = totalSize / 2
	}
	fmt.Printf("Total Size: %v\n", totalSize)
	fmt.Printf("Target Size: %v\n", targetSize)

	files, err := store.GetFiles()
	if err != nil {
		return nil, fmt.Errorf("error getting files: %w", err)
	}
	partitions = make([][]types.FileMetadata, 0)
	partitionSize := int64(0)
	partitionFiles := make([]types.FileMetadata, 0)
	overSizedFiles := make([]types.FileMetadata, 0)
	overSizedSize := int64(0)
	for _, file := range files {
		if partitionSize+file.Size > targetSize {
			//fmt.Printf("Partition Size: %v\n", humanize.Bytes(uint64(partitionSize)))
			partitions = append(partitions, partitionFiles)
			partitionFiles = make([]types.FileMetadata, 0)
			partitionSize = 0
		}
		if partitionSize < targetSize && partitionSize+file.Size < targetSize {
			partitionFiles = append(partitionFiles, file)
			partitionSize += file.Size
		} else {
			overSizedFiles = append(overSizedFiles, file)
			overSizedSize += file.Size
		}
	}
	if len(partitionFiles) > 0 {
		partitions = append(partitions, partitionFiles)
	}

	//for _, partition := range partitions {
	//	fmt.Printf("Partition File Count: %v\n", len(partition))
	//}
	fmt.Printf("Over Sized File Count: %v\n", len(overSizedFiles))
	fmt.Printf("Total Over Sized Size: %v\n", humanize.Bytes(uint64(overSizedSize)))

	return partitions, nil
}

// CalculatePartitionsLeftPack calculates the partitions efficiently by searching for files that fit the remaining space in each partition
func CalculatePartitionsLeftPack(store db.DB, targetSize int64) (partitions [][]types.FileMetadata, err error) {
	totalSize, err := store.GetTotalSize()
	if err != nil {
		return nil, fmt.Errorf("error getting total size: %w", err)
	}
	if targetSize <= 0 {
		targetSize = totalSize / 2
	}
	fmt.Printf("Total Size: %v\n", totalSize)
	fmt.Printf("Target Size: %v\n", targetSize)

	files, err := store.GetFiles()
	if err != nil {
		return nil, fmt.Errorf("error getting files: %w", err)
	}
	partitions = make([][]types.FileMetadata, 0)
	partitionSize := int64(0)
	partitionFiles := make([]types.FileMetadata, 0)
	overSizedFiles := make([]types.FileMetadata, 0)
	overSizedSize := int64(0)

	// sort files by size
	sort.SliceStable(files, func(i, j int) bool {
		return files[i].Size > files[j].Size
	})

	for _, file := range files {
		if file.Size > targetSize {
			overSizedFiles = append(overSizedFiles, file)
			overSizedSize += file.Size
			file.PartitionId = "-1"
		} else {
			// you've hit files that are smaller than the target size
			break
		}
	}

	partitionIndex := int64(0)
	// pick the largest file that fits in the partition's remaining space using sort.Search()
	// for loop counting down for the size of the files slice
	for i := len(files) - 1; i >= 0; i-- {
		index := indexOfLargestFittingFile(files, targetSize-partitionSize)
		if index == -1 {
			// no files fit in the partition so move on to the next partition
			partitions = append(partitions, partitionFiles)
			partitionFiles = make([]types.FileMetadata, 0)
			partitionSize = 0
			partitionIndex++
			index = indexOfLargestFittingFile(files, targetSize-partitionSize)
			if index == -1 {
				// no files fit in the new partition so break out of the loop
				break
			}
		}
		partitionFiles = append(partitionFiles, files[index])
		partitionSize += files[index].Size
		files[index].PartitionId = fmt.Sprintf("%d", partitionIndex)
		// remove the file from the slice
		files = append(files[:index], files[index+1:]...)
	}
	if len(partitionFiles) > 0 {
		partitions = append(partitions, partitionFiles)
	}
	fmt.Printf("Over Sized File Count: %v\n", len(overSizedFiles))
	fmt.Printf("Total Over Sized Size: %v\n", humanize.Bytes(uint64(overSizedSize)))
	return partitions, nil
}

func indexOfLargestFittingFile(files []types.FileMetadata, remainingSize int64) int {
	// find the index of the largest file that fits in the remaining space
	index := sort.Search(len(files), func(i int) bool {
		return files[i].Size < remainingSize
	})
	// if index is == len(files) then there are no files that fit
	if index == len(files) {
		return -1
	}
	return index
}