package partitioner import ( "fmt" "forever-files/db" "forever-files/types" "github.com/dustin/go-humanize" "sort" ) func CalculatePartitions(store db.DB, targetSize int64) (partitions [][]types.FileMetadata, err error) { totalSize, err := store.GetTotalSize() if err != nil { return nil, fmt.Errorf("error getting total size: %w", err) } if targetSize <= 0 { targetSize = totalSize / 2 } fmt.Printf("Total Size: %v\n", totalSize) fmt.Printf("Target Size: %v\n", targetSize) files, err := store.GetFiles() if err != nil { return nil, fmt.Errorf("error getting files: %w", err) } partitions = make([][]types.FileMetadata, 0) partitionSize := int64(0) partitionFiles := make([]types.FileMetadata, 0) overSizedFiles := make([]types.FileMetadata, 0) overSizedSize := int64(0) for _, file := range files { if partitionSize+file.Size > targetSize { //fmt.Printf("Partition Size: %v\n", humanize.Bytes(uint64(partitionSize))) partitions = append(partitions, partitionFiles) partitionFiles = make([]types.FileMetadata, 0) partitionSize = 0 } if partitionSize < targetSize && partitionSize+file.Size < targetSize { partitionFiles = append(partitionFiles, file) partitionSize += file.Size } else { overSizedFiles = append(overSizedFiles, file) overSizedSize += file.Size } } if len(partitionFiles) > 0 { partitions = append(partitions, partitionFiles) } //for _, partition := range partitions { // fmt.Printf("Partition File Count: %v\n", len(partition)) //} fmt.Printf("Over Sized File Count: %v\n", len(overSizedFiles)) fmt.Printf("Total Over Sized Size: %v\n", humanize.Bytes(uint64(overSizedSize))) return partitions, nil } // CalculatePartitionsLeftPack calculates the partitions efficiently by searching for files that fit the remaining space in each partition func CalculatePartitionsLeftPack(store db.DB, targetSize int64) (partitions [][]types.FileMetadata, err error) { totalSize, err := store.GetTotalSize() if err != nil { return nil, fmt.Errorf("error getting total size: %w", err) } if targetSize <= 0 { targetSize = totalSize / 2 } fmt.Printf("Total Size: %v\n", totalSize) fmt.Printf("Target Size: %v\n", targetSize) files, err := store.GetFiles() if err != nil { return nil, fmt.Errorf("error getting files: %w", err) } partitions = make([][]types.FileMetadata, 0) partitionSize := int64(0) partitionFiles := make([]types.FileMetadata, 0) overSizedFiles := make([]types.FileMetadata, 0) overSizedSize := int64(0) // sort files by size sort.SliceStable(files, func(i, j int) bool { return files[i].Size > files[j].Size }) for _, file := range files { if file.Size > targetSize { overSizedFiles = append(overSizedFiles, file) overSizedSize += file.Size file.PartitionId = "-1" } else { // you've hit files that are smaller than the target size break } } partitionIndex := int64(0) // pick the largest file that fits in the partition's remaining space using sort.Search() // for loop counting down for the size of the files slice for i := len(files) - 1; i >= 0; i-- { index := indexOfLargestFittingFile(files, targetSize-partitionSize) if index == -1 { // no files fit in the partition so move on to the next partition partitions = append(partitions, partitionFiles) partitionFiles = make([]types.FileMetadata, 0) partitionSize = 0 partitionIndex++ index = indexOfLargestFittingFile(files, targetSize-partitionSize) if index == -1 { // no files fit in the new partition so break out of the loop break } } partitionFiles = append(partitionFiles, files[index]) partitionSize += files[index].Size files[index].PartitionId = fmt.Sprintf("%d", partitionIndex) // remove the file from the slice files = append(files[:index], files[index+1:]...) } if len(partitionFiles) > 0 { partitions = append(partitions, partitionFiles) } fmt.Printf("Over Sized File Count: %v\n", len(overSizedFiles)) fmt.Printf("Total Over Sized Size: %v\n", humanize.Bytes(uint64(overSizedSize))) return partitions, nil } func indexOfLargestFittingFile(files []types.FileMetadata, remainingSize int64) int { // find the index of the largest file that fits in the remaining space index := sort.Search(len(files), func(i int) bool { return files[i].Size < remainingSize }) // if index is == len(files) then there are no files that fit if index == len(files) { return -1 } return index }