build out cli and add size and gather functionality

.gitignore

@@ -1,2 +1,3 @@
 forever
 forever-files
+/forever.exe

.idea/forever-files.iml

@@ -2,7 +2,9 @@
 <module type="WEB_MODULE" version="4">
   <component name="Go" enabled="true" />
   <component name="NewModuleRootManager">
-    <content url="file://$MODULE_DIR$" />
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/.idea/dataSources" />
+    </content>
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>

cmd/cli/cli.go

@@ -1,31 +1,20 @@
 package main
 
 import (
-	"context"
 	"fmt"
-	"github.com/urfave/cli/v2"
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/credentials/insecure"
 	"os"
-	"time"
+
+	"github.com/urfave/cli/v2"
 )
 
 func main() {
 	commands := []*cli.Command{
-		&cli.Command{
+		gather(),
-			Name:    "start",
+		plan(),
-			Aliases: []string{"s"},
-			Usage:   "Start the application service",
-			Action: func(c *cli.Context) error {
-				//d := daemon.NewDaemon()
-				//d.Start()
-				return nil
-			},
-		},
 	}
 
 	app := &cli.App{
-		Name:        "ForeverFiles",
+		Name:        "forever",
 		Usage:       "Create backups designed to last forever",
 		Version:     "v1.0.0",
 		Description: "ForeverFiles is a system for storing files forever.",
@@ -37,7 +26,7 @@ func main() {
 				Email: "mason@masonitestudios.com",
 			},
 		},
-		Copyright:              fmt.Sprintf("%v Masonite Studios LLC", time.Now().Year()),
+		Copyright:              "2024 Masonite Studios LLC",
 		UseShortOptionHandling: true,
 	}
 	err := app.Run(os.Args)
@@ -46,16 +35,3 @@ func main() {
 		return
 	}
 }
-
-func getConn(dialAddr string) (*grpc.ClientConn, error) {
-	conn, err := grpc.DialContext(
-		context.Background(),
-		dialAddr,
-		//grpc.WithBlock(),
-		grpc.WithTransportCredentials(insecure.NewCredentials()),
-	)
-	if err != nil {
-		return &grpc.ClientConn{}, fmt.Errorf("failed to dial server: %w", err)
-	}
-	return conn, nil
-}

cmd/cli/flags.go

@@ -0,0 +1,12 @@
+package main
+
+import "github.com/urfave/cli/v2"
+
+func baseDirFlag() *cli.StringFlag {
+	return &cli.StringFlag{
+		Name:    "baseDir",
+		Usage:   "The base directory to gather info from",
+		Aliases: []string{"b"},
+		Value:   ".",
+	}
+}

cmd/cli/gather.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"fmt"
+	"github.com/dustin/go-humanize"
+
+	"forever-files/db"
+	"forever-files/source"
+	"forever-files/types"
+	"github.com/urfave/cli/v2"
+)
+
+func gather() *cli.Command {
+	return &cli.Command{
+		Name:    "gather",
+		Aliases: []string{"g"},
+		Usage:   "Collects the files to be backed up and stores their info in the database",
+		Flags: []cli.Flag{
+			&cli.BoolFlag{
+				Name:    "reset",
+				Usage:   "Reset the database before gathering info",
+				Aliases: []string{"r"},
+				Action: func(c *cli.Context, reset bool) error {
+					if reset {
+						err := db.DeleteDB(types.AppName)
+						if err != nil {
+							return fmt.Errorf("error deleting db: %w", err)
+						}
+					}
+					return nil
+				},
+			},
+			baseDirFlag(),
+		},
+		Action: func(c *cli.Context) error {
+
+			store, err := db.NewDB(types.AppName)
+			if err != nil {
+				panic(fmt.Errorf("error creating db: %w", err))
+			}
+			defer store.Close()
+
+			err = store.Migrate()
+			if err != nil {
+				panic(fmt.Errorf("error migrating db: %w", err))
+			}
+
+			baseDir := c.String("baseDir")
+
+			err = source.GatherInfo(baseDir, store)
+			if err != nil {
+				return fmt.Errorf("error gathering info: %w", err)
+			}
+
+			fileCount, err := store.GetFileCount()
+			if err != nil {
+				return fmt.Errorf("error getting file count: %w", err)
+			}
+			totalSize, err := store.GetTotalSize()
+			if err != nil {
+				return fmt.Errorf("error getting total size: %w", err)
+			}
+			fmt.Printf("Total Files: %v\n", fileCount)
+			fmt.Printf("Total Size: %v\n", humanize.Bytes(uint64(totalSize)))
+
+			return nil
+		},
+	}
+}

cmd/cli/plan.go

@@ -0,0 +1,107 @@
+package main
+
+import (
+	"fmt"
+	"forever-files/db"
+	"forever-files/partitioner"
+	"forever-files/types"
+	"github.com/dustin/go-humanize"
+	"github.com/urfave/cli/v2"
+	"math"
+)
+
+func plan() *cli.Command {
+	return &cli.Command{
+		Name:    "plan",
+		Aliases: []string{"p"},
+		Usage:   "Reads the database and plans the partitions for the backup, stores the partitions in the database",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:    "targetSize",
+				Usage:   "The target size for each partition, valid options are DVD-SL, DVD-DL, BD-SL, BD-DL, BD-TL, BD-QL\nor you can provide any specific size e.g. 4.6GB, 8.1GB, 25GB, 50GB, 100GB, 128GB or 5MB!",
+				Aliases: []string{"s"},
+				Value:   "DVD-SL",
+			},
+			&cli.BoolFlag{
+				Name:    "verbose",
+				Usage:   "Print the files in each partition",
+				Aliases: []string{"v"},
+				Value:   false,
+			},
+		},
+		Action: func(c *cli.Context) error {
+
+			store, err := db.NewDB(types.AppName)
+			if err != nil {
+				panic(fmt.Errorf("error creating db: %w", err))
+			}
+			defer store.Close()
+
+			err = store.Migrate()
+			if err != nil {
+				panic(fmt.Errorf("error migrating db: %w", err))
+			}
+
+			size := int64(4600000000) // size of a single layer DVD
+			targetSize := c.String("targetSize")
+			switch targetSize {
+			case "DVD-SL":
+				size = 4600000000
+			case "DVD-DL":
+				size = 8100000000
+			case "BD-SL":
+				size = 25000000000
+			case "BD-DL":
+				size = 50000000000
+			case "BD-TL":
+				size = 100000000000
+			case "BD-QL":
+				size = 128000000000
+			default:
+				// try to parse the size from human-readable format
+				usize, err := humanize.ParseBytes(targetSize)
+				if err != nil {
+					fmt.Printf("invalid target size: %v\n", err)
+					fmt.Println("valid options are DVD-SL, DVD-DL, BD-SL, BD-DL, BD-TL, BD-QL")
+					fmt.Println("or you can provide any specific size e.g. 4.6GB, 8.1GB, 25GB, 50GB, 100GB, 128GB or 5MB!")
+					size = 4600000000
+				}
+				if usize > math.MaxInt64 {
+					fmt.Println("size is too large")
+					size = 4600000000
+				}
+				size = int64(usize)
+			}
+
+			fmt.Printf("Target Size: %v\n", humanize.Bytes(uint64(size)))
+			fmt.Println("Calculating partitions...")
+
+			partitions, err := partitioner.CalculatePartitions(store, size)
+			if err != nil {
+				return fmt.Errorf("error calculating partitions: %w", err)
+			}
+
+			for i, partition := range partitions {
+				partSize := int64(0)
+				if c.Bool("verbose") {
+					fmt.Printf("Partition %v:\n", i)
+				}
+				for _, file := range partition {
+					if c.Bool("verbose") {
+						fmt.Printf("%v/%v %v\n", file.Path, file.Name, humanize.Bytes(uint64(file.Size)))
+					}
+					partSize += file.Size
+					// save the planned partitions
+					file.PartitionId = fmt.Sprintf("%d", i)
+					err = store.StoreFilePartition(file)
+					if err != nil {
+						return fmt.Errorf("error storing file's partition: %w", err)
+					}
+				}
+				fmt.Printf("Partition %v Size: %v files %v\n", i, len(partition), humanize.Bytes(uint64(partSize)))
+			}
+
+			return nil
+		},
+	}
+}

db/db.go

@@ -17,6 +17,7 @@ type DB interface {
 	Migrate() error
 	StoreFile(fileMetadata types.FileMetadata) error
 	RemoveFile(fileMetadata types.FileMetadata) error
+	StoreFilePartition(fileMetadata types.FileMetadata) error
 	GetTotalSize() (int64, error)
 	GetFileCount() (int64, error)
 	GetFiles() ([]types.FileMetadata, error)
@@ -41,7 +42,8 @@ var migrations = []Migrations{
 				size INTEGER NOT NULL,
 				hash TEXT NOT NULL,
 				modifiedDate TIMESTAMP NOT NULL,
-				backedUp BOOLEAN NOT NULL
+				backedUp BOOLEAN NOT NULL,
+				partitionId TEXT DEFAULT ''
 				)`,
 	},
 	{
@@ -64,6 +66,16 @@ func NewDB(appName string) (DB, error) {
 	}, nil
 }
 
+func DeleteDB(appName string) error {
+	configPath := configdir.LocalConfig(appName)
+	dbPath := path.Join(configPath, "db", fmt.Sprintf("%v.db", appName))
+	err := os.Remove(dbPath)
+	if err != nil {
+		return fmt.Errorf("error deleting db | %w", err)
+	}
+	return nil
+}
+
 func (d *store) StoreFile(fileMetadata types.FileMetadata) error {
 	query := `INSERT INTO files (name, path, size, hash, modifiedDate, backedUp) VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT (name, path, hash) DO UPDATE SET size = ?, modifiedDate = ?, backedUp = ?`
 	_, err := d.db.Exec(
@@ -93,6 +105,21 @@ func (d *store) RemoveFile(fileMetadata types.FileMetadata) error {
 	return nil
 }
 
+func (d *store) StoreFilePartition(fileMetadata types.FileMetadata) error {
+	query := `UPDATE files SET partitionId = ? WHERE name = ? AND path = ? AND hash = ?`
+	_, err := d.db.Exec(
+		query,
+		fileMetadata.PartitionId,
+		fileMetadata.Name,
+		fileMetadata.Path,
+		fileMetadata.Hash,
+	)
+	if err != nil {
+		return fmt.Errorf("error storing file's partiition | %w", err)
+	}
+	return nil
+}
+
 func (d *store) GetTotalSize() (int64, error) {
 	var size int64
 	query := `SELECT SUM(size) FROM files`
@@ -115,7 +142,7 @@ func (d *store) GetFileCount() (int64, error) {
 
 func (d *store) GetFiles() ([]types.FileMetadata, error) {
 	var files []types.FileMetadata
-	query := `SELECT name, path, size, hash, modifiedDate, backedUp FROM files order by path, name`
+	query := `SELECT name, path, size, hash, modifiedDate, backedUp, partitionId FROM files order by path, name`
 	rows, err := d.db.Query(query)
 	if err != nil {
 		return nil, fmt.Errorf("error getting files | %w", err)
@@ -123,7 +150,7 @@ func (d *store) GetFiles() ([]types.FileMetadata, error) {
 	defer rows.Close()
 	for rows.Next() {
 		var file types.FileMetadata
-		err := rows.Scan(&file.Name, &file.Path, &file.Size, &file.Hash, &file.ModifiedDate, &file.BackedUp)
+		err := rows.Scan(&file.Name, &file.Path, &file.Size, &file.Hash, &file.ModifiedDate, &file.BackedUp, &file.PartitionId)
 		if err != nil {
 			return nil, fmt.Errorf("error scanning file | %w", err)
 		}
@@ -179,7 +206,7 @@ func createDBFileIfNotExist(appName string) (string, error) {
 	// set up the config directory
 	err := configdir.MakePath(configPath)
 	if err != nil {
-		panic(err)
+		return "", fmt.Errorf("error creating config directory: %w", err)
 	}
 
 	dbDirectoryPath := path.Join(configPath, "db")
@@ -187,7 +214,7 @@ func createDBFileIfNotExist(appName string) (string, error) {
 	// Set up the database
 	err = configdir.MakePath(dbDirectoryPath)
 	if err != nil {
-		panic(err)
+		return "", fmt.Errorf("error creating db directory: %w", err)
 	}
 
 	// If the file doesn't exist, create it, or append to the file

main.go

@@ -27,6 +27,7 @@ func main() {
 
 	source.GatherInfo(baseDir, store)
 	oneDVDSize := int64(4600000000)
+	//oneBRSize := int64(25000000000) // size of a small Blu-ray disc
 	partitions, err := partitioner.CalculatePartitions(store, oneDVDSize)
 	if err != nil {
 		panic(fmt.Errorf("error calculating partitions: %w", err))

partitioner/partitioner.go

@@ -25,11 +25,11 @@ func CalculatePartitions(store db.DB, targetSize int64) (partitions [][]types.Fi
 	partitions = make([][]types.FileMetadata, 0)
 	partitionSize := int64(0)
 	partitionFiles := make([]types.FileMetadata, 0)
-	leftOverFiles := make([]types.FileMetadata, 0)
+	overSizedFiles := make([]types.FileMetadata, 0)
-	leftOverSize := int64(0)
+	overSizedSize := int64(0)
 	for _, file := range files {
 		if partitionSize+file.Size > targetSize {
-			fmt.Printf("Partition Size: %v\n", humanize.Bytes(uint64(partitionSize)))
+			//fmt.Printf("Partition Size: %v\n", humanize.Bytes(uint64(partitionSize)))
 			partitions = append(partitions, partitionFiles)
 			partitionFiles = make([]types.FileMetadata, 0)
 			partitionSize = 0
@@ -38,16 +38,25 @@ func CalculatePartitions(store db.DB, targetSize int64) (partitions [][]types.Fi
 			partitionFiles = append(partitionFiles, file)
 			partitionSize += file.Size
 		} else {
-			leftOverFiles = append(leftOverFiles, file)
+			overSizedFiles = append(overSizedFiles, file)
-			leftOverSize += file.Size
+			overSizedSize += file.Size
 		}
 	}
-
+	if len(partitionFiles) > 0 {
-	for _, partition := range partitions {
+		partitions = append(partitions, partitionFiles)
-		fmt.Printf("Partition File Count: %v\n", len(partition))
 	}
-	fmt.Printf("Left Over File Count: %v\n", len(leftOverFiles))
+
-	fmt.Printf("Left Over Size: %v\n", humanize.Bytes(uint64(leftOverSize)))
+	//for _, partition := range partitions {
+	//	fmt.Printf("Partition File Count: %v\n", len(partition))
+	//}
+	fmt.Printf("Over Sized File Count: %v\n", len(overSizedFiles))
+	fmt.Printf("Total Over Sized Size: %v\n", humanize.Bytes(uint64(overSizedSize)))
 
 	return partitions, nil
 }
+
+// CalculatePartitionsLeftPack calculates the partitions efficiently by searching for files that fit the remaining space in each partition
+func CalculatePartitionsLeftPack(store db.DB, targetSize int64) (partitions [][]types.FileMetadata, err error) {
+	// TODO: implement this function
+	return nil, nil
+}

source/source.go

@@ -18,11 +18,12 @@ import (
 // - file hash
 // - modified date
 
-func GatherInfo(path string, db db.DB) {
+func GatherInfo(path string, db db.DB) error {
 	err := walkDir(path, db)
 	if err != nil {
-		log.Fatal(err)
+		return fmt.Errorf("error walking directory: %w", err)
 	}
+	return nil
 }
 
 func walkDir(dirPath string, db db.DB) error {

types/types.go

@@ -12,5 +12,6 @@ type FileMetadata struct {
 	Size         int64
 	Hash         []byte
 	ModifiedDate time.Time
+	PartitionId  string
 	BackedUp     bool
 }