payne8/forever-files
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

build out cli and add size and gather functionality

Browse Source
This commit is contained in:
Mason Payne
2024-06-05 00:03:05 -06:00
parent 6507518cd7
commit b3bbd2e5d1
11 changed files with 254 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,2 +1,3 @@
forever
forever-files
/forever.exe

4
.idea/forever-files.iml generated
View File

@ -2,7 +2,9 @@
<module type="WEB_MODULE" version="4">
<component name="Go" enabled="true" />
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<content url="file://$MODULE_DIR$">
<excludeFolder url="file://$MODULE_DIR$/.idea/dataSources" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>

36
cmd/cli/cli.go
View File

@ -1,31 +1,20 @@
package main
import (
"context"
"fmt"
"github.com/urfave/cli/v2"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"os"
"time"
"github.com/urfave/cli/v2"
)
func main() {
commands := []*cli.Command{
&cli.Command{
Name: "start",
Aliases: []string{"s"},
Usage: "Start the application service",
Action: func(c *cli.Context) error {
//d := daemon.NewDaemon()
//d.Start()
return nil
},
},
gather(),
plan(),
}
app := &cli.App{
Name: "ForeverFiles",
Name: "forever",
Usage: "Create backups designed to last forever",
Version: "v1.0.0",
Description: "ForeverFiles is a system for storing files forever.",
@ -37,7 +26,7 @@ func main() {
Email: "mason@masonitestudios.com",
},
},
Copyright: fmt.Sprintf("%v Masonite Studios LLC", time.Now().Year()),
Copyright: "2024 Masonite Studios LLC",
UseShortOptionHandling: true,
}
err := app.Run(os.Args)
@ -46,16 +35,3 @@ func main() {
return
}
}
func getConn(dialAddr string) (*grpc.ClientConn, error) {
conn, err := grpc.DialContext(
context.Background(),
dialAddr,
//grpc.WithBlock(),
grpc.WithTransportCredentials(insecure.NewCredentials()),
)
if err != nil {
return &grpc.ClientConn{}, fmt.Errorf("failed to dial server: %w", err)
}
return conn, nil
}

12
cmd/cli/flags.go Normal file
View File

@ -0,0 +1,12 @@
package main
import "github.com/urfave/cli/v2"
func baseDirFlag() *cli.StringFlag {
return &cli.StringFlag{
Name: "baseDir",
Usage: "The base directory to gather info from",
Aliases: []string{"b"},
Value: ".",
}
}

69
cmd/cli/gather.go Normal file
View File

@ -0,0 +1,69 @@
package main
import (
"fmt"
"github.com/dustin/go-humanize"
"forever-files/db"
"forever-files/source"
"forever-files/types"
"github.com/urfave/cli/v2"
)
func gather() *cli.Command {
return &cli.Command{
Name: "gather",
Aliases: []string{"g"},
Usage: "Collects the files to be backed up and stores their info in the database",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "reset",
Usage: "Reset the database before gathering info",
Aliases: []string{"r"},
Action: func(c *cli.Context, reset bool) error {
if reset {
err := db.DeleteDB(types.AppName)
if err != nil {
return fmt.Errorf("error deleting db: %w", err)
}
}
return nil
},
},
baseDirFlag(),
},
Action: func(c *cli.Context) error {
store, err := db.NewDB(types.AppName)
if err != nil {
panic(fmt.Errorf("error creating db: %w", err))
}
defer store.Close()
err = store.Migrate()
if err != nil {
panic(fmt.Errorf("error migrating db: %w", err))
}
baseDir := c.String("baseDir")
err = source.GatherInfo(baseDir, store)
if err != nil {
return fmt.Errorf("error gathering info: %w", err)
}
fileCount, err := store.GetFileCount()
if err != nil {
return fmt.Errorf("error getting file count: %w", err)
}
totalSize, err := store.GetTotalSize()
if err != nil {
return fmt.Errorf("error getting total size: %w", err)
}
fmt.Printf("Total Files: %v\n", fileCount)
fmt.Printf("Total Size: %v\n", humanize.Bytes(uint64(totalSize)))
return nil
},
}
}

107
cmd/cli/plan.go Normal file
View File

@ -0,0 +1,107 @@
package main
import (
"fmt"
"forever-files/db"
"forever-files/partitioner"
"forever-files/types"
"github.com/dustin/go-humanize"
"github.com/urfave/cli/v2"
"math"
)
func plan() *cli.Command {
return &cli.Command{
Name: "plan",
Aliases: []string{"p"},
Usage: "Reads the database and plans the partitions for the backup, stores the partitions in the database",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "targetSize",
Usage: "The target size for each partition, valid options are DVD-SL, DVD-DL, BD-SL, BD-DL, BD-TL, BD-QL\nor you can provide any specific size e.g. 4.6GB, 8.1GB, 25GB, 50GB, 100GB, 128GB or 5MB!",
Aliases: []string{"s"},
Value: "DVD-SL",
},
&cli.BoolFlag{
Name: "verbose",
Usage: "Print the files in each partition",
Aliases: []string{"v"},
Value: false,
},
},
Action: func(c *cli.Context) error {
store, err := db.NewDB(types.AppName)
if err != nil {
panic(fmt.Errorf("error creating db: %w", err))
}
defer store.Close()
err = store.Migrate()
if err != nil {
panic(fmt.Errorf("error migrating db: %w", err))
}
size := int64(4600000000) // size of a single layer DVD
targetSize := c.String("targetSize")
switch targetSize {
case "DVD-SL":
size = 4600000000
case "DVD-DL":
size = 8100000000
case "BD-SL":
size = 25000000000
case "BD-DL":
size = 50000000000
case "BD-TL":
size = 100000000000
case "BD-QL":
size = 128000000000
default:
// try to parse the size from human-readable format
usize, err := humanize.ParseBytes(targetSize)
if err != nil {
fmt.Printf("invalid target size: %v\n", err)
fmt.Println("valid options are DVD-SL, DVD-DL, BD-SL, BD-DL, BD-TL, BD-QL")
fmt.Println("or you can provide any specific size e.g. 4.6GB, 8.1GB, 25GB, 50GB, 100GB, 128GB or 5MB!")
size = 4600000000
}
if usize > math.MaxInt64 {
fmt.Println("size is too large")
size = 4600000000
}
size = int64(usize)
}
fmt.Printf("Target Size: %v\n", humanize.Bytes(uint64(size)))
fmt.Println("Calculating partitions...")
partitions, err := partitioner.CalculatePartitions(store, size)
if err != nil {
return fmt.Errorf("error calculating partitions: %w", err)
}
for i, partition := range partitions {
partSize := int64(0)
if c.Bool("verbose") {
fmt.Printf("Partition %v:\n", i)
}
for _, file := range partition {
if c.Bool("verbose") {
fmt.Printf("%v/%v %v\n", file.Path, file.Name, humanize.Bytes(uint64(file.Size)))
}
partSize += file.Size
// save the planned partitions
file.PartitionId = fmt.Sprintf("%d", i)
err = store.StoreFilePartition(file)
if err != nil {
return fmt.Errorf("error storing file's partition: %w", err)
}
}
fmt.Printf("Partition %v Size: %v files %v\n", i, len(partition), humanize.Bytes(uint64(partSize)))
}
return nil
},
}
}

37
db/db.go
View File

@ -17,6 +17,7 @@ type DB interface {
Migrate() error
StoreFile(fileMetadata types.FileMetadata) error
RemoveFile(fileMetadata types.FileMetadata) error
StoreFilePartition(fileMetadata types.FileMetadata) error
GetTotalSize() (int64, error)
GetFileCount() (int64, error)
GetFiles() ([]types.FileMetadata, error)
@ -41,7 +42,8 @@ var migrations = []Migrations{
size INTEGER NOT NULL,
hash TEXT NOT NULL,
modifiedDate TIMESTAMP NOT NULL,
backedUp BOOLEAN NOT NULL
backedUp BOOLEAN NOT NULL,
partitionId TEXT DEFAULT ''
)`,
},
{
@ -64,6 +66,16 @@ func NewDB(appName string) (DB, error) {
}, nil
}
func DeleteDB(appName string) error {
configPath := configdir.LocalConfig(appName)
dbPath := path.Join(configPath, "db", fmt.Sprintf("%v.db", appName))
err := os.Remove(dbPath)
if err != nil {
return fmt.Errorf("error deleting db | %w", err)
}
return nil
}
func (d *store) StoreFile(fileMetadata types.FileMetadata) error {
query := `INSERT INTO files (name, path, size, hash, modifiedDate, backedUp) VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT (name, path, hash) DO UPDATE SET size = ?, modifiedDate = ?, backedUp = ?`
_, err := d.db.Exec(
@ -93,6 +105,21 @@ func (d *store) RemoveFile(fileMetadata types.FileMetadata) error {
return nil
}
func (d *store) StoreFilePartition(fileMetadata types.FileMetadata) error {
query := `UPDATE files SET partitionId = ? WHERE name = ? AND path = ? AND hash = ?`
_, err := d.db.Exec(
query,
fileMetadata.PartitionId,
fileMetadata.Name,
fileMetadata.Path,
fileMetadata.Hash,
)
if err != nil {
return fmt.Errorf("error storing file's partiition | %w", err)
}
return nil
}
func (d *store) GetTotalSize() (int64, error) {
var size int64
query := `SELECT SUM(size) FROM files`
@ -115,7 +142,7 @@ func (d *store) GetFileCount() (int64, error) {
func (d *store) GetFiles() ([]types.FileMetadata, error) {
var files []types.FileMetadata
query := `SELECT name, path, size, hash, modifiedDate, backedUp FROM files order by path, name`
query := `SELECT name, path, size, hash, modifiedDate, backedUp, partitionId FROM files order by path, name`
rows, err := d.db.Query(query)
if err != nil {
return nil, fmt.Errorf("error getting files | %w", err)
@ -123,7 +150,7 @@ func (d *store) GetFiles() ([]types.FileMetadata, error) {
defer rows.Close()
for rows.Next() {
var file types.FileMetadata
err := rows.Scan(&file.Name, &file.Path, &file.Size, &file.Hash, &file.ModifiedDate, &file.BackedUp)
err := rows.Scan(&file.Name, &file.Path, &file.Size, &file.Hash, &file.ModifiedDate, &file.BackedUp, &file.PartitionId)
if err != nil {
return nil, fmt.Errorf("error scanning file | %w", err)
}
@ -179,7 +206,7 @@ func createDBFileIfNotExist(appName string) (string, error) {
// set up the config directory
err := configdir.MakePath(configPath)
if err != nil {
panic(err)
return "", fmt.Errorf("error creating config directory: %w", err)
}
dbDirectoryPath := path.Join(configPath, "db")
@ -187,7 +214,7 @@ func createDBFileIfNotExist(appName string) (string, error) {
// Set up the database
err = configdir.MakePath(dbDirectoryPath)
if err != nil {
panic(err)
return "", fmt.Errorf("error creating db directory: %w", err)
}
// If the file doesn't exist, create it, or append to the file

1
main.go
View File

@ -27,6 +27,7 @@ func main() {
source.GatherInfo(baseDir, store)
oneDVDSize := int64(4600000000)
//oneBRSize := int64(25000000000) // size of a small Blu-ray disc
partitions, err := partitioner.CalculatePartitions(store, oneDVDSize)
if err != nil {
panic(fmt.Errorf("error calculating partitions: %w", err))

29
partitioner/partitioner.go
View File

@ -25,11 +25,11 @@ func CalculatePartitions(store db.DB, targetSize int64) (partitions [][]types.Fi
partitions = make([][]types.FileMetadata, 0)
partitionSize := int64(0)
partitionFiles := make([]types.FileMetadata, 0)
leftOverFiles := make([]types.FileMetadata, 0)
leftOverSize := int64(0)
overSizedFiles := make([]types.FileMetadata, 0)
overSizedSize := int64(0)
for _, file := range files {
if partitionSize+file.Size > targetSize {
fmt.Printf("Partition Size: %v\n", humanize.Bytes(uint64(partitionSize)))
//fmt.Printf("Partition Size: %v\n", humanize.Bytes(uint64(partitionSize)))
partitions = append(partitions, partitionFiles)
partitionFiles = make([]types.FileMetadata, 0)
partitionSize = 0
@ -38,16 +38,25 @@ func CalculatePartitions(store db.DB, targetSize int64) (partitions [][]types.Fi
partitionFiles = append(partitionFiles, file)
partitionSize += file.Size
} else {
leftOverFiles = append(leftOverFiles, file)
leftOverSize += file.Size
overSizedFiles = append(overSizedFiles, file)
overSizedSize += file.Size
}
}
for _, partition := range partitions {
fmt.Printf("Partition File Count: %v\n", len(partition))
if len(partitionFiles) > 0 {
partitions = append(partitions, partitionFiles)
}
fmt.Printf("Left Over File Count: %v\n", len(leftOverFiles))
fmt.Printf("Left Over Size: %v\n", humanize.Bytes(uint64(leftOverSize)))
//for _, partition := range partitions {
// fmt.Printf("Partition File Count: %v\n", len(partition))
//}
fmt.Printf("Over Sized File Count: %v\n", len(overSizedFiles))
fmt.Printf("Total Over Sized Size: %v\n", humanize.Bytes(uint64(overSizedSize)))
return partitions, nil
}
// CalculatePartitionsLeftPack calculates the partitions efficiently by searching for files that fit the remaining space in each partition
func CalculatePartitionsLeftPack(store db.DB, targetSize int64) (partitions [][]types.FileMetadata, err error) {
// TODO: implement this function
return nil, nil
}

5
source/source.go
View File

@ -18,11 +18,12 @@ import (
// - file hash
// - modified date
func GatherInfo(path string, db db.DB) {
func GatherInfo(path string, db db.DB) error {
err := walkDir(path, db)
if err != nil {
log.Fatal(err)
return fmt.Errorf("error walking directory: %w", err)
}
return nil
}
func walkDir(dirPath string, db db.DB) error {

1
types/types.go
View File

@ -12,5 +12,6 @@ type FileMetadata struct {
Size int64
Hash []byte
ModifiedDate time.Time
PartitionId string
BackedUp bool
}