add collecting of filemetadata for a given source

.idea/dataSources.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="DataSourceManagerImpl" format="xml" multifile-model="true">
+    <data-source source="LOCAL" name="ForeverFiles.db" uuid="e957d09a-a7fb-406a-a58c-3019b7380d7d">
+      <driver-ref>sqlite.xerial</driver-ref>
+      <synchronize>true</synchronize>
+      <jdbc-driver>org.sqlite.JDBC</jdbc-driver>
+      <jdbc-url>jdbc:sqlite:$USER_HOME$/AppData/Roaming/ForeverFiles/db/ForeverFiles.db</jdbc-url>
+      <working-dir>$ProjectFileDir$</working-dir>
+    </data-source>
+  </component>
+</project>

.idea/sqldialects.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="SqlDialectMappings">
+    <file url="file://$PROJECT_DIR$/db/db.go" dialect="GenericSQL" />
+    <file url="PROJECT" dialect="SQLite" />
+  </component>
+</project>

db/db.go

@@ -0,0 +1,154 @@
+package db
+
+import (
+	"database/sql"
+	"fmt"
+	"os"
+	"path"
+
+	"forever-files/types"
+
+	"github.com/kirsle/configdir"
+	_ "github.com/mattn/go-sqlite3"
+)
+
+type DB interface {
+	Close() error
+	Migrate() error
+	StoreFile(fileMetadata types.FileMetadata) error
+	RemoveFile(fileMetadata types.FileMetadata) error
+}
+
+type store struct {
+	db *sql.DB
+}
+
+type Migrations struct {
+	name  string
+	query string
+}
+
+var migrations = []Migrations{
+	{
+		name: "001-sourceFiles",
+		query: `CREATE TABLE IF NOT EXISTS files (
+				id INTEGER PRIMARY KEY,
+				name TEXT NOT NULL,
+				path TEXT NOT NULL,
+				size INTEGER NOT NULL,
+				hash TEXT NOT NULL,
+				modifiedDate TIMESTAMP NOT NULL,
+				backedUp BOOLEAN NOT NULL
+				)`,
+	},
+	{
+		name:  "002-fileUniqueConstraint",
+		query: `CREATE UNIQUE INDEX IF NOT EXISTS file_unique ON files (name, path, hash)`,
+	},
+}
+
+func NewDB(appName string) (DB, error) {
+	dbPath, err := createDBFileIfNotExist(appName)
+	if err != nil {
+		return nil, fmt.Errorf("error creating db file %w", err)
+	}
+	dbSQL, err := sql.Open("sqlite3", dbPath)
+	if err != nil {
+		return nil, fmt.Errorf("error opening db | %w", err)
+	}
+	return &store{
+		db: dbSQL,
+	}, nil
+
+	return &store{}, nil
+}
+
+func (d *store) StoreFile(fileMetadata types.FileMetadata) error {
+	query := `INSERT INTO files (name, path, size, hash, modifiedDate, backedUp) VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT (name, path, hash) DO UPDATE SET size = ?, modifiedDate = ?, backedUp = ?`
+	_, err := d.db.Exec(query, fileMetadata.Name, fileMetadata.Path, fileMetadata.Size, fileMetadata.Hash, fileMetadata.ModifiedDate, fileMetadata.BackedUp, fileMetadata.Size, fileMetadata.ModifiedDate, fileMetadata.BackedUp)
+	if err != nil {
+		return fmt.Errorf("error storing file metadata | %w", err)
+	}
+	return nil
+}
+
+func (d *store) RemoveFile(fileMetadata types.FileMetadata) error {
+	query := `DELETE FROM files WHERE name = ? AND path = ? AND hash = ?`
+	_, err := d.db.Exec(query, fileMetadata.Name, fileMetadata.Path, fileMetadata.Hash)
+	if err != nil {
+		return fmt.Errorf("error removing file metadata | %w", err)
+	}
+	return nil
+}
+
+func (d *store) Close() error {
+	return d.db.Close()
+}
+
+func (d *store) Migrate() error {
+	// check if migration table exists
+	var migrationsCheck string
+	//goland:noinspection SqlResolve
+	err := d.db.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name='migrations'").Scan(&migrationsCheck)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			_, err := d.db.Exec("CREATE TABLE migrations (name TEXT NOT NULL)")
+			if err != nil {
+				return fmt.Errorf("error creating migrations table | %w", err)
+			}
+		} else {
+			return fmt.Errorf("error checking if migrations table exists | %w", err)
+		}
+	}
+
+	for _, migration := range migrations {
+		var migrationInHistory string
+		err = d.db.QueryRow("SELECT name FROM migrations WHERE name = ?", migration.name).Scan(&migrationInHistory)
+		if err != nil {
+			if err == sql.ErrNoRows {
+				_, err := d.db.Exec(migration.query)
+				if err != nil {
+					return fmt.Errorf("error running migration: %s | %w", migration.name, err)
+				}
+				_, err = d.db.Exec("INSERT INTO migrations (name) VALUES (?)", migration.name)
+				if err != nil {
+					return fmt.Errorf("error inserting migration: %s into migrations table | %w", migration.name, err)
+				}
+			} else {
+				return fmt.Errorf("error checking if migration: %s has been run | %w", migration.name, err)
+			}
+		}
+	}
+	return nil
+}
+
+func createDBFileIfNotExist(appName string) (string, error) {
+	configPath := configdir.LocalConfig(appName)
+
+	// set up the config directory
+	err := configdir.MakePath(configPath)
+	if err != nil {
+		panic(err)
+	}
+
+	dbDirectoryPath := path.Join(configPath, "db")
+	dbPath := path.Join(configPath, "db", fmt.Sprintf("%v.db", appName))
+	// Set up the database
+	err = configdir.MakePath(dbDirectoryPath)
+	if err != nil {
+		panic(err)
+	}
+
+	// If the file doesn't exist, create it, or append to the file
+	f, err := os.OpenFile(dbPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return "", fmt.Errorf("error opening file: %v", err)
+	}
+	defer func(f *os.File) {
+		err := f.Close()
+		if err != nil {
+			fmt.Println("error closing file")
+		}
+	}(f)
+	return dbPath, nil
+}

go.mod

@@ -3,6 +3,8 @@ module forever-files
 go 1.19
 
 require (
+	github.com/kirsle/configdir v0.0.0-20170128060238-e45d2f54772f
+	github.com/mattn/go-sqlite3 v1.14.17
 	github.com/urfave/cli/v2 v2.25.5
 	google.golang.org/grpc v1.55.0
 )

go.sum

@@ -5,6 +5,10 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/kirsle/configdir v0.0.0-20170128060238-e45d2f54772f h1:dKccXx7xA56UNqOcFIbuqFjAWPVtP688j5QMgmo6OHU=
+github.com/kirsle/configdir v0.0.0-20170128060238-e45d2f54772f/go.mod h1:4rEELDSfUAlBSyUjPG0JnaNGjf13JySHFeRdD/3dLP0=
+github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
+github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/urfave/cli/v2 v2.25.5 h1:d0NIAyhh5shGscroL7ek/Ya9QYQE0KNabJgiUinIQkc=

main.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"fmt"
+	"forever-files/db"
+	"forever-files/source"
+	"forever-files/types"
+)
+
+func main() {
+
+	fmt.Printf("%v\n", types.AppName)
+
+	store, err := db.NewDB(types.AppName)
+	if err != nil {
+		panic(fmt.Errorf("error creating db: %w", err))
+	}
+
+	err = store.Migrate()
+	if err != nil {
+		panic(fmt.Errorf("error migrating db: %w", err))
+	}
+
+	source.GatherInfo("C:\\Users\\gomas\\Nextcloud", store)
+}

source/source.go

@@ -0,0 +1,86 @@
+package source
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"forever-files/db"
+	"forever-files/types"
+	"io"
+	"log"
+	"os"
+	"path"
+)
+
+// the purpose of this package is to gather information about the source files for the backup
+// it will store the information in a database
+// information to gather:
+// - file name
+// - file path
+// - file size
+// - file hash
+// - modified date
+
+func GatherInfo(path string, db db.DB) {
+	err := walkDir(path, db)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func walkDir(dirPath string, db db.DB) error {
+	// get list of files in directory
+	directoryEntries, err := os.ReadDir(dirPath)
+	if err != nil {
+		return fmt.Errorf("error reading directory: %w", err)
+	}
+	for _, entry := range directoryEntries {
+		if entry.IsDir() {
+			err = walkDir(path.Join(dirPath, entry.Name()), db)
+			if err != nil {
+				return fmt.Errorf("error walking directory: %w", err)
+			}
+		} else {
+			// gather info
+			fileInfo, err := entry.Info()
+			if err != nil {
+				log.Default().Printf("error getting file info: %v", err)
+				continue
+			}
+			hash, err := hashFile(path.Join(dirPath, entry.Name()))
+			if err != nil {
+				log.Default().Printf("error hashing file: %v", err)
+				continue
+			}
+			// store info
+			fmt.Printf("Name: %v, Size: %v, Modified Date: %v, Hash: %v\n", fileInfo.Name(), fileInfo.Size(), fileInfo.ModTime(), hash)
+			err = db.StoreFile(types.FileMetadata{
+				Name:         fileInfo.Name(),
+				Path:         dirPath,
+				Size:         fileInfo.Size(),
+				Hash:         hash,
+				ModifiedDate: fileInfo.ModTime(),
+				BackedUp:     false,
+			})
+			if err != nil {
+				log.Default().Printf("error storing file metadata: %v", err)
+				continue
+			}
+		}
+	}
+	return nil
+}
+
+func hashFile(filePath string) ([]byte, error) {
+	file, err := os.Open(filePath)
+	if err != nil {
+		return []byte{}, fmt.Errorf("error opening file for hashing: %w", err)
+	}
+	defer file.Close()
+
+	h := sha256.New()
+	if _, err := io.Copy(h, file); err != nil {
+		return []byte{}, fmt.Errorf("error hashing file: %w", err)
+	}
+
+	return h.Sum(nil), nil
+}

types/types.go

@@ -0,0 +1,16 @@
+package types
+
+import "time"
+
+const (
+	AppName = "ForeverFiles"
+)
+
+type FileMetadata struct {
+	Name         string
+	Path         string
+	Size         int64
+	Hash         []byte
+	ModifiedDate time.Time
+	BackedUp     bool
+}