Nuke the import command

1 files changed, 76 insertions, 224 deletions

diff --git a/main.go b/main.go
index ac2d894..c78f01e 100644
--- a/main.go
+++ b/main.go
@@ -103,6 +103,48 @@ func dbCollectStrings(query string, a ...any) ([]string, error) {
 	return result, nil
 }
 
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+func idForDirectoryPath(tx *sql.Tx, path []string, create bool) (int64, error) {
+	var parent sql.NullInt64
+	for _, name := range path {
+		if err := tx.QueryRow(`SELECT id FROM node
+			WHERE parent IS ? AND name = ? AND sha1 IS NULL`,
+			parent, name).Scan(&parent); err == nil {
+			continue
+		} else if !errors.Is(err, sql.ErrNoRows) {
+			return 0, err
+		} else if !create {
+			return 0, err
+		}
+
+		// This fails when trying to override a leaf node.
+		// That needs special handling.
+		if result, err := tx.Exec(
+			`INSERT INTO node(parent, name) VALUES (?, ?)`,
+			parent, name); err != nil {
+			return 0, err
+		} else if id, err := result.LastInsertId(); err != nil {
+			return 0, err
+		} else {
+			parent = sql.NullInt64{Int64: id, Valid: true}
+		}
+	}
+	return parent.Int64, nil
+}
+
+func decodeWebPath(path string) []string {
+	// Relative paths could be handled differently,
+	// but right now, they're assumed to start at the root.
+	result := []string{}
+	for _, crumb := range strings.Split(path, "/") {
+		if crumb != "" {
+			result = append(result, crumb)
+		}
+	}
+	return result
+}
+
 // --- Semaphore ---------------------------------------------------------------
 
 type semaphore chan struct{}
@@ -1068,222 +1110,7 @@ func cmdWeb(args []string) error {
 	return s.ListenAndServe()
 }
 
-// --- Import ------------------------------------------------------------------
-
-func idForDirectoryPath(tx *sql.Tx, path []string, create bool) (int64, error) {
-	var parent sql.NullInt64
-	for _, name := range path {
-		if err := tx.QueryRow(`SELECT id FROM node
-			WHERE parent IS ? AND name = ? AND sha1 IS NULL`,
-			parent, name).Scan(&parent); err == nil {
-			continue
-		} else if !errors.Is(err, sql.ErrNoRows) {
-			return 0, err
-		} else if !create {
-			return 0, err
-		}
-
-		// This fails when trying to override a leaf node.
-		// That needs special handling.
-		if result, err := tx.Exec(
-			`INSERT INTO node(parent, name) VALUES (?, ?)`,
-			parent, name); err != nil {
-			return 0, err
-		} else if id, err := result.LastInsertId(); err != nil {
-			return 0, err
-		} else {
-			parent = sql.NullInt64{Int64: id, Valid: true}
-		}
-	}
-	return parent.Int64, nil
-}
-
-func decodeWebPath(path string) []string {
-	// Relative paths could be handled differently,
-	// but right now, they're assumed to start at the root.
-	result := []string{}
-	for _, crumb := range strings.Split(path, "/") {
-		if crumb != "" {
-			result = append(result, crumb)
-		}
-	}
-	return result
-}
-
-// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
-type directoryManager struct {
-	cache map[string]int64 // Unix-style paths to directory.id
-}
-
-func (dm *directoryManager) IDForDirectoryPath(
-	tx *sql.Tx, path string) (int64, error) {
-	path = filepath.ToSlash(filepath.Clean(path))
-	list := decodeWebPath(path)
-	if len(list) == 0 {
-		return 0, nil
-	}
-
-	if dm.cache == nil {
-		dm.cache = make(map[string]int64)
-	} else if id, ok := dm.cache[path]; ok {
-		return id, nil
-	}
-
-	id, err := idForDirectoryPath(tx, list, true)
-	if err != nil {
-		return 0, err
-	}
-	dm.cache[path] = id
-	return id, nil
-}
-
-func isImage(path string) (bool, error) {
-	out, err := exec.Command("xdg-mime", "query", "filetype", path).Output()
-	if err != nil {
-		return false, err
-	}
-
-	return bytes.HasPrefix(out, []byte("image/")), nil
-}
-
-func pingImage(path string) (int, int, error) {
-	out, err := exec.Command("identify", "-limit", "thread", "1", "-ping",
-		"-format", "%w %h", path+"[0]").Output()
-	if err != nil {
-		return 0, 0, err
-	}
-
-	var w, h int
-	_, err = fmt.Fscanf(bytes.NewReader(out), "%d %d", &w, &h)
-	return w, h, err
-}
-
-type importer struct {
-	dm      directoryManager
-	dmMutex sync.Mutex
-}
-
-func (i *importer) Import(path string) error {
-	// The input may be a relative path, and we want to remember it as such,
-	// but symlinks for the images must be absolute.
-	absPath, err := filepath.Abs(path)
-	if err != nil {
-		return err
-	}
-
-	// Skip videos, which ImageMagick can process, but we don't want it to,
-	// so that they're not converted 1:1 to WebP.
-	pathIsImage, err := isImage(path)
-	if err != nil {
-		return err
-	}
-	if !pathIsImage {
-		return nil
-	}
-
-	width, height, err := pingImage(path)
-	if err != nil {
-		return err
-	}
-
-	f, err := os.Open(path)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	s, err := f.Stat()
-	if err != nil {
-		return err
-	}
-
-	hash := sha1.New()
-	_, err = io.CopyBuffer(hash, f, make([]byte, 65536))
-	if err != nil {
-		return err
-	}
-
-	hexSHA1 := hex.EncodeToString(hash.Sum(nil))
-	pathImage := imagePath(hexSHA1)
-	imageDirname, _ := filepath.Split(pathImage)
-	if err := os.MkdirAll(imageDirname, 0755); err != nil {
-		return err
-	}
-	if err := os.Symlink(absPath, pathImage); err != nil &&
-		!errors.Is(err, fs.ErrExist) {
-		return err
-	}
-
-	// The directoryManager isn't thread-safe.
-	// This lock also simulates a timeout-less BEGIN EXCLUSIVE.
-	i.dmMutex.Lock()
-	defer i.dmMutex.Unlock()
-
-	tx, err := db.Begin()
-	if err != nil {
-		return err
-	}
-	defer tx.Rollback()
-
-	if _, err = tx.Exec(`INSERT INTO image(sha1, width, height) VALUES (?, ?, ?)
-		ON CONFLICT(sha1) DO NOTHING`, hexSHA1, width, height); err != nil {
-		return err
-	}
-
-	// XXX: The directoryManager's cache is questionable here,
-	// if only because it keeps entries even when transactions fail.
-	dbDirname, dbBasename := filepath.Split(path)
-	dbParent, err := i.dm.IDForDirectoryPath(tx, dbDirname)
-	if err != nil {
-		return err
-	}
-
-	// FIXME: This disallows any entries directly in the root.
-	_, err = tx.Exec(`INSERT INTO node(parent, name, mtime, sha1)
-		VALUES (?, ?, ?, ?) ON CONFLICT DO
-		UPDATE SET mtime = excluded.mtime, sha1 = excluded.sha1`,
-		dbParent, dbBasename, s.ModTime().Unix(), hexSHA1)
-	if err != nil {
-		return err
-	}
-
-	return tx.Commit()
-}
-
-// cmdImport adds files to the "node" table.
-// TODO: Consider making this copy rather than symlink images.
-func cmdImport(args []string) error {
-	if len(args) < 1 {
-		return errors.New("usage: GD ROOT...")
-	}
-	if err := openDB(args[0]); err != nil {
-		return err
-	}
-
-	// Make the first step collecting all the paths,
-	// in order to show more useful progress information.
-	paths := []string{}
-	cb := func(path string, d fs.DirEntry, err error) error {
-		if err != nil || d.IsDir() {
-			return err
-		}
-		paths = append(paths, path)
-		return nil
-	}
-	for _, name := range args[1:] {
-		if err := filepath.WalkDir(name, cb); err != nil {
-			return err
-		}
-	}
-
-	i := importer{}
-	return parallelize(paths, func(path string) (string, error) {
-		return "", i.Import(path)
-	})
-}
-
-// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+// --- Sync --------------------------------------------------------------------
 
 type syncFileInfo struct {
 	dbID     int64  // DB node ID, or zero if there was none
@@ -1310,9 +1137,7 @@ type syncContext struct {
 }
 
 func syncPrintf(c *syncContext, format string, v ...any) {
-	c.pb.Stop()
-	log.Printf(format+"\n", v...)
-	c.pb.Update()
+	c.pb.Interrupt(func() { log.Printf(format+"\n", v...) })
 }
 
 // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
@@ -1391,10 +1216,31 @@ func syncGetFiles(fsPath string) (files []syncFile, err error) {
 
 // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 
+func syncIsImage(path string) (bool, error) {
+	out, err := exec.Command("xdg-mime", "query", "filetype", path).Output()
+	if err != nil {
+		return false, err
+	}
+
+	return bytes.HasPrefix(out, []byte("image/")), nil
+}
+
+func syncPingImage(path string) (int, int, error) {
+	out, err := exec.Command("identify", "-limit", "thread", "1", "-ping",
+		"-format", "%w %h", path+"[0]").Output()
+	if err != nil {
+		return 0, 0, err
+	}
+
+	var w, h int
+	_, err = fmt.Fscanf(bytes.NewReader(out), "%d %d", &w, &h)
+	return w, h, err
+}
+
 func syncProcess(c *syncContext, info *syncFileInfo) error {
 	// Skip videos, which ImageMagick can process, but we don't want it to,
 	// so that they're not converted 1:1 to WebP.
-	pathIsImage, err := isImage(info.fsPath)
+	pathIsImage, err := syncIsImage(info.fsPath)
 	if err != nil {
 		return err
 	}
@@ -1402,7 +1248,7 @@ func syncProcess(c *syncContext, info *syncFileInfo) error {
 		return nil
 	}
 
-	info.width, info.height, err = pingImage(info.fsPath)
+	info.width, info.height, err = syncPingImage(info.fsPath)
 	if err != nil {
 		return err
 	}
@@ -1497,6 +1343,7 @@ func syncImage(c *syncContext, info syncFileInfo) error {
 	}
 
 	for {
+		// TODO: Make it possible to copy or reflink (ioctl FICLONE).
 		err := os.Symlink(info.fsPath, path)
 		if !errors.Is(err, fs.ErrExist) {
 			return err
@@ -1534,6 +1381,7 @@ func syncPostProcess(c *syncContext, info syncFileInfo) error {
 		}
 
 		// D → 0, F → 0
+		// TODO: Make it possible to disable removal (for copying only?)
 		return syncDispose(c, info.dbID, false /*keepNode*/)
 
 	case info.dbID == 0:
@@ -1599,6 +1447,7 @@ func syncDirectoryPair(c *syncContext, dbParent int64, fsPath string,
 
 	case fs == nil:
 		// D → 0, F → 0
+		// TODO: Make it possible to disable removal (for copying only?)
 		return syncDispose(c, db.dbID, false /*keepNode*/)
 
 	case db.dbIsDir() && fs.fsIsDir:
@@ -1649,7 +1498,7 @@ func syncDirectory(c *syncContext, dbParent int64, fsPath string) error {
 		fs = nil
 	}
 
-	// Convert differences to a more convenient form for processing.
+	// Convert differences to a form more convenient for processing.
 	iDB, iFS, pairs := 0, 0, []syncPair{}
 	for iDB < len(db) && iFS < len(fs) {
 		if db[iDB].dbName == fs[iFS].fsName {
@@ -1683,6 +1532,10 @@ func syncDirectory(c *syncContext, dbParent int64, fsPath string) error {
 }
 
 func syncRoot(c *syncContext, fsPath string) error {
+	// TODO: Support synchronizing individual files.
+	// This can only be treated as 0 → F, F → F, or D → F, that is,
+	// a variation on current syncEnqueue(), but dbParent must be nullable.
+
 	// Figure out a database root (not trying to convert F → D on conflict,
 	// also because we don't know yet if the argument is a directory).
 	//
@@ -1835,7 +1688,7 @@ func cmdSync(args []string) error {
 	// In case of a failure during processing, the only retained side effects
 	// on the filesystem tree are:
 	//  - Fixing dead symlinks to images.
-	//  - Creating symlinks to images that aren't necessary.
+	//  - Creating symlinks to images that aren't used by anything.
 	tx, err := db.BeginTx(ctx, nil)
 	if err != nil {
 		return err
@@ -2374,7 +2227,6 @@ var commands = map[string]struct {
 }{
 	"init":      {cmdInit},
 	"web":       {cmdWeb},
-	"import":    {cmdImport},
 	"tag":       {cmdTag},
 	"sync":      {cmdSync},
 	"remove":    {cmdRemove},