From 2e862c3e36cfdb8c656dae6ec3a72e23566c55b8 Mon Sep 17 00:00:00 2001 From: Přemysl Eric Janouch Date: Tue, 26 Dec 2023 01:32:51 +0100 Subject: Nuke the import command --- main.go | 300 ++++++++++++++++------------------------------------------------ 1 file changed, 76 insertions(+), 224 deletions(-) (limited to 'main.go') diff --git a/main.go b/main.go index ac2d894..c78f01e 100644 --- a/main.go +++ b/main.go @@ -103,6 +103,48 @@ func dbCollectStrings(query string, a ...any) ([]string, error) { return result, nil } +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +func idForDirectoryPath(tx *sql.Tx, path []string, create bool) (int64, error) { + var parent sql.NullInt64 + for _, name := range path { + if err := tx.QueryRow(`SELECT id FROM node + WHERE parent IS ? AND name = ? AND sha1 IS NULL`, + parent, name).Scan(&parent); err == nil { + continue + } else if !errors.Is(err, sql.ErrNoRows) { + return 0, err + } else if !create { + return 0, err + } + + // This fails when trying to override a leaf node. + // That needs special handling. + if result, err := tx.Exec( + `INSERT INTO node(parent, name) VALUES (?, ?)`, + parent, name); err != nil { + return 0, err + } else if id, err := result.LastInsertId(); err != nil { + return 0, err + } else { + parent = sql.NullInt64{Int64: id, Valid: true} + } + } + return parent.Int64, nil +} + +func decodeWebPath(path string) []string { + // Relative paths could be handled differently, + // but right now, they're assumed to start at the root. + result := []string{} + for _, crumb := range strings.Split(path, "/") { + if crumb != "" { + result = append(result, crumb) + } + } + return result +} + // --- Semaphore --------------------------------------------------------------- type semaphore chan struct{} @@ -1068,222 +1110,7 @@ func cmdWeb(args []string) error { return s.ListenAndServe() } -// --- Import ------------------------------------------------------------------ - -func idForDirectoryPath(tx *sql.Tx, path []string, create bool) (int64, error) { - var parent sql.NullInt64 - for _, name := range path { - if err := tx.QueryRow(`SELECT id FROM node - WHERE parent IS ? AND name = ? AND sha1 IS NULL`, - parent, name).Scan(&parent); err == nil { - continue - } else if !errors.Is(err, sql.ErrNoRows) { - return 0, err - } else if !create { - return 0, err - } - - // This fails when trying to override a leaf node. - // That needs special handling. - if result, err := tx.Exec( - `INSERT INTO node(parent, name) VALUES (?, ?)`, - parent, name); err != nil { - return 0, err - } else if id, err := result.LastInsertId(); err != nil { - return 0, err - } else { - parent = sql.NullInt64{Int64: id, Valid: true} - } - } - return parent.Int64, nil -} - -func decodeWebPath(path string) []string { - // Relative paths could be handled differently, - // but right now, they're assumed to start at the root. - result := []string{} - for _, crumb := range strings.Split(path, "/") { - if crumb != "" { - result = append(result, crumb) - } - } - return result -} - -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -type directoryManager struct { - cache map[string]int64 // Unix-style paths to directory.id -} - -func (dm *directoryManager) IDForDirectoryPath( - tx *sql.Tx, path string) (int64, error) { - path = filepath.ToSlash(filepath.Clean(path)) - list := decodeWebPath(path) - if len(list) == 0 { - return 0, nil - } - - if dm.cache == nil { - dm.cache = make(map[string]int64) - } else if id, ok := dm.cache[path]; ok { - return id, nil - } - - id, err := idForDirectoryPath(tx, list, true) - if err != nil { - return 0, err - } - dm.cache[path] = id - return id, nil -} - -func isImage(path string) (bool, error) { - out, err := exec.Command("xdg-mime", "query", "filetype", path).Output() - if err != nil { - return false, err - } - - return bytes.HasPrefix(out, []byte("image/")), nil -} - -func pingImage(path string) (int, int, error) { - out, err := exec.Command("identify", "-limit", "thread", "1", "-ping", - "-format", "%w %h", path+"[0]").Output() - if err != nil { - return 0, 0, err - } - - var w, h int - _, err = fmt.Fscanf(bytes.NewReader(out), "%d %d", &w, &h) - return w, h, err -} - -type importer struct { - dm directoryManager - dmMutex sync.Mutex -} - -func (i *importer) Import(path string) error { - // The input may be a relative path, and we want to remember it as such, - // but symlinks for the images must be absolute. - absPath, err := filepath.Abs(path) - if err != nil { - return err - } - - // Skip videos, which ImageMagick can process, but we don't want it to, - // so that they're not converted 1:1 to WebP. - pathIsImage, err := isImage(path) - if err != nil { - return err - } - if !pathIsImage { - return nil - } - - width, height, err := pingImage(path) - if err != nil { - return err - } - - f, err := os.Open(path) - if err != nil { - return err - } - defer f.Close() - - s, err := f.Stat() - if err != nil { - return err - } - - hash := sha1.New() - _, err = io.CopyBuffer(hash, f, make([]byte, 65536)) - if err != nil { - return err - } - - hexSHA1 := hex.EncodeToString(hash.Sum(nil)) - pathImage := imagePath(hexSHA1) - imageDirname, _ := filepath.Split(pathImage) - if err := os.MkdirAll(imageDirname, 0755); err != nil { - return err - } - if err := os.Symlink(absPath, pathImage); err != nil && - !errors.Is(err, fs.ErrExist) { - return err - } - - // The directoryManager isn't thread-safe. - // This lock also simulates a timeout-less BEGIN EXCLUSIVE. - i.dmMutex.Lock() - defer i.dmMutex.Unlock() - - tx, err := db.Begin() - if err != nil { - return err - } - defer tx.Rollback() - - if _, err = tx.Exec(`INSERT INTO image(sha1, width, height) VALUES (?, ?, ?) - ON CONFLICT(sha1) DO NOTHING`, hexSHA1, width, height); err != nil { - return err - } - - // XXX: The directoryManager's cache is questionable here, - // if only because it keeps entries even when transactions fail. - dbDirname, dbBasename := filepath.Split(path) - dbParent, err := i.dm.IDForDirectoryPath(tx, dbDirname) - if err != nil { - return err - } - - // FIXME: This disallows any entries directly in the root. - _, err = tx.Exec(`INSERT INTO node(parent, name, mtime, sha1) - VALUES (?, ?, ?, ?) ON CONFLICT DO - UPDATE SET mtime = excluded.mtime, sha1 = excluded.sha1`, - dbParent, dbBasename, s.ModTime().Unix(), hexSHA1) - if err != nil { - return err - } - - return tx.Commit() -} - -// cmdImport adds files to the "node" table. -// TODO: Consider making this copy rather than symlink images. -func cmdImport(args []string) error { - if len(args) < 1 { - return errors.New("usage: GD ROOT...") - } - if err := openDB(args[0]); err != nil { - return err - } - - // Make the first step collecting all the paths, - // in order to show more useful progress information. - paths := []string{} - cb := func(path string, d fs.DirEntry, err error) error { - if err != nil || d.IsDir() { - return err - } - paths = append(paths, path) - return nil - } - for _, name := range args[1:] { - if err := filepath.WalkDir(name, cb); err != nil { - return err - } - } - - i := importer{} - return parallelize(paths, func(path string) (string, error) { - return "", i.Import(path) - }) -} - -// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// --- Sync -------------------------------------------------------------------- type syncFileInfo struct { dbID int64 // DB node ID, or zero if there was none @@ -1310,9 +1137,7 @@ type syncContext struct { } func syncPrintf(c *syncContext, format string, v ...any) { - c.pb.Stop() - log.Printf(format+"\n", v...) - c.pb.Update() + c.pb.Interrupt(func() { log.Printf(format+"\n", v...) }) } // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -1391,10 +1216,31 @@ func syncGetFiles(fsPath string) (files []syncFile, err error) { // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +func syncIsImage(path string) (bool, error) { + out, err := exec.Command("xdg-mime", "query", "filetype", path).Output() + if err != nil { + return false, err + } + + return bytes.HasPrefix(out, []byte("image/")), nil +} + +func syncPingImage(path string) (int, int, error) { + out, err := exec.Command("identify", "-limit", "thread", "1", "-ping", + "-format", "%w %h", path+"[0]").Output() + if err != nil { + return 0, 0, err + } + + var w, h int + _, err = fmt.Fscanf(bytes.NewReader(out), "%d %d", &w, &h) + return w, h, err +} + func syncProcess(c *syncContext, info *syncFileInfo) error { // Skip videos, which ImageMagick can process, but we don't want it to, // so that they're not converted 1:1 to WebP. - pathIsImage, err := isImage(info.fsPath) + pathIsImage, err := syncIsImage(info.fsPath) if err != nil { return err } @@ -1402,7 +1248,7 @@ func syncProcess(c *syncContext, info *syncFileInfo) error { return nil } - info.width, info.height, err = pingImage(info.fsPath) + info.width, info.height, err = syncPingImage(info.fsPath) if err != nil { return err } @@ -1497,6 +1343,7 @@ func syncImage(c *syncContext, info syncFileInfo) error { } for { + // TODO: Make it possible to copy or reflink (ioctl FICLONE). err := os.Symlink(info.fsPath, path) if !errors.Is(err, fs.ErrExist) { return err @@ -1534,6 +1381,7 @@ func syncPostProcess(c *syncContext, info syncFileInfo) error { } // D → 0, F → 0 + // TODO: Make it possible to disable removal (for copying only?) return syncDispose(c, info.dbID, false /*keepNode*/) case info.dbID == 0: @@ -1599,6 +1447,7 @@ func syncDirectoryPair(c *syncContext, dbParent int64, fsPath string, case fs == nil: // D → 0, F → 0 + // TODO: Make it possible to disable removal (for copying only?) return syncDispose(c, db.dbID, false /*keepNode*/) case db.dbIsDir() && fs.fsIsDir: @@ -1649,7 +1498,7 @@ func syncDirectory(c *syncContext, dbParent int64, fsPath string) error { fs = nil } - // Convert differences to a more convenient form for processing. + // Convert differences to a form more convenient for processing. iDB, iFS, pairs := 0, 0, []syncPair{} for iDB < len(db) && iFS < len(fs) { if db[iDB].dbName == fs[iFS].fsName { @@ -1683,6 +1532,10 @@ func syncDirectory(c *syncContext, dbParent int64, fsPath string) error { } func syncRoot(c *syncContext, fsPath string) error { + // TODO: Support synchronizing individual files. + // This can only be treated as 0 → F, F → F, or D → F, that is, + // a variation on current syncEnqueue(), but dbParent must be nullable. + // Figure out a database root (not trying to convert F → D on conflict, // also because we don't know yet if the argument is a directory). // @@ -1835,7 +1688,7 @@ func cmdSync(args []string) error { // In case of a failure during processing, the only retained side effects // on the filesystem tree are: // - Fixing dead symlinks to images. - // - Creating symlinks to images that aren't necessary. + // - Creating symlinks to images that aren't used by anything. tx, err := db.BeginTx(ctx, nil) if err != nil { return err @@ -2374,7 +2227,6 @@ var commands = map[string]struct { }{ "init": {cmdInit}, "web": {cmdWeb}, - "import": {cmdImport}, "tag": {cmdTag}, "sync": {cmdSync}, "remove": {cmdRemove}, -- cgit v1.2.3-70-g09d2