summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--main.go300
-rwxr-xr-xtest.sh1
2 files changed, 76 insertions, 225 deletions
diff --git a/main.go b/main.go
index ac2d894..c78f01e 100644
--- a/main.go
+++ b/main.go
@@ -103,6 +103,48 @@ func dbCollectStrings(query string, a ...any) ([]string, error) {
return result, nil
}
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+func idForDirectoryPath(tx *sql.Tx, path []string, create bool) (int64, error) {
+ var parent sql.NullInt64
+ for _, name := range path {
+ if err := tx.QueryRow(`SELECT id FROM node
+ WHERE parent IS ? AND name = ? AND sha1 IS NULL`,
+ parent, name).Scan(&parent); err == nil {
+ continue
+ } else if !errors.Is(err, sql.ErrNoRows) {
+ return 0, err
+ } else if !create {
+ return 0, err
+ }
+
+ // This fails when trying to override a leaf node.
+ // That needs special handling.
+ if result, err := tx.Exec(
+ `INSERT INTO node(parent, name) VALUES (?, ?)`,
+ parent, name); err != nil {
+ return 0, err
+ } else if id, err := result.LastInsertId(); err != nil {
+ return 0, err
+ } else {
+ parent = sql.NullInt64{Int64: id, Valid: true}
+ }
+ }
+ return parent.Int64, nil
+}
+
+func decodeWebPath(path string) []string {
+ // Relative paths could be handled differently,
+ // but right now, they're assumed to start at the root.
+ result := []string{}
+ for _, crumb := range strings.Split(path, "/") {
+ if crumb != "" {
+ result = append(result, crumb)
+ }
+ }
+ return result
+}
+
// --- Semaphore ---------------------------------------------------------------
type semaphore chan struct{}
@@ -1068,222 +1110,7 @@ func cmdWeb(args []string) error {
return s.ListenAndServe()
}
-// --- Import ------------------------------------------------------------------
-
-func idForDirectoryPath(tx *sql.Tx, path []string, create bool) (int64, error) {
- var parent sql.NullInt64
- for _, name := range path {
- if err := tx.QueryRow(`SELECT id FROM node
- WHERE parent IS ? AND name = ? AND sha1 IS NULL`,
- parent, name).Scan(&parent); err == nil {
- continue
- } else if !errors.Is(err, sql.ErrNoRows) {
- return 0, err
- } else if !create {
- return 0, err
- }
-
- // This fails when trying to override a leaf node.
- // That needs special handling.
- if result, err := tx.Exec(
- `INSERT INTO node(parent, name) VALUES (?, ?)`,
- parent, name); err != nil {
- return 0, err
- } else if id, err := result.LastInsertId(); err != nil {
- return 0, err
- } else {
- parent = sql.NullInt64{Int64: id, Valid: true}
- }
- }
- return parent.Int64, nil
-}
-
-func decodeWebPath(path string) []string {
- // Relative paths could be handled differently,
- // but right now, they're assumed to start at the root.
- result := []string{}
- for _, crumb := range strings.Split(path, "/") {
- if crumb != "" {
- result = append(result, crumb)
- }
- }
- return result
-}
-
-// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
-type directoryManager struct {
- cache map[string]int64 // Unix-style paths to directory.id
-}
-
-func (dm *directoryManager) IDForDirectoryPath(
- tx *sql.Tx, path string) (int64, error) {
- path = filepath.ToSlash(filepath.Clean(path))
- list := decodeWebPath(path)
- if len(list) == 0 {
- return 0, nil
- }
-
- if dm.cache == nil {
- dm.cache = make(map[string]int64)
- } else if id, ok := dm.cache[path]; ok {
- return id, nil
- }
-
- id, err := idForDirectoryPath(tx, list, true)
- if err != nil {
- return 0, err
- }
- dm.cache[path] = id
- return id, nil
-}
-
-func isImage(path string) (bool, error) {
- out, err := exec.Command("xdg-mime", "query", "filetype", path).Output()
- if err != nil {
- return false, err
- }
-
- return bytes.HasPrefix(out, []byte("image/")), nil
-}
-
-func pingImage(path string) (int, int, error) {
- out, err := exec.Command("identify", "-limit", "thread", "1", "-ping",
- "-format", "%w %h", path+"[0]").Output()
- if err != nil {
- return 0, 0, err
- }
-
- var w, h int
- _, err = fmt.Fscanf(bytes.NewReader(out), "%d %d", &w, &h)
- return w, h, err
-}
-
-type importer struct {
- dm directoryManager
- dmMutex sync.Mutex
-}
-
-func (i *importer) Import(path string) error {
- // The input may be a relative path, and we want to remember it as such,
- // but symlinks for the images must be absolute.
- absPath, err := filepath.Abs(path)
- if err != nil {
- return err
- }
-
- // Skip videos, which ImageMagick can process, but we don't want it to,
- // so that they're not converted 1:1 to WebP.
- pathIsImage, err := isImage(path)
- if err != nil {
- return err
- }
- if !pathIsImage {
- return nil
- }
-
- width, height, err := pingImage(path)
- if err != nil {
- return err
- }
-
- f, err := os.Open(path)
- if err != nil {
- return err
- }
- defer f.Close()
-
- s, err := f.Stat()
- if err != nil {
- return err
- }
-
- hash := sha1.New()
- _, err = io.CopyBuffer(hash, f, make([]byte, 65536))
- if err != nil {
- return err
- }
-
- hexSHA1 := hex.EncodeToString(hash.Sum(nil))
- pathImage := imagePath(hexSHA1)
- imageDirname, _ := filepath.Split(pathImage)
- if err := os.MkdirAll(imageDirname, 0755); err != nil {
- return err
- }
- if err := os.Symlink(absPath, pathImage); err != nil &&
- !errors.Is(err, fs.ErrExist) {
- return err
- }
-
- // The directoryManager isn't thread-safe.
- // This lock also simulates a timeout-less BEGIN EXCLUSIVE.
- i.dmMutex.Lock()
- defer i.dmMutex.Unlock()
-
- tx, err := db.Begin()
- if err != nil {
- return err
- }
- defer tx.Rollback()
-
- if _, err = tx.Exec(`INSERT INTO image(sha1, width, height) VALUES (?, ?, ?)
- ON CONFLICT(sha1) DO NOTHING`, hexSHA1, width, height); err != nil {
- return err
- }
-
- // XXX: The directoryManager's cache is questionable here,
- // if only because it keeps entries even when transactions fail.
- dbDirname, dbBasename := filepath.Split(path)
- dbParent, err := i.dm.IDForDirectoryPath(tx, dbDirname)
- if err != nil {
- return err
- }
-
- // FIXME: This disallows any entries directly in the root.
- _, err = tx.Exec(`INSERT INTO node(parent, name, mtime, sha1)
- VALUES (?, ?, ?, ?) ON CONFLICT DO
- UPDATE SET mtime = excluded.mtime, sha1 = excluded.sha1`,
- dbParent, dbBasename, s.ModTime().Unix(), hexSHA1)
- if err != nil {
- return err
- }
-
- return tx.Commit()
-}
-
-// cmdImport adds files to the "node" table.
-// TODO: Consider making this copy rather than symlink images.
-func cmdImport(args []string) error {
- if len(args) < 1 {
- return errors.New("usage: GD ROOT...")
- }
- if err := openDB(args[0]); err != nil {
- return err
- }
-
- // Make the first step collecting all the paths,
- // in order to show more useful progress information.
- paths := []string{}
- cb := func(path string, d fs.DirEntry, err error) error {
- if err != nil || d.IsDir() {
- return err
- }
- paths = append(paths, path)
- return nil
- }
- for _, name := range args[1:] {
- if err := filepath.WalkDir(name, cb); err != nil {
- return err
- }
- }
-
- i := importer{}
- return parallelize(paths, func(path string) (string, error) {
- return "", i.Import(path)
- })
-}
-
-// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+// --- Sync --------------------------------------------------------------------
type syncFileInfo struct {
dbID int64 // DB node ID, or zero if there was none
@@ -1310,9 +1137,7 @@ type syncContext struct {
}
func syncPrintf(c *syncContext, format string, v ...any) {
- c.pb.Stop()
- log.Printf(format+"\n", v...)
- c.pb.Update()
+ c.pb.Interrupt(func() { log.Printf(format+"\n", v...) })
}
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
@@ -1391,10 +1216,31 @@ func syncGetFiles(fsPath string) (files []syncFile, err error) {
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+func syncIsImage(path string) (bool, error) {
+ out, err := exec.Command("xdg-mime", "query", "filetype", path).Output()
+ if err != nil {
+ return false, err
+ }
+
+ return bytes.HasPrefix(out, []byte("image/")), nil
+}
+
+func syncPingImage(path string) (int, int, error) {
+ out, err := exec.Command("identify", "-limit", "thread", "1", "-ping",
+ "-format", "%w %h", path+"[0]").Output()
+ if err != nil {
+ return 0, 0, err
+ }
+
+ var w, h int
+ _, err = fmt.Fscanf(bytes.NewReader(out), "%d %d", &w, &h)
+ return w, h, err
+}
+
func syncProcess(c *syncContext, info *syncFileInfo) error {
// Skip videos, which ImageMagick can process, but we don't want it to,
// so that they're not converted 1:1 to WebP.
- pathIsImage, err := isImage(info.fsPath)
+ pathIsImage, err := syncIsImage(info.fsPath)
if err != nil {
return err
}
@@ -1402,7 +1248,7 @@ func syncProcess(c *syncContext, info *syncFileInfo) error {
return nil
}
- info.width, info.height, err = pingImage(info.fsPath)
+ info.width, info.height, err = syncPingImage(info.fsPath)
if err != nil {
return err
}
@@ -1497,6 +1343,7 @@ func syncImage(c *syncContext, info syncFileInfo) error {
}
for {
+ // TODO: Make it possible to copy or reflink (ioctl FICLONE).
err := os.Symlink(info.fsPath, path)
if !errors.Is(err, fs.ErrExist) {
return err
@@ -1534,6 +1381,7 @@ func syncPostProcess(c *syncContext, info syncFileInfo) error {
}
// D → 0, F → 0
+ // TODO: Make it possible to disable removal (for copying only?)
return syncDispose(c, info.dbID, false /*keepNode*/)
case info.dbID == 0:
@@ -1599,6 +1447,7 @@ func syncDirectoryPair(c *syncContext, dbParent int64, fsPath string,
case fs == nil:
// D → 0, F → 0
+ // TODO: Make it possible to disable removal (for copying only?)
return syncDispose(c, db.dbID, false /*keepNode*/)
case db.dbIsDir() && fs.fsIsDir:
@@ -1649,7 +1498,7 @@ func syncDirectory(c *syncContext, dbParent int64, fsPath string) error {
fs = nil
}
- // Convert differences to a more convenient form for processing.
+ // Convert differences to a form more convenient for processing.
iDB, iFS, pairs := 0, 0, []syncPair{}
for iDB < len(db) && iFS < len(fs) {
if db[iDB].dbName == fs[iFS].fsName {
@@ -1683,6 +1532,10 @@ func syncDirectory(c *syncContext, dbParent int64, fsPath string) error {
}
func syncRoot(c *syncContext, fsPath string) error {
+ // TODO: Support synchronizing individual files.
+ // This can only be treated as 0 → F, F → F, or D → F, that is,
+ // a variation on current syncEnqueue(), but dbParent must be nullable.
+
// Figure out a database root (not trying to convert F → D on conflict,
// also because we don't know yet if the argument is a directory).
//
@@ -1835,7 +1688,7 @@ func cmdSync(args []string) error {
// In case of a failure during processing, the only retained side effects
// on the filesystem tree are:
// - Fixing dead symlinks to images.
- // - Creating symlinks to images that aren't necessary.
+ // - Creating symlinks to images that aren't used by anything.
tx, err := db.BeginTx(ctx, nil)
if err != nil {
return err
@@ -2374,7 +2227,6 @@ var commands = map[string]struct {
}{
"init": {cmdInit},
"web": {cmdWeb},
- "import": {cmdImport},
"tag": {cmdTag},
"sync": {cmdSync},
"remove": {cmdRemove},
diff --git a/test.sh b/test.sh
index dd0da83..bf2fedf 100755
--- a/test.sh
+++ b/test.sh
@@ -7,7 +7,6 @@ mkdir -p $target
cp -ra $HOME/Pictures/Anime $input
./gallery init $target
-#./gallery import $target $input
./gallery sync $target $input
./gallery thumbnail $target
./gallery dhash $target
generated by cgit v1.2.3-70-g09d2 (git 2.50.0) at 2025-10-31 01:47:15 +0000