aboutsummaryrefslogtreecommitdiff
path: root/main.go
diff options
context:
space:
mode:
Diffstat (limited to 'main.go')
-rw-r--r--main.go2497
1 files changed, 2497 insertions, 0 deletions
diff --git a/main.go b/main.go
new file mode 100644
index 0000000..95bb502
--- /dev/null
+++ b/main.go
@@ -0,0 +1,2497 @@
+package main
+
+import (
+ "bufio"
+ "bytes"
+ "context"
+ "crypto/sha1"
+ "database/sql"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "flag"
+ "fmt"
+ "html/template"
+ "image"
+ "image/color"
+ "io"
+ "io/fs"
+ "log"
+ "math"
+ "math/bits"
+ "net"
+ "net/http"
+ "os"
+ "os/exec"
+ "os/signal"
+ "path/filepath"
+ "regexp"
+ "runtime"
+ "slices"
+ "sort"
+ "strconv"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "syscall"
+ "time"
+
+ "github.com/mattn/go-sqlite3"
+ "golang.org/x/image/draw"
+ "golang.org/x/image/webp"
+)
+
+var (
+ db *sql.DB // sqlite database
+ galleryDirectory string // gallery directory
+
+ // taskSemaphore limits parallel computations.
+ taskSemaphore semaphore
+)
+
+const (
+ nameOfDB = "gallery.db"
+ nameOfImageRoot = "images"
+ nameOfThumbRoot = "thumbs"
+)
+
+func hammingDistance(a, b int64) int {
+ return bits.OnesCount64(uint64(a) ^ uint64(b))
+}
+
+func init() {
+ sql.Register("sqlite3_custom", &sqlite3.SQLiteDriver{
+ ConnectHook: func(conn *sqlite3.SQLiteConn) error {
+ return conn.RegisterFunc("hamming", hammingDistance, true /*pure*/)
+ },
+ })
+}
+
+func openDB(directory string) error {
+ var err error
+ db, err = sql.Open("sqlite3_custom", "file:"+filepath.Join(directory,
+ nameOfDB+"?_foreign_keys=1&_busy_timeout=1000"))
+ galleryDirectory = directory
+ return err
+}
+
+func imagePath(sha1 string) string {
+ return filepath.Join(galleryDirectory,
+ nameOfImageRoot, sha1[:2], sha1)
+}
+
+func thumbPath(sha1 string) string {
+ return filepath.Join(galleryDirectory,
+ nameOfThumbRoot, sha1[:2], sha1+".webp")
+}
+
+func dbCollectStrings(query string, a ...any) ([]string, error) {
+ rows, err := db.Query(query, a...)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+
+ result := []string{}
+ for rows.Next() {
+ var s string
+ if err := rows.Scan(&s); err != nil {
+ return nil, err
+ }
+ result = append(result, s)
+ }
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+ return result, nil
+}
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+func idForDirectoryPath(tx *sql.Tx, path []string, create bool) (int64, error) {
+ var parent sql.NullInt64
+ for _, name := range path {
+ if err := tx.QueryRow(`SELECT id FROM node
+ WHERE parent IS ? AND name = ? AND sha1 IS NULL`,
+ parent, name).Scan(&parent); err == nil {
+ continue
+ } else if !errors.Is(err, sql.ErrNoRows) {
+ return 0, err
+ } else if !create {
+ return 0, err
+ }
+
+ // This fails when trying to override a leaf node.
+ // That needs special handling.
+ if result, err := tx.Exec(
+ `INSERT INTO node(parent, name) VALUES (?, ?)`,
+ parent, name); err != nil {
+ return 0, err
+ } else if id, err := result.LastInsertId(); err != nil {
+ return 0, err
+ } else {
+ parent = sql.NullInt64{Int64: id, Valid: true}
+ }
+ }
+ return parent.Int64, nil
+}
+
+func decodeWebPath(path string) []string {
+ // Relative paths could be handled differently,
+ // but right now, they're assumed to start at the root.
+ result := []string{}
+ for _, crumb := range strings.Split(path, "/") {
+ if crumb != "" {
+ result = append(result, crumb)
+ }
+ }
+ return result
+}
+
+// --- Semaphore ---------------------------------------------------------------
+
+type semaphore chan struct{}
+
+func newSemaphore(size int) semaphore { return make(chan struct{}, size) }
+func (s semaphore) release() { <-s }
+
+func (s semaphore) acquire(ctx context.Context) error {
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case s <- struct{}{}:
+ }
+
+ // Give priority to context cancellation.
+ select {
+ case <-ctx.Done():
+ s.release()
+ return ctx.Err()
+ default:
+ }
+ return nil
+}
+
+// --- Progress bar ------------------------------------------------------------
+
+type progressBar struct {
+ sync.Mutex
+ current int
+ target int
+}
+
+func newProgressBar(target int) *progressBar {
+ pb := &progressBar{current: 0, target: target}
+ pb.Update()
+ return pb
+}
+
+func (pb *progressBar) Stop() {
+ // The minimum thing that works: just print a newline.
+ os.Stdout.WriteString("\n")
+}
+
+func (pb *progressBar) Update() {
+ if pb.target < 0 {
+ fmt.Printf("\r%d/?", pb.current)
+ return
+ }
+
+ var fraction int
+ if pb.target != 0 {
+ fraction = int(float32(pb.current) / float32(pb.target) * 100)
+ }
+
+ target := fmt.Sprintf("%d", pb.target)
+ fmt.Printf("\r%*d/%s (%2d%%)", len(target), pb.current, target, fraction)
+}
+
+func (pb *progressBar) Step() {
+ pb.Lock()
+ defer pb.Unlock()
+
+ pb.current++
+ pb.Update()
+}
+
+func (pb *progressBar) Interrupt(callback func()) {
+ pb.Lock()
+ defer pb.Unlock()
+ pb.Stop()
+ defer pb.Update()
+
+ callback()
+}
+
+// --- Parallelization ---------------------------------------------------------
+
+type parallelFunc func(item string) (message string, err error)
+
+// parallelize runs the callback in parallel on a list of strings,
+// reporting progress and any non-fatal messages.
+func parallelize(strings []string, callback parallelFunc) error {
+ pb := newProgressBar(len(strings))
+ defer pb.Stop()
+
+ ctx, cancel := context.WithCancelCause(context.Background())
+ wg := sync.WaitGroup{}
+ for _, item := range strings {
+ if taskSemaphore.acquire(ctx) != nil {
+ break
+ }
+
+ wg.Add(1)
+ go func(item string) {
+ defer taskSemaphore.release()
+ defer wg.Done()
+ if message, err := callback(item); err != nil {
+ cancel(err)
+ } else if message != "" {
+ pb.Interrupt(func() { log.Printf("%s: %s\n", item, message) })
+ }
+ pb.Step()
+ }(item)
+ }
+ wg.Wait()
+ if ctx.Err() != nil {
+ return context.Cause(ctx)
+ }
+ return nil
+}
+
+// --- Initialization ----------------------------------------------------------
+
+// cmdInit initializes a "gallery directory" that contains gallery.sqlite,
+// images, thumbs.
+func cmdInit(fs *flag.FlagSet, args []string) error {
+ if err := fs.Parse(args); err != nil {
+ return err
+ }
+ if fs.NArg() != 1 {
+ return errWrongUsage
+ }
+ if err := openDB(fs.Arg(0)); err != nil {
+ return err
+ }
+
+ if _, err := db.Exec(initializeSQL); err != nil {
+ return err
+ }
+
+ // XXX: There's technically no reason to keep images as symlinks,
+ // we might just keep absolute paths in the database as well.
+ if err := os.MkdirAll(
+ filepath.Join(galleryDirectory, nameOfImageRoot), 0755); err != nil {
+ return err
+ }
+ if err := os.MkdirAll(
+ filepath.Join(galleryDirectory, nameOfThumbRoot), 0755); err != nil {
+ return err
+ }
+ return nil
+}
+
+// --- Web ---------------------------------------------------------------------
+
+var hashRE = regexp.MustCompile(`^/.*?/([0-9a-f]{40})$`)
+var staticHandler http.Handler
+
+var page = template.Must(template.New("/").Parse(`<!DOCTYPE html><html><head>
+ <title>Gallery</title>
+ <meta charset="utf-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <link rel=stylesheet href=style.css>
+</head><body>
+ <noscript>This is a web application, and requires Javascript.</noscript>
+ <script src=mithril.js></script>
+ <script src=gallery.js></script>
+</body></html>`))
+
+func handleRequest(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/" {
+ staticHandler.ServeHTTP(w, r)
+ return
+ }
+ if err := page.Execute(w, nil); err != nil {
+ log.Println(err)
+ }
+}
+
+func handleImages(w http.ResponseWriter, r *http.Request) {
+ if m := hashRE.FindStringSubmatch(r.URL.Path); m == nil {
+ http.NotFound(w, r)
+ } else {
+ http.ServeFile(w, r, imagePath(m[1]))
+ }
+}
+
+func handleThumbs(w http.ResponseWriter, r *http.Request) {
+ if m := hashRE.FindStringSubmatch(r.URL.Path); m == nil {
+ http.NotFound(w, r)
+ } else {
+ http.ServeFile(w, r, thumbPath(m[1]))
+ }
+}
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+func getSubdirectories(tx *sql.Tx, parent int64) (names []string, err error) {
+ return dbCollectStrings(`SELECT name FROM node
+ WHERE IFNULL(parent, 0) = ? AND sha1 IS NULL`, parent)
+}
+
+type webEntry struct {
+ SHA1 string `json:"sha1"`
+ Name string `json:"name"`
+ Modified int64 `json:"modified"`
+ ThumbW int64 `json:"thumbW"`
+ ThumbH int64 `json:"thumbH"`
+}
+
+func getSubentries(tx *sql.Tx, parent int64) (entries []webEntry, err error) {
+ rows, err := tx.Query(`
+ SELECT i.sha1, n.name, n.mtime, IFNULL(i.thumbw, 0), IFNULL(i.thumbh, 0)
+ FROM node AS n
+ JOIN image AS i ON n.sha1 = i.sha1
+ WHERE n.parent = ?`, parent)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+
+ entries = []webEntry{}
+ for rows.Next() {
+ var e webEntry
+ if err = rows.Scan(
+ &e.SHA1, &e.Name, &e.Modified, &e.ThumbW, &e.ThumbH); err != nil {
+ return nil, err
+ }
+ entries = append(entries, e)
+ }
+ return entries, rows.Err()
+}
+
+func handleAPIBrowse(w http.ResponseWriter, r *http.Request) {
+ var params struct {
+ Path string
+ }
+ if err := json.NewDecoder(r.Body).Decode(&params); err != nil {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+
+ var result struct {
+ Subdirectories []string `json:"subdirectories"`
+ Entries []webEntry `json:"entries"`
+ }
+
+ tx, err := db.Begin()
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ defer tx.Rollback()
+
+ parent, err := idForDirectoryPath(tx, decodeWebPath(params.Path), false)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusNotFound)
+ return
+ }
+
+ result.Subdirectories, err = getSubdirectories(tx, parent)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ result.Entries, err = getSubentries(tx, parent)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ if err := json.NewEncoder(w).Encode(result); err != nil {
+ log.Println(err)
+ }
+}
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+type webTagNamespace struct {
+ Description string `json:"description"`
+ Tags map[string]int64 `json:"tags"`
+}
+
+func getTags(nsID int64) (result map[string]int64, err error) {
+ rows, err := db.Query(`
+ SELECT t.name, COUNT(ta.tag) AS count
+ FROM tag AS t
+ LEFT JOIN tag_assignment AS ta ON t.id = ta.tag
+ WHERE t.space = ?
+ GROUP BY t.id`, nsID)
+ if err != nil {
+ return
+ }
+ defer rows.Close()
+
+ result = make(map[string]int64)
+ for rows.Next() {
+ var (
+ name string
+ count int64
+ )
+ if err = rows.Scan(&name, &count); err != nil {
+ return
+ }
+ result[name] = count
+ }
+ return result, rows.Err()
+}
+
+func getTagNamespaces(match *string) (
+ result map[string]webTagNamespace, err error) {
+ var rows *sql.Rows
+ if match != nil {
+ rows, err = db.Query(`SELECT id, name, IFNULL(description, '')
+ FROM tag_space WHERE name = ?`, *match)
+ } else {
+ rows, err = db.Query(`SELECT id, name, IFNULL(description, '')
+ FROM tag_space`)
+ }
+ if err != nil {
+ return
+ }
+ defer rows.Close()
+
+ result = make(map[string]webTagNamespace)
+ for rows.Next() {
+ var (
+ id int64
+ name string
+ ns webTagNamespace
+ )
+ if err = rows.Scan(&id, &name, &ns.Description); err != nil {
+ return
+ }
+ if ns.Tags, err = getTags(id); err != nil {
+ return
+ }
+ result[name] = ns
+ }
+ return result, rows.Err()
+}
+
+func handleAPITags(w http.ResponseWriter, r *http.Request) {
+ var params struct {
+ Namespace *string
+ }
+ if err := json.NewDecoder(r.Body).Decode(&params); err != nil {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+
+ result, err := getTagNamespaces(params.Namespace)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+
+ if err := json.NewEncoder(w).Encode(result); err != nil {
+ log.Println(err)
+ }
+}
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+type webDuplicateImage struct {
+ SHA1 string `json:"sha1"`
+ ThumbW int64 `json:"thumbW"`
+ ThumbH int64 `json:"thumbH"`
+ Occurences int64 `json:"occurences"`
+}
+
+// A hamming distance of zero (direct dhash match) will be more than sufficient.
+const duplicatesCTE = `WITH
+ duplicated(dhash, count) AS (
+ SELECT dhash, COUNT(*) AS count FROM image
+ WHERE dhash IS NOT NULL
+ GROUP BY dhash HAVING count > 1
+ ),
+ multipathed(sha1, count) AS (
+ SELECT n.sha1, COUNT(*) AS count FROM node AS n
+ JOIN image AS i ON i.sha1 = n.sha1
+ WHERE i.dhash IS NULL
+ OR i.dhash NOT IN (SELECT dhash FROM duplicated)
+ GROUP BY n.sha1 HAVING count > 1
+ )
+`
+
+func getDuplicatesSimilar(stmt *sql.Stmt, dhash int64) (
+ result []webDuplicateImage, err error) {
+ rows, err := stmt.Query(dhash)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+
+ result = []webDuplicateImage{}
+ for rows.Next() {
+ var image webDuplicateImage
+ if err = rows.Scan(&image.SHA1, &image.ThumbW, &image.ThumbH,
+ &image.Occurences); err != nil {
+ return nil, err
+ }
+ result = append(result, image)
+ }
+ return result, rows.Err()
+}
+
+func getDuplicates1(result [][]webDuplicateImage) (
+ [][]webDuplicateImage, error) {
+ stmt, err := db.Prepare(`
+ SELECT i.sha1, IFNULL(i.thumbw, 0), IFNULL(i.thumbh, 0),
+ COUNT(*) AS occurences
+ FROM image AS i
+ JOIN node AS n ON n.sha1 = i.sha1
+ WHERE i.dhash = ?
+ GROUP BY n.sha1`)
+ if err != nil {
+ return nil, err
+ }
+ defer stmt.Close()
+
+ rows, err := db.Query(duplicatesCTE + `SELECT dhash FROM duplicated`)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+
+ for rows.Next() {
+ var (
+ group []webDuplicateImage
+ dhash int64
+ )
+ if err = rows.Scan(&dhash); err != nil {
+ return nil, err
+ }
+ if group, err = getDuplicatesSimilar(stmt, dhash); err != nil {
+ return nil, err
+ }
+ result = append(result, group)
+ }
+ return result, rows.Err()
+}
+
+func getDuplicates2(result [][]webDuplicateImage) (
+ [][]webDuplicateImage, error) {
+ stmt, err := db.Prepare(`
+ SELECT i.sha1, IFNULL(i.thumbw, 0), IFNULL(i.thumbh, 0),
+ COUNT(*) AS occurences
+ FROM image AS i
+ JOIN node AS n ON n.sha1 = i.sha1
+ WHERE i.sha1 = ?
+ GROUP BY n.sha1`)
+ if err != nil {
+ return nil, err
+ }
+ defer stmt.Close()
+
+ rows, err := db.Query(duplicatesCTE + `SELECT sha1 FROM multipathed`)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+
+ for rows.Next() {
+ var (
+ image webDuplicateImage
+ sha1 string
+ )
+ if err = rows.Scan(&sha1); err != nil {
+ return nil, err
+ }
+ if err := stmt.QueryRow(sha1).Scan(&image.SHA1,
+ &image.ThumbW, &image.ThumbH, &image.Occurences); err != nil {
+ return nil, err
+ }
+ result = append(result, []webDuplicateImage{image})
+ }
+ return result, rows.Err()
+}
+
+func handleAPIDuplicates(w http.ResponseWriter, r *http.Request) {
+ var params struct{}
+ if err := json.NewDecoder(r.Body).Decode(&params); err != nil {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+
+ var (
+ result = [][]webDuplicateImage{}
+ err error
+ )
+ if result, err = getDuplicates1(result); err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ if result, err = getDuplicates2(result); err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+
+ if err := json.NewEncoder(w).Encode(result); err != nil {
+ log.Println(err)
+ }
+}
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+type webOrphanImage struct {
+ SHA1 string `json:"sha1"`
+ ThumbW int64 `json:"thumbW"`
+ ThumbH int64 `json:"thumbH"`
+ Tags int64 `json:"tags"`
+}
+
+type webOrphan struct {
+ webOrphanImage
+ LastPath string `json:"lastPath"`
+ Replacement *webOrphanImage `json:"replacement"`
+}
+
+func getOrphanReplacement(webPath string) (*webOrphanImage, error) {
+ tx, err := db.Begin()
+ if err != nil {
+ return nil, err
+ }
+ defer tx.Rollback()
+
+ path := decodeWebPath(webPath)
+ if len(path) == 0 {
+ return nil, nil
+ }
+
+ parent, err := idForDirectoryPath(tx, path[:len(path)-1], false)
+ if err != nil {
+ return nil, err
+ }
+
+ var image webOrphanImage
+ err = db.QueryRow(`SELECT i.sha1,
+ IFNULL(i.thumbw, 0), IFNULL(i.thumbh, 0), COUNT(ta.sha1) AS tags
+ FROM node AS n
+ JOIN image AS i ON n.sha1 = i.sha1
+ LEFT JOIN tag_assignment AS ta ON n.sha1 = ta.sha1
+ WHERE n.parent = ? AND n.name = ?
+ GROUP BY n.sha1`, parent, path[len(path)-1]).Scan(
+ &image.SHA1, &image.ThumbW, &image.ThumbH, &image.Tags)
+ if errors.Is(err, sql.ErrNoRows) {
+ return nil, nil
+ } else if err != nil {
+ return nil, err
+ }
+ return &image, nil
+}
+
+func getOrphans() (result []webOrphan, err error) {
+ rows, err := db.Query(`SELECT o.sha1, o.path,
+ IFNULL(i.thumbw, 0), IFNULL(i.thumbh, 0), COUNT(ta.sha1) AS tags
+ FROM orphan AS o
+ JOIN image AS i ON o.sha1 = i.sha1
+ LEFT JOIN tag_assignment AS ta ON o.sha1 = ta.sha1
+ GROUP BY o.sha1`)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+
+ result = []webOrphan{}
+ for rows.Next() {
+ var orphan webOrphan
+ if err = rows.Scan(&orphan.SHA1, &orphan.LastPath,
+ &orphan.ThumbW, &orphan.ThumbH, &orphan.Tags); err != nil {
+ return nil, err
+ }
+
+ orphan.Replacement, err = getOrphanReplacement(orphan.LastPath)
+ if err != nil {
+ return nil, err
+ }
+
+ result = append(result, orphan)
+ }
+ return result, rows.Err()
+}
+
+func handleAPIOrphans(w http.ResponseWriter, r *http.Request) {
+ var params struct{}
+ if err := json.NewDecoder(r.Body).Decode(&params); err != nil {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+
+ result, err := getOrphans()
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+
+ if err := json.NewEncoder(w).Encode(result); err != nil {
+ log.Println(err)
+ }
+}
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+func getImageDimensions(sha1 string) (w int64, h int64, err error) {
+ err = db.QueryRow(`SELECT width, height FROM image WHERE sha1 = ?`,
+ sha1).Scan(&w, &h)
+ return
+}
+
+func getImagePaths(sha1 string) (paths []string, err error) {
+ rows, err := db.Query(`WITH RECURSIVE paths(parent, path) AS (
+ SELECT parent, name AS path FROM node WHERE sha1 = ?
+ UNION ALL
+ SELECT n.parent, n.name || '/' || p.path
+ FROM node AS n JOIN paths AS p ON n.id = p.parent
+ ) SELECT path FROM paths WHERE parent IS NULL`, sha1)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+
+ paths = []string{}
+ for rows.Next() {
+ var path string
+ if err := rows.Scan(&path); err != nil {
+ return nil, err
+ }
+ paths = append(paths, path)
+ }
+ return paths, rows.Err()
+}
+
+func getImageTags(sha1 string) (map[string]map[string]float32, error) {
+ rows, err := db.Query(`
+ SELECT ts.name, t.name, ta.weight FROM tag_assignment AS ta
+ JOIN tag AS t ON t.id = ta.tag
+ JOIN tag_space AS ts ON ts.id = t.space
+ WHERE ta.sha1 = ?`, sha1)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+
+ result := make(map[string]map[string]float32)
+ for rows.Next() {
+ var (
+ space, tag string
+ weight float32
+ )
+ if err := rows.Scan(&space, &tag, &weight); err != nil {
+ return nil, err
+ }
+
+ tags := result[space]
+ if tags == nil {
+ tags = make(map[string]float32)
+ result[space] = tags
+ }
+ tags[tag] = weight
+ }
+ return result, rows.Err()
+}
+
+func handleAPIInfo(w http.ResponseWriter, r *http.Request) {
+ var params struct {
+ SHA1 string
+ }
+ if err := json.NewDecoder(r.Body).Decode(&params); err != nil {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+
+ var result struct {
+ Width int64 `json:"width"`
+ Height int64 `json:"height"`
+ Paths []string `json:"paths"`
+ Tags map[string]map[string]float32 `json:"tags"`
+ }
+
+ var err error
+ result.Width, result.Height, err = getImageDimensions(params.SHA1)
+ if errors.Is(err, sql.ErrNoRows) {
+ http.Error(w, err.Error(), http.StatusNotFound)
+ return
+ } else if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+
+ result.Paths, err = getImagePaths(params.SHA1)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ result.Tags, err = getImageTags(params.SHA1)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ if err := json.NewEncoder(w).Encode(result); err != nil {
+ log.Println(err)
+ }
+}
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+type webSimilarImage struct {
+ SHA1 string `json:"sha1"`
+ PixelsRatio float32 `json:"pixelsRatio"`
+ ThumbW int64 `json:"thumbW"`
+ ThumbH int64 `json:"thumbH"`
+ Paths []string `json:"paths"`
+}
+
+func getSimilar(sha1 string, dhash int64, pixels int64, distance int) (
+ result []webSimilarImage, err error) {
+ // For distance ∈ {0, 1}, this query is quite inefficient.
+ // In exchange, it's generic.
+ //
+ // If there's a dhash, there should also be thumbnail dimensions,
+ // so not bothering with IFNULL on them.
+ rows, err := db.Query(`
+ SELECT sha1, width * height, IFNULL(thumbw, 0), IFNULL(thumbh, 0)
+ FROM image WHERE sha1 <> ? AND dhash IS NOT NULL
+ AND hamming(dhash, ?) = ?`, sha1, dhash, distance)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+
+ result = []webSimilarImage{}
+ for rows.Next() {
+ var (
+ match webSimilarImage
+ matchPixels int64
+ )
+ if err = rows.Scan(&match.SHA1,
+ &matchPixels, &match.ThumbW, &match.ThumbH); err != nil {
+ return nil, err
+ }
+ if match.Paths, err = getImagePaths(match.SHA1); err != nil {
+ return nil, err
+ }
+ match.PixelsRatio = float32(matchPixels) / float32(pixels)
+ result = append(result, match)
+ }
+ return result, rows.Err()
+}
+
+func getSimilarGroups(sha1 string, dhash int64, pixels int64,
+ output map[string][]webSimilarImage) error {
+ var err error
+ for distance := 0; distance <= 1; distance++ {
+ output[fmt.Sprintf("Perceptual distance %d", distance)], err =
+ getSimilar(sha1, dhash, pixels, distance)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func handleAPISimilar(w http.ResponseWriter, r *http.Request) {
+ var params struct {
+ SHA1 string
+ }
+ if err := json.NewDecoder(r.Body).Decode(&params); err != nil {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+
+ var result struct {
+ Info webSimilarImage `json:"info"`
+ Groups map[string][]webSimilarImage `json:"groups"`
+ }
+
+ result.Info = webSimilarImage{SHA1: params.SHA1, PixelsRatio: 1}
+ if paths, err := getImagePaths(params.SHA1); err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ } else {
+ result.Info.Paths = paths
+ }
+
+ var (
+ width, height int64
+ dhash sql.NullInt64
+ )
+ err := db.QueryRow(`
+ SELECT width, height, dhash, IFNULL(thumbw, 0), IFNULL(thumbh, 0)
+ FROM image WHERE sha1 = ?`, params.SHA1).Scan(&width, &height, &dhash,
+ &result.Info.ThumbW, &result.Info.ThumbH)
+ if errors.Is(err, sql.ErrNoRows) {
+ http.Error(w, err.Error(), http.StatusNotFound)
+ return
+ } else if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+
+ result.Groups = make(map[string][]webSimilarImage)
+ if dhash.Valid {
+ if err := getSimilarGroups(
+ params.SHA1, dhash.Int64, width*height, result.Groups); err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ }
+
+ if err := json.NewEncoder(w).Encode(result); err != nil {
+ log.Println(err)
+ }
+}
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+// NOTE: AND will mean MULTIPLY(IFNULL(ta.weight, 0)) per SHA1.
+const searchCTE = `WITH
+ matches(sha1, thumbw, thumbh, score) AS (
+ SELECT i.sha1, i.thumbw, i.thumbh, ta.weight AS score
+ FROM tag_assignment AS ta
+ JOIN image AS i ON i.sha1 = ta.sha1
+ WHERE ta.tag = ?
+ ),
+ supertags(tag) AS (
+ SELECT DISTINCT ta.tag
+ FROM tag_assignment AS ta
+ JOIN matches AS m ON m.sha1 = ta.sha1
+ ),
+ scoredtags(tag, score) AS (
+ -- The cross join is a deliberate optimization,
+ -- and this query may still be really slow.
+ SELECT st.tag, AVG(IFNULL(ta.weight, 0)) AS score
+ FROM matches AS m
+ CROSS JOIN supertags AS st
+ LEFT JOIN tag_assignment AS ta
+ ON ta.sha1 = m.sha1 AND ta.tag = st.tag
+ GROUP BY st.tag
+ -- Using the column alias doesn't fail, but it also doesn't work.
+ HAVING AVG(IFNULL(ta.weight, 0)) >= 0.01
+ )
+`
+
+type webTagMatch struct {
+ SHA1 string `json:"sha1"`
+ ThumbW int64 `json:"thumbW"`
+ ThumbH int64 `json:"thumbH"`
+ Score float32 `json:"score"`
+}
+
+func getTagMatches(tag int64) (matches []webTagMatch, err error) {
+ rows, err := db.Query(searchCTE+`
+ SELECT sha1, IFNULL(thumbw, 0), IFNULL(thumbh, 0), score
+ FROM matches`, tag)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+
+ matches = []webTagMatch{}
+ for rows.Next() {
+ var match webTagMatch
+ if err = rows.Scan(&match.SHA1,
+ &match.ThumbW, &match.ThumbH, &match.Score); err != nil {
+ return nil, err
+ }
+ matches = append(matches, match)
+ }
+ return matches, rows.Err()
+}
+
+type webTagRelated struct {
+ Tag string `json:"tag"`
+ Score float32 `json:"score"`
+}
+
+func getTagRelated(tag int64) (result map[string][]webTagRelated, err error) {
+ rows, err := db.Query(searchCTE+`
+ SELECT ts.name, t.name, st.score FROM scoredtags AS st
+ JOIN tag AS t ON st.tag = t.id
+ JOIN tag_space AS ts ON ts.id = t.space
+ ORDER BY st.score DESC`, tag)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+
+ result = make(map[string][]webTagRelated)
+ for rows.Next() {
+ var (
+ space string
+ r webTagRelated
+ )
+ if err = rows.Scan(&space, &r.Tag, &r.Score); err != nil {
+ return nil, err
+ }
+ result[space] = append(result[space], r)
+ }
+ return result, rows.Err()
+}
+
+func handleAPISearch(w http.ResponseWriter, r *http.Request) {
+ var params struct {
+ Query string
+ }
+ if err := json.NewDecoder(r.Body).Decode(&params); err != nil {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+
+ var result struct {
+ Matches []webTagMatch `json:"matches"`
+ Related map[string][]webTagRelated `json:"related"`
+ }
+
+ space, tag, _ := strings.Cut(params.Query, ":")
+
+ var tagID int64
+ err := db.QueryRow(`
+ SELECT t.id FROM tag AS t
+ JOIN tag_space AS ts ON t.space = ts.id
+ WHERE ts.name = ? AND t.name = ?`, space, tag).Scan(&tagID)
+ if errors.Is(err, sql.ErrNoRows) {
+ http.Error(w, err.Error(), http.StatusNotFound)
+ return
+ } else if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+
+ if result.Matches, err = getTagMatches(tagID); err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ if result.Related, err = getTagRelated(tagID); err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+
+ if err := json.NewEncoder(w).Encode(result); err != nil {
+ log.Println(err)
+ }
+}
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+// cmdWeb runs a web UI against GD on ADDRESS.
+func cmdWeb(fs *flag.FlagSet, args []string) error {
+ if err := fs.Parse(args); err != nil {
+ return err
+ }
+ if fs.NArg() != 2 {
+ return errWrongUsage
+ }
+ if err := openDB(fs.Arg(0)); err != nil {
+ return err
+ }
+
+ address := fs.Arg(1)
+
+ // This separation is not strictly necessary,
+ // but having an elementary level of security doesn't hurt either.
+ staticHandler = http.FileServer(http.Dir("public"))
+
+ http.HandleFunc("/", handleRequest)
+ http.HandleFunc("/image/", handleImages)
+ http.HandleFunc("/thumb/", handleThumbs)
+ http.HandleFunc("/api/browse", handleAPIBrowse)
+ http.HandleFunc("/api/tags", handleAPITags)
+ http.HandleFunc("/api/duplicates", handleAPIDuplicates)
+ http.HandleFunc("/api/orphans", handleAPIOrphans)
+ http.HandleFunc("/api/info", handleAPIInfo)
+ http.HandleFunc("/api/similar", handleAPISimilar)
+ http.HandleFunc("/api/search", handleAPISearch)
+
+ host, port, err := net.SplitHostPort(address)
+ if err != nil {
+ log.Println(err)
+ } else if host == "" {
+ log.Println("http://" + net.JoinHostPort("localhost", port))
+ } else {
+ log.Println("http://" + address)
+ }
+
+ s := &http.Server{
+ Addr: address,
+ ReadTimeout: 60 * time.Second,
+ WriteTimeout: 60 * time.Second,
+ MaxHeaderBytes: 32 << 10,
+ }
+ return s.ListenAndServe()
+}
+
+// --- Sync --------------------------------------------------------------------
+
+type syncFileInfo struct {
+ dbID int64 // DB node ID, or zero if there was none
+ dbParent int64 // where the file was to be stored
+ dbName string // the name under which it was to be stored
+ fsPath string // symlink target
+ fsMtime int64 // last modified Unix timestamp, used a bit like an ID
+
+ err error // any processing error
+ sha1 string // raw content hash, empty to skip file
+ width int // image width in pixels
+ height int // image height in pixels
+}
+
+type syncContext struct {
+ ctx context.Context
+ tx *sql.Tx
+ info chan syncFileInfo
+ pb *progressBar
+
+ stmtOrphan *sql.Stmt
+ stmtDisposeSub *sql.Stmt
+ stmtDisposeAll *sql.Stmt
+
+ // linked tracks which image hashes we've checked so far in the run.
+ linked map[string]struct{}
+}
+
+func syncPrintf(c *syncContext, format string, v ...any) {
+ c.pb.Interrupt(func() { log.Printf(format+"\n", v...) })
+}
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+type syncNode struct {
+ dbID int64
+ dbName string
+ dbMtime int64
+ dbSHA1 string
+}
+
+func (n *syncNode) dbIsDir() bool { return n.dbSHA1 == "" }
+
+type syncFile struct {
+ fsName string
+ fsMtime int64
+ fsIsDir bool
+}
+
+type syncPair struct {
+ db *syncNode
+ fs *syncFile
+}
+
+// syncGetNodes returns direct children of a DB node, ordered by name.
+// SQLite, like Go, compares strings byte-wise by default.
+func syncGetNodes(tx *sql.Tx, dbParent int64) (nodes []syncNode, err error) {
+ // This works even for the root, which doesn't exist as a DB node.
+ rows, err := tx.Query(`SELECT id, name, IFNULL(mtime, 0), IFNULL(sha1, '')
+ FROM node WHERE IFNULL(parent, 0) = ? ORDER BY name`, dbParent)
+ if err != nil {
+ return
+ }
+ defer rows.Close()
+
+ for rows.Next() {
+ var node syncNode
+ if err = rows.Scan(&node.dbID,
+ &node.dbName, &node.dbMtime, &node.dbSHA1); err != nil {
+ return
+ }
+ nodes = append(nodes, node)
+ }
+ return nodes, rows.Err()
+}
+
+// syncGetFiles returns direct children of a FS directory, ordered by name.
+func syncGetFiles(fsPath string) (files []syncFile, err error) {
+ dir, err := os.Open(fsPath)
+ if err != nil {
+ return
+ }
+ defer dir.Close()
+
+ entries, err := dir.ReadDir(0)
+ if err != nil {
+ return
+ }
+
+ for _, entry := range entries {
+ info, err := entry.Info()
+ if err != nil {
+ return files, err
+ }
+
+ files = append(files, syncFile{
+ fsName: entry.Name(),
+ fsMtime: info.ModTime().Unix(),
+ fsIsDir: entry.IsDir(),
+ })
+ }
+ sort.Slice(files,
+ func(a, b int) bool { return files[a].fsName < files[b].fsName })
+ return
+}
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+func syncIsImage(path string) (bool, error) {
+ out, err := exec.Command("xdg-mime", "query", "filetype", path).Output()
+ if err != nil {
+ return false, err
+ }
+
+ return bytes.HasPrefix(out, []byte("image/")), nil
+}
+
+func syncPingImage(path string) (int, int, error) {
+ out, err := exec.Command("magick", "identify", "-limit", "thread", "1",
+ "-ping", "-format", "%w %h", path+"[0]").Output()
+ if err != nil {
+ return 0, 0, err
+ }
+
+ var w, h int
+ _, err = fmt.Fscanf(bytes.NewReader(out), "%d %d", &w, &h)
+ return w, h, err
+}
+
+func syncProcess(c *syncContext, info *syncFileInfo) error {
+ // Skip videos, which ImageMagick can process, but we don't want it to,
+ // so that they're not converted 1:1 to WebP.
+ pathIsImage, err := syncIsImage(info.fsPath)
+ if err != nil {
+ return err
+ }
+ if !pathIsImage {
+ return nil
+ }
+
+ info.width, info.height, err = syncPingImage(info.fsPath)
+ if err != nil {
+ return err
+ }
+
+ f, err := os.Open(info.fsPath)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ // We could make this at least somewhat interruptible by c.ctx,
+ // though it would still work poorly.
+ hash := sha1.New()
+ _, err = io.CopyBuffer(hash, f, make([]byte, 65536))
+ if err != nil {
+ return err
+ }
+
+ info.sha1 = hex.EncodeToString(hash.Sum(nil))
+ return nil
+}
+
+// syncEnqueue runs file scanning, which can be CPU and I/O expensive,
+// in parallel. The goroutine only touches the filesystem, read-only.
+func syncEnqueue(c *syncContext, info syncFileInfo) error {
+ if err := taskSemaphore.acquire(c.ctx); err != nil {
+ return err
+ }
+
+ go func(info syncFileInfo) {
+ defer taskSemaphore.release()
+ info.err = syncProcess(c, &info)
+ c.info <- info
+ }(info)
+ return nil
+}
+
+// syncDequeue flushes the result queue of finished asynchronous tasks.
+func syncDequeue(c *syncContext) error {
+ for {
+ select {
+ case <-c.ctx.Done():
+ return c.ctx.Err()
+ case info := <-c.info:
+ if err := syncPostProcess(c, info); err != nil {
+ return err
+ }
+ default:
+ return nil
+ }
+ }
+}
+
+// syncDispose creates orphan records for the entire subtree given by nodeID
+// as appropriate, then deletes all nodes within the subtree. The subtree root
+// node is not deleted if "keepNode" is true.
+//
+// Orphans keep their thumbnail files, as evidence.
+func syncDispose(c *syncContext, nodeID int64, keepNode bool) error {
+ if _, err := c.stmtOrphan.Exec(nodeID); err != nil {
+ return err
+ }
+
+ if keepNode {
+ if _, err := c.stmtDisposeSub.Exec(nodeID); err != nil {
+ return err
+ }
+ } else {
+ if _, err := c.stmtDisposeAll.Exec(nodeID); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func syncImageResave(c *syncContext, path string, target string) error {
+ dirname, _ := filepath.Split(path)
+ if err := os.MkdirAll(dirname, 0755); err != nil {
+ return err
+ }
+
+ for {
+ // Try to remove anything standing in the way.
+ err := os.Remove(path)
+ if err != nil && !errors.Is(err, os.ErrNotExist) {
+ return err
+ }
+
+ // TODO: Make it possible to copy or reflink (ioctl FICLONE).
+ err = os.Symlink(target, path)
+ if err == nil || !errors.Is(err, fs.ErrExist) {
+ return err
+ }
+ }
+}
+
+func syncImageSave(c *syncContext, sha1 string, target string) error {
+ if _, ok := c.linked[sha1]; ok {
+ return nil
+ }
+
+ ok, path := false, imagePath(sha1)
+ if link, err := os.Readlink(path); err == nil {
+ ok = link == target
+ } else {
+ // If it exists, but it is not a symlink, let it be.
+ // Even though it may not be a regular file.
+ ok = errors.Is(err, syscall.EINVAL)
+ }
+
+ if !ok {
+ if err := syncImageResave(c, path, target); err != nil {
+ return err
+ }
+ }
+
+ c.linked[sha1] = struct{}{}
+ return nil
+}
+
+func syncImage(c *syncContext, info syncFileInfo) error {
+ if _, err := c.tx.Exec(`INSERT INTO image(sha1, width, height)
+ VALUES (?, ?, ?) ON CONFLICT(sha1) DO NOTHING`,
+ info.sha1, info.width, info.height); err != nil {
+ return err
+ }
+
+ return syncImageSave(c, info.sha1, info.fsPath)
+}
+
+func syncPostProcess(c *syncContext, info syncFileInfo) error {
+ defer c.pb.Step()
+
+ // TODO: When replacing an image node (whether it has or doesn't have
+ // other links to keep it alive), we could offer copying all tags,
+ // though this needs another table to track it.
+ // (If it's equivalent enough, the dhash will stay the same,
+ // so user can resolve this through the duplicates feature.)
+ switch {
+ case info.err != nil:
+ // * → error
+ if ee, ok := info.err.(*exec.ExitError); ok {
+ syncPrintf(c, "%s: %s", info.fsPath, ee.Stderr)
+ } else {
+ return info.err
+ }
+ fallthrough
+
+ case info.sha1 == "":
+ // 0 → 0
+ if info.dbID == 0 {
+ return nil
+ }
+
+ // D → 0, F → 0
+ // TODO: Make it possible to disable removal (for copying only?)
+ return syncDispose(c, info.dbID, false /*keepNode*/)
+
+ case info.dbID == 0:
+ // 0 → F
+ if err := syncImage(c, info); err != nil {
+ return err
+ }
+ if _, err := c.tx.Exec(`INSERT INTO node(parent, name, mtime, sha1)
+ VALUES (?, ?, ?, ?)`,
+ info.dbParent, info.dbName, info.fsMtime, info.sha1); err != nil {
+ return err
+ }
+ return nil
+
+ default:
+ // D → F, F → F (this statement is a no-op with the latter)
+ if err := syncDispose(c, info.dbID, true /*keepNode*/); err != nil {
+ return err
+ }
+
+ // Even if the hash didn't change, see comment in syncDirectoryPair().
+ if err := syncImage(c, info); err != nil {
+ return err
+ }
+ if _, err := c.tx.Exec(`UPDATE node SET mtime = ?, sha1 = ?
+ WHERE id = ?`, info.fsMtime, info.sha1, info.dbID); err != nil {
+ return err
+ }
+ return nil
+ }
+}
+
+func syncDirectoryPair(c *syncContext, dbParent int64, fsPath string,
+ pair syncPair) error {
+ db, fs, fsInfo := pair.db, pair.fs, syncFileInfo{dbParent: dbParent}
+ if db != nil {
+ fsInfo.dbID = db.dbID
+ }
+ if fs != nil {
+ fsInfo.dbName = fs.fsName
+ fsInfo.fsPath = filepath.Join(fsPath, fs.fsName)
+ fsInfo.fsMtime = fs.fsMtime
+ }
+
+ switch {
+ case db == nil && fs == nil:
+ // 0 → 0, unreachable.
+
+ case db == nil && fs.fsIsDir:
+ // 0 → D
+ var id int64
+ if result, err := c.tx.Exec(`INSERT INTO node(parent, name)
+ VALUES (?, ?)`, dbParent, fs.fsName); err != nil {
+ return err
+ } else if id, err = result.LastInsertId(); err != nil {
+ return err
+ }
+ return syncDirectory(c, id, fsInfo.fsPath)
+
+ case db == nil:
+ // 0 → F (or 0 → 0)
+ return syncEnqueue(c, fsInfo)
+
+ case fs == nil:
+ // D → 0, F → 0
+ // TODO: Make it possible to disable removal (for copying only?)
+ return syncDispose(c, db.dbID, false /*keepNode*/)
+
+ case db.dbIsDir() && fs.fsIsDir:
+ // D → D
+ return syncDirectory(c, db.dbID, fsInfo.fsPath)
+
+ case db.dbIsDir():
+ // D → F (or D → 0)
+ return syncEnqueue(c, fsInfo)
+
+ case fs.fsIsDir:
+ // F → D
+ if err := syncDispose(c, db.dbID, true /*keepNode*/); err != nil {
+ return err
+ }
+ if _, err := c.tx.Exec(`UPDATE node
+ SET mtime = NULL, sha1 = NULL WHERE id = ?`, db.dbID); err != nil {
+ return err
+ }
+ return syncDirectory(c, db.dbID, fsInfo.fsPath)
+
+ case db.dbMtime != fs.fsMtime:
+ // F → F (or F → 0)
+ // Assuming that any content modifications change the timestamp.
+ return syncEnqueue(c, fsInfo)
+
+ default:
+ // F → F
+ // Try to fix symlinks, to handle the following situations:
+ // 1. Image A occurs in paths 1 and 2, we use a symlink to path 1,
+ // and path 1 is removed from the filesystem:
+ // path 2 would not resolve if the mtime didn't change.
+ // 2. Image A occurs in paths 1 and 2, we use a symlink to path 1,
+ // and path 1 is changed:
+ // path 2 would resolve to the wrong file.
+ // This may relink images with multiple occurences unnecessarily,
+ // but it will always fix the roots that are being synced.
+ if err := syncImageSave(c, db.dbSHA1, fsInfo.fsPath); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func syncDirectory(c *syncContext, dbParent int64, fsPath string) error {
+ db, err := syncGetNodes(c.tx, dbParent)
+ if err != nil {
+ return err
+ }
+
+ fs, err := syncGetFiles(fsPath)
+ if err != nil {
+ return err
+ }
+
+ // This would not be fatal, but it has annoying consequences.
+ if _, ok := slices.BinarySearchFunc(fs, syncFile{fsName: nameOfDB},
+ func(a, b syncFile) int {
+ return strings.Compare(a.fsName, b.fsName)
+ }); ok {
+ syncPrintf(c, "%s may be a gallery directory, treating as empty",
+ fsPath)
+ fs = nil
+ }
+
+ // Convert differences to a form more convenient for processing.
+ iDB, iFS, pairs := 0, 0, []syncPair{}
+ for iDB < len(db) && iFS < len(fs) {
+ if db[iDB].dbName == fs[iFS].fsName {
+ pairs = append(pairs, syncPair{&db[iDB], &fs[iFS]})
+ iDB++
+ iFS++
+ } else if db[iDB].dbName < fs[iFS].fsName {
+ pairs = append(pairs, syncPair{&db[iDB], nil})
+ iDB++
+ } else {
+ pairs = append(pairs, syncPair{nil, &fs[iFS]})
+ iFS++
+ }
+ }
+ for i := range db[iDB:] {
+ pairs = append(pairs, syncPair{&db[iDB+i], nil})
+ }
+ for i := range fs[iFS:] {
+ pairs = append(pairs, syncPair{nil, &fs[iFS+i]})
+ }
+
+ for _, pair := range pairs {
+ if err := syncDequeue(c); err != nil {
+ return err
+ }
+ if err := syncDirectoryPair(c, dbParent, fsPath, pair); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func syncRoot(c *syncContext, dbPath []string, fsPath string) error {
+ // TODO: Support synchronizing individual files.
+ // This can only be treated as 0 → F, F → F, or D → F, that is,
+ // a variation on current syncEnqueue(), but dbParent must be nullable.
+
+ // Figure out a database root (not trying to convert F → D on conflict,
+ // also because we don't know yet if the argument is a directory).
+ //
+ // Synchronizing F → D or * → F are special cases not worth implementing.
+ dbParent, err := idForDirectoryPath(c.tx, dbPath, true)
+ if err != nil {
+ return err
+ }
+ if err := syncDirectory(c, dbParent, fsPath); err != nil {
+ return err
+ }
+
+ // Wait for all tasks to finish, and process the results of their work.
+ for i := 0; i < cap(taskSemaphore); i++ {
+ if err := taskSemaphore.acquire(c.ctx); err != nil {
+ return err
+ }
+ }
+ if err := syncDequeue(c); err != nil {
+ return err
+ }
+
+ // This is not our semaphore, so prepare it for the next user.
+ for i := 0; i < cap(taskSemaphore); i++ {
+ taskSemaphore.release()
+ }
+
+ // Delete empty directories, from the bottom of the tree up to,
+ // but not including, the inserted root.
+ //
+ // We need to do this at the end due to our recursive handling,
+ // as well as because of asynchronous file filtering.
+ stmt, err := c.tx.Prepare(`
+ WITH RECURSIVE subtree(id, parent, sha1, level) AS (
+ SELECT id, parent, sha1, 1 FROM node WHERE id = ?
+ UNION ALL
+ SELECT n.id, n.parent, n.sha1, s.level + 1
+ FROM node AS n JOIN subtree AS s ON n.parent = s.id
+ ) DELETE FROM node WHERE id IN (
+ SELECT id FROM subtree WHERE level <> 1 AND sha1 IS NULL
+ AND id NOT IN (SELECT parent FROM node WHERE parent IS NOT NULL)
+ )`)
+ if err != nil {
+ return err
+ }
+
+ for {
+ if result, err := stmt.Exec(dbParent); err != nil {
+ return err
+ } else if n, err := result.RowsAffected(); err != nil {
+ return err
+ } else if n == 0 {
+ return nil
+ }
+ }
+}
+
+type syncPath struct {
+ db []string // database path, in terms of nodes
+ fs string // normalized filesystem path
+}
+
+// syncResolveRoots normalizes filesystem paths given in command line arguments,
+// and figures out a database path for each. Duplicates are skipped or rejected.
+func syncResolveRoots(args []string, fullpaths bool) (
+ roots []*syncPath, err error) {
+ for i := range args {
+ fs, err := filepath.Abs(filepath.Clean(args[i]))
+ if err != nil {
+ return nil, err
+ }
+
+ roots = append(roots,
+ &syncPath{decodeWebPath(filepath.ToSlash(fs)), fs})
+ }
+
+ if fullpaths {
+ // Filter out duplicates. In this case, they're just duplicated work.
+ slices.SortFunc(roots, func(a, b *syncPath) int {
+ return strings.Compare(a.fs, b.fs)
+ })
+ roots = slices.CompactFunc(roots, func(a, b *syncPath) bool {
+ if a.fs != b.fs && !strings.HasPrefix(b.fs, a.fs+"/") {
+ return false
+ }
+ log.Printf("asking to sync path twice: %s\n", b.fs)
+ return true
+ })
+ } else {
+ // Keep just the basenames.
+ for _, path := range roots {
+ if len(path.db) > 0 {
+ path.db = path.db[len(path.db)-1:]
+ }
+ }
+
+ // Different filesystem paths mapping to the same DB location
+ // are definitely a problem we would like to avoid,
+ // otherwise we don't care.
+ slices.SortFunc(roots, func(a, b *syncPath) int {
+ return slices.Compare(a.db, b.db)
+ })
+ for i := 1; i < len(roots); i++ {
+ if slices.Equal(roots[i-1].db, roots[i].db) {
+ return nil, fmt.Errorf("duplicate root: %v", roots[i].db)
+ }
+ }
+ }
+ return
+}
+
+const disposeCTE = `WITH RECURSIVE
+ root(id, sha1, parent, path) AS (
+ SELECT id, sha1, parent, name FROM node WHERE id = ?
+ UNION ALL
+ SELECT r.id, r.sha1, n.parent, n.name || '/' || r.path
+ FROM node AS n JOIN root AS r ON n.id = r.parent
+ ),
+ children(id, sha1, path, level) AS (
+ SELECT id, sha1, path, 1 FROM root WHERE parent IS NULL
+ UNION ALL
+ SELECT n.id, n.sha1, c.path || '/' || n.name, c.level + 1
+ FROM node AS n JOIN children AS c ON n.parent = c.id
+ ),
+ removed(sha1, count, path) AS (
+ SELECT sha1, COUNT(*) AS count, MIN(path) AS path
+ FROM children
+ GROUP BY sha1
+ ),
+ orphaned(sha1, path, count, total) AS (
+ SELECT r.sha1, r.path, r.count, COUNT(*) AS total
+ FROM removed AS r
+ JOIN node ON node.sha1 = r.sha1
+ GROUP BY node.sha1
+ HAVING count = total
+ )`
+
+// cmdSync ensures the given (sub)roots are accurately reflected
+// in the database.
+func cmdSync(fs *flag.FlagSet, args []string) error {
+ fullpaths := fs.Bool("fullpaths", false, "don't basename arguments")
+ if err := fs.Parse(args); err != nil {
+ return err
+ }
+ if fs.NArg() < 2 {
+ return errWrongUsage
+ }
+ if err := openDB(fs.Arg(0)); err != nil {
+ return err
+ }
+
+ roots, err := syncResolveRoots(fs.Args()[1:], *fullpaths)
+ if err != nil {
+ return err
+ }
+
+ ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
+ defer stop()
+
+ // In case of a failure during processing, the only retained side effects
+ // on the filesystem tree are:
+ // - Fixing dead symlinks to images.
+ // - Creating symlinks to images that aren't used by anything.
+ tx, err := db.BeginTx(ctx, nil)
+ if err != nil {
+ return err
+ }
+ defer tx.Rollback()
+
+ // Mild hack: upgrade the transaction to a write one straight away,
+ // in order to rule out deadlocks (preventable failure).
+ if _, err := tx.Exec(`END TRANSACTION;
+ BEGIN IMMEDIATE TRANSACTION`); err != nil {
+ return err
+ }
+
+ c := syncContext{ctx: ctx, tx: tx, pb: newProgressBar(-1),
+ linked: make(map[string]struct{})}
+ defer c.pb.Stop()
+
+ if c.stmtOrphan, err = c.tx.Prepare(disposeCTE + `
+ INSERT OR IGNORE INTO orphan(sha1, path)
+ SELECT sha1, path FROM orphaned`); err != nil {
+ return err
+ }
+ if c.stmtDisposeSub, err = c.tx.Prepare(disposeCTE + `
+ DELETE FROM node WHERE id
+ IN (SELECT DISTINCT id FROM children WHERE level <> 1)`); err != nil {
+ return err
+ }
+ if c.stmtDisposeAll, err = c.tx.Prepare(disposeCTE + `
+ DELETE FROM node WHERE id
+ IN (SELECT DISTINCT id FROM children)`); err != nil {
+ return err
+ }
+
+ // Info tasks take a position in the task semaphore channel.
+ // then fill the info channel.
+ //
+ // Immediately after syncDequeue(), the info channel is empty,
+ // but the semaphore might be full.
+ //
+ // By having at least one position in the info channel,
+ // we allow at least one info task to run to semaphore release,
+ // so that syncEnqueue() doesn't deadlock.
+ //
+ // By making it the same size as the semaphore,
+ // the end of this function doesn't need to dequeue while waiting.
+ // It also prevents goroutine leaks despite leaving them running--
+ // once they finish their job, they're gone,
+ // and eventually the info channel would get garbage collected.
+ //
+ // The additional slot is there to handle the one result
+ // that may be placed while syncEnqueue() waits for the semaphore,
+ // i.e., it is for the result of the task that syncEnqueue() spawns.
+ c.info = make(chan syncFileInfo, cap(taskSemaphore)+1)
+
+ for _, root := range roots {
+ if err := syncRoot(&c, root.db, root.fs); err != nil {
+ return err
+ }
+ }
+ return tx.Commit()
+}
+
+// --- Removal -----------------------------------------------------------------
+
+// cmdRemove is for manual removal of subtrees from the database.
+// Beware that inputs are database, not filesystem paths.
+func cmdRemove(fs *flag.FlagSet, args []string) error {
+ if err := fs.Parse(args); err != nil {
+ return err
+ }
+ if fs.NArg() < 2 {
+ return errWrongUsage
+ }
+ if err := openDB(fs.Arg(0)); err != nil {
+ return err
+ }
+
+ tx, err := db.BeginTx(context.Background(), nil)
+ if err != nil {
+ return err
+ }
+ defer tx.Rollback()
+
+ for _, path := range fs.Args()[1:] {
+ var id sql.NullInt64
+ for _, name := range decodeWebPath(path) {
+ if err := tx.QueryRow(`SELECT id FROM node
+ WHERE parent IS ? AND name = ?`,
+ id, name).Scan(&id); err != nil {
+ return err
+ }
+ }
+ if id.Int64 == 0 {
+ return errors.New("can't remove root")
+ }
+
+ if _, err = tx.Exec(disposeCTE+`
+ INSERT OR IGNORE INTO orphan(sha1, path)
+ SELECT sha1, path FROM orphaned`, id); err != nil {
+ return err
+ }
+ if _, err = tx.Exec(disposeCTE+`
+ DELETE FROM node WHERE id
+ IN (SELECT DISTINCT id FROM children)`, id); err != nil {
+ return err
+ }
+ }
+ return tx.Commit()
+}
+
+// --- Tagging -----------------------------------------------------------------
+
+// cmdTag mass imports tags from data passed on stdin as a TSV
+// of SHA1 TAG WEIGHT entries.
+func cmdTag(fs *flag.FlagSet, args []string) error {
+ if err := fs.Parse(args); err != nil {
+ return err
+ }
+ if fs.NArg() < 2 || fs.NArg() > 3 {
+ return errWrongUsage
+ }
+ if err := openDB(fs.Arg(0)); err != nil {
+ return err
+ }
+
+ space := fs.Arg(1)
+
+ var description sql.NullString
+ if fs.NArg() >= 3 {
+ description = sql.NullString{String: fs.Arg(2), Valid: true}
+ }
+
+ // Note that starting as a write transaction prevents deadlocks.
+ // Imports are rare, and just bulk load data, so this scope is fine.
+ tx, err := db.Begin()
+ if err != nil {
+ return err
+ }
+ defer tx.Rollback()
+
+ if _, err := tx.Exec(`INSERT OR IGNORE INTO tag_space(name, description)
+ VALUES (?, ?)`, space, description); err != nil {
+ return err
+ }
+
+ var spaceID int64
+ if err := tx.QueryRow(`SELECT id FROM tag_space WHERE name = ?`,
+ space).Scan(&spaceID); err != nil {
+ return err
+ }
+
+ // XXX: It might make sense to pre-erase all tag assignments within
+ // the given space for that image, the first time we see it:
+ //
+ // DELETE FROM tag_assignment
+ // WHERE sha1 = ? AND tag IN (SELECT id FROM tag WHERE space = ?)
+ //
+ // or even just clear the tag space completely:
+ //
+ // DELETE FROM tag_assignment
+ // WHERE tag IN (SELECT id FROM tag WHERE space = ?);
+ // DELETE FROM tag WHERE space = ?;
+ stmt, err := tx.Prepare(`INSERT INTO tag_assignment(sha1, tag, weight)
+ VALUES (?, (SELECT id FROM tag WHERE space = ? AND name = ?), ?)
+ ON CONFLICT DO UPDATE SET weight = ?`)
+ if err != nil {
+ return err
+ }
+
+ scanner := bufio.NewScanner(os.Stdin)
+ for scanner.Scan() {
+ fields := strings.Split(scanner.Text(), "\t")
+ if len(fields) != 3 {
+ return errors.New("invalid input format")
+ }
+
+ sha1, tag := fields[0], fields[1]
+ weight, err := strconv.ParseFloat(fields[2], 64)
+ if err != nil {
+ return err
+ }
+
+ if _, err := tx.Exec(
+ `INSERT OR IGNORE INTO tag(space, name) VALUES (?, ?);`,
+ spaceID, tag); err != nil {
+ return nil
+ }
+ if _, err := stmt.Exec(sha1, spaceID, tag, weight, weight); err != nil {
+ log.Printf("%s: %s\n", sha1, err)
+ }
+ }
+ if err := scanner.Err(); err != nil {
+ return err
+ }
+ return tx.Commit()
+}
+
+// --- Check -------------------------------------------------------------------
+
+func isValidSHA1(hash string) bool {
+ if len(hash) != sha1.Size*2 || strings.ToLower(hash) != hash {
+ return false
+ }
+ if _, err := hex.DecodeString(hash); err != nil {
+ return false
+ }
+ return true
+}
+
+func hashesToFileListing(root, suffix string, hashes []string) []string {
+ // Note that we're semi-duplicating {image,thumb}Path().
+ paths := []string{root}
+ for _, hash := range hashes {
+ dir := filepath.Join(root, hash[:2])
+ paths = append(paths, dir, filepath.Join(dir, hash+suffix))
+ }
+ slices.Sort(paths)
+ return slices.Compact(paths)
+}
+
+func collectFileListing(root string) (paths []string, err error) {
+ err = filepath.WalkDir(root,
+ func(path string, d fs.DirEntry, err error) error {
+ paths = append(paths, path)
+ return err
+ })
+
+ // Even though it should already be sorted somehow.
+ slices.Sort(paths)
+ return
+}
+
+func checkFiles(root, suffix string, hashes []string) (bool, []string, error) {
+ db := hashesToFileListing(root, suffix, hashes)
+ fs, err := collectFileListing(root)
+ if err != nil {
+ return false, nil, err
+ }
+
+ iDB, iFS, ok, intersection := 0, 0, true, []string{}
+ for iDB < len(db) && iFS < len(fs) {
+ if db[iDB] == fs[iFS] {
+ intersection = append(intersection, db[iDB])
+ iDB++
+ iFS++
+ } else if db[iDB] < fs[iFS] {
+ ok = false
+ fmt.Printf("only in DB: %s\n", db[iDB])
+ iDB++
+ } else {
+ ok = false
+ fmt.Printf("only in FS: %s\n", fs[iFS])
+ iFS++
+ }
+ }
+ for _, path := range db[iDB:] {
+ ok = false
+ fmt.Printf("only in DB: %s\n", path)
+ }
+ for _, path := range fs[iFS:] {
+ ok = false
+ fmt.Printf("only in FS: %s\n", path)
+ }
+ return ok, intersection, nil
+}
+
+func checkHash(path string) (message string, err error) {
+ f, err := os.Open(path)
+ if err != nil {
+ return err.Error(), nil
+ }
+ defer f.Close()
+
+ // We get 2 levels of parent directories in here, just filter them out.
+ if fi, err := f.Stat(); err != nil {
+ return err.Error(), nil
+ } else if fi.IsDir() {
+ return "", nil
+ }
+
+ hash := sha1.New()
+ _, err = io.CopyBuffer(hash, f, make([]byte, 65536))
+ if err != nil {
+ return err.Error(), nil
+ }
+
+ sha1 := hex.EncodeToString(hash.Sum(nil))
+ if sha1 != filepath.Base(path) {
+ return fmt.Sprintf("mismatch, found %s", sha1), nil
+ }
+ return "", nil
+}
+
+func checkHashes(paths []string) (bool, error) {
+ log.Println("checking image hashes")
+ var failed atomic.Bool
+ err := parallelize(paths, func(path string) (string, error) {
+ message, err := checkHash(path)
+ if message != "" {
+ failed.Store(true)
+ }
+ return message, err
+ })
+ return !failed.Load(), err
+}
+
+// cmdCheck carries out various database consistency checks.
+func cmdCheck(fs *flag.FlagSet, args []string) error {
+ full := fs.Bool("full", false, "verify image hashes")
+ if err := fs.Parse(args); err != nil {
+ return err
+ }
+ if fs.NArg() != 1 {
+ return errWrongUsage
+ }
+ if err := openDB(fs.Arg(0)); err != nil {
+ return err
+ }
+
+ // Check if hashes are in the right format.
+ log.Println("checking image hashes")
+
+ allSHA1, err := dbCollectStrings(`SELECT sha1 FROM image`)
+ if err != nil {
+ return err
+ }
+
+ ok := true
+ for _, hash := range allSHA1 {
+ if !isValidSHA1(hash) {
+ ok = false
+ fmt.Printf("invalid image SHA1: %s\n", hash)
+ }
+ }
+
+ // This is, rather obviously, just a strict subset.
+ // Although it doesn't run in the same transaction.
+ thumbSHA1, err := dbCollectStrings(`SELECT sha1 FROM image
+ WHERE thumbw IS NOT NULL OR thumbh IS NOT NULL`)
+ if err != nil {
+ return err
+ }
+
+ // This somewhat duplicates {image,thumb}Path().
+ log.Println("checking SQL against filesystem")
+ okImages, intersection, err := checkFiles(
+ filepath.Join(galleryDirectory, nameOfImageRoot), "", allSHA1)
+ if err != nil {
+ return err
+ }
+
+ okThumbs, _, err := checkFiles(
+ filepath.Join(galleryDirectory, nameOfThumbRoot), ".webp", thumbSHA1)
+ if err != nil {
+ return err
+ }
+ if !okImages || !okThumbs {
+ ok = false
+ }
+
+ log.Println("checking for dead symlinks")
+ for _, path := range intersection {
+ if _, err := os.Stat(path); err != nil {
+ ok = false
+ fmt.Printf("%s: %s\n", path, err)
+ }
+ }
+
+ if *full {
+ if ok2, err := checkHashes(intersection); err != nil {
+ return err
+ } else if !ok2 {
+ ok = false
+ }
+ }
+
+ if !ok {
+ return errors.New("detected inconsistencies")
+ }
+ return nil
+}
+
+// --- Thumbnailing ------------------------------------------------------------
+
+func identifyThumbnail(path string) (w, h int, err error) {
+ f, err := os.Open(path)
+ if err != nil {
+ return
+ }
+ defer f.Close()
+
+ config, err := webp.DecodeConfig(f)
+ if err != nil {
+ return
+ }
+ return config.Width, config.Height, nil
+}
+
+func makeThumbnail(load bool, pathImage, pathThumb string) (
+ w, h int, err error) {
+ if load {
+ if w, h, err = identifyThumbnail(pathThumb); err == nil {
+ return
+ }
+ }
+
+ thumbDirname, _ := filepath.Split(pathThumb)
+ if err := os.MkdirAll(thumbDirname, 0755); err != nil {
+ return 0, 0, err
+ }
+
+ // Create a normalized thumbnail. Since we don't particularly need
+ // any complex processing, such as surrounding of metadata,
+ // simply push it through ImageMagick.
+ //
+ // - http://www.ericbrasseur.org/gamma.html
+ // - https://www.imagemagick.org/Usage/thumbnails/
+ // - https://imagemagick.org/script/command-line-options.php#layers
+ //
+ // "info:" output is written for each frame, which is why we delete
+ // all of them but the first one beforehands.
+ //
+ // TODO: See if we can optimize resulting WebP animations.
+ // (Do -layers optimize* apply to this format at all?)
+ cmd := exec.Command("magick", "-limit", "thread", "1", pathImage,
+ "-coalesce", "-colorspace", "RGB", "-auto-orient", "-strip",
+ "-resize", "256x128>", "-colorspace", "sRGB",
+ "-format", "%w %h", "+write", pathThumb, "-delete", "1--1", "info:")
+
+ out, err := cmd.Output()
+ if err != nil {
+ return 0, 0, err
+ }
+
+ _, err = fmt.Fscanf(bytes.NewReader(out), "%d %d", &w, &h)
+ return w, h, err
+}
+
+// cmdThumbnail generates missing thumbnails, in parallel.
+func cmdThumbnail(fs *flag.FlagSet, args []string) error {
+ load := fs.Bool("load", false, "try to load existing thumbnail files")
+ if err := fs.Parse(args); err != nil {
+ return err
+ }
+ if fs.NArg() < 1 {
+ return errWrongUsage
+ }
+ if err := openDB(fs.Arg(0)); err != nil {
+ return err
+ }
+
+ hexSHA1 := fs.Args()[1:]
+ if len(hexSHA1) == 0 {
+ // Get all unique images in the database with no thumbnail.
+ var err error
+ hexSHA1, err = dbCollectStrings(`SELECT sha1 FROM image
+ WHERE thumbw IS NULL OR thumbh IS NULL`)
+ if err != nil {
+ return err
+ }
+ }
+
+ stmt, err := db.Prepare(
+ `UPDATE image SET thumbw = ?, thumbh = ? WHERE sha1 = ?`)
+ if err != nil {
+ return err
+ }
+ defer stmt.Close()
+
+ var mu sync.Mutex
+ return parallelize(hexSHA1, func(sha1 string) (message string, err error) {
+ pathImage := imagePath(sha1)
+ pathThumb := thumbPath(sha1)
+ w, h, err := makeThumbnail(*load, pathImage, pathThumb)
+ if err != nil {
+ if ee, ok := err.(*exec.ExitError); ok {
+ return string(ee.Stderr), nil
+ }
+ return "", err
+ }
+
+ mu.Lock()
+ defer mu.Unlock()
+ _, err = stmt.Exec(w, h, sha1)
+ return "", err
+ })
+}
+
+// --- Perceptual hash ---------------------------------------------------------
+
+type linearImage struct {
+ img image.Image
+}
+
+func newLinearImage(img image.Image) *linearImage {
+ return &linearImage{img: img}
+}
+
+func (l *linearImage) ColorModel() color.Model { return l.img.ColorModel() }
+func (l *linearImage) Bounds() image.Rectangle { return l.img.Bounds() }
+
+func unSRGB(c uint32) uint8 {
+ n := float64(c) / 0xffff
+ if n <= 0.04045 {
+ return uint8(n * (255.0 / 12.92))
+ }
+ return uint8(math.Pow((n+0.055)/(1.055), 2.4) * 255.0)
+}
+
+func (l *linearImage) At(x, y int) color.Color {
+ r, g, b, a := l.img.At(x, y).RGBA()
+ return color.RGBA{
+ R: unSRGB(r), G: unSRGB(g), B: unSRGB(b), A: uint8(a >> 8)}
+}
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+// isWebPAnimation returns whether the given ReadSeeker starts a WebP animation.
+// See https://developers.google.com/speed/webp/docs/riff_container
+func isWebPAnimation(rs io.ReadSeeker) (bool, error) {
+ b := make([]byte, 21)
+ if _, err := rs.Read(b); err != nil {
+ return false, err
+ }
+ if _, err := rs.Seek(0, io.SeekStart); err != nil {
+ return false, err
+ }
+
+ return bytes.Equal(b[:4], []byte("RIFF")) &&
+ bytes.Equal(b[8:16], []byte("WEBPVP8X")) &&
+ b[20]&0b00000010 != 0, nil
+}
+
+var errIsAnimation = errors.New("cannot perceptually hash animations")
+
+func dhashWebP(rs io.ReadSeeker) (uint64, error) {
+ if a, err := isWebPAnimation(rs); err != nil {
+ return 0, err
+ } else if a {
+ return 0, errIsAnimation
+ }
+
+ // Doing this entire thing in Go is SLOW, but convenient.
+ source, err := webp.Decode(rs)
+ if err != nil {
+ return 0, err
+ }
+
+ var (
+ linear = newLinearImage(source)
+ resized = image.NewNRGBA64(image.Rect(0, 0, 9, 8))
+ )
+ draw.CatmullRom.Scale(resized, resized.Bounds(),
+ linear, linear.Bounds(), draw.Src, nil)
+
+ var hash uint64
+ for y := 0; y < 8; y++ {
+ var grey [9]float32
+ for x := 0; x < 9; x++ {
+ rgba := resized.NRGBA64At(x, y)
+ grey[x] = 0.2126*float32(rgba.R) +
+ 0.7152*float32(rgba.G) +
+ 0.0722*float32(rgba.B)
+ }
+
+ var row uint64
+ if grey[0] < grey[1] {
+ row |= 1 << 7
+ }
+ if grey[1] < grey[2] {
+ row |= 1 << 6
+ }
+ if grey[2] < grey[3] {
+ row |= 1 << 5
+ }
+ if grey[3] < grey[4] {
+ row |= 1 << 4
+ }
+ if grey[4] < grey[5] {
+ row |= 1 << 3
+ }
+ if grey[5] < grey[6] {
+ row |= 1 << 2
+ }
+ if grey[6] < grey[7] {
+ row |= 1 << 1
+ }
+ if grey[7] < grey[8] {
+ row |= 1 << 0
+ }
+ hash = hash<<8 | row
+ }
+ return hash, nil
+}
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+func makeDhash(sha1 string) (uint64, error) {
+ pathThumb := thumbPath(sha1)
+ f, err := os.Open(pathThumb)
+ if err != nil {
+ return 0, err
+ }
+ defer f.Close()
+ return dhashWebP(f)
+}
+
+// cmdDhash computes perceptual hashes from thumbnails.
+func cmdDhash(fs *flag.FlagSet, args []string) error {
+ if err := fs.Parse(args); err != nil {
+ return err
+ }
+ if fs.NArg() < 1 {
+ return errWrongUsage
+ }
+ if err := openDB(fs.Arg(0)); err != nil {
+ return err
+ }
+
+ hexSHA1 := fs.Args()[1:]
+ if len(hexSHA1) == 0 {
+ var err error
+ hexSHA1, err = dbCollectStrings(`SELECT sha1 FROM image
+ WHERE thumbw IS NOT NULL AND thumbh IS NOT NULL AND dhash IS NULL`)
+ if err != nil {
+ return err
+ }
+ }
+
+ stmt, err := db.Prepare(`UPDATE image SET dhash = ? WHERE sha1 = ?`)
+ if err != nil {
+ return err
+ }
+ defer stmt.Close()
+
+ var mu sync.Mutex
+ return parallelize(hexSHA1, func(sha1 string) (message string, err error) {
+ hash, err := makeDhash(sha1)
+ if errors.Is(err, errIsAnimation) {
+ // Ignoring this common condition.
+ return "", nil
+ } else if err != nil {
+ return err.Error(), nil
+ }
+
+ mu.Lock()
+ defer mu.Unlock()
+ _, err = stmt.Exec(int64(hash), sha1)
+ return "", err
+ })
+}
+
+// --- Main --------------------------------------------------------------------
+
+var errWrongUsage = errors.New("wrong usage")
+
+var commands = map[string]struct {
+ handler func(*flag.FlagSet, []string) error
+ usage string
+ function string
+}{
+ "init": {cmdInit, "GD", "Initialize a database."},
+ "web": {cmdWeb, "GD ADDRESS", "Launch a web interface."},
+ "tag": {cmdTag, "GD SPACE [DESCRIPTION]", "Import tags."},
+ "sync": {cmdSync, "GD ROOT...", "Synchronise with the filesystem."},
+ "remove": {cmdRemove, "GD PATH...", "Remove database subtrees."},
+ "check": {cmdCheck, "GD", "Run consistency checks."},
+ "thumbnail": {cmdThumbnail, "GD [SHA1...]", "Generate thumbnails."},
+ "dhash": {cmdDhash, "GD [SHA1...]", "Compute perceptual hashes."},
+}
+
+func usage() {
+ f := flag.CommandLine.Output()
+ fmt.Fprintf(f, "Usage: %s COMMAND [ARG...]\n", os.Args[0])
+ flag.PrintDefaults()
+
+ // The alphabetic ordering is unfortunate, but tolerable.
+ keys := []string{}
+ for key := range commands {
+ keys = append(keys, key)
+ }
+ sort.Strings(keys)
+
+ fmt.Fprintf(f, "\nCommands:\n")
+ for _, key := range keys {
+ fmt.Fprintf(f, " %s [OPTION...] %s\n \t%s\n",
+ key, commands[key].usage, commands[key].function)
+ }
+}
+
+func main() {
+ // This implements the -h switch for us by default.
+ // The rest of the handling here closely follows what flag does internally.
+ flag.Usage = usage
+ flag.Parse()
+ if flag.NArg() < 1 {
+ flag.Usage()
+ os.Exit(2)
+ }
+
+ cmd, ok := commands[flag.Arg(0)]
+ if !ok {
+ fmt.Fprintf(flag.CommandLine.Output(),
+ "unknown command: %s\n", flag.Arg(0))
+ flag.Usage()
+ os.Exit(2)
+ }
+
+ fs := flag.NewFlagSet(flag.Arg(0), flag.ExitOnError)
+ fs.Usage = func() {
+ fmt.Fprintf(fs.Output(),
+ "Usage: %s [OPTION...] %s\n%s\n",
+ fs.Name(), cmd.usage, cmd.function)
+ fs.PrintDefaults()
+ }
+
+ taskSemaphore = newSemaphore(runtime.NumCPU())
+ err := cmd.handler(fs, flag.Args()[1:])
+
+ // Note that the database object has a closing finalizer,
+ // we just additionally print any errors coming from there.
+ if db != nil {
+ if err := db.Close(); err != nil {
+ log.Println(err)
+ }
+ }
+
+ if errors.Is(err, errWrongUsage) {
+ fs.Usage()
+ os.Exit(2)
+ } else if err != nil {
+ log.Fatalln(err)
+ }
+}
generated by cgit v1.2.3-70-g09d2 (git 2.47.0) at 2024-11-07 07:30:15 +0000