WIP: Add an optional deployment stageHEADmaster

1 files changed, 196 insertions, 40 deletions

diff --git a/acid.go b/acid.go
index f938d02..a9589b8 100644
--- a/acid.go
+++ b/acid.go
@@ -21,6 +21,7 @@ import (
 	"os"
 	"os/exec"
 	"os/signal"
+	"path/filepath"
 	"sort"
 	"strconv"
 	"strings"
@@ -30,6 +31,7 @@ import (
 	"time"
 
 	_ "github.com/mattn/go-sqlite3"
+	"github.com/pkg/sftp"
 	"golang.org/x/crypto/ssh"
 	"gopkg.in/yaml.v3"
 )
@@ -96,6 +98,7 @@ func (cf *ConfigProject) AutomaticRunners() (runners []string) {
 type ConfigProjectRunner struct {
 	Setup   string `yaml:"setup"`   // project setup script (SSH)
 	Build   string `yaml:"build"`   // project build script (SSH)
+	Deploy  string `yaml:"deploy"`  // project deploy script (local)
 	Timeout string `yaml:"timeout"` // timeout duration
 }
 
@@ -153,7 +156,8 @@ func giteaNewRequest(ctx context.Context, method, path string, body io.Reader) (
 func getTasks(ctx context.Context, query string, args ...any) ([]Task, error) {
 	rows, err := gDB.QueryContext(ctx, `
 		SELECT id, owner, repo, hash, runner,
-			state, detail, notified, runlog, tasklog FROM task `+query, args...)
+			state, detail, notified,
+			runlog, tasklog, deploylog FROM task `+query, args...)
 	if err != nil {
 		return nil, err
 	}
@@ -163,7 +167,8 @@ func getTasks(ctx context.Context, query string, args ...any) ([]Task, error) {
 	for rows.Next() {
 		var t Task
 		err := rows.Scan(&t.ID, &t.Owner, &t.Repo, &t.Hash, &t.Runner,
-			&t.State, &t.Detail, &t.Notified, &t.RunLog, &t.TaskLog)
+			&t.State, &t.Detail, &t.Notified,
+			&t.RunLog, &t.TaskLog, &t.DeployLog)
 		if err != nil {
 			return nil, err
 		}
@@ -259,6 +264,10 @@ var templateTask = template.Must(template.New("tasks").Parse(`
 <h2>Task log</h2>
 <pre>{{printf "%s" .TaskLog}}</pre>
 {{end}}
+{{if .DeployLog}}
+<h2>Deploy log</h2>
+<pre>{{printf "%s" .DeployLog}}</pre>
+{{end}}
 </table>
 </body>
 </html>
@@ -302,9 +311,12 @@ func handleTask(w http.ResponseWriter, r *http.Request) {
 		defer rt.RunLog.mu.Unlock()
 		rt.TaskLog.mu.Lock()
 		defer rt.TaskLog.mu.Unlock()
+		rt.DeployLog.mu.Lock()
+		defer rt.DeployLog.mu.Unlock()
 
 		task.RunLog = rt.RunLog.b
 		task.TaskLog = rt.TaskLog.b
+		task.DeployLog = rt.DeployLog.b
 	}()
 
 	if err := templateTask.Execute(w, &task); err != nil {
@@ -823,30 +835,32 @@ type RunningTask struct {
 	Runner        ConfigRunner
 	ProjectRunner ConfigProjectRunner
 
-	RunLog  terminalWriter
-	TaskLog terminalWriter
+	RunLog    terminalWriter
+	TaskLog   terminalWriter
+	DeployLog terminalWriter
 }
 
 func executorUpdate(rt *RunningTask) error {
-	rt.RunLog.mu.Lock()
-	defer rt.RunLog.mu.Unlock()
-	rt.DB.RunLog = bytes.Clone(rt.RunLog.b)
-	if rt.DB.RunLog == nil {
-		rt.DB.RunLog = []byte{}
-	}
-
-	rt.TaskLog.mu.Lock()
-	defer rt.TaskLog.mu.Unlock()
-	rt.DB.TaskLog = bytes.Clone(rt.TaskLog.b)
-	if rt.DB.TaskLog == nil {
-		rt.DB.TaskLog = []byte{}
+	for _, i := range []struct {
+		tw  *terminalWriter
+		log *[]byte
+	}{
+		{&rt.RunLog, &rt.DB.RunLog},
+		{&rt.TaskLog, &rt.DB.TaskLog},
+		{&rt.DeployLog, &rt.DB.DeployLog},
+	} {
+		i.tw.mu.Lock()
+		defer i.tw.mu.Unlock()
+		if *i.log = bytes.Clone(i.tw.b); *i.log == nil {
+			*i.log = []byte{}
+		}
 	}
 
 	_, err := gDB.ExecContext(context.Background(), `UPDATE task
-		SET state = ?, detail = ?, notified = ?, runlog = ?, tasklog = ?
-		WHERE id = ?`,
-		rt.DB.State, rt.DB.Detail, rt.DB.Notified, rt.DB.RunLog, rt.DB.TaskLog,
-		rt.DB.ID)
+		SET state = ?, detail = ?, notified = ?,
+		runlog = ?, tasklog = ?, deploylog = ? WHERE id = ?`,
+		rt.DB.State, rt.DB.Detail, rt.DB.Notified,
+		rt.DB.RunLog, rt.DB.TaskLog, rt.DB.DeployLog, rt.DB.ID)
 	if err == nil {
 		notifierAwaken()
 	}
@@ -899,6 +913,81 @@ func executorConnect(
 	}
 }
 
+func executorDownloadFile(sc *sftp.Client, remotePath, localPath string) error {
+	src, err := sc.Open(remotePath)
+	if err != nil {
+		return fmt.Errorf("failed to open remote file %s: %w", remotePath, err)
+	}
+	defer src.Close()
+
+	dst, err := os.Create(localPath)
+	if err != nil {
+		return fmt.Errorf("failed to create local file %s: %w", localPath, err)
+	}
+	defer dst.Close()
+
+	if _, err = io.Copy(dst, src); err != nil {
+		return fmt.Errorf("failed to copy file from remote %s to local %s: %w",
+			remotePath, localPath, err)
+	}
+	return nil
+}
+
+func executorDownload(client *ssh.Client, remoteRoot, localRoot string) error {
+	sc, err := sftp.NewClient(client)
+	if err != nil {
+		return err
+	}
+	defer sc.Close()
+
+	// TODO(p): Is it okay if the remote root is a relative path?
+	// In particular in relation to the filepath.Rel below.
+	walker := sc.Walk(remoteRoot)
+	for walker.Step() {
+		if walker.Err() != nil {
+			return walker.Err()
+		}
+
+		relativePath, err := filepath.Rel(remoteRoot, walker.Path())
+		if err != nil {
+			return err
+		}
+
+		localPath := filepath.Join(localRoot, relativePath)
+		if walker.Stat().IsDir() {
+			if err = os.MkdirAll(localPath, os.ModePerm); err != nil {
+				return fmt.Errorf("failed to create local directory %s: %w",
+					localPath, err)
+			}
+		} else {
+			err = executorDownloadFile(sc, walker.Path(), localPath)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func executorDeploy(
+	ctx context.Context, client *ssh.Client, rt *RunningTask) error {
+	// TODO(p): Ensure that this directory exists with the right rights,
+	// and that it is otherwise empty.
+	dir := "/var/tmp/acid-deploy"
+
+	// TODO(p): What is the remote working directory?
+	err := executorDownload(client, "acid-deploy", dir)
+	if err != nil {
+		return err
+	}
+
+	// TODO(p): Run it locally, with that directory as the working directory.
+	_ = rt.ProjectRunner.Setup
+
+	// TODO(p): Also don't forget to select on ctx.Done().
+	return nil
+}
+
 func executorRunTask(ctx context.Context, task Task) error {
 	rt := &RunningTask{DB: task}
 
@@ -1069,6 +1158,13 @@ func executorRunTask(ctx context.Context, task Task) error {
 	}
 	defer client.Close()
 
+	// This is so that it doesn't stay hanging, in particular within
+	// the sftp package, which uses context.Background() everywhere.
+	go func() {
+		<-ctxRunner.Done()
+		client.Close()
+	}()
+
 	session, err := client.NewSession()
 	if err != nil {
 		fmt.Fprintf(&rt.TaskLog, "%s\n", err)
@@ -1117,6 +1213,12 @@ func executorRunTask(ctx context.Context, task Task) error {
 		// in particular when it's on the same machine.
 	}
 
+	if err == nil && rt.ProjectRunner.Deploy != "" {
+		// TODO(p): Distinguish "Deployment failed" from "Scripts failed".
+		// The err also needs to go to the DeployLog.
+		err = executorDeploy(ctxRunner, client, rt)
+	}
+
 	gRunningMutex.Lock()
 	defer gRunningMutex.Unlock()
 
@@ -1208,11 +1310,12 @@ type Task struct {
 	Hash   string
 	Runner string
 
-	State    taskState
-	Detail   string
-	Notified int64
-	RunLog   []byte
-	TaskLog  []byte
+	State     taskState
+	Detail    string
+	Notified  int64
+	RunLog    []byte
+	TaskLog   []byte
+	DeployLog []byte
 }
 
 func (t *Task) FullName() string { return t.Owner + "/" + t.Repo }
@@ -1242,26 +1345,46 @@ func (t *Task) CloneURL() string {
 	return fmt.Sprintf("%s/%s/%s.git", gConfig.Gitea, t.Owner, t.Repo)
 }
 
-const schemaSQL = `
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+const initializeSQL = `
+PRAGMA schema.application_id = 0x61636964;   -- "acid" in big endian
+
 CREATE TABLE IF NOT EXISTS task(
-	id       INTEGER NOT NULL,  -- unique ID
+	id        INTEGER NOT NULL,  -- unique ID
 
-	owner    TEXT NOT NULL,     -- Gitea username
-	repo     TEXT NOT NULL,     -- Gitea repository name
-	hash     TEXT NOT NULL,     -- commit hash
-	runner   TEXT NOT NULL,     -- the runner to use
+	owner     TEXT NOT NULL,     -- Gitea username
+	repo      TEXT NOT NULL,     -- Gitea repository name
+	hash      TEXT NOT NULL,     -- commit hash
+	runner    TEXT NOT NULL,     -- the runner to use
 
-	state    INTEGER NOT NULL DEFAULT 0,    -- task state
-	detail   TEXT    NOT NULL DEFAULT '',   -- task state detail
-	notified INTEGER NOT NULL DEFAULT 0,    -- Gitea knows the state
-	runlog   BLOB    NOT NULL DEFAULT x'',  -- combined task runner output
-	tasklog  BLOB    NOT NULL DEFAULT x'',  -- combined task SSH output
+	state     INTEGER NOT NULL DEFAULT 0,    -- task state
+	detail    TEXT    NOT NULL DEFAULT '',   -- task state detail
+	notified  INTEGER NOT NULL DEFAULT 0,    -- Gitea knows the state
+	runlog    BLOB    NOT NULL DEFAULT x'',  -- combined task runner output
+	tasklog   BLOB    NOT NULL DEFAULT x'',  -- combined task SSH output
+	deploylog BLOB    NOT NULL DEFAULT x'',  -- deployment output
 
 	PRIMARY KEY (id)
 ) STRICT;
 `
 
-func openDB(path string) error {
+func dbEnsureColumn(tx *sql.Tx, table, column, definition string) error {
+	var count int64
+	if err := tx.QueryRow(
+		`SELECT count(*) FROM pragma_table_info(?) WHERE name = ?`,
+		table, column).Scan(&count); err != nil {
+		return err
+	} else if count == 1 {
+		return nil
+	}
+
+	_, err := tx.Exec(
+		`ALTER TABLE ` + table + ` ADD COLUMN ` + column + ` ` + definition)
+	return err
+}
+
+func dbOpen(path string) error {
 	var err error
 	gDB, err = sql.Open("sqlite3",
 		"file:"+path+"?_foreign_keys=1&_busy_timeout=1000")
@@ -1269,10 +1392,43 @@ func openDB(path string) error {
 		return err
 	}
 
-	_, err = gDB.Exec(schemaSQL)
-	return err
+	tx, err := gDB.BeginTx(context.Background(), nil)
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+
+	var version int64
+	if err = tx.QueryRow(
+		`PRAGMA schema.user_version`).Scan(&version); err != nil {
+		return err
+	}
+
+	switch version {
+	case 0:
+		if _, err = tx.Exec(initializeSQL); err != nil {
+			return err
+		}
+
+		// We had not initially set a database schema version,
+		// so we're stuck checking this column even on new databases.
+		if err = dbEnsureColumn(tx,
+			`task`, `deploylog`, `BLOB NOT NULL DEFAULT x''`); err != nil {
+			return err
+		}
+		break
+	case 1:
+		// The next migration goes here, remember to increment the number below.
+	}
+
+	if _, err = tx.Exec("PRAGMA schema.user_version = ?", 1); err != nil {
+		return err
+	}
+	return tx.Commit()
 }
 
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
 // callRPC forwards command line commands to a running server.
 func callRPC(args []string) error {
 	body, err := json.Marshal(args)
@@ -1334,7 +1490,7 @@ func main() {
 		return
 	}
 
-	if err := openDB(gConfig.DB); err != nil {
+	if err := dbOpen(gConfig.DB); err != nil {
 		log.Fatalln(err)
 	}
 	defer gDB.Close()