diff options
Diffstat (limited to 'acid.go')
| -rw-r--r-- | acid.go | 236 |
1 files changed, 196 insertions, 40 deletions
@@ -21,6 +21,7 @@ import ( "os" "os/exec" "os/signal" + "path/filepath" "sort" "strconv" "strings" @@ -30,6 +31,7 @@ import ( "time" _ "github.com/mattn/go-sqlite3" + "github.com/pkg/sftp" "golang.org/x/crypto/ssh" "gopkg.in/yaml.v3" ) @@ -96,6 +98,7 @@ func (cf *ConfigProject) AutomaticRunners() (runners []string) { type ConfigProjectRunner struct { Setup string `yaml:"setup"` // project setup script (SSH) Build string `yaml:"build"` // project build script (SSH) + Deploy string `yaml:"deploy"` // project deploy script (local) Timeout string `yaml:"timeout"` // timeout duration } @@ -153,7 +156,8 @@ func giteaNewRequest(ctx context.Context, method, path string, body io.Reader) ( func getTasks(ctx context.Context, query string, args ...any) ([]Task, error) { rows, err := gDB.QueryContext(ctx, ` SELECT id, owner, repo, hash, runner, - state, detail, notified, runlog, tasklog FROM task `+query, args...) + state, detail, notified, + runlog, tasklog, deploylog FROM task `+query, args...) if err != nil { return nil, err } @@ -163,7 +167,8 @@ func getTasks(ctx context.Context, query string, args ...any) ([]Task, error) { for rows.Next() { var t Task err := rows.Scan(&t.ID, &t.Owner, &t.Repo, &t.Hash, &t.Runner, - &t.State, &t.Detail, &t.Notified, &t.RunLog, &t.TaskLog) + &t.State, &t.Detail, &t.Notified, + &t.RunLog, &t.TaskLog, &t.DeployLog) if err != nil { return nil, err } @@ -259,6 +264,10 @@ var templateTask = template.Must(template.New("tasks").Parse(` <h2>Task log</h2> <pre>{{printf "%s" .TaskLog}}</pre> {{end}} +{{if .DeployLog}} +<h2>Deploy log</h2> +<pre>{{printf "%s" .DeployLog}}</pre> +{{end}} </table> </body> </html> @@ -302,9 +311,12 @@ func handleTask(w http.ResponseWriter, r *http.Request) { defer rt.RunLog.mu.Unlock() rt.TaskLog.mu.Lock() defer rt.TaskLog.mu.Unlock() + rt.DeployLog.mu.Lock() + defer rt.DeployLog.mu.Unlock() task.RunLog = rt.RunLog.b task.TaskLog = rt.TaskLog.b + task.DeployLog = rt.DeployLog.b }() if err := templateTask.Execute(w, &task); err != nil { @@ -823,30 +835,32 @@ type RunningTask struct { Runner ConfigRunner ProjectRunner ConfigProjectRunner - RunLog terminalWriter - TaskLog terminalWriter + RunLog terminalWriter + TaskLog terminalWriter + DeployLog terminalWriter } func executorUpdate(rt *RunningTask) error { - rt.RunLog.mu.Lock() - defer rt.RunLog.mu.Unlock() - rt.DB.RunLog = bytes.Clone(rt.RunLog.b) - if rt.DB.RunLog == nil { - rt.DB.RunLog = []byte{} - } - - rt.TaskLog.mu.Lock() - defer rt.TaskLog.mu.Unlock() - rt.DB.TaskLog = bytes.Clone(rt.TaskLog.b) - if rt.DB.TaskLog == nil { - rt.DB.TaskLog = []byte{} + for _, i := range []struct { + tw *terminalWriter + log *[]byte + }{ + {&rt.RunLog, &rt.DB.RunLog}, + {&rt.TaskLog, &rt.DB.TaskLog}, + {&rt.DeployLog, &rt.DB.DeployLog}, + } { + i.tw.mu.Lock() + defer i.tw.mu.Unlock() + if *i.log = bytes.Clone(i.tw.b); *i.log == nil { + *i.log = []byte{} + } } _, err := gDB.ExecContext(context.Background(), `UPDATE task - SET state = ?, detail = ?, notified = ?, runlog = ?, tasklog = ? - WHERE id = ?`, - rt.DB.State, rt.DB.Detail, rt.DB.Notified, rt.DB.RunLog, rt.DB.TaskLog, - rt.DB.ID) + SET state = ?, detail = ?, notified = ?, + runlog = ?, tasklog = ?, deploylog = ? WHERE id = ?`, + rt.DB.State, rt.DB.Detail, rt.DB.Notified, + rt.DB.RunLog, rt.DB.TaskLog, rt.DB.DeployLog, rt.DB.ID) if err == nil { notifierAwaken() } @@ -899,6 +913,81 @@ func executorConnect( } } +func executorDownloadFile(sc *sftp.Client, remotePath, localPath string) error { + src, err := sc.Open(remotePath) + if err != nil { + return fmt.Errorf("failed to open remote file %s: %w", remotePath, err) + } + defer src.Close() + + dst, err := os.Create(localPath) + if err != nil { + return fmt.Errorf("failed to create local file %s: %w", localPath, err) + } + defer dst.Close() + + if _, err = io.Copy(dst, src); err != nil { + return fmt.Errorf("failed to copy file from remote %s to local %s: %w", + remotePath, localPath, err) + } + return nil +} + +func executorDownload(client *ssh.Client, remoteRoot, localRoot string) error { + sc, err := sftp.NewClient(client) + if err != nil { + return err + } + defer sc.Close() + + // TODO(p): Is it okay if the remote root is a relative path? + // In particular in relation to the filepath.Rel below. + walker := sc.Walk(remoteRoot) + for walker.Step() { + if walker.Err() != nil { + return walker.Err() + } + + relativePath, err := filepath.Rel(remoteRoot, walker.Path()) + if err != nil { + return err + } + + localPath := filepath.Join(localRoot, relativePath) + if walker.Stat().IsDir() { + if err = os.MkdirAll(localPath, os.ModePerm); err != nil { + return fmt.Errorf("failed to create local directory %s: %w", + localPath, err) + } + } else { + err = executorDownloadFile(sc, walker.Path(), localPath) + if err != nil { + return err + } + } + } + return nil +} + +func executorDeploy( + ctx context.Context, client *ssh.Client, rt *RunningTask) error { + // TODO(p): Ensure that this directory exists with the right rights, + // and that it is otherwise empty. + dir := "/var/tmp/acid-deploy" + + // TODO(p): What is the remote working directory? + err := executorDownload(client, "acid-deploy", dir) + if err != nil { + return err + } + + // TODO(p): Run it locally, with that directory as the working directory. + _ = rt.ProjectRunner.Setup + + // TODO(p): Also don't forget to select on ctx.Done(). + return nil +} + func executorRunTask(ctx context.Context, task Task) error { rt := &RunningTask{DB: task} @@ -1069,6 +1158,13 @@ func executorRunTask(ctx context.Context, task Task) error { } defer client.Close() + // This is so that it doesn't stay hanging, in particular within + // the sftp package, which uses context.Background() everywhere. + go func() { + <-ctxRunner.Done() + client.Close() + }() + session, err := client.NewSession() if err != nil { fmt.Fprintf(&rt.TaskLog, "%s\n", err) @@ -1117,6 +1213,12 @@ func executorRunTask(ctx context.Context, task Task) error { // in particular when it's on the same machine. } + if err == nil && rt.ProjectRunner.Deploy != "" { + // TODO(p): Distinguish "Deployment failed" from "Scripts failed". + // The err also needs to go to the DeployLog. + err = executorDeploy(ctxRunner, client, rt) + } + gRunningMutex.Lock() defer gRunningMutex.Unlock() @@ -1208,11 +1310,12 @@ type Task struct { Hash string Runner string - State taskState - Detail string - Notified int64 - RunLog []byte - TaskLog []byte + State taskState + Detail string + Notified int64 + RunLog []byte + TaskLog []byte + DeployLog []byte } func (t *Task) FullName() string { return t.Owner + "/" + t.Repo } @@ -1242,26 +1345,46 @@ func (t *Task) CloneURL() string { return fmt.Sprintf("%s/%s/%s.git", gConfig.Gitea, t.Owner, t.Repo) } -const schemaSQL = ` +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +const initializeSQL = ` +PRAGMA schema.application_id = 0x61636964; -- "acid" in big endian + CREATE TABLE IF NOT EXISTS task( - id INTEGER NOT NULL, -- unique ID + id INTEGER NOT NULL, -- unique ID - owner TEXT NOT NULL, -- Gitea username - repo TEXT NOT NULL, -- Gitea repository name - hash TEXT NOT NULL, -- commit hash - runner TEXT NOT NULL, -- the runner to use + owner TEXT NOT NULL, -- Gitea username + repo TEXT NOT NULL, -- Gitea repository name + hash TEXT NOT NULL, -- commit hash + runner TEXT NOT NULL, -- the runner to use - state INTEGER NOT NULL DEFAULT 0, -- task state - detail TEXT NOT NULL DEFAULT '', -- task state detail - notified INTEGER NOT NULL DEFAULT 0, -- Gitea knows the state - runlog BLOB NOT NULL DEFAULT x'', -- combined task runner output - tasklog BLOB NOT NULL DEFAULT x'', -- combined task SSH output + state INTEGER NOT NULL DEFAULT 0, -- task state + detail TEXT NOT NULL DEFAULT '', -- task state detail + notified INTEGER NOT NULL DEFAULT 0, -- Gitea knows the state + runlog BLOB NOT NULL DEFAULT x'', -- combined task runner output + tasklog BLOB NOT NULL DEFAULT x'', -- combined task SSH output + deploylog BLOB NOT NULL DEFAULT x'', -- deployment output PRIMARY KEY (id) ) STRICT; ` -func openDB(path string) error { +func dbEnsureColumn(tx *sql.Tx, table, column, definition string) error { + var count int64 + if err := tx.QueryRow( + `SELECT count(*) FROM pragma_table_info(?) WHERE name = ?`, + table, column).Scan(&count); err != nil { + return err + } else if count == 1 { + return nil + } + + _, err := tx.Exec( + `ALTER TABLE ` + table + ` ADD COLUMN ` + column + ` ` + definition) + return err +} + +func dbOpen(path string) error { var err error gDB, err = sql.Open("sqlite3", "file:"+path+"?_foreign_keys=1&_busy_timeout=1000") @@ -1269,10 +1392,43 @@ func openDB(path string) error { return err } - _, err = gDB.Exec(schemaSQL) - return err + tx, err := gDB.BeginTx(context.Background(), nil) + if err != nil { + return err + } + defer tx.Rollback() + + var version int64 + if err = tx.QueryRow( + `PRAGMA schema.user_version`).Scan(&version); err != nil { + return err + } + + switch version { + case 0: + if _, err = tx.Exec(initializeSQL); err != nil { + return err + } + + // We had not initially set a database schema version, + // so we're stuck checking this column even on new databases. + if err = dbEnsureColumn(tx, + `task`, `deploylog`, `BLOB NOT NULL DEFAULT x''`); err != nil { + return err + } + break + case 1: + // The next migration goes here, remember to increment the number below. + } + + if _, err = tx.Exec("PRAGMA schema.user_version = ?", 1); err != nil { + return err + } + return tx.Commit() } +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + // callRPC forwards command line commands to a running server. func callRPC(args []string) error { body, err := json.Marshal(args) @@ -1334,7 +1490,7 @@ func main() { return } - if err := openDB(gConfig.DB); err != nil { + if err := dbOpen(gConfig.DB); err != nil { log.Fatalln(err) } defer gDB.Close() |