From 7732392a963dcdb7bf1a0d47777b1f62c2332474 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 29 Apr 2022 20:23:48 +0800 Subject: [PATCH 001/408] Add bots --- models/bots/runner.go | 108 +++++++ models/bots/task.go | 131 +++++++++ models/migrations/v216.go | 52 ++++ modules/actions/gitea/action.go | 79 ++++++ modules/actions/gitea/gitea.go | 60 ++++ modules/actions/gitea/planner.go | 265 +++++++++++++++++ modules/actions/gitea/workflow.go | 377 +++++++++++++++++++++++++ modules/actions/gitea/workflow_test.go | 100 +++++++ modules/actions/github/github.go | 165 +++++++++++ modules/actions/runner/runner.go | 27 ++ modules/context/response.go | 11 + modules/notification/bots/bots.go | 209 ++++++++++++++ modules/notification/notification.go | 2 + routers/api/bots/bots.go | 149 ++++++++++ routers/init.go | 4 + services/bots/bots.go | 62 ++++ 16 files changed, 1801 insertions(+) create mode 100644 models/bots/runner.go create mode 100644 models/bots/task.go create mode 100644 models/migrations/v216.go create mode 100644 modules/actions/gitea/action.go create mode 100644 modules/actions/gitea/gitea.go create mode 100644 modules/actions/gitea/planner.go create mode 100644 modules/actions/gitea/workflow.go create mode 100644 modules/actions/gitea/workflow_test.go create mode 100644 modules/actions/github/github.go create mode 100644 modules/actions/runner/runner.go create mode 100644 modules/notification/bots/bots.go create mode 100644 routers/api/bots/bots.go create mode 100644 services/bots/bots.go diff --git a/models/bots/runner.go b/models/bots/runner.go new file mode 100644 index 0000000000000..f09f89d8ea6d5 --- /dev/null +++ b/models/bots/runner.go @@ -0,0 +1,108 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package bots + +import ( + "fmt" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/timeutil" + "xorm.io/builder" +) + +// ErrRunnerNotExist represents an error for bot runner not exist +type ErrRunnerNotExist struct { + UUID string +} + +func (err ErrRunnerNotExist) Error() string { + return fmt.Sprintf("Bot runner [%s] is not exist", err.UUID) +} + +// Runner represents runner machines +type Runner struct { + ID int64 + UUID string `xorm:"CHAR(36) UNIQUE"` + Name string `xorm:"VARCHAR(32) UNIQUE"` + OS string `xorm:"VARCHAR(16) index"` // the runner running os + Arch string `xorm:"VARCHAR(16) index"` // the runner running architecture + Type string `xorm:"VARCHAR(16)"` + OwnerID int64 `xorm:"index"` // org level runner, 0 means system + RepoID int64 `xorm:"index"` // repo level runner, if orgid also is zero, then it's a global + Description string `xorm:"TEXT"` + Base int // 0 native 1 docker 2 virtual machine + RepoRange string // glob match which repositories could use this runner + Token string + LastOnline timeutil.TimeStamp `xorm:"index"` + Created timeutil.TimeStamp `xorm:"created"` +} + +func (Runner) TableName() string { + return "actions_runner" +} + +func init() { + db.RegisterModel(&Runner{}) +} + +type GetRunnerOptions struct { + RepoID int64 + OwnerID int64 +} + +func (opts GetRunnerOptions) toCond() builder.Cond { + cond := builder.NewCond() + if opts.RepoID > 0 { + cond = cond.And(builder.Eq{"repo_id": opts.RepoID}) + } + if opts.OwnerID > 0 { + cond = cond.And(builder.Eq{"owner_id": opts.OwnerID}) + } + cond = cond.Or(builder.Eq{"repo_id": 0, "owner_id": 0}) + return cond +} + +// GetUsableRunner returns the usable runner +func GetUsableRunner(opts GetRunnerOptions) (*Runner, error) { + var runner Runner + has, err := db.GetEngine(db.DefaultContext). + Where(opts.toCond()). + Asc("last_online"). + Get(&runner) + if err != nil { + return nil, err + } + if !has { + return nil, ErrRunnerNotExist{} + } + + return &runner, nil +} + +// GetRunnerByUUID returns a bot runner via uuid +func GetRunnerByUUID(uuid string) (*Runner, error) { + var runner Runner + has, err := db.GetEngine(db.DefaultContext).Where("uuid=?", uuid).Get(&runner) + if err != nil { + return nil, err + } else if !has { + return nil, ErrRunnerNotExist{ + UUID: uuid, + } + } + return &runner, nil +} + +// FindRunnersByRepoID returns all workers for the repository +func FindRunnersByRepoID(repoID int64) ([]*Runner, error) { + var runners []*Runner + err := db.GetEngine(db.DefaultContext).Where("repo_id=? OR repo_id=0", repoID). + Find(&runners) + if err != nil { + return nil, err + } + err = db.GetEngine(db.DefaultContext).Join("INNER", "repository", "repository.owner_id = bot_runner.owner_id").Find(&runners) + return runners, err +} diff --git a/models/bots/task.go b/models/bots/task.go new file mode 100644 index 0000000000000..65eb9de77b5ad --- /dev/null +++ b/models/bots/task.go @@ -0,0 +1,131 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package bots + +import ( + "errors" + "fmt" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/webhook" + "code.gitea.io/gitea/modules/timeutil" + + "github.com/google/uuid" +) + +// TaskStatus represents a task status +type TaskStatus int + +// enumerate all the statuses of bot task +const ( + TaskPending TaskStatus = iota // wait for assign + TaskAssigned // assigned to a runner + TaskRunning // running + TaskFailed + TaskFinished + TaskCanceled + TaskTimeout +) + +// Task represnets bot tasks +type Task struct { + ID int64 + UUID string `xorm:"CHAR(36)"` + RepoID int64 `xorm:"index"` + TriggerUserID int64 + Ref string + CommitSHA string + Event webhook.HookEventType + Token string // token for this task + Grant string // permissions for this task + EventPayload string `xorm:"LONGTEXT"` + RunnerID int64 `xorm:"index"` + Status TaskStatus `xorm:"index"` + Created timeutil.TimeStamp `xorm:"created"` + StartTime timeutil.TimeStamp + EndTime timeutil.TimeStamp + Updated timeutil.TimeStamp `xorm:"updated"` +} + +// TableName represents a bot task +func (Task) TableName() string { + return "actions_task" +} + +// InsertTask inserts a bot task +func InsertTask(t *Task) error { + if t.UUID == "" { + t.UUID = uuid.New().String() + } + return db.Insert(db.DefaultContext, t) +} + +// UpdateTask updates bot task +func UpdateTask(t *Task, cols ...string) error { + _, err := db.GetEngine(db.DefaultContext).ID(t.ID).Cols(cols...).Update(t) + return err +} + +// ErrTaskNotExist represents an error for bot task not exist +type ErrTaskNotExist struct { + UUID string +} + +func (err ErrTaskNotExist) Error() string { + return fmt.Sprintf("Bot task [%s] is not exist", err.UUID) +} + +// GetTaskByUUID gets bot task by uuid +func GetTaskByUUID(taskUUID string) (*Task, error) { + var task Task + has, err := db.GetEngine(db.DefaultContext).Where("uuid=?", taskUUID).Get(&task) + if err != nil { + return nil, err + } else if !has { + return nil, ErrTaskNotExist{ + UUID: taskUUID, + } + } + return &task, nil +} + +// GetCurTask return the task for the bot +func GetCurTask(runnerID int64) (*Task, error) { + var tasks []Task + // FIXME: for test, just return all tasks + err := db.GetEngine(db.DefaultContext).Where("status=?", TaskPending).Find(&tasks) + // err := x.Where("runner_id = ?", botID). + // And("status=?", BotTaskPending). + // Find(&tasks) + if err != nil { + return nil, err + } + if len(tasks) == 0 { + return nil, nil + } + return &tasks[0], err +} + +// AssignTaskToRunner assign a task to a runner +func AssignTaskToRunner(taskID int64, runnerID int64) error { + cnt, err := db.GetEngine(db.DefaultContext). + Where("runner_id=0"). + And("id=?", taskID). + Cols("runner_id"). + Update(&Task{ + RunnerID: runnerID, + }) + if err != nil { + return err + } + if cnt != 1 { + return errors.New("assign faild") + } + return nil +} + +type TaskStage struct{} + +type StageStep struct{} diff --git a/models/migrations/v216.go b/models/migrations/v216.go new file mode 100644 index 0000000000000..114cc6c4c7156 --- /dev/null +++ b/models/migrations/v216.go @@ -0,0 +1,52 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/xorm" +) + +func addBotTables(x *xorm.Engine) error { + type BotRunner struct { + ID int64 + UUID string `xorm:"CHAR(36) UNIQUE"` + Name string `xorm:"VARCHAR(32) UNIQUE"` + OS string `xorm:"VARCHAR(16) index"` // the runner running os + Arch string `xorm:"VARCHAR(16) index"` // the runner running architecture + Type string `xorm:"VARCHAR(16)"` + OwnerID int64 `xorm:"index"` // org level runner, 0 means system + RepoID int64 `xorm:"index"` // repo level runner, if orgid also is zero, then it's a global + Description string `xorm:"TEXT"` + Base int // 0 native 1 docker 2 virtual machine + RepoRange string // glob match which repositories could use this runner + Token string + LastOnline timeutil.TimeStamp + Created timeutil.TimeStamp `xorm:"created"` + } + + type BotTask struct { + ID int64 + UUID string `xorm:"CHAR(36)"` + RepoID int64 `xorm:"index"` + Type string `xorm:"VARCHAR(16)"` // 0 commit 1 pullrequest + Ref string + CommitSHA string + Event string + Token string // token for this task + Grant string // permissions for this task + EventPayload string `xorm:"LONGTEXT"` + RunnerID int64 `xorm:"index"` + Status int + Content string `xorm:"LONGTEXT"` + Created timeutil.TimeStamp `xorm:"created"` + StartTime timeutil.TimeStamp + EndTime timeutil.TimeStamp + Updated timeutil.TimeStamp `xorm:"updated"` + } + + return x.Sync2(new(BotRunner), new(BotTask)) +} diff --git a/modules/actions/gitea/action.go b/modules/actions/gitea/action.go new file mode 100644 index 0000000000000..ef4a4a41dda36 --- /dev/null +++ b/modules/actions/gitea/action.go @@ -0,0 +1,79 @@ +package gitea + +import ( + "fmt" + "io" + "strings" + + "gopkg.in/yaml.v3" +) + +// ActionRunsUsing is the type of runner for the action +type ActionRunsUsing string + +func (a *ActionRunsUsing) UnmarshalYAML(unmarshal func(interface{}) error) error { + var using string + if err := unmarshal(&using); err != nil { + return err + } + + // Force input to lowercase for case insensitive comparison + format := ActionRunsUsing(strings.ToLower(using)) + switch format { + case ActionRunsUsingNode12, ActionRunsUsingDocker: + *a = format + default: + return fmt.Errorf(fmt.Sprintf("The runs.using key in action.yml must be one of: %v, got %s", []string{ + ActionRunsUsingDocker, + ActionRunsUsingNode12, + }, format)) + } + return nil +} + +const ( + // ActionRunsUsingNode12 for running with node12 + ActionRunsUsingNode12 = "node12" + // ActionRunsUsingDocker for running with docker + ActionRunsUsingDocker = "docker" +) + +// Action describes a metadata file for GitHub actions. The metadata filename must be either action.yml or action.yaml. The data in the metadata file defines the inputs, outputs and main entrypoint for your action. +type Action struct { + Name string `yaml:"name"` + Author string `yaml:"author"` + Description string `yaml:"description"` + Inputs map[string]Input `yaml:"inputs"` + Outputs map[string]Output `yaml:"outputs"` + Runs struct { + Using ActionRunsUsing `yaml:"using"` + Env map[string]string `yaml:"env"` + Main string `yaml:"main"` + Image string `yaml:"image"` + Entrypoint []string `yaml:"entrypoint"` + Args []string `yaml:"args"` + } `yaml:"runs"` + Branding struct { + Color string `yaml:"color"` + Icon string `yaml:"icon"` + } `yaml:"branding"` +} + +// Input parameters allow you to specify data that the action expects to use during runtime. GitHub stores input parameters as environment variables. Input ids with uppercase letters are converted to lowercase during runtime. We recommended using lowercase input ids. +type Input struct { + Description string `yaml:"description"` + Required bool `yaml:"required"` + Default string `yaml:"default"` +} + +// Output parameters allow you to declare data that an action sets. Actions that run later in a workflow can use the output data set in previously run actions. For example, if you had an action that performed the addition of two inputs (x + y = z), the action could output the sum (z) for other actions to use as an input. +type Output struct { + Description string `yaml:"description"` +} + +// ReadAction reads an action from a reader +func ReadAction(in io.Reader) (*Action, error) { + a := new(Action) + err := yaml.NewDecoder(in).Decode(a) + return a, err +} diff --git a/modules/actions/gitea/gitea.go b/modules/actions/gitea/gitea.go new file mode 100644 index 0000000000000..8f7c0835e5dec --- /dev/null +++ b/modules/actions/gitea/gitea.go @@ -0,0 +1,60 @@ +package gitea + +import ( + "code.gitea.io/gitea/models/webhook" + "code.gitea.io/gitea/modules/bot/runner" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/json" +) + +func init() { + runner.RegisterRunnerType(new(GiteaRunner)) +} + +type GiteaRunner struct { +} + +func (gw *GiteaRunner) Name() string { + return "gitea" +} + +func (gw *GiteaRunner) Detect(commit *git.Commit, event webhook.HookEventType, ref string) (bool, string, error) { + tree, err := commit.SubTree(".gitea/workflow") + if err != nil { + return false, "", err + } + entries, err := tree.ListEntries() + if err != nil { + return false, "", err + } + + var wfs []*Workflow + for _, entry := range entries { + blob := entry.Blob() + rd, err := blob.DataAsync() + if err != nil { + return false, "", err + } + defer rd.Close() + wf, err := ReadWorkflow(rd) + if err != nil { + log.Error("ReadWorkflow file %s failed: %v", entry.Name(), err) + continue + } + + // FIXME: we have to convert the event type to github known name + if !util.IsStringInSlice(string(event), wf.On()) { + continue + } + + wfs = append(wfs, wf) + } + + wfBs, err := json.Marshal(wfs) + if err != nil { + return false, "", err + } + return true, string(wfBs), nil +} diff --git a/modules/actions/gitea/planner.go b/modules/actions/gitea/planner.go new file mode 100644 index 0000000000000..6d80e79d49c5b --- /dev/null +++ b/modules/actions/gitea/planner.go @@ -0,0 +1,265 @@ +package gitea + +import ( + "io" + "io/ioutil" + "math" + "os" + "path/filepath" + "sort" + + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +// WorkflowPlanner contains methods for creating plans +type WorkflowPlanner interface { + PlanEvent(eventName string) *Plan + PlanJob(jobName string) *Plan + GetEvents() []string +} + +// Plan contains a list of stages to run in series +type Plan struct { + Stages []*Stage +} + +// Stage contains a list of runs to execute in parallel +type Stage struct { + Runs []*Run +} + +// Run represents a job from a workflow that needs to be run +type Run struct { + Workflow *Workflow + JobID string +} + +func (r *Run) String() string { + jobName := r.Job().Name + if jobName == "" { + jobName = r.JobID + } + return jobName +} + +// Job returns the job for this Run +func (r *Run) Job() *Job { + return r.Workflow.GetJob(r.JobID) +} + +// NewWorkflowPlanner will load a specific workflow or all workflows from a directory +func NewWorkflowPlanner(path string) (WorkflowPlanner, error) { + fi, err := os.Stat(path) + if err != nil { + return nil, err + } + + var files []os.FileInfo + var dirname string + + if fi.IsDir() { + log.Debugf("Loading workflows from '%s'", path) + dirname = path + files, err = ioutil.ReadDir(path) + } else { + log.Debugf("Loading workflow '%s'", path) + dirname, err = filepath.Abs(filepath.Dir(path)) + files = []os.FileInfo{fi} + } + if err != nil { + return nil, err + } + + wp := new(workflowPlanner) + for _, file := range files { + ext := filepath.Ext(file.Name()) + if ext == ".yml" || ext == ".yaml" { + f, err := os.Open(filepath.Join(dirname, file.Name())) + if err != nil { + return nil, err + } + + log.Debugf("Reading workflow '%s'", f.Name()) + workflow, err := ReadWorkflow(f) + if err != nil { + f.Close() + if err == io.EOF { + return nil, errors.WithMessagef(err, "unable to read workflow, %s file is empty", file.Name()) + } + return nil, err + } + if workflow.Name == "" { + workflow.Name = file.Name() + } + wp.workflows = append(wp.workflows, workflow) + f.Close() + } + } + + return wp, nil +} + +type workflowPlanner struct { + workflows []*Workflow +} + +// PlanEvent builds a new list of runs to execute in parallel for an event name +func (wp *workflowPlanner) PlanEvent(eventName string) *Plan { + plan := new(Plan) + if len(wp.workflows) == 0 { + log.Debugf("no events found for workflow: %s", eventName) + } + + for _, w := range wp.workflows { + for _, e := range w.When().Events { + if e.Type == eventName { + plan.mergeStages(createStages(w, w.GetJobIDs()...)) + } + } + } + return plan +} + +// PlanJob builds a new run to execute in parallel for a job name +func (wp *workflowPlanner) PlanJob(jobName string) *Plan { + plan := new(Plan) + if len(wp.workflows) == 0 { + log.Debugf("no jobs found for workflow: %s", jobName) + } + + for _, w := range wp.workflows { + plan.mergeStages(createStages(w, jobName)) + } + return plan +} + +// GetEvents gets all the events in the workflows file +func (wp *workflowPlanner) GetEvents() []string { + events := make([]string, 0) + for _, w := range wp.workflows { + found := false + for _, e := range events { + for _, we := range w.When().Events { + if e == we.Type { + found = true + break + } + } + if found { + break + } + } + + if !found { + for _, evt := range w.When().Events { + events = append(events, evt.Type) + } + } + } + + // sort the list based on depth of dependencies + sort.Slice(events, func(i, j int) bool { + return events[i] < events[j] + }) + + return events +} + +// MaxRunNameLen determines the max name length of all jobs +func (p *Plan) MaxRunNameLen() int { + maxRunNameLen := 0 + for _, stage := range p.Stages { + for _, run := range stage.Runs { + runNameLen := len(run.String()) + if runNameLen > maxRunNameLen { + maxRunNameLen = runNameLen + } + } + } + return maxRunNameLen +} + +// GetJobIDs will get all the job names in the stage +func (s *Stage) GetJobIDs() []string { + names := make([]string, 0) + for _, r := range s.Runs { + names = append(names, r.JobID) + } + return names +} + +// Merge stages with existing stages in plan +func (p *Plan) mergeStages(stages []*Stage) { + newStages := make([]*Stage, int(math.Max(float64(len(p.Stages)), float64(len(stages))))) + for i := 0; i < len(newStages); i++ { + newStages[i] = new(Stage) + if i >= len(p.Stages) { + newStages[i].Runs = append(newStages[i].Runs, stages[i].Runs...) + } else if i >= len(stages) { + newStages[i].Runs = append(newStages[i].Runs, p.Stages[i].Runs...) + } else { + newStages[i].Runs = append(newStages[i].Runs, p.Stages[i].Runs...) + newStages[i].Runs = append(newStages[i].Runs, stages[i].Runs...) + } + } + p.Stages = newStages +} + +func createStages(w *Workflow, jobIDs ...string) []*Stage { + // first, build a list of all the necessary jobs to run, and their dependencies + jobDependencies := make(map[string][]string) + for len(jobIDs) > 0 { + newJobIDs := make([]string, 0) + for _, jID := range jobIDs { + // make sure we haven't visited this job yet + if _, ok := jobDependencies[jID]; !ok { + if job := w.GetJob(jID); job != nil { + jobDependencies[jID] = job.Needs() + newJobIDs = append(newJobIDs, job.Needs()...) + } + } + } + jobIDs = newJobIDs + } + + // next, build an execution graph + stages := make([]*Stage, 0) + for len(jobDependencies) > 0 { + stage := new(Stage) + for jID, jDeps := range jobDependencies { + // make sure all deps are in the graph already + if listInStages(jDeps, stages...) { + stage.Runs = append(stage.Runs, &Run{ + Workflow: w, + JobID: jID, + }) + delete(jobDependencies, jID) + } + } + if len(stage.Runs) == 0 { + log.Fatalf("Unable to build dependency graph!") + } + stages = append(stages, stage) + } + + return stages +} + +// return true iff all strings in srcList exist in at least one of the stages +func listInStages(srcList []string, stages ...*Stage) bool { + for _, src := range srcList { + found := false + for _, stage := range stages { + for _, search := range stage.GetJobIDs() { + if src == search { + found = true + } + } + } + if !found { + return false + } + } + return true +} diff --git a/modules/actions/gitea/workflow.go b/modules/actions/gitea/workflow.go new file mode 100644 index 0000000000000..2a8a5d04f037b --- /dev/null +++ b/modules/actions/gitea/workflow.go @@ -0,0 +1,377 @@ +package gitea + +import ( + "fmt" + "io" + "reflect" + "regexp" + "strings" + + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" +) + +// Workflow is the structure of the files in .github/workflows +type Workflow struct { + Name string `yaml:"name"` + RawWhen yaml.Node `yaml:"when"` + Env map[string]string `yaml:"env"` + Jobs map[string]*Job `yaml:"jobs"` + Defaults Defaults `yaml:"defaults"` +} + +type Event struct { + Type string + Ref string +} + +type When struct { + Events []Event +} + +func (w *When) Match(tp string) bool { + for _, evt := range w.Events { + if strings.EqualFold(tp, evt.Type) { + return true + } + } + return false +} + +// When events for the workflow +func (w *Workflow) When() *When { + switch w.RawWhen.Kind { + case yaml.ScalarNode: + var val string + err := w.RawWhen.Decode(&val) + if err != nil { + log.Fatal(err) + } + return &When{ + Events: []Event{ + { + Type: val, + }, + }, + } + case yaml.SequenceNode: + var vals []string + err := w.RawWhen.Decode(&vals) + if err != nil { + log.Fatal(err) + } + var when When + for _, val := range vals { + when.Events = append(when.Events, Event{ + Type: val, + }) + } + return &when + case yaml.MappingNode: + var val map[string]interface{} + err := w.RawWhen.Decode(&val) + if err != nil { + log.Fatal(err) + } + var keys []string + for k := range val { + keys = append(keys, k) + } + var when When + for _, val := range keys { + when.Events = append(when.Events, Event{ + Type: val, + }) + } + return &when + } + return nil +} + +// Job is the structure of one job in a workflow +type Job struct { + Name string `yaml:"name"` + RawNeeds yaml.Node `yaml:"needs"` + RawRunsOn yaml.Node `yaml:"runs-on"` + Env map[string]string `yaml:"env"` + If string `yaml:"if"` + Steps []*Step `yaml:"steps"` + TimeoutMinutes int64 `yaml:"timeout-minutes"` + Services map[string]*ContainerSpec `yaml:"services"` + Strategy *Strategy `yaml:"strategy"` + RawContainer yaml.Node `yaml:"container"` + Defaults Defaults `yaml:"defaults"` +} + +// Strategy for the job +type Strategy struct { + FailFast bool `yaml:"fail-fast"` + MaxParallel int `yaml:"max-parallel"` + Matrix map[string][]interface{} `yaml:"matrix"` +} + +// Default settings that will apply to all steps in the job or workflow +type Defaults struct { + Run RunDefaults `yaml:"run"` +} + +// Defaults for all run steps in the job or workflow +type RunDefaults struct { + Shell string `yaml:"shell"` + WorkingDirectory string `yaml:"working-directory"` +} + +// Container details for the job +func (j *Job) Container() *ContainerSpec { + var val *ContainerSpec + switch j.RawContainer.Kind { + case yaml.ScalarNode: + val = new(ContainerSpec) + err := j.RawContainer.Decode(&val.Image) + if err != nil { + log.Fatal(err) + } + case yaml.MappingNode: + val = new(ContainerSpec) + err := j.RawContainer.Decode(val) + if err != nil { + log.Fatal(err) + } + } + return val +} + +// Needs list for Job +func (j *Job) Needs() []string { + + switch j.RawNeeds.Kind { + case yaml.ScalarNode: + var val string + err := j.RawNeeds.Decode(&val) + if err != nil { + log.Fatal(err) + } + return []string{val} + case yaml.SequenceNode: + var val []string + err := j.RawNeeds.Decode(&val) + if err != nil { + log.Fatal(err) + } + return val + } + return nil +} + +// RunsOn list for Job +func (j *Job) RunsOn() []string { + + switch j.RawRunsOn.Kind { + case yaml.ScalarNode: + var val string + err := j.RawRunsOn.Decode(&val) + if err != nil { + log.Fatal(err) + } + return []string{val} + case yaml.SequenceNode: + var val []string + err := j.RawRunsOn.Decode(&val) + if err != nil { + log.Fatal(err) + } + return val + } + return nil +} + +// GetMatrixes returns the matrix cross product +func (j *Job) GetMatrixes() []map[string]interface{} { + matrixes := make([]map[string]interface{}, 0) + /*if j.Strategy != nil { + includes := make([]map[string]interface{}, 0) + for _, v := range j.Strategy.Matrix["include"] { + includes = append(includes, v.(map[string]interface{})) + } + delete(j.Strategy.Matrix, "include") + + excludes := make([]map[string]interface{}, 0) + for _, v := range j.Strategy.Matrix["exclude"] { + excludes = append(excludes, v.(map[string]interface{})) + } + delete(j.Strategy.Matrix, "exclude") + + matrixProduct := common.CartesianProduct(j.Strategy.Matrix) + + MATRIX: + for _, matrix := range matrixProduct { + for _, exclude := range excludes { + if commonKeysMatch(matrix, exclude) { + log.Debugf("Skipping matrix '%v' due to exclude '%v'", matrix, exclude) + continue MATRIX + } + } + matrixes = append(matrixes, matrix) + } + for _, include := range includes { + log.Debugf("Adding include '%v'", include) + matrixes = append(matrixes, include) + } + + } else { + matrixes = append(matrixes, make(map[string]interface{})) + }*/ + return matrixes +} + +func commonKeysMatch(a map[string]interface{}, b map[string]interface{}) bool { + for aKey, aVal := range a { + if bVal, ok := b[aKey]; ok && !reflect.DeepEqual(aVal, bVal) { + return false + } + } + return true +} + +// ContainerSpec is the specification of the container to use for the job +type ContainerSpec struct { + Image string `yaml:"image"` + Env map[string]string `yaml:"env"` + Ports []string `yaml:"ports"` + Volumes []string `yaml:"volumes"` + Options string `yaml:"options"` + Entrypoint string + Args string + Name string + Reuse bool +} + +// Step is the structure of one step in a job +type Step struct { + ID string `yaml:"id"` + If string `yaml:"if"` + Name string `yaml:"name"` + Uses string `yaml:"uses"` + Run string `yaml:"run"` + WorkingDirectory string `yaml:"working-directory"` + Shell string `yaml:"shell"` + Env map[string]string `yaml:"env"` + With map[string]string `yaml:"with"` + ContinueOnError bool `yaml:"continue-on-error"` + TimeoutMinutes int64 `yaml:"timeout-minutes"` +} + +// String gets the name of step +func (s *Step) String() string { + if s.Name != "" { + return s.Name + } else if s.Uses != "" { + return s.Uses + } else if s.Run != "" { + return s.Run + } + return s.ID +} + +// GetEnv gets the env for a step +func (s *Step) GetEnv() map[string]string { + rtnEnv := make(map[string]string) + for k, v := range s.Env { + rtnEnv[k] = v + } + for k, v := range s.With { + envKey := regexp.MustCompile("[^A-Z0-9-]").ReplaceAllString(strings.ToUpper(k), "_") + envKey = fmt.Sprintf("INPUT_%s", strings.ToUpper(envKey)) + rtnEnv[envKey] = v + } + return rtnEnv +} + +// ShellCommand returns the command for the shell +func (s *Step) ShellCommand() string { + shellCommand := "" + + switch s.Shell { + case "", "bash": + shellCommand = "bash --noprofile --norc -eo pipefail {0}" + case "pwsh": + shellCommand = "pwsh -command \"& '{0}'\"" + case "python": + shellCommand = "python {0}" + case "sh": + shellCommand = "sh -e -c {0}" + case "cmd": + shellCommand = "%ComSpec% /D /E:ON /V:OFF /S /C \"CALL \"{0}\"\"" + case "powershell": + shellCommand = "powershell -command \"& '{0}'\"" + default: + shellCommand = s.Shell + } + return shellCommand +} + +// StepType describes what type of step we are about to run +type StepType int + +const ( + // StepTypeRun is all steps that have a `run` attribute + StepTypeRun StepType = iota + + //StepTypeUsesDockerURL is all steps that have a `uses` that is of the form `docker://...` + StepTypeUsesDockerURL + + //StepTypeUsesActionLocal is all steps that have a `uses` that is a local action in a subdirectory + StepTypeUsesActionLocal + + //StepTypeUsesActionRemote is all steps that have a `uses` that is a reference to a github repo + StepTypeUsesActionRemote +) + +// Type returns the type of the step +func (s *Step) Type() StepType { + if s.Run != "" { + return StepTypeRun + } else if strings.HasPrefix(s.Uses, "docker://") { + return StepTypeUsesDockerURL + } else if strings.HasPrefix(s.Uses, "./") { + return StepTypeUsesActionLocal + } + return StepTypeUsesActionRemote +} + +// ReadWorkflow returns a list of jobs for a given workflow file reader +func ReadWorkflow(in io.Reader) (*Workflow, error) { + w := new(Workflow) + err := yaml.NewDecoder(in).Decode(w) + return w, err +} + +// GetJob will get a job by name in the workflow +func (w *Workflow) GetJob(jobID string) *Job { + for id, j := range w.Jobs { + if jobID == id { + if j.Name == "" { + j.Name = id + } + return j + } + } + return nil +} + +// GetJobIDs will get all the job names in the workflow +func (w *Workflow) GetJobIDs() []string { + ids := make([]string, 0) + for id := range w.Jobs { + ids = append(ids, id) + } + return ids +} + +func (w *Workflow) On() []string { + var evts []string + for _, job := range w.Jobs { + evts = append(evts, job.RunsOn()...) + } + return evts +} \ No newline at end of file diff --git a/modules/actions/gitea/workflow_test.go b/modules/actions/gitea/workflow_test.go new file mode 100644 index 0000000000000..6df4a15e75d93 --- /dev/null +++ b/modules/actions/gitea/workflow_test.go @@ -0,0 +1,100 @@ +package gitea + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestReadWorkflow_StringEvent(t *testing.T) { + yaml := ` +name: local-action-docker-url +on: push + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: ./actions/docker-url +` + + workflow, err := ReadWorkflow(strings.NewReader(yaml)) + assert.NoError(t, err, "read workflow should succeed") + + assert.Len(t, workflow.On(), 1) + assert.Contains(t, workflow.On(), "push") +} + +func TestReadWorkflow_ListEvent(t *testing.T) { + yaml := ` +name: local-action-docker-url +on: [push, pull_request] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: ./actions/docker-url +` + + workflow, err := ReadWorkflow(strings.NewReader(yaml)) + assert.NoError(t, err, "read workflow should succeed") + + assert.Len(t, workflow.On(), 2) + assert.Contains(t, workflow.On(), "push") + assert.Contains(t, workflow.On(), "pull_request") +} + +func TestReadWorkflow_MapEvent(t *testing.T) { + yaml := ` +name: local-action-docker-url +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: ./actions/docker-url +` + + workflow, err := ReadWorkflow(strings.NewReader(yaml)) + assert.NoError(t, err, "read workflow should succeed") + assert.Len(t, workflow.On(), 2) + assert.Contains(t, workflow.On(), "push") + assert.Contains(t, workflow.On(), "pull_request") +} + +func TestReadWorkflow_StringContainer(t *testing.T) { + yaml := ` +name: local-action-docker-url + +jobs: + test: + container: nginx:latest + runs-on: ubuntu-latest + steps: + - uses: ./actions/docker-url + test2: + container: + image: nginx:latest + env: + foo: bar + runs-on: ubuntu-latest + steps: + - uses: ./actions/docker-url +` + + workflow, err := ReadWorkflow(strings.NewReader(yaml)) + assert.NoError(t, err, "read workflow should succeed") + assert.Len(t, workflow.Jobs, 2) + assert.Contains(t, workflow.Jobs["test"].Container().Image, "nginx:latest") + assert.Contains(t, workflow.Jobs["test2"].Container().Image, "nginx:latest") + assert.Contains(t, workflow.Jobs["test2"].Container().Env["foo"], "bar") +} diff --git a/modules/actions/github/github.go b/modules/actions/github/github.go new file mode 100644 index 0000000000000..b39539fa0da19 --- /dev/null +++ b/modules/actions/github/github.go @@ -0,0 +1,165 @@ +package github + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + + bot_model "code.gitea.io/gitea/models/bot" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/models/webhook" + "code.gitea.io/gitea/modules/bot/runner" + "code.gitea.io/gitea/modules/git" + + //"code.gitea.io/gitea/modules/log" + //"code.gitea.io/gitea/modules/util" + + "github.com/nektos/act/pkg/model" + act_runner "github.com/nektos/act/pkg/runner" +) + +func init() { + runner.RegisterRunnerType(new(GithubRunner)) +} + +type GithubRunner struct { +} + +func (gw *GithubRunner) Name() string { + return "github" +} + +func (gw *GithubRunner) Detect(commit *git.Commit, event webhook.HookEventType, ref string) (bool, string, error) { + tree, err := commit.SubTree(".github/workflow") + if err != nil { + return false, "", err + } + entries, err := tree.ListEntries() + if err != nil { + return false, "", err + } + + var content = make(map[string]string) + for _, entry := range entries { + blob := entry.Blob() + rd, err := blob.DataAsync() + if err != nil { + return false, "", err + } + + bs, err := io.ReadAll(rd) + rd.Close() + if err != nil { + return false, "", err + } + content[entry.Name()] = string(bs) + } + + res, err := json.Marshal(content) + if err != nil { + return false, "", err + } + return true, string(res), nil +} + +func (gw *GithubRunner) Run(task *bot_model.Task) error { + tmpDir, err := os.MkdirTemp("", fmt.Sprintf("%d", task.ID)) + if err != nil { + return err + } + + var files = make(map[string]string) + if err := json.Unmarshal([]byte(task.Content), &files); err != nil { + return err + } + for name, content := range files { + f, err := os.Create(filepath.Join(tmpDir, name)) + if err != nil { + return err + } + if _, err := f.WriteString(content); err != nil { + f.Close() + return err + } + f.Close() + } + + repo, err := repo_model.GetRepositoryByID(task.RepoID) + if err != nil { + return err + } + + evtFilePath := filepath.Join(tmpDir, "event.json") + evtFile, err := os.Create(evtFilePath) + if err != nil { + return err + } + + if _, err := evtFile.WriteString(task.EventPayload); err != nil { + evtFile.Close() + return err + } + evtFile.Close() + + planner, err := model.NewWorkflowPlanner(tmpDir, false) + if err != nil { + return err + } + plan := planner.PlanEvent(task.Event) + + actor, err := user_model.GetUserByID(task.TriggerUserID) + if err != nil { + return err + } + + // run the plan + config := &act_runner.Config{ + Actor: actor.LoginName, + EventName: task.Event, + EventPath: evtFilePath, + DefaultBranch: repo.DefaultBranch, + /*ForcePull: input.forcePull, + ForceRebuild: input.forceRebuild, + ReuseContainers: input.reuseContainers, + Workdir: input.Workdir(), + BindWorkdir: input.bindWorkdir, + LogOutput: !input.noOutput,*/ + //Env: envs, + Secrets: map[string]string{ + "token": "614e597274a527b6fcf6ddfe45def79430126f08", + }, + //InsecureSecrets: input.insecureSecrets,*/ + Platforms: map[string]string{ + "ubuntu-latest": "node:12-buster-slim", + "ubuntu-20.04": "node:12-buster-slim", + "ubuntu-18.04": "node:12-buster-slim", + }, + /*Privileged: input.privileged, + UsernsMode: input.usernsMode, + ContainerArchitecture: input.containerArchitecture, + ContainerDaemonSocket: input.containerDaemonSocket, + UseGitIgnore: input.useGitIgnore,*/ + GitHubInstance: "gitea.com", + /*ContainerCapAdd: input.containerCapAdd, + ContainerCapDrop: input.containerCapDrop, + AutoRemove: input.autoRemove, + ArtifactServerPath: input.artifactServerPath, + ArtifactServerPort: input.artifactServerPort,*/ + } + r, err := act_runner.New(config) + if err != nil { + return err + } + + //ctx, cancel := context.WithTimeout(context.Background(), ) + + executor := r.NewPlanExecutor(plan).Finally(func(ctx context.Context) error { + //cancel() + return nil + }) + return executor(context.Background()) +} diff --git a/modules/actions/runner/runner.go b/modules/actions/runner/runner.go new file mode 100644 index 0000000000000..2a9540ad769c2 --- /dev/null +++ b/modules/actions/runner/runner.go @@ -0,0 +1,27 @@ +package runner + +import ( + bots_model "code.gitea.io/gitea/models/bots" + "code.gitea.io/gitea/models/webhook" + "code.gitea.io/gitea/modules/git" +) + +var runnerTypes = make(map[string]RunnerType) + +type RunnerType interface { + Name() string + Detect(commit *git.Commit, event webhook.HookEventType, ref string) (bool, string, error) + Run(task *bots_model.Task) error +} + +func RegisterRunnerType(runnerType RunnerType) { + runnerTypes[runnerType.Name()] = runnerType +} + +func GetRunnerType(name string) RunnerType { + return runnerTypes[name] +} + +func GetRunnerTypes() map[string]RunnerType { + return runnerTypes +} diff --git a/modules/context/response.go b/modules/context/response.go index 112964dbe14cd..24844baa081e6 100644 --- a/modules/context/response.go +++ b/modules/context/response.go @@ -5,6 +5,9 @@ package context import ( + "bufio" + "errors" + "net" "net/http" ) @@ -84,6 +87,14 @@ func (r *Response) Before(f func(ResponseWriter)) { r.befores = append(r.befores, f) } +func (r *Response) Hijack() (net.Conn, *bufio.ReadWriter, error) { + if h, ok := r.ResponseWriter.(http.Hijacker); ok { + return h.Hijack() + } + + return nil, nil, errors.New("unimplemented http.Hijacker ") +} + // NewResponse creates a response func NewResponse(resp http.ResponseWriter) *Response { if v, ok := resp.(*Response); ok { diff --git a/modules/notification/bots/bots.go b/modules/notification/bots/bots.go new file mode 100644 index 0000000000000..8fc31ed3cf563 --- /dev/null +++ b/modules/notification/bots/bots.go @@ -0,0 +1,209 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package bots + +import ( + "context" + "encoding/json" + "fmt" + + "code.gitea.io/gitea/models" + bots_model "code.gitea.io/gitea/models/bots" + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/perm" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/models/webhook" + "code.gitea.io/gitea/modules/convert" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/notification/base" + "code.gitea.io/gitea/modules/process" + "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/setting" + api "code.gitea.io/gitea/modules/structs" + bots_service "code.gitea.io/gitea/services/bots" +) + +type botsNotifier struct { + base.NullNotifier +} + +var _ base.Notifier = &botsNotifier{} + +// NewNotifier create a new botsNotifier notifier +func NewNotifier() base.Notifier { + return &botsNotifier{} +} + +func notifyIssue(issue *models.Issue, doer *user_model.User, evt webhook.HookEventType, payload string) { + err := issue.LoadRepo(db.DefaultContext) + if err != nil { + log.Error("issue.LoadRepo: %v", err) + return + } + if issue.Repo.IsEmpty || issue.Repo.IsArchived { + return + } + + ref := issue.Ref + if ref == "" { + ref = issue.Repo.DefaultBranch + } + + gitRepo, err := git.OpenRepository(context.Background(), issue.Repo.RepoPath()) + if err != nil { + log.Error("issue.LoadRepo: %v", err) + return + } + defer gitRepo.Close() + + // Get the commit object for the ref + commit, err := gitRepo.GetCommit(ref) + if err != nil { + log.Error("gitRepo.GetCommit: %v", err) + return + } + + task := bots_model.Task{ + RepoID: issue.RepoID, + TriggerUserID: doer.ID, + Event: evt, + EventPayload: payload, + Status: bots_model.TaskPending, + Ref: ref, + CommitSHA: commit.ID.String(), + } + + if err := bots_model.InsertTask(&task); err != nil { + log.Error("InsertBotTask: %v", err) + } else { + bots_service.PushToQueue(&task) + } +} + +// TODO: implement all events +func (a *botsNotifier) NotifyNewIssue(issue *models.Issue, mentions []*user_model.User) { + payload := map[string]interface{}{ + "issue": map[string]interface{}{ + "number": issue.Index, + }, + } + bs, err := json.Marshal(payload) + if err != nil { + log.Error("NotifyNewIssue: %v", err) + return + } + notifyIssue(issue, issue.Poster, webhook.HookEventIssues, string(bs)) +} + +// NotifyIssueChangeStatus notifies close or reopen issue to notifiers +func (a *botsNotifier) NotifyIssueChangeStatus(doer *user_model.User, issue *models.Issue, actionComment *models.Comment, closeOrReopen bool) { +} + +func (a *botsNotifier) NotifyIssueChangeLabels(doer *user_model.User, issue *models.Issue, + addedLabels []*models.Label, removedLabels []*models.Label, +) { + payload := map[string]interface{}{ + "issue": map[string]interface{}{ + "number": issue.Index, + }, + } + bs, err := json.Marshal(payload) + if err != nil { + log.Error("NotifyNewIssue: %v", err) + return + } + notifyIssue(issue, doer, webhook.HookEventIssueLabel, string(bs)) +} + +// NotifyCreateIssueComment notifies comment on an issue to notifiers +func (a *botsNotifier) NotifyCreateIssueComment(doer *user_model.User, repo *repo_model.Repository, + issue *models.Issue, comment *models.Comment, mentions []*user_model.User) { +} + +func (a *botsNotifier) NotifyNewPullRequest(pull *models.PullRequest, mentions []*user_model.User) { +} + +func (a *botsNotifier) NotifyRenameRepository(doer *user_model.User, repo *repo_model.Repository, oldRepoName string) { +} + +func (a *botsNotifier) NotifyTransferRepository(doer *user_model.User, repo *repo_model.Repository, oldOwnerName string) { +} + +func (a *botsNotifier) NotifyCreateRepository(doer *user_model.User, u *user_model.User, repo *repo_model.Repository) { +} + +func (a *botsNotifier) NotifyForkRepository(doer *user_model.User, oldRepo, repo *repo_model.Repository) { +} + +func (a *botsNotifier) NotifyPullRequestReview(pr *models.PullRequest, review *models.Review, comment *models.Comment, mentions []*user_model.User) { +} + +func (*botsNotifier) NotifyMergePullRequest(pr *models.PullRequest, doer *user_model.User) { +} + +func (a *botsNotifier) NotifyPushCommits(pusher *user_model.User, repo *repo_model.Repository, opts *repository.PushUpdateOptions, commits *repository.PushCommits) { + ctx, _, finished := process.GetManager().AddContext(graceful.GetManager().HammerContext(), fmt.Sprintf("webhook.NotifyPushCommits User: %s[%d] in %s[%d]", pusher.Name, pusher.ID, repo.FullName(), repo.ID)) + defer finished() + + apiPusher := convert.ToUser(pusher, nil) + apiCommits, apiHeadCommit, err := commits.ToAPIPayloadCommits(ctx, repo.RepoPath(), repo.HTMLURL()) + if err != nil { + log.Error("commits.ToAPIPayloadCommits failed: %v", err) + return + } + + payload := &api.PushPayload{ + Ref: opts.RefFullName, + Before: opts.OldCommitID, + After: opts.NewCommitID, + CompareURL: setting.AppURL + commits.CompareURL, + Commits: apiCommits, + HeadCommit: apiHeadCommit, + Repo: convert.ToRepo(repo, perm.AccessModeOwner), + Pusher: apiPusher, + Sender: apiPusher, + } + + bs, err := json.Marshal(payload) + if err != nil { + log.Error("json.Marshal(payload) failed: %v", err) + return + } + + task := bots_model.Task{ + RepoID: repo.ID, + TriggerUserID: pusher.ID, + Event: webhook.HookEventPush, + EventPayload: string(bs), + Status: bots_model.TaskPending, + } + + if err := bots_model.InsertTask(&task); err != nil { + log.Error("InsertBotTask: %v", err) + } else { + bots_service.PushToQueue(&task) + } +} + +func (a *botsNotifier) NotifyCreateRef(doer *user_model.User, repo *repo_model.Repository, refType, refFullName, refID string) { +} + +func (a *botsNotifier) NotifyDeleteRef(doer *user_model.User, repo *repo_model.Repository, refType, refFullName string) { +} + +func (a *botsNotifier) NotifySyncPushCommits(pusher *user_model.User, repo *repo_model.Repository, opts *repository.PushUpdateOptions, commits *repository.PushCommits) { +} + +func (a *botsNotifier) NotifySyncCreateRef(doer *user_model.User, repo *repo_model.Repository, refType, refFullName, refID string) { +} + +func (a *botsNotifier) NotifySyncDeleteRef(doer *user_model.User, repo *repo_model.Repository, refType, refFullName string) { +} + +func (a *botsNotifier) NotifyNewRelease(rel *models.Release) { +} diff --git a/modules/notification/notification.go b/modules/notification/notification.go index a117a60815b47..e122dcf604dbe 100644 --- a/modules/notification/notification.go +++ b/modules/notification/notification.go @@ -13,6 +13,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/notification/action" "code.gitea.io/gitea/modules/notification/base" + "code.gitea.io/gitea/modules/notification/bots" "code.gitea.io/gitea/modules/notification/indexer" "code.gitea.io/gitea/modules/notification/mail" "code.gitea.io/gitea/modules/notification/mirror" @@ -40,6 +41,7 @@ func NewContext() { RegisterNotifier(webhook.NewNotifier()) RegisterNotifier(action.NewNotifier()) RegisterNotifier(mirror.NewNotifier()) + RegisterNotifier(bots.NewNotifier()) } // NotifyNewWikiPage notifies creating new wiki pages to notifiers diff --git a/routers/api/bots/bots.go b/routers/api/bots/bots.go new file mode 100644 index 0000000000000..0e91c19617782 --- /dev/null +++ b/routers/api/bots/bots.go @@ -0,0 +1,149 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package bots + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + "time" + + bots_model "code.gitea.io/gitea/models/bots" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/web" + + "github.com/gorilla/websocket" +) + +func Routes() *web.Route { + r := web.NewRoute() + r.Get("/", Serve) + return r +} + +var upgrader = websocket.Upgrader{ + ReadBufferSize: 4096, + WriteBufferSize: 4096, + EnableCompression: true, + CheckOrigin: func(r *http.Request) bool { + return true + }, +} + +var pongWait = 60 * time.Second + +type Message struct { + Version int // + Type int // message type, 1 register 2 error + RunnerUUID string // runner uuid + ErrCode int // error code + ErrContent string // errors message +} + +func Serve(w http.ResponseWriter, r *http.Request) { + log.Trace("websocket init request begin from %s", r.RemoteAddr) + c, err := upgrader.Upgrade(w, r, nil) + if err != nil { + log.Error("websocket upgrade failed: %v", err) + return + } + defer c.Close() + log.Trace("websocket upgrade from %s successfully", r.RemoteAddr) + + c.SetReadDeadline(time.Now().Add(pongWait)) + c.SetPongHandler(func(string) error { c.SetReadDeadline(time.Now().Add(pongWait)); return nil }) + +MESSAGE_BUMP: + for { + // read log from client + mt, message, err := c.ReadMessage() + if err != nil { + if websocket.IsCloseError(err, websocket.CloseAbnormalClosure) || + websocket.IsCloseError(err, websocket.CloseNormalClosure) { + c.Close() + break + } + if !strings.Contains(err.Error(), "i/o timeout") { + log.Error("websocket[%s] read failed: %#v", r.RemoteAddr, err) + } + break + } else { + log.Trace("websocket[%s] received message: %s", r.RemoteAddr, message) + } + + // read message first + var msg Message + if err = json.Unmarshal(message, &msg); err != nil { + log.Error("websocket[%s] unmarshal failed: %#v", r.RemoteAddr, err) + break + } + + switch msg.Version { + case 1: + switch msg.Type { + case 1: + log.Info("websocket[%s] registered", r.RemoteAddr) + runner, err := bots_model.GetRunnerByUUID(msg.RunnerUUID) + if err != nil { + if !errors.Is(err, bots_model.ErrRunnerNotExist{}) { + log.Error("websocket[%s] get runner [%s] failed: %v", r.RemoteAddr, msg.RunnerUUID, err) + break + } + err = c.WriteMessage(mt, message) + if err != nil { + log.Error("websocket[%s] sent message failed: %v", r.RemoteAddr, err) + break + } + } else { + fmt.Printf("-----%v\n", runner) + // TODO: handle read message + err = c.WriteMessage(mt, message) + if err != nil { + log.Error("websocket[%s] sent message failed: %v", r.RemoteAddr, err) + break + } + } + default: + returnMsg := Message{ + Version: 1, + Type: 2, + ErrCode: 1, + ErrContent: "type is not supported", + } + bs, err := json.Marshal(&returnMsg) + if err != nil { + log.Error("websocket[%s] marshal message failed: %v", r.RemoteAddr, err) + break MESSAGE_BUMP + } + err = c.WriteMessage(mt, bs) + if err != nil { + log.Error("websocket[%s] sent message failed: %v", r.RemoteAddr, err) + } + break MESSAGE_BUMP + } + default: + returnMsg := Message{ + Version: 1, + Type: 2, + ErrCode: 1, + ErrContent: "version is not supported", + } + bs, err := json.Marshal(&returnMsg) + if err != nil { + log.Error("websocket[%s] marshal message failed: %v", r.RemoteAddr, err) + break MESSAGE_BUMP + } + err = c.WriteMessage(mt, bs) + if err != nil { + log.Error("websocket[%s] sent message failed: %v", r.RemoteAddr, err) + } + break MESSAGE_BUMP + } + + // TODO: find new task and send to client + } +} diff --git a/routers/init.go b/routers/init.go index fecc5c439c6de..80c44bf61b7f8 100644 --- a/routers/init.go +++ b/routers/init.go @@ -31,6 +31,7 @@ import ( "code.gitea.io/gitea/modules/translation" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web" + bots_router "code.gitea.io/gitea/routers/api/bots" packages_router "code.gitea.io/gitea/routers/api/packages" apiv1 "code.gitea.io/gitea/routers/api/v1" "code.gitea.io/gitea/routers/common" @@ -39,6 +40,7 @@ import ( "code.gitea.io/gitea/services/auth" "code.gitea.io/gitea/services/auth/source/oauth2" "code.gitea.io/gitea/services/automerge" + bots_service "code.gitea.io/gitea/services/bots" "code.gitea.io/gitea/services/cron" "code.gitea.io/gitea/services/mailer" markup_service "code.gitea.io/gitea/services/markup" @@ -160,6 +162,7 @@ func GlobalInitInstalled(ctx context.Context) { mustInit(pull_service.Init) mustInit(automerge.Init) mustInit(task.Init) + mustInit(bots_service.Init) mustInit(repo_migrations.Init) eventsource.GetManager().Init() @@ -195,5 +198,6 @@ func NormalRoutes(ctx context.Context) *web.Route { // This implements the OCI API (Note this is not preceded by /api but is instead /v2) r.Mount("/v2", packages_router.ContainerRoutes(ctx)) } + r.Mount("/api/actions", bots_router.Routes()) return r } diff --git a/services/bots/bots.go b/services/bots/bots.go new file mode 100644 index 0000000000000..39701105658bb --- /dev/null +++ b/services/bots/bots.go @@ -0,0 +1,62 @@ +// Copyright 2022 Gitea. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package bots + +import ( + "fmt" + + bots_model "code.gitea.io/gitea/models/bots" + "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/queue" + //"code.gitea.io/gitea/modules/json" +) + +// taskQueue is a global queue of tasks +var taskQueue queue.Queue + +// PushToQueue +func PushToQueue(task *bots_model.Task) { + taskQueue.Push(task) +} + +// Dispatch assign a task to a runner +func Dispatch(task *bots_model.Task) (*bots_model.Runner, error) { + runner, err := bots_model.GetUsableRunner(bots_model.GetRunnerOptions{ + RepoID: task.RepoID, + }) + if err != nil { + return nil, err + } + + return runner, bots_model.AssignTaskToRunner(task.ID, runner.ID) +} + +// Init will start the service to get all unfinished tasks and run them +func Init() error { + taskQueue = queue.CreateQueue("actions_task", handle, &bots_model.Task{}) + if taskQueue == nil { + return fmt.Errorf("Unable to create Task Queue") + } + + go graceful.GetManager().RunWithShutdownFns(taskQueue.Run) + + return nil +} + +func handle(data ...queue.Data) []queue.Data { + var unhandled []queue.Data + for _, datum := range data { + task := datum.(*bots_model.Task) + runner, err := Dispatch(task) + if err != nil { + log.Error("Run task failed: %v", err) + unhandled = append(unhandled, task) + } else { + log.Trace("task %v assigned to %s", task.UUID, runner.UUID) + } + } + return unhandled +} From 5a479bb03478e943b09c7dd0bd993a3785142aca Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 3 May 2022 16:03:24 +0800 Subject: [PATCH 002/408] Add builds UI --- models/bots/runner.go | 3 +- models/bots/task.go | 177 +++++++++++++++++++++--- models/bots/task_list.go | 37 +++++ models/migrations/v216.go | 51 ++++--- models/repo/repo.go | 4 + models/repo/repo_unit.go | 2 +- models/unit/unit.go | 14 ++ modules/context/context.go | 1 + modules/context/repo.go | 1 + modules/notification/bots/bots.go | 57 ++++++++ options/locale/locale_en-US.ini | 4 + routers/api/bots/bots.go | 51 ++++++- routers/web/repo/builds/builds.go | 99 +++++++++++++ routers/web/repo/setting.go | 9 ++ routers/web/web.go | 9 ++ services/forms/repo_form.go | 1 + templates/repo/builds/build_list.tmpl | 38 +++++ templates/repo/builds/list.tmpl | 35 +++++ templates/repo/builds/openclose.tmpl | 10 ++ templates/repo/builds/view.tmpl | 16 +++ templates/repo/builds/view_content.tmpl | 8 ++ templates/repo/header.tmpl | 9 ++ templates/repo/settings/options.tmpl | 13 ++ 23 files changed, 599 insertions(+), 50 deletions(-) create mode 100644 models/bots/task_list.go create mode 100644 routers/web/repo/builds/builds.go create mode 100644 templates/repo/builds/build_list.tmpl create mode 100644 templates/repo/builds/list.tmpl create mode 100644 templates/repo/builds/openclose.tmpl create mode 100644 templates/repo/builds/view.tmpl create mode 100644 templates/repo/builds/view_content.tmpl diff --git a/models/bots/runner.go b/models/bots/runner.go index f09f89d8ea6d5..dd97b92866c63 100644 --- a/models/bots/runner.go +++ b/models/bots/runner.go @@ -9,6 +9,7 @@ import ( "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/modules/timeutil" + "xorm.io/builder" ) @@ -40,7 +41,7 @@ type Runner struct { } func (Runner) TableName() string { - return "actions_runner" + return "bots_runner" } func init() { diff --git a/models/bots/task.go b/models/bots/task.go index 65eb9de77b5ad..73a9da867c996 100644 --- a/models/bots/task.go +++ b/models/bots/task.go @@ -5,16 +5,26 @@ package bots import ( + "context" "errors" "fmt" "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/models/webhook" "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" "github.com/google/uuid" + "xorm.io/builder" ) +func init() { + db.RegisterModel(new(Task)) + db.RegisterModel(new(BuildIndex)) +} + // TaskStatus represents a task status type TaskStatus int @@ -31,27 +41,70 @@ const ( // Task represnets bot tasks type Task struct { - ID int64 - UUID string `xorm:"CHAR(36)"` - RepoID int64 `xorm:"index"` - TriggerUserID int64 - Ref string - CommitSHA string - Event webhook.HookEventType - Token string // token for this task - Grant string // permissions for this task - EventPayload string `xorm:"LONGTEXT"` - RunnerID int64 `xorm:"index"` - Status TaskStatus `xorm:"index"` - Created timeutil.TimeStamp `xorm:"created"` - StartTime timeutil.TimeStamp - EndTime timeutil.TimeStamp - Updated timeutil.TimeStamp `xorm:"updated"` + ID int64 + Title string + UUID string `xorm:"CHAR(36)"` + Index int64 `xorm:"index unique(repo_index)"` + RepoID int64 `xorm:"index unique(repo_index)"` + TriggerUserID int64 + TriggerUser *user_model.User `xorm:"-"` + Ref string + CommitSHA string + Event webhook.HookEventType + Token string // token for this task + Grant string // permissions for this task + EventPayload string `xorm:"LONGTEXT"` + RunnerID int64 `xorm:"index"` + Status TaskStatus `xorm:"index"` + WorkflowsStatuses map[string]map[string]TaskStatus `xorm:"LONGTEXT"` + Created timeutil.TimeStamp `xorm:"created"` + StartTime timeutil.TimeStamp + EndTime timeutil.TimeStamp + Updated timeutil.TimeStamp `xorm:"updated"` +} + +func (t *Task) IsPending() bool { + return t.Status == TaskPending || t.Status == TaskAssigned +} + +func (t *Task) IsRunning() bool { + return t.Status == TaskRunning +} + +func (t *Task) IsFailed() bool { + return t.Status == TaskFailed || t.Status == TaskCanceled || t.Status == TaskTimeout +} + +func (t *Task) IsSuccess() bool { + return t.Status == TaskFinished } // TableName represents a bot task func (Task) TableName() string { - return "actions_task" + return "bots_task" +} + +func (t *Task) HTMLURL() string { + return fmt.Sprintf("") +} + +func updateRepoBuildsNumbers(ctx context.Context, repo *repo_model.Repository) error { + _, err := db.GetEngine(ctx).ID(repo.ID). + SetExpr("num_builds", + builder.Select("count(*)").From("bots_task"). + Where(builder.Eq{"repo_id": repo.ID}), + ). + SetExpr("num_closed_builds", + builder.Select("count(*)").From("bots_task"). + Where(builder.Eq{ + "repo_id": repo.ID, + }.And( + builder.In("status", TaskFailed, TaskCanceled, TaskTimeout, TaskFinished), + ), + ), + ). + Update(repo) + return err } // InsertTask inserts a bot task @@ -59,7 +112,27 @@ func InsertTask(t *Task) error { if t.UUID == "" { t.UUID = uuid.New().String() } - return db.Insert(db.DefaultContext, t) + index, err := db.GetNextResourceIndex("build_index", t.RepoID) + if err != nil { + return err + } + t.Index = index + + ctx, commiter, err := db.TxContext() + if err != nil { + return err + } + defer commiter.Close() + + if err := db.Insert(ctx, t); err != nil { + return err + } + + if err := updateRepoBuildsNumbers(ctx, &repo_model.Repository{ID: t.RepoID}); err != nil { + return err + } + + return commiter.Commit() } // UpdateTask updates bot task @@ -70,7 +143,9 @@ func UpdateTask(t *Task, cols ...string) error { // ErrTaskNotExist represents an error for bot task not exist type ErrTaskNotExist struct { - UUID string + RepoID int64 + Index int64 + UUID string } func (err ErrTaskNotExist) Error() string { @@ -91,8 +166,8 @@ func GetTaskByUUID(taskUUID string) (*Task, error) { return &task, nil } -// GetCurTask return the task for the bot -func GetCurTask(runnerID int64) (*Task, error) { +// GetCurTaskByID return the task for the bot +func GetCurTaskByID(runnerID int64) (*Task, error) { var tasks []Task // FIXME: for test, just return all tasks err := db.GetEngine(db.DefaultContext).Where("status=?", TaskPending).Find(&tasks) @@ -108,6 +183,31 @@ func GetCurTask(runnerID int64) (*Task, error) { return &tasks[0], err } +// GetCurTaskByUUID return the task for the bot +func GetCurTaskByUUID(runnerUUID string) (*Task, error) { + runner, err := GetRunnerByUUID(runnerUUID) + if err != nil { + return nil, err + } + return GetCurTaskByID(runner.ID) +} + +func GetTaskByRepoAndIndex(repoID, index int64) (*Task, error) { + var task Task + has, err := db.GetEngine(db.DefaultContext).Where("repo_id=?", repoID). + And("`index` = ?", index). + Get(&task) + if err != nil { + return nil, err + } else if !has { + return nil, ErrTaskNotExist{ + RepoID: repoID, + Index: index, + } + } + return &task, nil +} + // AssignTaskToRunner assign a task to a runner func AssignTaskToRunner(taskID int64, runnerID int64) error { cnt, err := db.GetEngine(db.DefaultContext). @@ -126,6 +226,41 @@ func AssignTaskToRunner(taskID int64, runnerID int64) error { return nil } +type FindTaskOptions struct { + db.ListOptions + RepoID int64 + IsClosed util.OptionalBool +} + +func (opts FindTaskOptions) toConds() builder.Cond { + cond := builder.NewCond() + if opts.RepoID > 0 { + cond = cond.And(builder.Eq{"repo_id": opts.RepoID}) + } + if opts.IsClosed.IsTrue() { + cond = cond.And(builder.Expr("status IN (?,?,?,?)", TaskCanceled, TaskFailed, TaskTimeout, TaskFinished)) + } else if opts.IsClosed.IsFalse() { + cond = cond.And(builder.Expr("status IN (?,?,?)", TaskPending, TaskAssigned, TaskRunning)) + } + return cond +} + +func FindTasks(opts FindTaskOptions) (TaskList, error) { + sess := db.GetEngine(db.DefaultContext).Where(opts.toConds()) + if opts.ListOptions.PageSize > 0 { + skip, take := opts.GetSkipTake() + sess.Limit(take, skip) + } + var tasks []*Task + return tasks, sess.Find(&tasks) +} + +func CountTasks(opts FindTaskOptions) (int64, error) { + return db.GetEngine(db.DefaultContext).Table("bots_task").Where(opts.toConds()).Count() +} + type TaskStage struct{} type StageStep struct{} + +type BuildIndex db.ResourceIndex diff --git a/models/bots/task_list.go b/models/bots/task_list.go new file mode 100644 index 0000000000000..351e334f7d7bc --- /dev/null +++ b/models/bots/task_list.go @@ -0,0 +1,37 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package bots + +import ( + "code.gitea.io/gitea/models/db" + user_model "code.gitea.io/gitea/models/user" +) + +type TaskList []*Task + +// GetUserIDs returns a slice of user's id +func (tasks TaskList) GetUserIDs() []int64 { + userIDsMap := make(map[int64]struct{}) + for _, task := range tasks { + userIDsMap[task.TriggerUserID] = struct{}{} + } + userIDs := make([]int64, 0, len(userIDsMap)) + for userID := range userIDsMap { + userIDs = append(userIDs, userID) + } + return userIDs +} + +func (tasks TaskList) LoadTriggerUser() error { + userIDs := tasks.GetUserIDs() + users := make(map[int64]*user_model.User, len(userIDs)) + if err := db.GetEngine(db.DefaultContext).In("id", userIDs).Find(&users); err != nil { + return err + } + for _, task := range tasks { + task.TriggerUser = users[task.TriggerUserID] + } + return nil +} diff --git a/models/migrations/v216.go b/models/migrations/v216.go index 114cc6c4c7156..fd0d700574482 100644 --- a/models/migrations/v216.go +++ b/models/migrations/v216.go @@ -5,13 +5,14 @@ package migrations import ( + "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/modules/timeutil" "xorm.io/xorm" ) func addBotTables(x *xorm.Engine) error { - type BotRunner struct { + type BotsRunner struct { ID int64 UUID string `xorm:"CHAR(36) UNIQUE"` Name string `xorm:"VARCHAR(32) UNIQUE"` @@ -24,29 +25,37 @@ func addBotTables(x *xorm.Engine) error { Base int // 0 native 1 docker 2 virtual machine RepoRange string // glob match which repositories could use this runner Token string - LastOnline timeutil.TimeStamp + LastOnline timeutil.TimeStamp `xorm:"index"` Created timeutil.TimeStamp `xorm:"created"` } - type BotTask struct { - ID int64 - UUID string `xorm:"CHAR(36)"` - RepoID int64 `xorm:"index"` - Type string `xorm:"VARCHAR(16)"` // 0 commit 1 pullrequest - Ref string - CommitSHA string - Event string - Token string // token for this task - Grant string // permissions for this task - EventPayload string `xorm:"LONGTEXT"` - RunnerID int64 `xorm:"index"` - Status int - Content string `xorm:"LONGTEXT"` - Created timeutil.TimeStamp `xorm:"created"` - StartTime timeutil.TimeStamp - EndTime timeutil.TimeStamp - Updated timeutil.TimeStamp `xorm:"updated"` + type BotsTask struct { + ID int64 + Title string + UUID string `xorm:"CHAR(36)"` + Index int64 `xorm:"index unique(repo_index)"` + RepoID int64 `xorm:"index unique(repo_index)"` + TriggerUserID int64 + Ref string + CommitSHA string + Event string + Token string // token for this task + Grant string // permissions for this task + EventPayload string `xorm:"LONGTEXT"` + RunnerID int64 `xorm:"index"` + Status int `xorm:"index"` + Created timeutil.TimeStamp `xorm:"created"` + StartTime timeutil.TimeStamp + EndTime timeutil.TimeStamp + Updated timeutil.TimeStamp `xorm:"updated"` } - return x.Sync2(new(BotRunner), new(BotTask)) + type Repository struct { + NumBuilds int `xorm:"NOT NULL DEFAULT 0"` + NumClosedBuilds int `xorm:"NOT NULL DEFAULT 0"` + } + + type BuildIndex db.ResourceIndex + + return x.Sync2(new(BotsRunner), new(BotsTask), new(Repository), new(BuildIndex)) } diff --git a/models/repo/repo.go b/models/repo/repo.go index a3dac8383f7bc..37d8cffd60a07 100644 --- a/models/repo/repo.go +++ b/models/repo/repo.go @@ -142,6 +142,9 @@ type Repository struct { NumProjects int `xorm:"NOT NULL DEFAULT 0"` NumClosedProjects int `xorm:"NOT NULL DEFAULT 0"` NumOpenProjects int `xorm:"-"` + NumBuilds int `xorm:"NOT NULL DEFAULT 0"` + NumClosedBuilds int `xorm:"NOT NULL DEFAULT 0"` + NumOpenBuilds int `xorm:"-"` IsPrivate bool `xorm:"INDEX"` IsEmpty bool `xorm:"INDEX"` @@ -234,6 +237,7 @@ func (repo *Repository) AfterLoad() { repo.NumOpenPulls = repo.NumPulls - repo.NumClosedPulls repo.NumOpenMilestones = repo.NumMilestones - repo.NumClosedMilestones repo.NumOpenProjects = repo.NumProjects - repo.NumClosedProjects + repo.NumOpenBuilds = repo.NumBuilds - repo.NumClosedBuilds } // LoadAttributes loads attributes of the repository. diff --git a/models/repo/repo_unit.go b/models/repo/repo_unit.go index c21c1e7a79e2a..c4090255cbaa0 100644 --- a/models/repo/repo_unit.go +++ b/models/repo/repo_unit.go @@ -175,7 +175,7 @@ func (r *RepoUnit) BeforeSet(colName string, val xorm.Cell) { r.Config = new(PullRequestsConfig) case unit.TypeIssues: r.Config = new(IssuesConfig) - case unit.TypeCode, unit.TypeReleases, unit.TypeWiki, unit.TypeProjects, unit.TypePackages: + case unit.TypeCode, unit.TypeReleases, unit.TypeWiki, unit.TypeProjects, unit.TypePackages, unit.TypeBuilds: fallthrough default: r.Config = new(UnitConfig) diff --git a/models/unit/unit.go b/models/unit/unit.go index b83bd61831a44..672494be9894d 100644 --- a/models/unit/unit.go +++ b/models/unit/unit.go @@ -28,6 +28,7 @@ const ( TypeExternalTracker // 7 ExternalTracker TypeProjects // 8 Kanban board TypePackages // 9 Packages + TypeBuilds // 10 Builds ) // Value returns integer value for unit type @@ -55,6 +56,8 @@ func (u Type) String() string { return "TypeProjects" case TypePackages: return "TypePackages" + case TypeBuilds: + return "TypeBuilds" } return fmt.Sprintf("Unknown Type %d", u) } @@ -78,6 +81,7 @@ var ( TypeExternalTracker, TypeProjects, TypePackages, + TypeBuilds, } // DefaultRepoUnits contains the default unit types @@ -289,6 +293,15 @@ var ( perm.AccessModeRead, } + UnitBuilds = Unit{ + TypeBuilds, + "repo.builds", + "/builds", + "repo.builds.desc", + 7, + perm.AccessModeOwner, + } + // Units contains all the units Units = map[Type]Unit{ TypeCode: UnitCode, @@ -300,6 +313,7 @@ var ( TypeExternalWiki: UnitExternalWiki, TypeProjects: UnitProjects, TypePackages: UnitPackages, + TypeBuilds: UnitBuilds, } ) diff --git a/modules/context/context.go b/modules/context/context.go index 47368bb280584..78d3472b673cd 100644 --- a/modules/context/context.go +++ b/modules/context/context.go @@ -807,6 +807,7 @@ func Contexter(ctx context.Context) func(next http.Handler) http.Handler { ctx.Data["UnitIssuesGlobalDisabled"] = unit.TypeIssues.UnitGlobalDisabled() ctx.Data["UnitPullsGlobalDisabled"] = unit.TypePullRequests.UnitGlobalDisabled() ctx.Data["UnitProjectsGlobalDisabled"] = unit.TypeProjects.UnitGlobalDisabled() + ctx.Data["UnitBuildsGlobalDisabled"] = unit.TypeBuilds.UnitGlobalDisabled() ctx.Data["locale"] = locale ctx.Data["AllLangs"] = translation.AllLangs() diff --git a/modules/context/repo.go b/modules/context/repo.go index 1a83c49e95a26..5408991d728d3 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -1043,6 +1043,7 @@ func UnitTypes() func(ctx *Context) { ctx.Data["UnitTypeExternalTracker"] = unit_model.TypeExternalTracker ctx.Data["UnitTypeProjects"] = unit_model.TypeProjects ctx.Data["UnitTypePackages"] = unit_model.TypePackages + ctx.Data["UnitTypeBuilds"] = unit_model.TypeBuilds } } diff --git a/modules/notification/bots/bots.go b/modules/notification/bots/bots.go index 8fc31ed3cf563..a6dec9f449940 100644 --- a/modules/notification/bots/bots.go +++ b/modules/notification/bots/bots.go @@ -39,6 +39,28 @@ func NewNotifier() base.Notifier { return &botsNotifier{} } +func detectWorkflows(commit *git.Commit, event webhook.HookEventType, ref string) (bool, error) { + tree, err := commit.SubTree(".github/workflows") + if _, ok := err.(git.ErrNotExist); ok { + tree, err = commit.SubTree(".gitea/workflows") + } + if _, ok := err.(git.ErrNotExist); ok { + return false, nil + } + if err != nil { + return false, err + } + + entries, err := tree.ListEntries() + if err != nil { + return false, err + } + + log.Trace("detected %s has %d entries", commit.ID, len(entries)) + + return len(entries) > 0, nil +} + func notifyIssue(issue *models.Issue, doer *user_model.User, evt webhook.HookEventType, payload string) { err := issue.LoadRepo(db.DefaultContext) if err != nil { @@ -68,7 +90,18 @@ func notifyIssue(issue *models.Issue, doer *user_model.User, evt webhook.HookEve return } + hasWorkflows, err := detectWorkflows(commit, evt, ref) + if err != nil { + log.Error("detectWorkflows: %v", err) + return + } + if !hasWorkflows { + log.Trace("repo %s with commit %s couldn't find workflows", issue.Repo.RepoPath(), commit.ID) + return + } + task := bots_model.Task{ + Title: commit.CommitMessage, RepoID: issue.RepoID, TriggerUserID: doer.ID, Event: evt, @@ -157,6 +190,29 @@ func (a *botsNotifier) NotifyPushCommits(pusher *user_model.User, repo *repo_mod return } + gitRepo, err := git.OpenRepository(ctx, repo.RepoPath()) + if err != nil { + log.Error("commits.ToAPIPayloadCommits failed: %v", err) + return + } + defer gitRepo.Close() + + commit, err := gitRepo.GetCommit(commits.HeadCommit.Sha1) + if err != nil { + log.Error("commits.ToAPIPayloadCommits failed: %v", err) + return + } + + hasWorkflows, err := detectWorkflows(commit, webhook.HookEventPush, opts.RefFullName) + if err != nil { + log.Error("detectWorkflows: %v", err) + return + } + if !hasWorkflows { + log.Trace("repo %s with commit %s couldn't find workflows", repo.RepoPath(), commit.ID) + return + } + payload := &api.PushPayload{ Ref: opts.RefFullName, Before: opts.OldCommitID, @@ -176,6 +232,7 @@ func (a *botsNotifier) NotifyPushCommits(pusher *user_model.User, repo *repo_mod } task := bots_model.Task{ + Title: commit.Message(), RepoID: repo.ID, TriggerUserID: pusher.ID, Event: webhook.HookEventPush, diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 8503cb78d712f..7cfaac0efd065 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1219,6 +1219,10 @@ projects.open = Open projects.close = Close projects.board.assigned_to = Assigned to +builds = Builds +builds.desc = Manage builds +builds.opened_by = opened %[1]s by %[2]s + issues.desc = Organize bug reports, tasks and milestones. issues.filter_assignees = Filter Assignee issues.filter_milestones = Filter Milestone diff --git a/routers/api/bots/bots.go b/routers/api/bots/bots.go index 0e91c19617782..32fdc75673f34 100644 --- a/routers/api/bots/bots.go +++ b/routers/api/bots/bots.go @@ -37,11 +37,13 @@ var upgrader = websocket.Upgrader{ var pongWait = 60 * time.Second type Message struct { - Version int // - Type int // message type, 1 register 2 error - RunnerUUID string // runner uuid - ErrCode int // error code - ErrContent string // errors message + Version int // + Type int // message type, 1 register 2 error 3 task 4 no task + RunnerUUID string // runner uuid + ErrCode int // error code + ErrContent string // errors message + EventName string + EventPayload string } func Serve(w http.ResponseWriter, r *http.Request) { @@ -112,7 +114,7 @@ MESSAGE_BUMP: Version: 1, Type: 2, ErrCode: 1, - ErrContent: "type is not supported", + ErrContent: fmt.Sprintf("message type %d is not supported", msg.Type), } bs, err := json.Marshal(&returnMsg) if err != nil { @@ -145,5 +147,42 @@ MESSAGE_BUMP: } // TODO: find new task and send to client + task, err := bots_model.GetCurTaskByUUID(msg.RunnerUUID) + if err != nil { + log.Error("websocket[%s] get task failed: %v", r.RemoteAddr, err) + break + } + if task == nil { + returnMsg := Message{ + Version: 1, + Type: 4, + } + bs, err := json.Marshal(&returnMsg) + if err != nil { + log.Error("websocket[%s] marshal message failed: %v", r.RemoteAddr, err) + break MESSAGE_BUMP + } + err = c.WriteMessage(mt, bs) + if err != nil { + log.Error("websocket[%s] sent message failed: %v", r.RemoteAddr, err) + } + } else { + returnMsg := Message{ + Version: 1, + Type: 3, + EventName: task.Event.Event(), + EventPayload: task.EventPayload, + } + bs, err := json.Marshal(&returnMsg) + if err != nil { + log.Error("websocket[%s] marshal message failed: %v", r.RemoteAddr, err) + break + } + err = c.WriteMessage(mt, bs) + if err != nil { + log.Error("websocket[%s] sent message failed: %v", r.RemoteAddr, err) + } + } + } } diff --git a/routers/web/repo/builds/builds.go b/routers/web/repo/builds/builds.go new file mode 100644 index 0000000000000..f7d8be4a58173 --- /dev/null +++ b/routers/web/repo/builds/builds.go @@ -0,0 +1,99 @@ +// Copyright 2018 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package builds + +import ( + "net/http" + + bots_model "code.gitea.io/gitea/models/bots" + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/unit" + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/convert" + "code.gitea.io/gitea/modules/util" +) + +const ( + tplListBuilds base.TplName = "repo/builds/list" + tplViewBuild base.TplName = "repo/builds/view" +) + +// MustEnableBuilds check if builds are enabled in settings +func MustEnableBuilds(ctx *context.Context) { + if unit.TypeBuilds.UnitGlobalDisabled() { + ctx.NotFound("EnableTypeBuilds", nil) + return + } + + if ctx.Repo.Repository != nil { + if !ctx.Repo.CanRead(unit.TypeBuilds) { + ctx.NotFound("MustEnableBuilds", nil) + return + } + } +} + +func List(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("repo.builds") + ctx.Data["PageIsBuildList"] = true + + page := ctx.FormInt("page") + if page <= 0 { + page = 1 + } + + opts := bots_model.FindTaskOptions{ + ListOptions: db.ListOptions{ + Page: page, + PageSize: convert.ToCorrectPageSize(ctx.FormInt("limit")), + }, + RepoID: ctx.Repo.Repository.ID, + } + if ctx.FormString("state") == "closed" { + opts.IsClosed = util.OptionalBoolTrue + } else { + opts.IsClosed = util.OptionalBoolFalse + } + tasks, err := bots_model.FindTasks(opts) + if err != nil { + ctx.Error(http.StatusInternalServerError, err.Error()) + return + } + + if err := tasks.LoadTriggerUser(); err != nil { + ctx.Error(http.StatusInternalServerError, err.Error()) + return + } + + total, err := bots_model.CountTasks(opts) + if err != nil { + ctx.Error(http.StatusInternalServerError, err.Error()) + return + } + + ctx.Data["Tasks"] = tasks + + pager := context.NewPagination(int(total), opts.PageSize, opts.Page, 5) + pager.SetDefaultParams(ctx) + ctx.Data["Page"] = pager + + ctx.HTML(http.StatusOK, tplListBuilds) +} + +func ViewBuild(ctx *context.Context) { + index := ctx.ParamsInt64("index") + task, err := bots_model.GetTaskByRepoAndIndex(ctx.Repo.Repository.ID, index) + if err != nil { + ctx.Error(http.StatusInternalServerError, err.Error()) + return + } + + ctx.Data["Title"] = task.Title + " - " + ctx.Tr("repo.builds") + ctx.Data["PageIsBuildList"] = true + ctx.Data["Build"] = task + + ctx.HTML(http.StatusOK, tplViewBuild) +} diff --git a/routers/web/repo/setting.go b/routers/web/repo/setting.go index 2b5691ce88501..69088e18216f0 100644 --- a/routers/web/repo/setting.go +++ b/routers/web/repo/setting.go @@ -489,6 +489,15 @@ func SettingsPost(ctx *context.Context) { deleteUnitTypes = append(deleteUnitTypes, unit_model.TypePackages) } + if form.EnableBuilds && !unit_model.TypeBuilds.UnitGlobalDisabled() { + units = append(units, repo_model.RepoUnit{ + RepoID: repo.ID, + Type: unit_model.TypeBuilds, + }) + } else if !unit_model.TypeBuilds.UnitGlobalDisabled() { + deleteUnitTypes = append(deleteUnitTypes, unit_model.TypeBuilds) + } + if form.EnablePulls && !unit_model.TypePullRequests.UnitGlobalDisabled() { units = append(units, repo_model.RepoUnit{ RepoID: repo.ID, diff --git a/routers/web/web.go b/routers/web/web.go index fe5007abb76ac..2dff1d3544eb2 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -35,6 +35,7 @@ import ( "code.gitea.io/gitea/routers/web/misc" "code.gitea.io/gitea/routers/web/org" "code.gitea.io/gitea/routers/web/repo" + "code.gitea.io/gitea/routers/web/repo/builds" "code.gitea.io/gitea/routers/web/user" user_setting "code.gitea.io/gitea/routers/web/user/setting" "code.gitea.io/gitea/routers/web/user/setting/security" @@ -665,6 +666,7 @@ func RegisterRoutes(m *web.Route) { reqRepoIssuesOrPullsReader := context.RequireRepoReaderOr(unit.TypeIssues, unit.TypePullRequests) reqRepoProjectsReader := context.RequireRepoReader(unit.TypeProjects) reqRepoProjectsWriter := context.RequireRepoWriter(unit.TypeProjects) + reqRepoBuildsReader := context.RequireRepoReader(unit.TypeBuilds) reqPackageAccess := func(accessMode perm.AccessMode) func(ctx *context.Context) { return func(ctx *context.Context) { @@ -1169,6 +1171,13 @@ func RegisterRoutes(m *web.Route) { }, reqRepoProjectsWriter, context.RepoMustNotBeArchived()) }, reqRepoProjectsReader, repo.MustEnableProjects) + m.Group("/builds", func() { + m.Get("", builds.List) + m.Group("/{index}", func() { + m.Get("", builds.ViewBuild) + }) + }, reqRepoBuildsReader, builds.MustEnableBuilds) + m.Group("/wiki", func() { m.Combo("/"). Get(repo.Wiki). diff --git a/services/forms/repo_form.go b/services/forms/repo_form.go index 64f47aadd5cae..6652178cf86a8 100644 --- a/services/forms/repo_form.go +++ b/services/forms/repo_form.go @@ -148,6 +148,7 @@ type RepoSettingForm struct { EnableProjects bool EnablePackages bool EnablePulls bool + EnableBuilds bool PullsIgnoreWhitespace bool PullsAllowMerge bool PullsAllowRebase bool diff --git a/templates/repo/builds/build_list.tmpl b/templates/repo/builds/build_list.tmpl new file mode 100644 index 0000000000000..55d177cbcbb6d --- /dev/null +++ b/templates/repo/builds/build_list.tmpl @@ -0,0 +1,38 @@ +
+ {{range .Tasks}} +
  • +
    + {{if $.CanWriteIssuesOrPulls}} +
    + + +
    + {{end}} +
    + {{if .IsPending}} + + {{end}} + {{if .IsRunning}} + + {{end}} + {{if .IsSuccess}} + + {{end}} + {{if .IsFailed}} + + {{end}} +
    +
    +
    +
    + + #{{.Index}} {{.Title}} + + {{ $timeStr := TimeSinceUnix .Updated $.i18n.Lang }} + {{$.i18n.Tr "repo.builds.opened_by" $timeStr (.TriggerUser.GetDisplayName | Escape) | Safe}} +
    +
    +
  • + {{end}} +
    +{{template "base/paginate" .}} diff --git a/templates/repo/builds/list.tmpl b/templates/repo/builds/list.tmpl new file mode 100644 index 0000000000000..7692536e7b2ea --- /dev/null +++ b/templates/repo/builds/list.tmpl @@ -0,0 +1,35 @@ +{{template "base/head" .}} +
    + {{template "repo/header" .}} +
    +
    +
    + {{template "repo/builds/openclose" .}} +
    +
    +
    +
    + {{template "repo/builds/openclose" .}} +
    +
    + {{/* Ten wide does not cope well and makes the columns stack. + This seems to be related to jQuery's hide/show: in fact, switching + issue-actions and issue-filters and having this ten wide will show + this one correctly, but not the other one. */}} +
    + +
    +
    + {{template "repo/builds/build_list" .}} +
    +
    +{{template "base/footer" .}} diff --git a/templates/repo/builds/openclose.tmpl b/templates/repo/builds/openclose.tmpl new file mode 100644 index 0000000000000..aa8c621c01796 --- /dev/null +++ b/templates/repo/builds/openclose.tmpl @@ -0,0 +1,10 @@ + diff --git a/templates/repo/builds/view.tmpl b/templates/repo/builds/view.tmpl new file mode 100644 index 0000000000000..8b01e586dc1e0 --- /dev/null +++ b/templates/repo/builds/view.tmpl @@ -0,0 +1,16 @@ +{{template "base/head" .}} +
    + {{template "repo/header" .}} +
    +
    +
    +

    + {{RenderIssueTitle $.Context .Build.Title $.RepoLink $.Repository.ComposeMetas}} + #{{.Build.Index}} +

    +
    +
    + {{template "repo/builds/view_content" .}} +
    +
    +{{template "base/footer" .}} diff --git a/templates/repo/builds/view_content.tmpl b/templates/repo/builds/view_content.tmpl new file mode 100644 index 0000000000000..7dc3ca9ac8c03 --- /dev/null +++ b/templates/repo/builds/view_content.tmpl @@ -0,0 +1,8 @@ +
    + {{if .Flash}} +
    + {{template "base/alert" .}} +
    + {{end}} + +
    diff --git a/templates/repo/header.tmpl b/templates/repo/header.tmpl index e6bd839f5704e..e166acae1e528 100644 --- a/templates/repo/header.tmpl +++ b/templates/repo/header.tmpl @@ -183,6 +183,15 @@ {{end}} + {{ if and (not .UnitBuildsGlobalDisabled) (.Permission.CanRead $.UnitTypeBuilds)}} + + {{svg "octicon-git-builds"}} {{.i18n.Tr "repo.builds"}} + {{if .Repository.NumOpenBuilds}} + {{CountFmt .Repository.NumOpenBuilds}} + {{end}} + + {{end}} + {{if .Permission.CanRead $.UnitTypePackages}} {{svg "octicon-package"}} {{.locale.Tr "packages.title"}} diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index 3c5996e903a73..d515f2414160d 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -420,6 +420,19 @@ + {{$isBuildsEnabled := .Repository.UnitEnabled $.UnitTypeBuilds}} +
    + + {{if .UnitTypeBuilds.UnitGlobalDisabled}} +
    + {{else}} +
    + {{end}} + + +
    +
    + {{if not .IsMirror}}
    {{$pullRequestEnabled := .Repository.UnitEnabled $.UnitTypePullRequests}} From 2c4f6fd42f5f4d6ca2682b83a1def3bb624ffae6 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 5 May 2022 00:39:20 +0800 Subject: [PATCH 003/408] add most tables --- go.mod | 1 + models/bots/build.go | 289 +++++++++++++++ models/bots/build_job.go | 42 +++ models/bots/{task_list.go => build_list.go} | 14 +- models/bots/build_log.go | 43 +++ models/bots/task.go | 266 -------------- models/migrations/v216.go | 9 +- modules/actions/gitea/action.go | 79 ---- modules/actions/gitea/gitea.go | 60 ---- modules/actions/gitea/planner.go | 265 -------------- modules/actions/gitea/workflow.go | 377 -------------------- modules/actions/gitea/workflow_test.go | 100 ------ modules/actions/github/github.go | 165 --------- modules/actions/runner/runner.go | 27 -- modules/bots/bots.go | 61 ++++ modules/notification/bots/bots.go | 94 ++--- routers/api/bots/bots.go | 2 +- routers/web/repo/builds/builds.go | 66 +++- routers/web/web.go | 1 + services/bots/bots.go | 28 +- templates/repo/builds/build_list.tmpl | 17 +- templates/repo/builds/status.tmpl | 14 + templates/repo/builds/view.tmpl | 1 + templates/repo/builds/view_content.tmpl | 25 +- templates/repo/builds/view_left.tmpl | 18 + 25 files changed, 598 insertions(+), 1466 deletions(-) create mode 100644 models/bots/build.go create mode 100644 models/bots/build_job.go rename models/bots/{task_list.go => build_list.go} (73%) create mode 100644 models/bots/build_log.go delete mode 100644 models/bots/task.go delete mode 100644 modules/actions/gitea/action.go delete mode 100644 modules/actions/gitea/gitea.go delete mode 100644 modules/actions/gitea/planner.go delete mode 100644 modules/actions/gitea/workflow.go delete mode 100644 modules/actions/gitea/workflow_test.go delete mode 100644 modules/actions/github/github.go delete mode 100644 modules/actions/runner/runner.go create mode 100644 modules/bots/bots.go create mode 100644 templates/repo/builds/status.tmpl create mode 100644 templates/repo/builds/view_left.tmpl diff --git a/go.mod b/go.mod index ca8c79c689003..5e6529e0e4416 100644 --- a/go.mod +++ b/go.mod @@ -233,6 +233,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mrjones/oauth v0.0.0-20190623134757-126b35219450 // indirect github.com/mschoch/smat v0.2.0 // indirect + github.com/nektos/act v0.2.26 // indirect github.com/nwaples/rardecode v1.1.3 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect diff --git a/models/bots/build.go b/models/bots/build.go new file mode 100644 index 0000000000000..cd24205370f31 --- /dev/null +++ b/models/bots/build.go @@ -0,0 +1,289 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package bots + +import ( + "context" + "errors" + "fmt" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/models/webhook" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" + + "github.com/google/uuid" + "xorm.io/builder" +) + +func init() { + db.RegisterModel(new(Build)) + db.RegisterModel(new(BuildIndex)) +} + +// BuildStatus represents a build status +type BuildStatus int + +// enumerate all the statuses of bot build +const ( + BuildPending BuildStatus = iota // wait for assign + BuildAssigned // assigned to a runner + BuildRunning // running + BuildFailed + BuildFinished + BuildCanceled + BuildTimeout +) + +func (status BuildStatus) IsPending() bool { + return status == BuildPending || status == BuildAssigned +} + +func (status BuildStatus) IsRunning() bool { + return status == BuildRunning +} + +func (status BuildStatus) IsFailed() bool { + return status == BuildFailed || status == BuildCanceled || status == BuildTimeout +} + +func (status BuildStatus) IsSuccess() bool { + return status == BuildFinished +} + +// Build represnets bot build task +type Build struct { + ID int64 + Title string + UUID string `xorm:"CHAR(36)"` + Index int64 `xorm:"index unique(repo_index)"` + RepoID int64 `xorm:"index unique(repo_index)"` + TriggerUserID int64 + TriggerUser *user_model.User `xorm:"-"` + Ref string + CommitSHA string + Event webhook.HookEventType + Token string // token for this task + Grant string // permissions for this task + EventPayload string `xorm:"LONGTEXT"` + RunnerID int64 `xorm:"index"` + Status BuildStatus `xorm:"index"` + Created timeutil.TimeStamp `xorm:"created"` + StartTime timeutil.TimeStamp + EndTime timeutil.TimeStamp + Updated timeutil.TimeStamp `xorm:"updated"` +} + +// TableName represents a bot build +func (Build) TableName() string { + return "bots_build" +} + +func (t *Build) HTMLURL() string { + return fmt.Sprintf("") +} + +func updateRepoBuildsNumbers(ctx context.Context, repo *repo_model.Repository) error { + _, err := db.GetEngine(ctx).ID(repo.ID). + SetExpr("num_builds", + builder.Select("count(*)").From("bots_build"). + Where(builder.Eq{"repo_id": repo.ID}), + ). + SetExpr("num_closed_builds", + builder.Select("count(*)").From("bots_build"). + Where(builder.Eq{ + "repo_id": repo.ID, + }.And( + builder.In("status", BuildFailed, BuildCanceled, BuildTimeout, BuildFinished), + ), + ), + ). + Update(repo) + return err +} + +// InsertBuild inserts a bot build task +func InsertBuild(t *Build, workflowsStatuses map[string]map[string]BuildStatus) error { + if t.UUID == "" { + t.UUID = uuid.New().String() + } + index, err := db.GetNextResourceIndex("bots_build_index", t.RepoID) + if err != nil { + return err + } + t.Index = index + + ctx, commiter, err := db.TxContext() + if err != nil { + return err + } + defer commiter.Close() + + if err := db.Insert(ctx, t); err != nil { + return err + } + + if err := updateRepoBuildsNumbers(ctx, &repo_model.Repository{ID: t.RepoID}); err != nil { + return err + } + + var buildJobs []BuildJob + for filename, workflow := range workflowsStatuses { + for job, status := range workflow { + buildJobs = append(buildJobs, BuildJob{ + BuildID: t.ID, + Filename: filename, + Jobname: job, + Status: status, + }) + } + } + if err := db.Insert(ctx, buildJobs); err != nil { + return err + } + + if err := commiter.Commit(); err != nil { + return err + } + + if err := CreateBuildLog(t.ID); err != nil { + log.Error("create build log for %d table failed, will try it again when received logs", t.ID) + } + return nil +} + +// UpdateBuild updates bot build +func UpdateBuild(t *Build, cols ...string) error { + _, err := db.GetEngine(db.DefaultContext).ID(t.ID).Cols(cols...).Update(t) + return err +} + +// ErrBuildNotExist represents an error for bot build not exist +type ErrBuildNotExist struct { + RepoID int64 + Index int64 + UUID string +} + +func (err ErrBuildNotExist) Error() string { + return fmt.Sprintf("Bot build [%s] is not exist", err.UUID) +} + +// GetBuildByUUID gets bot build by uuid +func GetBuildByUUID(buildUUID string) (*Build, error) { + var build Build + has, err := db.GetEngine(db.DefaultContext).Where("uuid=?", buildUUID).Get(&build) + if err != nil { + return nil, err + } else if !has { + return nil, ErrBuildNotExist{ + UUID: buildUUID, + } + } + return &build, nil +} + +// GetCurBuildByID return the build for the bot +func GetCurBuildByID(runnerID int64) (*Build, error) { + var builds []Build + err := db.GetEngine(db.DefaultContext). + Where("runner_id=?", runnerID). + And("status=?", BuildPending). + Asc("created"). + Find(&builds) + if err != nil { + return nil, err + } + if len(builds) == 0 { + return nil, nil + } + return &builds[0], err +} + +// GetCurBuildByUUID return the task for the bot +func GetCurBuildByUUID(runnerUUID string) (*Build, error) { + runner, err := GetRunnerByUUID(runnerUUID) + if err != nil { + return nil, err + } + return GetCurBuildByID(runner.ID) +} + +func GetBuildByRepoAndIndex(repoID, index int64) (*Build, error) { + var build Build + has, err := db.GetEngine(db.DefaultContext).Where("repo_id=?", repoID). + And("`index` = ?", index). + Get(&build) + if err != nil { + return nil, err + } else if !has { + return nil, ErrBuildNotExist{ + RepoID: repoID, + Index: index, + } + } + return &build, nil +} + +// AssignBuildToRunner assign a build to a runner +func AssignBuildToRunner(buildID int64, runnerID int64) error { + cnt, err := db.GetEngine(db.DefaultContext). + Where("runner_id=0"). + And("id=?", buildID). + Cols("runner_id"). + Update(&Build{ + RunnerID: runnerID, + }) + if err != nil { + return err + } + if cnt != 1 { + return errors.New("assign faild") + } + return nil +} + +type FindBuildOptions struct { + db.ListOptions + RepoID int64 + IsClosed util.OptionalBool +} + +func (opts FindBuildOptions) toConds() builder.Cond { + cond := builder.NewCond() + if opts.RepoID > 0 { + cond = cond.And(builder.Eq{"repo_id": opts.RepoID}) + } + if opts.IsClosed.IsTrue() { + cond = cond.And(builder.Expr("status IN (?,?,?,?)", BuildCanceled, BuildFailed, BuildTimeout, BuildFinished)) + } else if opts.IsClosed.IsFalse() { + cond = cond.And(builder.Expr("status IN (?,?,?)", BuildPending, BuildAssigned, BuildRunning)) + } + return cond +} + +func FindBuilds(opts FindBuildOptions) (BuildList, error) { + sess := db.GetEngine(db.DefaultContext).Where(opts.toConds()) + if opts.ListOptions.PageSize > 0 { + skip, take := opts.GetSkipTake() + sess.Limit(take, skip) + } + var builds []*Build + return builds, sess.Find(&builds) +} + +func CountBuilds(opts FindBuildOptions) (int64, error) { + return db.GetEngine(db.DefaultContext).Table("bots_build").Where(opts.toConds()).Count() +} + +type BuildIndex db.ResourceIndex + +// TableName represents a bot build index +func (BuildIndex) TableName() string { + return "bots_build_index" +} diff --git a/models/bots/build_job.go b/models/bots/build_job.go new file mode 100644 index 0000000000000..62ec3b0fd0b11 --- /dev/null +++ b/models/bots/build_job.go @@ -0,0 +1,42 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package bots + +import ( + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/timeutil" +) + +type BuildJob struct { + ID int64 + BuildID int64 `xorm:"index"` + Filename string + Jobname string + Status BuildStatus + LogToFile bool // read log from database or from storage + Created timeutil.TimeStamp `xorm:"created"` +} + +func (bj BuildJob) TableName() string { + return "bots_build_job" +} + +func init() { + db.RegisterModel(new(BuildJob)) +} + +func GetBuildWorkflows(buildID int64) (map[string]map[string]*BuildJob, error) { + jobs := make(map[string]map[string]*BuildJob) + err := db.GetEngine(db.DefaultContext).Iterate(new(BuildJob), func(idx int, bean interface{}) error { + job := bean.(*BuildJob) + _, ok := jobs[job.Filename] + if !ok { + jobs[job.Filename] = make(map[string]*BuildJob) + } + jobs[job.Filename][job.Jobname] = job + return nil + }) + return jobs, err +} diff --git a/models/bots/task_list.go b/models/bots/build_list.go similarity index 73% rename from models/bots/task_list.go rename to models/bots/build_list.go index 351e334f7d7bc..fdc6ef66f004b 100644 --- a/models/bots/task_list.go +++ b/models/bots/build_list.go @@ -9,13 +9,13 @@ import ( user_model "code.gitea.io/gitea/models/user" ) -type TaskList []*Task +type BuildList []*Build // GetUserIDs returns a slice of user's id -func (tasks TaskList) GetUserIDs() []int64 { +func (builds BuildList) GetUserIDs() []int64 { userIDsMap := make(map[int64]struct{}) - for _, task := range tasks { - userIDsMap[task.TriggerUserID] = struct{}{} + for _, build := range builds { + userIDsMap[build.TriggerUserID] = struct{}{} } userIDs := make([]int64, 0, len(userIDsMap)) for userID := range userIDsMap { @@ -24,13 +24,13 @@ func (tasks TaskList) GetUserIDs() []int64 { return userIDs } -func (tasks TaskList) LoadTriggerUser() error { - userIDs := tasks.GetUserIDs() +func (builds BuildList) LoadTriggerUser() error { + userIDs := builds.GetUserIDs() users := make(map[int64]*user_model.User, len(userIDs)) if err := db.GetEngine(db.DefaultContext).In("id", userIDs).Find(&users); err != nil { return err } - for _, task := range tasks { + for _, task := range builds { task.TriggerUser = users[task.TriggerUserID] } return nil diff --git a/models/bots/build_log.go b/models/bots/build_log.go new file mode 100644 index 0000000000000..93d3b695e0381 --- /dev/null +++ b/models/bots/build_log.go @@ -0,0 +1,43 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package bots + +import ( + "fmt" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/timeutil" +) + +// BuildLog represents a build's log, every build has a standalone table +type BuildLog struct { + ID int64 + BuildJobID int64 `xorm:"index"` + LineNumber int + Content string `xorm:"LONGTEXT"` + Created timeutil.TimeStamp `xorm:"created"` +} + +func init() { + db.RegisterModel(new(BuildLog)) +} + +func GetBuildLogTableName(buildID int64) string { + return fmt.Sprintf("bots_build_log_%d", buildID) +} + +// CreateBuildLog table for a build +func CreateBuildLog(buildID int64) error { + return db.GetEngine(db.DefaultContext). + Table(GetBuildLogTableName(buildID)). + Sync2(new(BuildLog)) +} + +func GetBuildLogs(buildID, jobID int64) (logs []BuildLog, err error) { + err = db.GetEngine(db.DefaultContext).Table(GetBuildLogTableName(buildID)). + Where("build_job_id=?", jobID). + Find(&logs) + return +} diff --git a/models/bots/task.go b/models/bots/task.go deleted file mode 100644 index 73a9da867c996..0000000000000 --- a/models/bots/task.go +++ /dev/null @@ -1,266 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package bots - -import ( - "context" - "errors" - "fmt" - - "code.gitea.io/gitea/models/db" - repo_model "code.gitea.io/gitea/models/repo" - user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/models/webhook" - "code.gitea.io/gitea/modules/timeutil" - "code.gitea.io/gitea/modules/util" - - "github.com/google/uuid" - "xorm.io/builder" -) - -func init() { - db.RegisterModel(new(Task)) - db.RegisterModel(new(BuildIndex)) -} - -// TaskStatus represents a task status -type TaskStatus int - -// enumerate all the statuses of bot task -const ( - TaskPending TaskStatus = iota // wait for assign - TaskAssigned // assigned to a runner - TaskRunning // running - TaskFailed - TaskFinished - TaskCanceled - TaskTimeout -) - -// Task represnets bot tasks -type Task struct { - ID int64 - Title string - UUID string `xorm:"CHAR(36)"` - Index int64 `xorm:"index unique(repo_index)"` - RepoID int64 `xorm:"index unique(repo_index)"` - TriggerUserID int64 - TriggerUser *user_model.User `xorm:"-"` - Ref string - CommitSHA string - Event webhook.HookEventType - Token string // token for this task - Grant string // permissions for this task - EventPayload string `xorm:"LONGTEXT"` - RunnerID int64 `xorm:"index"` - Status TaskStatus `xorm:"index"` - WorkflowsStatuses map[string]map[string]TaskStatus `xorm:"LONGTEXT"` - Created timeutil.TimeStamp `xorm:"created"` - StartTime timeutil.TimeStamp - EndTime timeutil.TimeStamp - Updated timeutil.TimeStamp `xorm:"updated"` -} - -func (t *Task) IsPending() bool { - return t.Status == TaskPending || t.Status == TaskAssigned -} - -func (t *Task) IsRunning() bool { - return t.Status == TaskRunning -} - -func (t *Task) IsFailed() bool { - return t.Status == TaskFailed || t.Status == TaskCanceled || t.Status == TaskTimeout -} - -func (t *Task) IsSuccess() bool { - return t.Status == TaskFinished -} - -// TableName represents a bot task -func (Task) TableName() string { - return "bots_task" -} - -func (t *Task) HTMLURL() string { - return fmt.Sprintf("") -} - -func updateRepoBuildsNumbers(ctx context.Context, repo *repo_model.Repository) error { - _, err := db.GetEngine(ctx).ID(repo.ID). - SetExpr("num_builds", - builder.Select("count(*)").From("bots_task"). - Where(builder.Eq{"repo_id": repo.ID}), - ). - SetExpr("num_closed_builds", - builder.Select("count(*)").From("bots_task"). - Where(builder.Eq{ - "repo_id": repo.ID, - }.And( - builder.In("status", TaskFailed, TaskCanceled, TaskTimeout, TaskFinished), - ), - ), - ). - Update(repo) - return err -} - -// InsertTask inserts a bot task -func InsertTask(t *Task) error { - if t.UUID == "" { - t.UUID = uuid.New().String() - } - index, err := db.GetNextResourceIndex("build_index", t.RepoID) - if err != nil { - return err - } - t.Index = index - - ctx, commiter, err := db.TxContext() - if err != nil { - return err - } - defer commiter.Close() - - if err := db.Insert(ctx, t); err != nil { - return err - } - - if err := updateRepoBuildsNumbers(ctx, &repo_model.Repository{ID: t.RepoID}); err != nil { - return err - } - - return commiter.Commit() -} - -// UpdateTask updates bot task -func UpdateTask(t *Task, cols ...string) error { - _, err := db.GetEngine(db.DefaultContext).ID(t.ID).Cols(cols...).Update(t) - return err -} - -// ErrTaskNotExist represents an error for bot task not exist -type ErrTaskNotExist struct { - RepoID int64 - Index int64 - UUID string -} - -func (err ErrTaskNotExist) Error() string { - return fmt.Sprintf("Bot task [%s] is not exist", err.UUID) -} - -// GetTaskByUUID gets bot task by uuid -func GetTaskByUUID(taskUUID string) (*Task, error) { - var task Task - has, err := db.GetEngine(db.DefaultContext).Where("uuid=?", taskUUID).Get(&task) - if err != nil { - return nil, err - } else if !has { - return nil, ErrTaskNotExist{ - UUID: taskUUID, - } - } - return &task, nil -} - -// GetCurTaskByID return the task for the bot -func GetCurTaskByID(runnerID int64) (*Task, error) { - var tasks []Task - // FIXME: for test, just return all tasks - err := db.GetEngine(db.DefaultContext).Where("status=?", TaskPending).Find(&tasks) - // err := x.Where("runner_id = ?", botID). - // And("status=?", BotTaskPending). - // Find(&tasks) - if err != nil { - return nil, err - } - if len(tasks) == 0 { - return nil, nil - } - return &tasks[0], err -} - -// GetCurTaskByUUID return the task for the bot -func GetCurTaskByUUID(runnerUUID string) (*Task, error) { - runner, err := GetRunnerByUUID(runnerUUID) - if err != nil { - return nil, err - } - return GetCurTaskByID(runner.ID) -} - -func GetTaskByRepoAndIndex(repoID, index int64) (*Task, error) { - var task Task - has, err := db.GetEngine(db.DefaultContext).Where("repo_id=?", repoID). - And("`index` = ?", index). - Get(&task) - if err != nil { - return nil, err - } else if !has { - return nil, ErrTaskNotExist{ - RepoID: repoID, - Index: index, - } - } - return &task, nil -} - -// AssignTaskToRunner assign a task to a runner -func AssignTaskToRunner(taskID int64, runnerID int64) error { - cnt, err := db.GetEngine(db.DefaultContext). - Where("runner_id=0"). - And("id=?", taskID). - Cols("runner_id"). - Update(&Task{ - RunnerID: runnerID, - }) - if err != nil { - return err - } - if cnt != 1 { - return errors.New("assign faild") - } - return nil -} - -type FindTaskOptions struct { - db.ListOptions - RepoID int64 - IsClosed util.OptionalBool -} - -func (opts FindTaskOptions) toConds() builder.Cond { - cond := builder.NewCond() - if opts.RepoID > 0 { - cond = cond.And(builder.Eq{"repo_id": opts.RepoID}) - } - if opts.IsClosed.IsTrue() { - cond = cond.And(builder.Expr("status IN (?,?,?,?)", TaskCanceled, TaskFailed, TaskTimeout, TaskFinished)) - } else if opts.IsClosed.IsFalse() { - cond = cond.And(builder.Expr("status IN (?,?,?)", TaskPending, TaskAssigned, TaskRunning)) - } - return cond -} - -func FindTasks(opts FindTaskOptions) (TaskList, error) { - sess := db.GetEngine(db.DefaultContext).Where(opts.toConds()) - if opts.ListOptions.PageSize > 0 { - skip, take := opts.GetSkipTake() - sess.Limit(take, skip) - } - var tasks []*Task - return tasks, sess.Find(&tasks) -} - -func CountTasks(opts FindTaskOptions) (int64, error) { - return db.GetEngine(db.DefaultContext).Table("bots_task").Where(opts.toConds()).Count() -} - -type TaskStage struct{} - -type StageStep struct{} - -type BuildIndex db.ResourceIndex diff --git a/models/migrations/v216.go b/models/migrations/v216.go index fd0d700574482..a8e1110d3f2e2 100644 --- a/models/migrations/v216.go +++ b/models/migrations/v216.go @@ -5,9 +5,6 @@ package migrations import ( - "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/modules/timeutil" - "xorm.io/xorm" ) @@ -29,7 +26,7 @@ func addBotTables(x *xorm.Engine) error { Created timeutil.TimeStamp `xorm:"created"` } - type BotsTask struct { + type BotsBuild struct { ID int64 Title string UUID string `xorm:"CHAR(36)"` @@ -55,7 +52,7 @@ func addBotTables(x *xorm.Engine) error { NumClosedBuilds int `xorm:"NOT NULL DEFAULT 0"` } - type BuildIndex db.ResourceIndex + type BotsBuildIndex db.ResourceIndex - return x.Sync2(new(BotsRunner), new(BotsTask), new(Repository), new(BuildIndex)) + return x.Sync2(new(BotsRunner), new(BotsBuild), new(Repository), new(BotsBuildIndex)) } diff --git a/modules/actions/gitea/action.go b/modules/actions/gitea/action.go deleted file mode 100644 index ef4a4a41dda36..0000000000000 --- a/modules/actions/gitea/action.go +++ /dev/null @@ -1,79 +0,0 @@ -package gitea - -import ( - "fmt" - "io" - "strings" - - "gopkg.in/yaml.v3" -) - -// ActionRunsUsing is the type of runner for the action -type ActionRunsUsing string - -func (a *ActionRunsUsing) UnmarshalYAML(unmarshal func(interface{}) error) error { - var using string - if err := unmarshal(&using); err != nil { - return err - } - - // Force input to lowercase for case insensitive comparison - format := ActionRunsUsing(strings.ToLower(using)) - switch format { - case ActionRunsUsingNode12, ActionRunsUsingDocker: - *a = format - default: - return fmt.Errorf(fmt.Sprintf("The runs.using key in action.yml must be one of: %v, got %s", []string{ - ActionRunsUsingDocker, - ActionRunsUsingNode12, - }, format)) - } - return nil -} - -const ( - // ActionRunsUsingNode12 for running with node12 - ActionRunsUsingNode12 = "node12" - // ActionRunsUsingDocker for running with docker - ActionRunsUsingDocker = "docker" -) - -// Action describes a metadata file for GitHub actions. The metadata filename must be either action.yml or action.yaml. The data in the metadata file defines the inputs, outputs and main entrypoint for your action. -type Action struct { - Name string `yaml:"name"` - Author string `yaml:"author"` - Description string `yaml:"description"` - Inputs map[string]Input `yaml:"inputs"` - Outputs map[string]Output `yaml:"outputs"` - Runs struct { - Using ActionRunsUsing `yaml:"using"` - Env map[string]string `yaml:"env"` - Main string `yaml:"main"` - Image string `yaml:"image"` - Entrypoint []string `yaml:"entrypoint"` - Args []string `yaml:"args"` - } `yaml:"runs"` - Branding struct { - Color string `yaml:"color"` - Icon string `yaml:"icon"` - } `yaml:"branding"` -} - -// Input parameters allow you to specify data that the action expects to use during runtime. GitHub stores input parameters as environment variables. Input ids with uppercase letters are converted to lowercase during runtime. We recommended using lowercase input ids. -type Input struct { - Description string `yaml:"description"` - Required bool `yaml:"required"` - Default string `yaml:"default"` -} - -// Output parameters allow you to declare data that an action sets. Actions that run later in a workflow can use the output data set in previously run actions. For example, if you had an action that performed the addition of two inputs (x + y = z), the action could output the sum (z) for other actions to use as an input. -type Output struct { - Description string `yaml:"description"` -} - -// ReadAction reads an action from a reader -func ReadAction(in io.Reader) (*Action, error) { - a := new(Action) - err := yaml.NewDecoder(in).Decode(a) - return a, err -} diff --git a/modules/actions/gitea/gitea.go b/modules/actions/gitea/gitea.go deleted file mode 100644 index 8f7c0835e5dec..0000000000000 --- a/modules/actions/gitea/gitea.go +++ /dev/null @@ -1,60 +0,0 @@ -package gitea - -import ( - "code.gitea.io/gitea/models/webhook" - "code.gitea.io/gitea/modules/bot/runner" - "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/util" - "code.gitea.io/gitea/modules/json" -) - -func init() { - runner.RegisterRunnerType(new(GiteaRunner)) -} - -type GiteaRunner struct { -} - -func (gw *GiteaRunner) Name() string { - return "gitea" -} - -func (gw *GiteaRunner) Detect(commit *git.Commit, event webhook.HookEventType, ref string) (bool, string, error) { - tree, err := commit.SubTree(".gitea/workflow") - if err != nil { - return false, "", err - } - entries, err := tree.ListEntries() - if err != nil { - return false, "", err - } - - var wfs []*Workflow - for _, entry := range entries { - blob := entry.Blob() - rd, err := blob.DataAsync() - if err != nil { - return false, "", err - } - defer rd.Close() - wf, err := ReadWorkflow(rd) - if err != nil { - log.Error("ReadWorkflow file %s failed: %v", entry.Name(), err) - continue - } - - // FIXME: we have to convert the event type to github known name - if !util.IsStringInSlice(string(event), wf.On()) { - continue - } - - wfs = append(wfs, wf) - } - - wfBs, err := json.Marshal(wfs) - if err != nil { - return false, "", err - } - return true, string(wfBs), nil -} diff --git a/modules/actions/gitea/planner.go b/modules/actions/gitea/planner.go deleted file mode 100644 index 6d80e79d49c5b..0000000000000 --- a/modules/actions/gitea/planner.go +++ /dev/null @@ -1,265 +0,0 @@ -package gitea - -import ( - "io" - "io/ioutil" - "math" - "os" - "path/filepath" - "sort" - - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" -) - -// WorkflowPlanner contains methods for creating plans -type WorkflowPlanner interface { - PlanEvent(eventName string) *Plan - PlanJob(jobName string) *Plan - GetEvents() []string -} - -// Plan contains a list of stages to run in series -type Plan struct { - Stages []*Stage -} - -// Stage contains a list of runs to execute in parallel -type Stage struct { - Runs []*Run -} - -// Run represents a job from a workflow that needs to be run -type Run struct { - Workflow *Workflow - JobID string -} - -func (r *Run) String() string { - jobName := r.Job().Name - if jobName == "" { - jobName = r.JobID - } - return jobName -} - -// Job returns the job for this Run -func (r *Run) Job() *Job { - return r.Workflow.GetJob(r.JobID) -} - -// NewWorkflowPlanner will load a specific workflow or all workflows from a directory -func NewWorkflowPlanner(path string) (WorkflowPlanner, error) { - fi, err := os.Stat(path) - if err != nil { - return nil, err - } - - var files []os.FileInfo - var dirname string - - if fi.IsDir() { - log.Debugf("Loading workflows from '%s'", path) - dirname = path - files, err = ioutil.ReadDir(path) - } else { - log.Debugf("Loading workflow '%s'", path) - dirname, err = filepath.Abs(filepath.Dir(path)) - files = []os.FileInfo{fi} - } - if err != nil { - return nil, err - } - - wp := new(workflowPlanner) - for _, file := range files { - ext := filepath.Ext(file.Name()) - if ext == ".yml" || ext == ".yaml" { - f, err := os.Open(filepath.Join(dirname, file.Name())) - if err != nil { - return nil, err - } - - log.Debugf("Reading workflow '%s'", f.Name()) - workflow, err := ReadWorkflow(f) - if err != nil { - f.Close() - if err == io.EOF { - return nil, errors.WithMessagef(err, "unable to read workflow, %s file is empty", file.Name()) - } - return nil, err - } - if workflow.Name == "" { - workflow.Name = file.Name() - } - wp.workflows = append(wp.workflows, workflow) - f.Close() - } - } - - return wp, nil -} - -type workflowPlanner struct { - workflows []*Workflow -} - -// PlanEvent builds a new list of runs to execute in parallel for an event name -func (wp *workflowPlanner) PlanEvent(eventName string) *Plan { - plan := new(Plan) - if len(wp.workflows) == 0 { - log.Debugf("no events found for workflow: %s", eventName) - } - - for _, w := range wp.workflows { - for _, e := range w.When().Events { - if e.Type == eventName { - plan.mergeStages(createStages(w, w.GetJobIDs()...)) - } - } - } - return plan -} - -// PlanJob builds a new run to execute in parallel for a job name -func (wp *workflowPlanner) PlanJob(jobName string) *Plan { - plan := new(Plan) - if len(wp.workflows) == 0 { - log.Debugf("no jobs found for workflow: %s", jobName) - } - - for _, w := range wp.workflows { - plan.mergeStages(createStages(w, jobName)) - } - return plan -} - -// GetEvents gets all the events in the workflows file -func (wp *workflowPlanner) GetEvents() []string { - events := make([]string, 0) - for _, w := range wp.workflows { - found := false - for _, e := range events { - for _, we := range w.When().Events { - if e == we.Type { - found = true - break - } - } - if found { - break - } - } - - if !found { - for _, evt := range w.When().Events { - events = append(events, evt.Type) - } - } - } - - // sort the list based on depth of dependencies - sort.Slice(events, func(i, j int) bool { - return events[i] < events[j] - }) - - return events -} - -// MaxRunNameLen determines the max name length of all jobs -func (p *Plan) MaxRunNameLen() int { - maxRunNameLen := 0 - for _, stage := range p.Stages { - for _, run := range stage.Runs { - runNameLen := len(run.String()) - if runNameLen > maxRunNameLen { - maxRunNameLen = runNameLen - } - } - } - return maxRunNameLen -} - -// GetJobIDs will get all the job names in the stage -func (s *Stage) GetJobIDs() []string { - names := make([]string, 0) - for _, r := range s.Runs { - names = append(names, r.JobID) - } - return names -} - -// Merge stages with existing stages in plan -func (p *Plan) mergeStages(stages []*Stage) { - newStages := make([]*Stage, int(math.Max(float64(len(p.Stages)), float64(len(stages))))) - for i := 0; i < len(newStages); i++ { - newStages[i] = new(Stage) - if i >= len(p.Stages) { - newStages[i].Runs = append(newStages[i].Runs, stages[i].Runs...) - } else if i >= len(stages) { - newStages[i].Runs = append(newStages[i].Runs, p.Stages[i].Runs...) - } else { - newStages[i].Runs = append(newStages[i].Runs, p.Stages[i].Runs...) - newStages[i].Runs = append(newStages[i].Runs, stages[i].Runs...) - } - } - p.Stages = newStages -} - -func createStages(w *Workflow, jobIDs ...string) []*Stage { - // first, build a list of all the necessary jobs to run, and their dependencies - jobDependencies := make(map[string][]string) - for len(jobIDs) > 0 { - newJobIDs := make([]string, 0) - for _, jID := range jobIDs { - // make sure we haven't visited this job yet - if _, ok := jobDependencies[jID]; !ok { - if job := w.GetJob(jID); job != nil { - jobDependencies[jID] = job.Needs() - newJobIDs = append(newJobIDs, job.Needs()...) - } - } - } - jobIDs = newJobIDs - } - - // next, build an execution graph - stages := make([]*Stage, 0) - for len(jobDependencies) > 0 { - stage := new(Stage) - for jID, jDeps := range jobDependencies { - // make sure all deps are in the graph already - if listInStages(jDeps, stages...) { - stage.Runs = append(stage.Runs, &Run{ - Workflow: w, - JobID: jID, - }) - delete(jobDependencies, jID) - } - } - if len(stage.Runs) == 0 { - log.Fatalf("Unable to build dependency graph!") - } - stages = append(stages, stage) - } - - return stages -} - -// return true iff all strings in srcList exist in at least one of the stages -func listInStages(srcList []string, stages ...*Stage) bool { - for _, src := range srcList { - found := false - for _, stage := range stages { - for _, search := range stage.GetJobIDs() { - if src == search { - found = true - } - } - } - if !found { - return false - } - } - return true -} diff --git a/modules/actions/gitea/workflow.go b/modules/actions/gitea/workflow.go deleted file mode 100644 index 2a8a5d04f037b..0000000000000 --- a/modules/actions/gitea/workflow.go +++ /dev/null @@ -1,377 +0,0 @@ -package gitea - -import ( - "fmt" - "io" - "reflect" - "regexp" - "strings" - - log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v3" -) - -// Workflow is the structure of the files in .github/workflows -type Workflow struct { - Name string `yaml:"name"` - RawWhen yaml.Node `yaml:"when"` - Env map[string]string `yaml:"env"` - Jobs map[string]*Job `yaml:"jobs"` - Defaults Defaults `yaml:"defaults"` -} - -type Event struct { - Type string - Ref string -} - -type When struct { - Events []Event -} - -func (w *When) Match(tp string) bool { - for _, evt := range w.Events { - if strings.EqualFold(tp, evt.Type) { - return true - } - } - return false -} - -// When events for the workflow -func (w *Workflow) When() *When { - switch w.RawWhen.Kind { - case yaml.ScalarNode: - var val string - err := w.RawWhen.Decode(&val) - if err != nil { - log.Fatal(err) - } - return &When{ - Events: []Event{ - { - Type: val, - }, - }, - } - case yaml.SequenceNode: - var vals []string - err := w.RawWhen.Decode(&vals) - if err != nil { - log.Fatal(err) - } - var when When - for _, val := range vals { - when.Events = append(when.Events, Event{ - Type: val, - }) - } - return &when - case yaml.MappingNode: - var val map[string]interface{} - err := w.RawWhen.Decode(&val) - if err != nil { - log.Fatal(err) - } - var keys []string - for k := range val { - keys = append(keys, k) - } - var when When - for _, val := range keys { - when.Events = append(when.Events, Event{ - Type: val, - }) - } - return &when - } - return nil -} - -// Job is the structure of one job in a workflow -type Job struct { - Name string `yaml:"name"` - RawNeeds yaml.Node `yaml:"needs"` - RawRunsOn yaml.Node `yaml:"runs-on"` - Env map[string]string `yaml:"env"` - If string `yaml:"if"` - Steps []*Step `yaml:"steps"` - TimeoutMinutes int64 `yaml:"timeout-minutes"` - Services map[string]*ContainerSpec `yaml:"services"` - Strategy *Strategy `yaml:"strategy"` - RawContainer yaml.Node `yaml:"container"` - Defaults Defaults `yaml:"defaults"` -} - -// Strategy for the job -type Strategy struct { - FailFast bool `yaml:"fail-fast"` - MaxParallel int `yaml:"max-parallel"` - Matrix map[string][]interface{} `yaml:"matrix"` -} - -// Default settings that will apply to all steps in the job or workflow -type Defaults struct { - Run RunDefaults `yaml:"run"` -} - -// Defaults for all run steps in the job or workflow -type RunDefaults struct { - Shell string `yaml:"shell"` - WorkingDirectory string `yaml:"working-directory"` -} - -// Container details for the job -func (j *Job) Container() *ContainerSpec { - var val *ContainerSpec - switch j.RawContainer.Kind { - case yaml.ScalarNode: - val = new(ContainerSpec) - err := j.RawContainer.Decode(&val.Image) - if err != nil { - log.Fatal(err) - } - case yaml.MappingNode: - val = new(ContainerSpec) - err := j.RawContainer.Decode(val) - if err != nil { - log.Fatal(err) - } - } - return val -} - -// Needs list for Job -func (j *Job) Needs() []string { - - switch j.RawNeeds.Kind { - case yaml.ScalarNode: - var val string - err := j.RawNeeds.Decode(&val) - if err != nil { - log.Fatal(err) - } - return []string{val} - case yaml.SequenceNode: - var val []string - err := j.RawNeeds.Decode(&val) - if err != nil { - log.Fatal(err) - } - return val - } - return nil -} - -// RunsOn list for Job -func (j *Job) RunsOn() []string { - - switch j.RawRunsOn.Kind { - case yaml.ScalarNode: - var val string - err := j.RawRunsOn.Decode(&val) - if err != nil { - log.Fatal(err) - } - return []string{val} - case yaml.SequenceNode: - var val []string - err := j.RawRunsOn.Decode(&val) - if err != nil { - log.Fatal(err) - } - return val - } - return nil -} - -// GetMatrixes returns the matrix cross product -func (j *Job) GetMatrixes() []map[string]interface{} { - matrixes := make([]map[string]interface{}, 0) - /*if j.Strategy != nil { - includes := make([]map[string]interface{}, 0) - for _, v := range j.Strategy.Matrix["include"] { - includes = append(includes, v.(map[string]interface{})) - } - delete(j.Strategy.Matrix, "include") - - excludes := make([]map[string]interface{}, 0) - for _, v := range j.Strategy.Matrix["exclude"] { - excludes = append(excludes, v.(map[string]interface{})) - } - delete(j.Strategy.Matrix, "exclude") - - matrixProduct := common.CartesianProduct(j.Strategy.Matrix) - - MATRIX: - for _, matrix := range matrixProduct { - for _, exclude := range excludes { - if commonKeysMatch(matrix, exclude) { - log.Debugf("Skipping matrix '%v' due to exclude '%v'", matrix, exclude) - continue MATRIX - } - } - matrixes = append(matrixes, matrix) - } - for _, include := range includes { - log.Debugf("Adding include '%v'", include) - matrixes = append(matrixes, include) - } - - } else { - matrixes = append(matrixes, make(map[string]interface{})) - }*/ - return matrixes -} - -func commonKeysMatch(a map[string]interface{}, b map[string]interface{}) bool { - for aKey, aVal := range a { - if bVal, ok := b[aKey]; ok && !reflect.DeepEqual(aVal, bVal) { - return false - } - } - return true -} - -// ContainerSpec is the specification of the container to use for the job -type ContainerSpec struct { - Image string `yaml:"image"` - Env map[string]string `yaml:"env"` - Ports []string `yaml:"ports"` - Volumes []string `yaml:"volumes"` - Options string `yaml:"options"` - Entrypoint string - Args string - Name string - Reuse bool -} - -// Step is the structure of one step in a job -type Step struct { - ID string `yaml:"id"` - If string `yaml:"if"` - Name string `yaml:"name"` - Uses string `yaml:"uses"` - Run string `yaml:"run"` - WorkingDirectory string `yaml:"working-directory"` - Shell string `yaml:"shell"` - Env map[string]string `yaml:"env"` - With map[string]string `yaml:"with"` - ContinueOnError bool `yaml:"continue-on-error"` - TimeoutMinutes int64 `yaml:"timeout-minutes"` -} - -// String gets the name of step -func (s *Step) String() string { - if s.Name != "" { - return s.Name - } else if s.Uses != "" { - return s.Uses - } else if s.Run != "" { - return s.Run - } - return s.ID -} - -// GetEnv gets the env for a step -func (s *Step) GetEnv() map[string]string { - rtnEnv := make(map[string]string) - for k, v := range s.Env { - rtnEnv[k] = v - } - for k, v := range s.With { - envKey := regexp.MustCompile("[^A-Z0-9-]").ReplaceAllString(strings.ToUpper(k), "_") - envKey = fmt.Sprintf("INPUT_%s", strings.ToUpper(envKey)) - rtnEnv[envKey] = v - } - return rtnEnv -} - -// ShellCommand returns the command for the shell -func (s *Step) ShellCommand() string { - shellCommand := "" - - switch s.Shell { - case "", "bash": - shellCommand = "bash --noprofile --norc -eo pipefail {0}" - case "pwsh": - shellCommand = "pwsh -command \"& '{0}'\"" - case "python": - shellCommand = "python {0}" - case "sh": - shellCommand = "sh -e -c {0}" - case "cmd": - shellCommand = "%ComSpec% /D /E:ON /V:OFF /S /C \"CALL \"{0}\"\"" - case "powershell": - shellCommand = "powershell -command \"& '{0}'\"" - default: - shellCommand = s.Shell - } - return shellCommand -} - -// StepType describes what type of step we are about to run -type StepType int - -const ( - // StepTypeRun is all steps that have a `run` attribute - StepTypeRun StepType = iota - - //StepTypeUsesDockerURL is all steps that have a `uses` that is of the form `docker://...` - StepTypeUsesDockerURL - - //StepTypeUsesActionLocal is all steps that have a `uses` that is a local action in a subdirectory - StepTypeUsesActionLocal - - //StepTypeUsesActionRemote is all steps that have a `uses` that is a reference to a github repo - StepTypeUsesActionRemote -) - -// Type returns the type of the step -func (s *Step) Type() StepType { - if s.Run != "" { - return StepTypeRun - } else if strings.HasPrefix(s.Uses, "docker://") { - return StepTypeUsesDockerURL - } else if strings.HasPrefix(s.Uses, "./") { - return StepTypeUsesActionLocal - } - return StepTypeUsesActionRemote -} - -// ReadWorkflow returns a list of jobs for a given workflow file reader -func ReadWorkflow(in io.Reader) (*Workflow, error) { - w := new(Workflow) - err := yaml.NewDecoder(in).Decode(w) - return w, err -} - -// GetJob will get a job by name in the workflow -func (w *Workflow) GetJob(jobID string) *Job { - for id, j := range w.Jobs { - if jobID == id { - if j.Name == "" { - j.Name = id - } - return j - } - } - return nil -} - -// GetJobIDs will get all the job names in the workflow -func (w *Workflow) GetJobIDs() []string { - ids := make([]string, 0) - for id := range w.Jobs { - ids = append(ids, id) - } - return ids -} - -func (w *Workflow) On() []string { - var evts []string - for _, job := range w.Jobs { - evts = append(evts, job.RunsOn()...) - } - return evts -} \ No newline at end of file diff --git a/modules/actions/gitea/workflow_test.go b/modules/actions/gitea/workflow_test.go deleted file mode 100644 index 6df4a15e75d93..0000000000000 --- a/modules/actions/gitea/workflow_test.go +++ /dev/null @@ -1,100 +0,0 @@ -package gitea - -import ( - "strings" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestReadWorkflow_StringEvent(t *testing.T) { - yaml := ` -name: local-action-docker-url -on: push - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: ./actions/docker-url -` - - workflow, err := ReadWorkflow(strings.NewReader(yaml)) - assert.NoError(t, err, "read workflow should succeed") - - assert.Len(t, workflow.On(), 1) - assert.Contains(t, workflow.On(), "push") -} - -func TestReadWorkflow_ListEvent(t *testing.T) { - yaml := ` -name: local-action-docker-url -on: [push, pull_request] - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: ./actions/docker-url -` - - workflow, err := ReadWorkflow(strings.NewReader(yaml)) - assert.NoError(t, err, "read workflow should succeed") - - assert.Len(t, workflow.On(), 2) - assert.Contains(t, workflow.On(), "push") - assert.Contains(t, workflow.On(), "pull_request") -} - -func TestReadWorkflow_MapEvent(t *testing.T) { - yaml := ` -name: local-action-docker-url -on: - push: - branches: - - master - pull_request: - branches: - - master - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: ./actions/docker-url -` - - workflow, err := ReadWorkflow(strings.NewReader(yaml)) - assert.NoError(t, err, "read workflow should succeed") - assert.Len(t, workflow.On(), 2) - assert.Contains(t, workflow.On(), "push") - assert.Contains(t, workflow.On(), "pull_request") -} - -func TestReadWorkflow_StringContainer(t *testing.T) { - yaml := ` -name: local-action-docker-url - -jobs: - test: - container: nginx:latest - runs-on: ubuntu-latest - steps: - - uses: ./actions/docker-url - test2: - container: - image: nginx:latest - env: - foo: bar - runs-on: ubuntu-latest - steps: - - uses: ./actions/docker-url -` - - workflow, err := ReadWorkflow(strings.NewReader(yaml)) - assert.NoError(t, err, "read workflow should succeed") - assert.Len(t, workflow.Jobs, 2) - assert.Contains(t, workflow.Jobs["test"].Container().Image, "nginx:latest") - assert.Contains(t, workflow.Jobs["test2"].Container().Image, "nginx:latest") - assert.Contains(t, workflow.Jobs["test2"].Container().Env["foo"], "bar") -} diff --git a/modules/actions/github/github.go b/modules/actions/github/github.go deleted file mode 100644 index b39539fa0da19..0000000000000 --- a/modules/actions/github/github.go +++ /dev/null @@ -1,165 +0,0 @@ -package github - -import ( - "context" - "encoding/json" - "fmt" - "io" - "os" - "path/filepath" - - bot_model "code.gitea.io/gitea/models/bot" - repo_model "code.gitea.io/gitea/models/repo" - user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/models/webhook" - "code.gitea.io/gitea/modules/bot/runner" - "code.gitea.io/gitea/modules/git" - - //"code.gitea.io/gitea/modules/log" - //"code.gitea.io/gitea/modules/util" - - "github.com/nektos/act/pkg/model" - act_runner "github.com/nektos/act/pkg/runner" -) - -func init() { - runner.RegisterRunnerType(new(GithubRunner)) -} - -type GithubRunner struct { -} - -func (gw *GithubRunner) Name() string { - return "github" -} - -func (gw *GithubRunner) Detect(commit *git.Commit, event webhook.HookEventType, ref string) (bool, string, error) { - tree, err := commit.SubTree(".github/workflow") - if err != nil { - return false, "", err - } - entries, err := tree.ListEntries() - if err != nil { - return false, "", err - } - - var content = make(map[string]string) - for _, entry := range entries { - blob := entry.Blob() - rd, err := blob.DataAsync() - if err != nil { - return false, "", err - } - - bs, err := io.ReadAll(rd) - rd.Close() - if err != nil { - return false, "", err - } - content[entry.Name()] = string(bs) - } - - res, err := json.Marshal(content) - if err != nil { - return false, "", err - } - return true, string(res), nil -} - -func (gw *GithubRunner) Run(task *bot_model.Task) error { - tmpDir, err := os.MkdirTemp("", fmt.Sprintf("%d", task.ID)) - if err != nil { - return err - } - - var files = make(map[string]string) - if err := json.Unmarshal([]byte(task.Content), &files); err != nil { - return err - } - for name, content := range files { - f, err := os.Create(filepath.Join(tmpDir, name)) - if err != nil { - return err - } - if _, err := f.WriteString(content); err != nil { - f.Close() - return err - } - f.Close() - } - - repo, err := repo_model.GetRepositoryByID(task.RepoID) - if err != nil { - return err - } - - evtFilePath := filepath.Join(tmpDir, "event.json") - evtFile, err := os.Create(evtFilePath) - if err != nil { - return err - } - - if _, err := evtFile.WriteString(task.EventPayload); err != nil { - evtFile.Close() - return err - } - evtFile.Close() - - planner, err := model.NewWorkflowPlanner(tmpDir, false) - if err != nil { - return err - } - plan := planner.PlanEvent(task.Event) - - actor, err := user_model.GetUserByID(task.TriggerUserID) - if err != nil { - return err - } - - // run the plan - config := &act_runner.Config{ - Actor: actor.LoginName, - EventName: task.Event, - EventPath: evtFilePath, - DefaultBranch: repo.DefaultBranch, - /*ForcePull: input.forcePull, - ForceRebuild: input.forceRebuild, - ReuseContainers: input.reuseContainers, - Workdir: input.Workdir(), - BindWorkdir: input.bindWorkdir, - LogOutput: !input.noOutput,*/ - //Env: envs, - Secrets: map[string]string{ - "token": "614e597274a527b6fcf6ddfe45def79430126f08", - }, - //InsecureSecrets: input.insecureSecrets,*/ - Platforms: map[string]string{ - "ubuntu-latest": "node:12-buster-slim", - "ubuntu-20.04": "node:12-buster-slim", - "ubuntu-18.04": "node:12-buster-slim", - }, - /*Privileged: input.privileged, - UsernsMode: input.usernsMode, - ContainerArchitecture: input.containerArchitecture, - ContainerDaemonSocket: input.containerDaemonSocket, - UseGitIgnore: input.useGitIgnore,*/ - GitHubInstance: "gitea.com", - /*ContainerCapAdd: input.containerCapAdd, - ContainerCapDrop: input.containerCapDrop, - AutoRemove: input.autoRemove, - ArtifactServerPath: input.artifactServerPath, - ArtifactServerPort: input.artifactServerPort,*/ - } - r, err := act_runner.New(config) - if err != nil { - return err - } - - //ctx, cancel := context.WithTimeout(context.Background(), ) - - executor := r.NewPlanExecutor(plan).Finally(func(ctx context.Context) error { - //cancel() - return nil - }) - return executor(context.Background()) -} diff --git a/modules/actions/runner/runner.go b/modules/actions/runner/runner.go deleted file mode 100644 index 2a9540ad769c2..0000000000000 --- a/modules/actions/runner/runner.go +++ /dev/null @@ -1,27 +0,0 @@ -package runner - -import ( - bots_model "code.gitea.io/gitea/models/bots" - "code.gitea.io/gitea/models/webhook" - "code.gitea.io/gitea/modules/git" -) - -var runnerTypes = make(map[string]RunnerType) - -type RunnerType interface { - Name() string - Detect(commit *git.Commit, event webhook.HookEventType, ref string) (bool, string, error) - Run(task *bots_model.Task) error -} - -func RegisterRunnerType(runnerType RunnerType) { - runnerTypes[runnerType.Name()] = runnerType -} - -func GetRunnerType(name string) RunnerType { - return runnerTypes[name] -} - -func GetRunnerTypes() map[string]RunnerType { - return runnerTypes -} diff --git a/modules/bots/bots.go b/modules/bots/bots.go new file mode 100644 index 0000000000000..a83732e06e396 --- /dev/null +++ b/modules/bots/bots.go @@ -0,0 +1,61 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package bots + +import ( + "strings" + + "code.gitea.io/gitea/models/webhook" + "code.gitea.io/gitea/modules/git" + + "github.com/nektos/act/pkg/model" +) + +func DetectWorkflows(commit *git.Commit, event webhook.HookEventType) (git.Entries, []map[string]*model.Job, error) { + tree, err := commit.SubTree(".github/workflows") + if _, ok := err.(git.ErrNotExist); ok { + tree, err = commit.SubTree(".gitea/workflows") + } + if _, ok := err.(git.ErrNotExist); ok { + return nil, nil, nil + } + if err != nil { + return nil, nil, err + } + + entries, err := tree.ListEntriesRecursive() + if err != nil { + return nil, nil, err + } + + matchedEntries := make(git.Entries, 0, len(entries)) + jobs := make([]map[string]*model.Job, 0, len(entries)) + + for _, entry := range entries { + if !strings.HasSuffix(entry.Name(), ".yml") && !strings.HasSuffix(entry.Name(), ".yaml") { + continue + } + f, err := entry.Blob().DataAsync() + if err != nil { + return nil, nil, err + } + workflow, err := model.ReadWorkflow(f) + if err != nil { + f.Close() + return nil, nil, err + } + + for _, e := range workflow.On() { + if e == event.Event() { + matchedEntries = append(matchedEntries, entry) + jobs = append(jobs, workflow.Jobs) + break + } + } + f.Close() + } + + return matchedEntries, jobs, nil +} diff --git a/modules/notification/bots/bots.go b/modules/notification/bots/bots.go index a6dec9f449940..f013e795f1f78 100644 --- a/modules/notification/bots/bots.go +++ b/modules/notification/bots/bots.go @@ -16,6 +16,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/models/webhook" + bots_module "code.gitea.io/gitea/modules/bots" "code.gitea.io/gitea/modules/convert" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/graceful" @@ -39,28 +40,6 @@ func NewNotifier() base.Notifier { return &botsNotifier{} } -func detectWorkflows(commit *git.Commit, event webhook.HookEventType, ref string) (bool, error) { - tree, err := commit.SubTree(".github/workflows") - if _, ok := err.(git.ErrNotExist); ok { - tree, err = commit.SubTree(".gitea/workflows") - } - if _, ok := err.(git.ErrNotExist); ok { - return false, nil - } - if err != nil { - return false, err - } - - entries, err := tree.ListEntries() - if err != nil { - return false, err - } - - log.Trace("detected %s has %d entries", commit.ID, len(entries)) - - return len(entries) > 0, nil -} - func notifyIssue(issue *models.Issue, doer *user_model.User, evt webhook.HookEventType, payload string) { err := issue.LoadRepo(db.DefaultContext) if err != nil { @@ -75,8 +54,11 @@ func notifyIssue(issue *models.Issue, doer *user_model.User, evt webhook.HookEve if ref == "" { ref = issue.Repo.DefaultBranch } + notify(issue.Repo, doer, payload, ref, evt) +} - gitRepo, err := git.OpenRepository(context.Background(), issue.Repo.RepoPath()) +func notify(repo *repo_model.Repository, doer *user_model.User, payload, ref string, evt webhook.HookEventType) { + gitRepo, err := git.OpenRepository(context.Background(), repo.RepoPath()) if err != nil { log.Error("issue.LoadRepo: %v", err) return @@ -90,31 +72,41 @@ func notifyIssue(issue *models.Issue, doer *user_model.User, evt webhook.HookEve return } - hasWorkflows, err := detectWorkflows(commit, evt, ref) + matchedEntries, jobs, err := bots_module.DetectWorkflows(commit, evt) if err != nil { log.Error("detectWorkflows: %v", err) return } - if !hasWorkflows { - log.Trace("repo %s with commit %s couldn't find workflows", issue.Repo.RepoPath(), commit.ID) + log.Trace("detected %s has %d entries", commit.ID, len(matchedEntries)) + if len(matchedEntries) == 0 { + log.Trace("repo %s with commit %s couldn't find workflows", repo.RepoPath(), commit.ID) return } - task := bots_model.Task{ - Title: commit.CommitMessage, - RepoID: issue.RepoID, + workflowsStatuses := make(map[string]map[string]bots_model.BuildStatus) + for i, entry := range matchedEntries { + taskStatuses := make(map[string]bots_model.BuildStatus) + for k := range jobs[i] { + taskStatuses[k] = bots_model.BuildPending + } + workflowsStatuses[entry.Name()] = taskStatuses + } + + build := bots_model.Build{ + Title: commit.Message(), + RepoID: repo.ID, TriggerUserID: doer.ID, Event: evt, EventPayload: payload, - Status: bots_model.TaskPending, + Status: bots_model.BuildPending, Ref: ref, CommitSHA: commit.ID.String(), } - if err := bots_model.InsertTask(&task); err != nil { + if err := bots_model.InsertBuild(&build, workflowsStatuses); err != nil { log.Error("InsertBotTask: %v", err) } else { - bots_service.PushToQueue(&task) + bots_service.PushToQueue(&build) } } @@ -190,29 +182,6 @@ func (a *botsNotifier) NotifyPushCommits(pusher *user_model.User, repo *repo_mod return } - gitRepo, err := git.OpenRepository(ctx, repo.RepoPath()) - if err != nil { - log.Error("commits.ToAPIPayloadCommits failed: %v", err) - return - } - defer gitRepo.Close() - - commit, err := gitRepo.GetCommit(commits.HeadCommit.Sha1) - if err != nil { - log.Error("commits.ToAPIPayloadCommits failed: %v", err) - return - } - - hasWorkflows, err := detectWorkflows(commit, webhook.HookEventPush, opts.RefFullName) - if err != nil { - log.Error("detectWorkflows: %v", err) - return - } - if !hasWorkflows { - log.Trace("repo %s with commit %s couldn't find workflows", repo.RepoPath(), commit.ID) - return - } - payload := &api.PushPayload{ Ref: opts.RefFullName, Before: opts.OldCommitID, @@ -231,20 +200,7 @@ func (a *botsNotifier) NotifyPushCommits(pusher *user_model.User, repo *repo_mod return } - task := bots_model.Task{ - Title: commit.Message(), - RepoID: repo.ID, - TriggerUserID: pusher.ID, - Event: webhook.HookEventPush, - EventPayload: string(bs), - Status: bots_model.TaskPending, - } - - if err := bots_model.InsertTask(&task); err != nil { - log.Error("InsertBotTask: %v", err) - } else { - bots_service.PushToQueue(&task) - } + notify(repo, pusher, string(bs), opts.RefFullName, webhook.HookEventPush) } func (a *botsNotifier) NotifyCreateRef(doer *user_model.User, repo *repo_model.Repository, refType, refFullName, refID string) { diff --git a/routers/api/bots/bots.go b/routers/api/bots/bots.go index 32fdc75673f34..5513caa801176 100644 --- a/routers/api/bots/bots.go +++ b/routers/api/bots/bots.go @@ -147,7 +147,7 @@ MESSAGE_BUMP: } // TODO: find new task and send to client - task, err := bots_model.GetCurTaskByUUID(msg.RunnerUUID) + task, err := bots_model.GetCurBuildByUUID(msg.RunnerUUID) if err != nil { log.Error("websocket[%s] get task failed: %v", r.RemoteAddr, err) break diff --git a/routers/web/repo/builds/builds.go b/routers/web/repo/builds/builds.go index f7d8be4a58173..7194347b25169 100644 --- a/routers/web/repo/builds/builds.go +++ b/routers/web/repo/builds/builds.go @@ -5,6 +5,7 @@ package builds import ( + "fmt" "net/http" bots_model "code.gitea.io/gitea/models/bots" @@ -45,7 +46,7 @@ func List(ctx *context.Context) { page = 1 } - opts := bots_model.FindTaskOptions{ + opts := bots_model.FindBuildOptions{ ListOptions: db.ListOptions{ Page: page, PageSize: convert.ToCorrectPageSize(ctx.FormInt("limit")), @@ -57,24 +58,24 @@ func List(ctx *context.Context) { } else { opts.IsClosed = util.OptionalBoolFalse } - tasks, err := bots_model.FindTasks(opts) + builds, err := bots_model.FindBuilds(opts) if err != nil { ctx.Error(http.StatusInternalServerError, err.Error()) return } - if err := tasks.LoadTriggerUser(); err != nil { + if err := builds.LoadTriggerUser(); err != nil { ctx.Error(http.StatusInternalServerError, err.Error()) return } - total, err := bots_model.CountTasks(opts) + total, err := bots_model.CountBuilds(opts) if err != nil { ctx.Error(http.StatusInternalServerError, err.Error()) return } - ctx.Data["Tasks"] = tasks + ctx.Data["Builds"] = builds pager := context.NewPagination(int(total), opts.PageSize, opts.Page, 5) pager.SetDefaultParams(ctx) @@ -85,15 +86,64 @@ func List(ctx *context.Context) { func ViewBuild(ctx *context.Context) { index := ctx.ParamsInt64("index") - task, err := bots_model.GetTaskByRepoAndIndex(ctx.Repo.Repository.ID, index) + build, err := bots_model.GetBuildByRepoAndIndex(ctx.Repo.Repository.ID, index) if err != nil { ctx.Error(http.StatusInternalServerError, err.Error()) return } - ctx.Data["Title"] = task.Title + " - " + ctx.Tr("repo.builds") + ctx.Data["Title"] = build.Title + " - " + ctx.Tr("repo.builds") ctx.Data["PageIsBuildList"] = true - ctx.Data["Build"] = task + ctx.Data["Build"] = build + statuses, err := bots_model.GetBuildWorkflows(build.ID) + if err != nil { + ctx.Error(http.StatusInternalServerError, err.Error()) + return + } + + ctx.Data["WorkflowsStatuses"] = statuses ctx.HTML(http.StatusOK, tplViewBuild) } + +func GetBuildJobLogs(ctx *context.Context) { + index := ctx.ParamsInt64("index") + build, err := bots_model.GetBuildByRepoAndIndex(ctx.Repo.Repository.ID, index) + if err != nil { + ctx.Error(http.StatusInternalServerError, err.Error()) + return + } + workflows, err := bots_model.GetBuildWorkflows(build.ID) + if err != nil { + ctx.Error(http.StatusInternalServerError, err.Error()) + return + } + var buildJob *bots_model.BuildJob + wf := ctx.Params("workflow") + jobname := ctx.Params("jobname") +LOOP_WORKFLOWS: + for workflow, jobs := range workflows { + if workflow == wf { + for _, job := range jobs { + if jobname == job.Jobname { + buildJob = job + break LOOP_WORKFLOWS + } + } + } + } + if buildJob == nil { + ctx.Error(http.StatusNotFound, fmt.Sprintf("workflow %s job %s not exist", wf, jobname)) + return + } + + // TODO: if buildJob.LogToFile is true, read the logs from the file + + logs, err := bots_model.GetBuildLogs(build.ID, buildJob.ID) + if err != nil { + ctx.Error(http.StatusInternalServerError, err.Error()) + return + } + + ctx.JSON(http.StatusOK, logs) +} diff --git a/routers/web/web.go b/routers/web/web.go index 2dff1d3544eb2..09b2c3f812272 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -1175,6 +1175,7 @@ func RegisterRoutes(m *web.Route) { m.Get("", builds.List) m.Group("/{index}", func() { m.Get("", builds.ViewBuild) + m.Get("/{workflow}/job/{jobname}/logs", builds.GetBuildJobLogs) }) }, reqRepoBuildsReader, builds.MustEnableBuilds) diff --git a/services/bots/bots.go b/services/bots/bots.go index 39701105658bb..98b94765355a8 100644 --- a/services/bots/bots.go +++ b/services/bots/bots.go @@ -14,16 +14,16 @@ import ( //"code.gitea.io/gitea/modules/json" ) -// taskQueue is a global queue of tasks -var taskQueue queue.Queue +// buildQueue is a global queue of bot build +var buildQueue queue.Queue // PushToQueue -func PushToQueue(task *bots_model.Task) { - taskQueue.Push(task) +func PushToQueue(task *bots_model.Build) { + buildQueue.Push(task) } // Dispatch assign a task to a runner -func Dispatch(task *bots_model.Task) (*bots_model.Runner, error) { +func Dispatch(task *bots_model.Build) (*bots_model.Runner, error) { runner, err := bots_model.GetUsableRunner(bots_model.GetRunnerOptions{ RepoID: task.RepoID, }) @@ -31,17 +31,17 @@ func Dispatch(task *bots_model.Task) (*bots_model.Runner, error) { return nil, err } - return runner, bots_model.AssignTaskToRunner(task.ID, runner.ID) + return runner, bots_model.AssignBuildToRunner(task.ID, runner.ID) } // Init will start the service to get all unfinished tasks and run them func Init() error { - taskQueue = queue.CreateQueue("actions_task", handle, &bots_model.Task{}) - if taskQueue == nil { + buildQueue = queue.CreateQueue("actions_task", handle, &bots_model.Build{}) + if buildQueue == nil { return fmt.Errorf("Unable to create Task Queue") } - go graceful.GetManager().RunWithShutdownFns(taskQueue.Run) + go graceful.GetManager().RunWithShutdownFns(buildQueue.Run) return nil } @@ -49,13 +49,13 @@ func Init() error { func handle(data ...queue.Data) []queue.Data { var unhandled []queue.Data for _, datum := range data { - task := datum.(*bots_model.Task) - runner, err := Dispatch(task) + build := datum.(*bots_model.Build) + runner, err := Dispatch(build) if err != nil { - log.Error("Run task failed: %v", err) - unhandled = append(unhandled, task) + log.Error("Run build failed: %v", err) + unhandled = append(unhandled, build) } else { - log.Trace("task %v assigned to %s", task.UUID, runner.UUID) + log.Trace("build %v assigned to %s", build.UUID, runner.UUID) } } return unhandled diff --git a/templates/repo/builds/build_list.tmpl b/templates/repo/builds/build_list.tmpl index 55d177cbcbb6d..84d3a785e6f9e 100644 --- a/templates/repo/builds/build_list.tmpl +++ b/templates/repo/builds/build_list.tmpl @@ -1,5 +1,5 @@
    - {{range .Tasks}} + {{range .Builds}}
  • {{if $.CanWriteIssuesOrPulls}} @@ -8,20 +8,7 @@
    {{end}} -
    - {{if .IsPending}} - - {{end}} - {{if .IsRunning}} - - {{end}} - {{if .IsSuccess}} - - {{end}} - {{if .IsFailed}} - - {{end}} -
    + {{template "repo/builds/status" .Status}}
  • diff --git a/templates/repo/builds/status.tmpl b/templates/repo/builds/status.tmpl new file mode 100644 index 0000000000000..a48a895dd686f --- /dev/null +++ b/templates/repo/builds/status.tmpl @@ -0,0 +1,14 @@ +
    + {{if .IsPending}} + + {{end}} + {{if .IsRunning}} + + {{end}} + {{if .IsSuccess}} + + {{end}} + {{if .IsFailed}} + + {{end}} +
    diff --git a/templates/repo/builds/view.tmpl b/templates/repo/builds/view.tmpl index 8b01e586dc1e0..512b03e2955fd 100644 --- a/templates/repo/builds/view.tmpl +++ b/templates/repo/builds/view.tmpl @@ -10,6 +10,7 @@
    + {{template "repo/builds/view_left" .}} {{template "repo/builds/view_content" .}}
    diff --git a/templates/repo/builds/view_content.tmpl b/templates/repo/builds/view_content.tmpl index 7dc3ca9ac8c03..c7015dbcff406 100644 --- a/templates/repo/builds/view_content.tmpl +++ b/templates/repo/builds/view_content.tmpl @@ -1,8 +1,19 @@ -
    - {{if .Flash}} -
    - {{template "base/alert" .}} -
    - {{end}} - +
    +
    +
    +
    +
    +

    Console Logs

    +
    +
    +
    
    +						
    1Initialized empty Git repository in /drone/src/.git/ +0s
    2+ git fetch origin +refs/heads/main: +0s
    3From https://github.com/go-gitea/gitea +20s
    4 * branch main -> FETCH_HEAD +20s
    5 * [new branch] main -> origin/main +20s
    6+ git checkout c8ec2261a99590f15699e9147a28e4b61c1c2ea5 -b main +20s
    7Switched to a new branch 'main' +20s
    Exit Code 0
    diff --git a/templates/repo/builds/view_left.tmpl b/templates/repo/builds/view_left.tmpl new file mode 100644 index 0000000000000..4dfee9ec37378 --- /dev/null +++ b/templates/repo/builds/view_left.tmpl @@ -0,0 +1,18 @@ +
    +
    +

    Menu

    +
    +
    +
    From a1bd3a8932069f6e3b171b77dfe6868d387eddc8 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 28 May 2022 17:47:20 +0800 Subject: [PATCH 004/408] Fix build --- models/migrations/v216.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/models/migrations/v216.go b/models/migrations/v216.go index a8e1110d3f2e2..23695cf5e412c 100644 --- a/models/migrations/v216.go +++ b/models/migrations/v216.go @@ -5,6 +5,9 @@ package migrations import ( + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/timeutil" + "xorm.io/xorm" ) From 2ea693cdd23a9d9892513d7e7fcf9c9bc6743de9 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 28 May 2022 22:00:03 +0800 Subject: [PATCH 005/408] update template --- templates/repo/builds/view.tmpl | 2 +- templates/repo/builds/view_content.tmpl | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/templates/repo/builds/view.tmpl b/templates/repo/builds/view.tmpl index 512b03e2955fd..da279aa97de93 100644 --- a/templates/repo/builds/view.tmpl +++ b/templates/repo/builds/view.tmpl @@ -5,7 +5,7 @@

    - {{RenderIssueTitle $.Context .Build.Title $.RepoLink $.Repository.ComposeMetas}} + {{template "repo/builds/status" .Build.Status}}{{RenderIssueTitle $.Context .Build.Title $.RepoLink $.Repository.ComposeMetas}} #{{.Build.Index}}

    diff --git a/templates/repo/builds/view_content.tmpl b/templates/repo/builds/view_content.tmpl index c7015dbcff406..095525bf608fd 100644 --- a/templates/repo/builds/view_content.tmpl +++ b/templates/repo/builds/view_content.tmpl @@ -8,12 +8,15 @@
    
    -						
    1Initialized empty Git repository in /drone/src/.git/ +
    + 1 + Initialized empty Git repository in /drone/src/.git/ 0s
    2+ git fetch origin +refs/heads/main: 0s
    3From https://github.com/go-gitea/gitea 20s
    4 * branch main -> FETCH_HEAD 20s
    5 * [new branch] main -> origin/main 20s
    6+ git checkout c8ec2261a99590f15699e9147a28e4b61c1c2ea5 -b main 20s
    7Switched to a new branch 'main' -20s
    Exit Code 0
    +20s
    +
    Exit Code 0
    From 931d8c2c2112c8e7c5a3df5673e3e2cdc19ee4ba Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 20 Jun 2022 16:31:54 +0800 Subject: [PATCH 006/408] add runners management ui --- models/bots/runner.go | 58 ++++++-- models/bots/runner_list.go | 84 +++++++++++ routers/api/bots/bots.go | 209 +++++++++++++++------------ routers/web/admin/runners.go | 129 +++++++++++++++++ routers/web/web.go | 7 + services/bots/bots.go | 2 +- services/forms/admin.go | 24 +++ templates/admin/navbar.tmpl | 3 + templates/admin/runner/edit.tmpl | 209 +++++++++++++++++++++++++++ templates/admin/runner/list.tmpl | 53 +++++++ templates/admin/runner/new.tmpl | 94 ++++++++++++ templates/repo/builds/status.tmpl | 26 ++-- templates/repo/builds/view_left.tmpl | 4 +- 13 files changed, 779 insertions(+), 123 deletions(-) create mode 100644 models/bots/runner_list.go create mode 100644 routers/web/admin/runners.go create mode 100644 templates/admin/runner/edit.tmpl create mode 100644 templates/admin/runner/list.tmpl create mode 100644 templates/admin/runner/new.tmpl diff --git a/models/bots/runner.go b/models/bots/runner.go index dd97b92866c63..24ac953492d46 100644 --- a/models/bots/runner.go +++ b/models/bots/runner.go @@ -8,6 +8,8 @@ import ( "fmt" "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/timeutil" "xorm.io/builder" @@ -25,16 +27,18 @@ func (err ErrRunnerNotExist) Error() string { // Runner represents runner machines type Runner struct { ID int64 - UUID string `xorm:"CHAR(36) UNIQUE"` - Name string `xorm:"VARCHAR(32) UNIQUE"` - OS string `xorm:"VARCHAR(16) index"` // the runner running os - Arch string `xorm:"VARCHAR(16) index"` // the runner running architecture - Type string `xorm:"VARCHAR(16)"` - OwnerID int64 `xorm:"index"` // org level runner, 0 means system - RepoID int64 `xorm:"index"` // repo level runner, if orgid also is zero, then it's a global - Description string `xorm:"TEXT"` - Base int // 0 native 1 docker 2 virtual machine - RepoRange string // glob match which repositories could use this runner + UUID string `xorm:"CHAR(36) UNIQUE"` + Name string `xorm:"VARCHAR(32) UNIQUE"` + OS string `xorm:"VARCHAR(16) index"` // the runner running os + Arch string `xorm:"VARCHAR(16) index"` // the runner running architecture + Type string `xorm:"VARCHAR(16)"` + OwnerID int64 `xorm:"index"` // org level runner, 0 means system + Owner *user_model.User `xorm:"-"` + RepoID int64 `xorm:"index"` // repo level runner, if orgid also is zero, then it's a global + Repo *repo_model.Repository `xorm:"-"` + Description string `xorm:"TEXT"` + Base int // 0 native 1 docker 2 virtual machine + RepoRange string // glob match which repositories could use this runner Token string LastOnline timeutil.TimeStamp `xorm:"index"` Created timeutil.TimeStamp `xorm:"created"` @@ -44,16 +48,28 @@ func (Runner) TableName() string { return "bots_runner" } +func (r *Runner) OwnType() string { + if r.OwnerID == 0 { + return "Global Type" + } + if r.RepoID == 0 { + return r.Owner.Name + } + + return r.Repo.FullName() +} + func init() { db.RegisterModel(&Runner{}) } -type GetRunnerOptions struct { +type FindRunnerOptions struct { + db.ListOptions RepoID int64 OwnerID int64 } -func (opts GetRunnerOptions) toCond() builder.Cond { +func (opts FindRunnerOptions) toCond() builder.Cond { cond := builder.NewCond() if opts.RepoID > 0 { cond = cond.And(builder.Eq{"repo_id": opts.RepoID}) @@ -65,8 +81,24 @@ func (opts GetRunnerOptions) toCond() builder.Cond { return cond } +func CountRunners(opts FindRunnerOptions) (int64, error) { + return db.GetEngine(db.DefaultContext). + Table("bots_runner"). + Where(opts.toCond()). + Count() +} + +func FindRunners(opts FindRunnerOptions) (runners RunnerList, err error) { + sess := db.GetEngine(db.DefaultContext). + Where(opts.toCond()) + if opts.Page > 0 { + sess.Limit(opts.PageSize, (opts.Page-1)*opts.PageSize) + } + return runners, sess.Find(&runners) +} + // GetUsableRunner returns the usable runner -func GetUsableRunner(opts GetRunnerOptions) (*Runner, error) { +func GetUsableRunner(opts FindRunnerOptions) (*Runner, error) { var runner Runner has, err := db.GetEngine(db.DefaultContext). Where(opts.toCond()). diff --git a/models/bots/runner_list.go b/models/bots/runner_list.go new file mode 100644 index 0000000000000..197865d8e2085 --- /dev/null +++ b/models/bots/runner_list.go @@ -0,0 +1,84 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package bots + +import ( + "context" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/container" +) + +type RunnerList []*Runner + +// GetUserIDs returns a slice of user's id +func (runners RunnerList) GetUserIDs() []int64 { + userIDsMap := make(map[int64]struct{}) + for _, runner := range runners { + if runner.OwnerID == 0 { + continue + } + userIDsMap[runner.OwnerID] = struct{}{} + } + userIDs := make([]int64, 0, len(userIDsMap)) + for userID := range userIDsMap { + userIDs = append(userIDs, userID) + } + return userIDs +} + +func (runners RunnerList) LoadOwners(ctx context.Context) error { + userIDs := runners.GetUserIDs() + users := make(map[int64]*user_model.User, len(userIDs)) + if err := db.GetEngine(ctx).In("id", userIDs).Find(&users); err != nil { + return err + } + for _, runner := range runners { + if runner.OwnerID > 0 && runner.Owner == nil { + runner.Owner = users[runner.OwnerID] + } + } + return nil +} + +func (runners RunnerList) getRepoIDs() []int64 { + repoIDs := make(map[int64]struct{}, len(runners)) + for _, runner := range runners { + if runner.RepoID == 0 { + continue + } + if _, ok := repoIDs[runner.RepoID]; !ok { + repoIDs[runner.RepoID] = struct{}{} + } + } + return container.KeysInt64(repoIDs) +} + +func (runners RunnerList) LoadRepos(ctx context.Context) error { + repoIDs := runners.getRepoIDs() + repos := make(map[int64]*repo_model.Repository, len(repoIDs)) + if err := db.GetEngine(ctx).In("id", repoIDs).Find(&repos); err != nil { + return err + } + + for _, runner := range runners { + if runner.RepoID > 0 && runner.Repo == nil { + runner.Repo = repos[runner.RepoID] + } + } + return nil +} + +func (runners RunnerList) LoadAttributes(ctx context.Context) error { + if err := runners.LoadOwners(ctx); err != nil { + return err + } + if err := runners.LoadRepos(ctx); err != nil { + return err + } + return nil +} diff --git a/routers/api/bots/bots.go b/routers/api/bots/bots.go index 5513caa801176..89d445c6f1d0c 100644 --- a/routers/api/bots/bots.go +++ b/routers/api/bots/bots.go @@ -14,6 +14,7 @@ import ( bots_model "code.gitea.io/gitea/models/bots" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/web" "github.com/gorilla/websocket" @@ -40,10 +41,113 @@ type Message struct { Version int // Type int // message type, 1 register 2 error 3 task 4 no task RunnerUUID string // runner uuid + BuildUUID string // build uuid ErrCode int // error code ErrContent string // errors message EventName string EventPayload string + JobID string // only run the special job, empty means run all the jobs +} + +const ( + version1 = 1 +) + +const ( + MsgTypeRegister = iota + 1 // register + MsgTypeError // error + MsgTypeRequestBuild // request build task + MsgTypeIdle // no task + MsgTypeBuildResult // build result + MsgTypeBuildJobResult // build job result +) + +func handleVersion1(r *http.Request, c *websocket.Conn, mt int, message []byte, msg *Message) error { + switch msg.Type { + case MsgTypeRegister: + log.Info("websocket[%s] registered", r.RemoteAddr) + runner, err := bots_model.GetRunnerByUUID(msg.RunnerUUID) + if err != nil { + if !errors.Is(err, bots_model.ErrRunnerNotExist{}) { + return fmt.Errorf("websocket[%s] get runner [%s] failed: %v", r.RemoteAddr, msg.RunnerUUID, err) + } + err = c.WriteMessage(mt, message) + if err != nil { + return fmt.Errorf("websocket[%s] sent message failed: %v", r.RemoteAddr, err) + } + } else { + fmt.Printf("-----%v\n", runner) + // TODO: handle read message + err = c.WriteMessage(mt, message) + if err != nil { + return fmt.Errorf("websocket[%s] sent message failed: %v", r.RemoteAddr, err) + } + } + case MsgTypeRequestBuild: + // TODO: find new task and send to client + build, err := bots_model.GetCurBuildByUUID(msg.RunnerUUID) + if err != nil { + return fmt.Errorf("websocket[%s] get task[%s] failed: %v", r.RemoteAddr, msg.RunnerUUID, err) + } + var returnMsg *Message + if build == nil { + time.Sleep(3 * time.Second) + returnMsg = &Message{ + Version: version1, + Type: MsgTypeIdle, + RunnerUUID: msg.RunnerUUID, + } + } else { + returnMsg = &Message{ + Version: version1, + Type: MsgTypeRequestBuild, + RunnerUUID: msg.RunnerUUID, + BuildUUID: build.UUID, + EventName: build.Event.Event(), + EventPayload: build.EventPayload, + } + } + bs, err := json.Marshal(&returnMsg) + if err != nil { + return fmt.Errorf("websocket[%s] marshal message failed: %v", r.RemoteAddr, err) + } + err = c.WriteMessage(mt, bs) + if err != nil { + return fmt.Errorf("websocket[%s] sent message failed: %v", r.RemoteAddr, err) + } + case MsgTypeBuildResult: + log.Info("websocket[%s] returned CI result: %v", r.RemoteAddr, msg) + build, err := bots_model.GetBuildByUUID(msg.BuildUUID) + if err != nil { + return fmt.Errorf("websocket[%s] get build by uuid failed: %v", r.RemoteAddr, err) + } + cols := []string{"status", "end_time"} + if msg.ErrCode == 0 { + build.Status = bots_model.BuildFinished + } else { + build.Status = bots_model.BuildFailed + } + build.EndTime = timeutil.TimeStampNow() + if err := bots_model.UpdateBuild(build, cols...); err != nil { + log.Error("websocket[%s] update build failed: %v", r.RemoteAddr, err) + } + default: + returnMsg := Message{ + Version: version1, + Type: MsgTypeError, + ErrCode: 1, + ErrContent: fmt.Sprintf("message type %d is not supported", msg.Type), + } + bs, err := json.Marshal(&returnMsg) + if err != nil { + return fmt.Errorf("websocket[%s] marshal message failed: %v", r.RemoteAddr, err) + } + err = c.WriteMessage(mt, bs) + if err != nil { + return fmt.Errorf("websocket[%s] sent message failed: %v", r.RemoteAddr, err) + } + } + return nil } func Serve(w http.ResponseWriter, r *http.Request) { @@ -59,24 +163,21 @@ func Serve(w http.ResponseWriter, r *http.Request) { c.SetReadDeadline(time.Now().Add(pongWait)) c.SetPongHandler(func(string) error { c.SetReadDeadline(time.Now().Add(pongWait)); return nil }) -MESSAGE_BUMP: for { - // read log from client + // read message from client mt, message, err := c.ReadMessage() if err != nil { if websocket.IsCloseError(err, websocket.CloseAbnormalClosure) || websocket.IsCloseError(err, websocket.CloseNormalClosure) { c.Close() - break - } - if !strings.Contains(err.Error(), "i/o timeout") { + } else if !strings.Contains(err.Error(), "i/o timeout") { log.Error("websocket[%s] read failed: %#v", r.RemoteAddr, err) } break - } else { - log.Trace("websocket[%s] received message: %s", r.RemoteAddr, message) } + log.Trace("websocket[%s] received message: %s", r.RemoteAddr, string(message)) + // read message first var msg Message if err = json.Unmarshal(message, &msg); err != nil { @@ -86,103 +187,25 @@ MESSAGE_BUMP: switch msg.Version { case 1: - switch msg.Type { - case 1: - log.Info("websocket[%s] registered", r.RemoteAddr) - runner, err := bots_model.GetRunnerByUUID(msg.RunnerUUID) - if err != nil { - if !errors.Is(err, bots_model.ErrRunnerNotExist{}) { - log.Error("websocket[%s] get runner [%s] failed: %v", r.RemoteAddr, msg.RunnerUUID, err) - break - } - err = c.WriteMessage(mt, message) - if err != nil { - log.Error("websocket[%s] sent message failed: %v", r.RemoteAddr, err) - break - } - } else { - fmt.Printf("-----%v\n", runner) - // TODO: handle read message - err = c.WriteMessage(mt, message) - if err != nil { - log.Error("websocket[%s] sent message failed: %v", r.RemoteAddr, err) - break - } - } - default: - returnMsg := Message{ - Version: 1, - Type: 2, - ErrCode: 1, - ErrContent: fmt.Sprintf("message type %d is not supported", msg.Type), - } - bs, err := json.Marshal(&returnMsg) - if err != nil { - log.Error("websocket[%s] marshal message failed: %v", r.RemoteAddr, err) - break MESSAGE_BUMP - } - err = c.WriteMessage(mt, bs) - if err != nil { - log.Error("websocket[%s] sent message failed: %v", r.RemoteAddr, err) - } - break MESSAGE_BUMP + if err := handleVersion1(r, c, mt, message, &msg); err != nil { + log.Error("%v", err) } default: returnMsg := Message{ Version: 1, - Type: 2, + Type: MsgTypeError, ErrCode: 1, ErrContent: "version is not supported", } bs, err := json.Marshal(&returnMsg) if err != nil { log.Error("websocket[%s] marshal message failed: %v", r.RemoteAddr, err) - break MESSAGE_BUMP - } - err = c.WriteMessage(mt, bs) - if err != nil { - log.Error("websocket[%s] sent message failed: %v", r.RemoteAddr, err) - } - break MESSAGE_BUMP - } - - // TODO: find new task and send to client - task, err := bots_model.GetCurBuildByUUID(msg.RunnerUUID) - if err != nil { - log.Error("websocket[%s] get task failed: %v", r.RemoteAddr, err) - break - } - if task == nil { - returnMsg := Message{ - Version: 1, - Type: 4, - } - bs, err := json.Marshal(&returnMsg) - if err != nil { - log.Error("websocket[%s] marshal message failed: %v", r.RemoteAddr, err) - break MESSAGE_BUMP - } - err = c.WriteMessage(mt, bs) - if err != nil { - log.Error("websocket[%s] sent message failed: %v", r.RemoteAddr, err) - } - } else { - returnMsg := Message{ - Version: 1, - Type: 3, - EventName: task.Event.Event(), - EventPayload: task.EventPayload, - } - bs, err := json.Marshal(&returnMsg) - if err != nil { - log.Error("websocket[%s] marshal message failed: %v", r.RemoteAddr, err) - break - } - err = c.WriteMessage(mt, bs) - if err != nil { - log.Error("websocket[%s] sent message failed: %v", r.RemoteAddr, err) + } else { + err = c.WriteMessage(mt, bs) + if err != nil { + log.Error("websocket[%s] sent message failed: %v", r.RemoteAddr, err) + } } } - } } diff --git a/routers/web/admin/runners.go b/routers/web/admin/runners.go new file mode 100644 index 0000000000000..952c71466b09c --- /dev/null +++ b/routers/web/admin/runners.go @@ -0,0 +1,129 @@ +// Copyright 2014 The Gogs Authors. All rights reserved. +// Copyright 2020 The Gitea Authors. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package admin + +import ( + "net/http" + "net/url" + + bots_model "code.gitea.io/gitea/models/bots" + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/setting" +) + +const ( + tplRunners base.TplName = "admin/runner/list" + tplRunnerNew base.TplName = "admin/runner/new" + tplRunnerEdit base.TplName = "admin/runner/edit" +) + +// Runners show all the runners +func Runners(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("admin.runners") + ctx.Data["PageIsAdmin"] = true + ctx.Data["PageIsAdminRunners"] = true + + page := ctx.FormInt("page") + if page <= 1 { + page = 1 + } + + opts := bots_model.FindRunnerOptions{ + ListOptions: db.ListOptions{ + Page: page, + PageSize: 100, + }, + } + + count, err := bots_model.CountRunners(opts) + if err != nil { + ctx.ServerError("SearchUsers", err) + return + } + + runners, err := bots_model.FindRunners(opts) + if err != nil { + ctx.ServerError("SearchUsers", err) + return + } + if err := runners.LoadAttributes(ctx); err != nil { + ctx.ServerError("LoadAttributes", err) + return + } + + ctx.Data["Runners"] = runners + ctx.Data["Total"] = count + + pager := context.NewPagination(int(count), opts.PageSize, opts.Page, 5) + ctx.Data["Page"] = pager + + ctx.HTML(http.StatusOK, tplRunners) +} + +// NewRunner render adding a new runner page +func NewRunner(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("admin.runners.new") + ctx.Data["PageIsAdmin"] = true + ctx.Data["PageIsAdminRunners"] = true + + ctx.HTML(http.StatusOK, tplRunnerNew) +} + +// NewRunnerPost response for adding a new runner +func NewRunnerPost(ctx *context.Context) { + // form := web.GetForm(ctx).(*forms.AdminCreateRunnerForm) + ctx.Data["Title"] = ctx.Tr("admin.runners.new") + ctx.Data["PageIsAdmin"] = true + ctx.Data["PageIsAdminRunners"] = true + + if ctx.HasError() { + ctx.HTML(http.StatusOK, tplRunnerNew) + return + } + + // ctx.Flash.Success(ctx.Tr("admin.runners.new_success", u.Name)) + // ctx.Redirect(setting.AppSubURL + "/admin/users/" + strconv.FormatInt(u.ID, 10)) +} + +// EditRunner show editing runner page +func EditRunner(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("admin.runners.edit") + ctx.Data["PageIsAdmin"] = true + ctx.Data["PageIsAdminRunners"] = true + + prepareUserInfo(ctx) + if ctx.Written() { + return + } + + ctx.HTML(http.StatusOK, tplUserEdit) +} + +// EditRunnerPost response for editing runner +func EditRunnerPost(ctx *context.Context) { + // form := web.GetForm(ctx).(*forms.AdminEditRunnerForm) + ctx.Data["Title"] = ctx.Tr("admin.runners.edit") + ctx.Data["PageIsAdmin"] = true + ctx.Data["PageIsAdminRunners"] = true + + if ctx.HasError() { + ctx.HTML(http.StatusOK, tplUserEdit) + return + } + + ctx.Flash.Success(ctx.Tr("admin.users.update_profile_success")) + ctx.Redirect(setting.AppSubURL + "/admin/users/" + url.PathEscape(ctx.Params(":userid"))) +} + +// DeleteRunner response for deleting a runner +func DeleteRunner(ctx *context.Context) { + ctx.Flash.Success(ctx.Tr("admin.runners.deletion_success")) + ctx.JSON(http.StatusOK, map[string]interface{}{ + "redirect": setting.AppSubURL + "/admin/runners", + }) +} diff --git a/routers/web/web.go b/routers/web/web.go index 09b2c3f812272..a1f601b9c44e4 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -624,6 +624,13 @@ func RegisterRoutes(m *web.Route) { return } }) + + m.Group("/runners", func() { + m.Get("", admin.Runners) + m.Combo("/new").Get(admin.NewRunner).Post(bindIgnErr(forms.AdminCreateRunnerForm{}), admin.NewRunnerPost) + m.Combo("/{runnerid}").Get(admin.EditRunner).Post(bindIgnErr(forms.AdminEditRunnerForm{}), admin.EditRunnerPost) + m.Post("/{runnerid}/delete", admin.DeleteRunner) + }) }, func(ctx *context.Context) { ctx.Data["EnableOAuth2"] = setting.OAuth2.Enable ctx.Data["EnablePackages"] = setting.Packages.Enabled diff --git a/services/bots/bots.go b/services/bots/bots.go index 98b94765355a8..f894a678ea18c 100644 --- a/services/bots/bots.go +++ b/services/bots/bots.go @@ -24,7 +24,7 @@ func PushToQueue(task *bots_model.Build) { // Dispatch assign a task to a runner func Dispatch(task *bots_model.Build) (*bots_model.Runner, error) { - runner, err := bots_model.GetUsableRunner(bots_model.GetRunnerOptions{ + runner, err := bots_model.GetUsableRunner(bots_model.FindRunnerOptions{ RepoID: task.RepoID, }) if err != nil { diff --git a/services/forms/admin.go b/services/forms/admin.go index 537b9f982cfe1..8c1ba0589ea13 100644 --- a/services/forms/admin.go +++ b/services/forms/admin.go @@ -71,3 +71,27 @@ func (f *AdminDashboardForm) Validate(req *http.Request, errs binding.Errors) bi ctx := context.GetContext(req) return middleware.Validate(errs, ctx.Data, f, ctx.Locale) } + +// AdminCreateRunnerForm form for admin to create runner +type AdminCreateRunnerForm struct { + Name string `binding:"Required"` + Type string +} + +// Validate validates form fields +func (f *AdminCreateRunnerForm) Validate(req *http.Request, errs binding.Errors) binding.Errors { + ctx := context.GetContext(req) + return middleware.Validate(errs, ctx.Data, f, ctx.Locale) +} + +// AdminEditRunnerForm form for admin to create runner +type AdminEditRunnerForm struct { + Name string `binding:"Required"` + Type string +} + +// Validate validates form fields +func (f *AdminEditRunnerForm) Validate(req *http.Request, errs binding.Errors) binding.Errors { + ctx := context.GetContext(req) + return middleware.Validate(errs, ctx.Data, f, ctx.Locale) +} diff --git a/templates/admin/navbar.tmpl b/templates/admin/navbar.tmpl index 1c8b12fc2fafc..d40814978a7fd 100644 --- a/templates/admin/navbar.tmpl +++ b/templates/admin/navbar.tmpl @@ -33,6 +33,9 @@ {{.locale.Tr "settings.applications"}} {{end}} + + {{.locale.Tr "admin.runners"}} + {{.locale.Tr "admin.config"}} diff --git a/templates/admin/runner/edit.tmpl b/templates/admin/runner/edit.tmpl new file mode 100644 index 0000000000000..1ee46f3077a3b --- /dev/null +++ b/templates/admin/runner/edit.tmpl @@ -0,0 +1,209 @@ +{{template "base/head" .}} +
    + {{template "admin/navbar" .}} +
    + {{template "base/alert" .}} +

    + {{.i18n.Tr "admin.users.edit_account"}} +

    +
    +
    + {{template "base/disable_form_autofill"}} + {{.CsrfTokenHtml}} +
    + + +
    + +
    + + +
    + +
    + + +
    + +
    + + +
    +
    + + +
    +
    + + +
    +
    + + +

    {{.i18n.Tr "admin.users.password_helper"}}

    +
    +
    + + +
    +
    + + +
    + +
    + +
    + + +

    {{.i18n.Tr "admin.users.max_repo_creation_desc"}}

    +
    + +
    + +
    +
    + + +
    +
    +
    +
    + + +
    +
    +
    +
    + + +
    +
    +
    +
    + + +
    +
    +
    +
    + + +
    +
    +
    +
    + + +
    +
    + {{if not .DisableRegularOrgCreation}} +
    +
    + + +
    +
    + {{end}} + + {{if .TwoFactorEnabled}} +
    +
    +
    + + +
    +
    + {{end}} + +
    + +
    + +
    {{.i18n.Tr "admin.users.delete_account"}}
    +
    +
    +
    + +

    + {{.i18n.Tr "settings.avatar"}} +

    +
    +
    + {{.CsrfTokenHtml}} + {{if not DisableGravatar}} +
    +
    + + +
    +
    +
    + + +
    + {{end}} + +
    +
    + + +
    +
    + +
    + + +
    + +
    + + {{$.i18n.Tr "settings.delete_current_avatar"}} +
    +
    +
    +
    +
    + + +{{template "base/footer" .}} diff --git a/templates/admin/runner/list.tmpl b/templates/admin/runner/list.tmpl new file mode 100644 index 0000000000000..04160a08e3c73 --- /dev/null +++ b/templates/admin/runner/list.tmpl @@ -0,0 +1,53 @@ +{{template "base/head" .}} +
    + {{template "admin/navbar" .}} +
    + {{template "base/alert" .}} +

    + {{.i18n.Tr "admin.runners.runner_manage_panel"}} ({{.i18n.Tr "admin.total" .Total}}) + +

    +
    +
    + +
    + + +
    +
    +
    +
    + + + + + + + + + + + + + {{range .Runners}} + + + + + + + + + {{end}} + +
    ID + {{.i18n.Tr "admin.runners.name"}} + {{.i18n.Tr "admin.runners.own_type"}}{{.i18n.Tr "admin.runners.uuid"}}{{.i18n.Tr "admin.runners.created"}}{{.i18n.Tr "admin.runners.edit"}}
    {{.ID}}{{.Name}}{{.OwnType}}{{.UUID}}{{.Created}}{{svg "octicon-pencil"}}
    +
    + + {{template "base/paginate" .}} +
    +
    +{{template "base/footer" .}} diff --git a/templates/admin/runner/new.tmpl b/templates/admin/runner/new.tmpl new file mode 100644 index 0000000000000..6e3861fa50269 --- /dev/null +++ b/templates/admin/runner/new.tmpl @@ -0,0 +1,94 @@ +{{template "base/head" .}} +
    + {{template "admin/navbar" .}} +
    + {{template "base/alert" .}} +

    + {{.i18n.Tr "admin.users.new_account"}} +

    +
    +
    + {{template "base/disable_form_autofill"}} + {{.CsrfTokenHtml}} + +
    + + +
    + +
    + + +
    + +
    + + +
    +
    + + +
    +
    + + +
    +
    + + +
    + +
    +
    + + +
    +
    + + + {{if .CanSendEmail}} +
    +
    + + +
    +
    + {{end}} + +
    + +
    +
    +
    +
    +
    +{{template "base/footer" .}} diff --git a/templates/repo/builds/status.tmpl b/templates/repo/builds/status.tmpl index a48a895dd686f..454c951eb131b 100644 --- a/templates/repo/builds/status.tmpl +++ b/templates/repo/builds/status.tmpl @@ -1,14 +1,12 @@ -
    - {{if .IsPending}} - - {{end}} - {{if .IsRunning}} - - {{end}} - {{if .IsSuccess}} - - {{end}} - {{if .IsFailed}} - - {{end}} -
    +{{if .IsPending}} + +{{end}} +{{if .IsRunning}} + +{{end}} +{{if .IsSuccess}} + +{{end}} +{{if .IsFailed}} + +{{end}} diff --git a/templates/repo/builds/view_left.tmpl b/templates/repo/builds/view_left.tmpl index 4dfee9ec37378..6e0b60fc9ed4c 100644 --- a/templates/repo/builds/view_left.tmpl +++ b/templates/repo/builds/view_left.tmpl @@ -4,8 +4,8 @@