forkjo/services/webhook/webhook.go

273 lines
7 KiB
Go
Raw Normal View History

// Copyright 2019 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package webhook
import (
"context"
"errors"
"fmt"
"html/template"
Store webhook event in database (#29145) Refactor the webhook logic, to have the type-dependent processing happen only in one place. --- 1. An event happens 2. It is pre-processed (depending on the webhook type) and its body is added to a task queue 3. When the task is processed, some more logic (depending on the webhook type as well) is applied to make an HTTP request This means that webhook-type dependant logic is needed in step 2 and 3. This is cumbersome and brittle to maintain. Updated webhook flow with this PR: 1. An event happens 2. It is stored as-is and added to a task queue 3. When the task is processed, the event is processed (depending on the webhook type) to make an HTTP request So the only webhook-type dependent logic happens in one place (step 3) which should be much more robust. - the raw event must be stored in the hooktask (until now, the pre-processed body was stored) - to ensure that previous hooktasks are correctly sent, a `payload_version` is added (version 1: the body has already been pre-process / version 2: the body is the raw event) So future webhook additions will only have to deal with creating an http.Request based on the raw event (no need to adjust the code in multiple places, like currently). Moreover since this processing happens when fetching from the task queue, it ensures that the queuing of new events (upon a `git push` for instance) does not get slowed down by a slow webhook. As a concrete example, the PR #19307 for custom webhooks, should be substantially smaller: - no need to change `services/webhook/deliver.go` - minimal change in `services/webhook/webhook.go` (add the new webhook to the map) - no need to change all the individual webhook files (since with this refactor the `*webhook_model.Webhook` is provided as argument) (cherry picked from commit 26653b196bd1d15c532af41f60351596dd4330bd) Conflicts: services/webhook/deliver_test.go trivial context conflict
2024-03-07 23:18:38 +01:00
"net/http"
"strings"
"code.gitea.io/gitea/models/db"
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
webhook_model "code.gitea.io/gitea/models/webhook"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/optional"
"code.gitea.io/gitea/modules/queue"
"code.gitea.io/gitea/modules/setting"
api "code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/util"
webhook_module "code.gitea.io/gitea/modules/webhook"
2024-03-21 13:23:27 +01:00
"code.gitea.io/gitea/services/forms"
2024-03-13 16:49:48 +01:00
"code.gitea.io/gitea/services/webhook/sourcehut"
"github.com/gobwas/glob"
)
2024-03-20 15:44:01 +01:00
type Handler interface {
Type() webhook_module.HookType
Metadata(*webhook_model.Webhook) any
// UnmarshalForm provides a function to bind the request to the form.
2024-03-21 13:23:27 +01:00
// If form implements the [binding.Validator] interface, the Validate method will be called
UnmarshalForm(bind func(form any)) forms.WebhookForm
2024-03-21 13:23:27 +01:00
NewRequest(context.Context, *webhook_model.Webhook, *webhook_model.HookTask) (req *http.Request, body []byte, err error)
Icon(size int) template.HTML
2024-03-21 13:23:27 +01:00
}
2024-03-20 15:44:01 +01:00
var webhookHandlers = []Handler{
defaultHandler{true},
defaultHandler{false},
gogsHandler{},
slackHandler{},
discordHandler{},
dingtalkHandler{},
telegramHandler{},
msteamsHandler{},
feishuHandler{},
matrixHandler{},
wechatworkHandler{},
packagistHandler{},
2024-03-13 16:49:48 +01:00
sourcehut.BuildsHandler{},
2024-03-20 15:44:01 +01:00
}
// GetWebhookHandler return the handler for a given webhook type (nil if not found)
func GetWebhookHandler(name webhook_module.HookType) Handler {
for _, h := range webhookHandlers {
if h.Type() == name {
return h
}
}
return nil
}
// List provides a list of the supported webhooks
func List() []Handler {
return webhookHandlers
}
// IsValidHookTaskType returns true if a webhook registered
func IsValidHookTaskType(name string) bool {
2024-03-20 15:44:01 +01:00
return GetWebhookHandler(name) != nil
}
// hookQueue is a global queue of web hooks
Rewrite queue (#24505) # ⚠️ Breaking Many deprecated queue config options are removed (actually, they should have been removed in 1.18/1.19). If you see the fatal message when starting Gitea: "Please update your app.ini to remove deprecated config options", please follow the error messages to remove these options from your app.ini. Example: ``` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].ISSUE_INDEXER_QUEUE_TYPE`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].UPDATE_BUFFER_LEN`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [F] Please update your app.ini to remove deprecated config options ``` Many options in `[queue]` are are dropped, including: `WRAP_IF_NECESSARY`, `MAX_ATTEMPTS`, `TIMEOUT`, `WORKERS`, `BLOCK_TIMEOUT`, `BOOST_TIMEOUT`, `BOOST_WORKERS`, they can be removed from app.ini. # The problem The old queue package has some legacy problems: * complexity: I doubt few people could tell how it works. * maintainability: Too many channels and mutex/cond are mixed together, too many different structs/interfaces depends each other. * stability: due to the complexity & maintainability, sometimes there are strange bugs and difficult to debug, and some code doesn't have test (indeed some code is difficult to test because a lot of things are mixed together). * general applicability: although it is called "queue", its behavior is not a well-known queue. * scalability: it doesn't seem easy to make it work with a cluster without breaking its behaviors. It came from some very old code to "avoid breaking", however, its technical debt is too heavy now. It's a good time to introduce a better "queue" package. # The new queue package It keeps using old config and concept as much as possible. * It only contains two major kinds of concepts: * The "base queue": channel, levelqueue, redis * They have the same abstraction, the same interface, and they are tested by the same testing code. * The "WokerPoolQueue", it uses the "base queue" to provide "worker pool" function, calls the "handler" to process the data in the base queue. * The new code doesn't do "PushBack" * Think about a queue with many workers, the "PushBack" can't guarantee the order for re-queued unhandled items, so in new code it just does "normal push" * The new code doesn't do "pause/resume" * The "pause/resume" was designed to handle some handler's failure: eg: document indexer (elasticsearch) is down * If a queue is paused for long time, either the producers blocks or the new items are dropped. * The new code doesn't do such "pause/resume" trick, it's not a common queue's behavior and it doesn't help much. * If there are unhandled items, the "push" function just blocks for a few seconds and then re-queue them and retry. * The new code doesn't do "worker booster" * Gitea's queue's handlers are light functions, the cost is only the go-routine, so it doesn't make sense to "boost" them. * The new code only use "max worker number" to limit the concurrent workers. * The new "Push" never blocks forever * Instead of creating more and more blocking goroutines, return an error is more friendly to the server and to the end user. There are more details in code comments: eg: the "Flush" problem, the strange "code.index" hanging problem, the "immediate" queue problem. Almost ready for review. TODO: * [x] add some necessary comments during review * [x] add some more tests if necessary * [x] update documents and config options * [x] test max worker / active worker * [x] re-run the CI tasks to see whether any test is flaky * [x] improve the `handleOldLengthConfiguration` to provide more friendly messages * [x] fine tune default config values (eg: length?) ## Code coverage: ![image](https://user-images.githubusercontent.com/2114189/236620635-55576955-f95d-4810-b12f-879026a3afdf.png)
2023-05-08 13:49:59 +02:00
var hookQueue *queue.WorkerPoolQueue[int64]
// getPayloadBranch returns branch for hook event, if applicable.
func getPayloadBranch(p api.Payloader) string {
switch pp := p.(type) {
case *api.CreatePayload:
if pp.RefType == "branch" {
return pp.Ref
}
case *api.DeletePayload:
if pp.RefType == "branch" {
return pp.Ref
}
case *api.PushPayload:
if strings.HasPrefix(pp.Ref, git.BranchPrefix) {
return pp.Ref[len(git.BranchPrefix):]
}
}
return ""
}
// EventSource represents the source of a webhook action. Repository and/or Owner must be set.
type EventSource struct {
Repository *repo_model.Repository
Owner *user_model.User
}
// handle delivers hook tasks
Rewrite queue (#24505) # ⚠️ Breaking Many deprecated queue config options are removed (actually, they should have been removed in 1.18/1.19). If you see the fatal message when starting Gitea: "Please update your app.ini to remove deprecated config options", please follow the error messages to remove these options from your app.ini. Example: ``` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].ISSUE_INDEXER_QUEUE_TYPE`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].UPDATE_BUFFER_LEN`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [F] Please update your app.ini to remove deprecated config options ``` Many options in `[queue]` are are dropped, including: `WRAP_IF_NECESSARY`, `MAX_ATTEMPTS`, `TIMEOUT`, `WORKERS`, `BLOCK_TIMEOUT`, `BOOST_TIMEOUT`, `BOOST_WORKERS`, they can be removed from app.ini. # The problem The old queue package has some legacy problems: * complexity: I doubt few people could tell how it works. * maintainability: Too many channels and mutex/cond are mixed together, too many different structs/interfaces depends each other. * stability: due to the complexity & maintainability, sometimes there are strange bugs and difficult to debug, and some code doesn't have test (indeed some code is difficult to test because a lot of things are mixed together). * general applicability: although it is called "queue", its behavior is not a well-known queue. * scalability: it doesn't seem easy to make it work with a cluster without breaking its behaviors. It came from some very old code to "avoid breaking", however, its technical debt is too heavy now. It's a good time to introduce a better "queue" package. # The new queue package It keeps using old config and concept as much as possible. * It only contains two major kinds of concepts: * The "base queue": channel, levelqueue, redis * They have the same abstraction, the same interface, and they are tested by the same testing code. * The "WokerPoolQueue", it uses the "base queue" to provide "worker pool" function, calls the "handler" to process the data in the base queue. * The new code doesn't do "PushBack" * Think about a queue with many workers, the "PushBack" can't guarantee the order for re-queued unhandled items, so in new code it just does "normal push" * The new code doesn't do "pause/resume" * The "pause/resume" was designed to handle some handler's failure: eg: document indexer (elasticsearch) is down * If a queue is paused for long time, either the producers blocks or the new items are dropped. * The new code doesn't do such "pause/resume" trick, it's not a common queue's behavior and it doesn't help much. * If there are unhandled items, the "push" function just blocks for a few seconds and then re-queue them and retry. * The new code doesn't do "worker booster" * Gitea's queue's handlers are light functions, the cost is only the go-routine, so it doesn't make sense to "boost" them. * The new code only use "max worker number" to limit the concurrent workers. * The new "Push" never blocks forever * Instead of creating more and more blocking goroutines, return an error is more friendly to the server and to the end user. There are more details in code comments: eg: the "Flush" problem, the strange "code.index" hanging problem, the "immediate" queue problem. Almost ready for review. TODO: * [x] add some necessary comments during review * [x] add some more tests if necessary * [x] update documents and config options * [x] test max worker / active worker * [x] re-run the CI tasks to see whether any test is flaky * [x] improve the `handleOldLengthConfiguration` to provide more friendly messages * [x] fine tune default config values (eg: length?) ## Code coverage: ![image](https://user-images.githubusercontent.com/2114189/236620635-55576955-f95d-4810-b12f-879026a3afdf.png)
2023-05-08 13:49:59 +02:00
func handler(items ...int64) []int64 {
ctx := graceful.GetManager().HammerContext()
Rewrite queue (#24505) # ⚠️ Breaking Many deprecated queue config options are removed (actually, they should have been removed in 1.18/1.19). If you see the fatal message when starting Gitea: "Please update your app.ini to remove deprecated config options", please follow the error messages to remove these options from your app.ini. Example: ``` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].ISSUE_INDEXER_QUEUE_TYPE`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].UPDATE_BUFFER_LEN`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [F] Please update your app.ini to remove deprecated config options ``` Many options in `[queue]` are are dropped, including: `WRAP_IF_NECESSARY`, `MAX_ATTEMPTS`, `TIMEOUT`, `WORKERS`, `BLOCK_TIMEOUT`, `BOOST_TIMEOUT`, `BOOST_WORKERS`, they can be removed from app.ini. # The problem The old queue package has some legacy problems: * complexity: I doubt few people could tell how it works. * maintainability: Too many channels and mutex/cond are mixed together, too many different structs/interfaces depends each other. * stability: due to the complexity & maintainability, sometimes there are strange bugs and difficult to debug, and some code doesn't have test (indeed some code is difficult to test because a lot of things are mixed together). * general applicability: although it is called "queue", its behavior is not a well-known queue. * scalability: it doesn't seem easy to make it work with a cluster without breaking its behaviors. It came from some very old code to "avoid breaking", however, its technical debt is too heavy now. It's a good time to introduce a better "queue" package. # The new queue package It keeps using old config and concept as much as possible. * It only contains two major kinds of concepts: * The "base queue": channel, levelqueue, redis * They have the same abstraction, the same interface, and they are tested by the same testing code. * The "WokerPoolQueue", it uses the "base queue" to provide "worker pool" function, calls the "handler" to process the data in the base queue. * The new code doesn't do "PushBack" * Think about a queue with many workers, the "PushBack" can't guarantee the order for re-queued unhandled items, so in new code it just does "normal push" * The new code doesn't do "pause/resume" * The "pause/resume" was designed to handle some handler's failure: eg: document indexer (elasticsearch) is down * If a queue is paused for long time, either the producers blocks or the new items are dropped. * The new code doesn't do such "pause/resume" trick, it's not a common queue's behavior and it doesn't help much. * If there are unhandled items, the "push" function just blocks for a few seconds and then re-queue them and retry. * The new code doesn't do "worker booster" * Gitea's queue's handlers are light functions, the cost is only the go-routine, so it doesn't make sense to "boost" them. * The new code only use "max worker number" to limit the concurrent workers. * The new "Push" never blocks forever * Instead of creating more and more blocking goroutines, return an error is more friendly to the server and to the end user. There are more details in code comments: eg: the "Flush" problem, the strange "code.index" hanging problem, the "immediate" queue problem. Almost ready for review. TODO: * [x] add some necessary comments during review * [x] add some more tests if necessary * [x] update documents and config options * [x] test max worker / active worker * [x] re-run the CI tasks to see whether any test is flaky * [x] improve the `handleOldLengthConfiguration` to provide more friendly messages * [x] fine tune default config values (eg: length?) ## Code coverage: ![image](https://user-images.githubusercontent.com/2114189/236620635-55576955-f95d-4810-b12f-879026a3afdf.png)
2023-05-08 13:49:59 +02:00
for _, taskID := range items {
task, err := webhook_model.GetHookTaskByID(ctx, taskID)
if err != nil {
if errors.Is(err, util.ErrNotExist) {
log.Warn("GetHookTaskByID[%d] warn: %v", taskID, err)
} else {
log.Error("GetHookTaskByID[%d] failed: %v", taskID, err)
}
continue
}
if task.IsDelivered {
// Already delivered in the meantime
log.Trace("Task[%d] has already been delivered", task.ID)
continue
}
if err := Deliver(ctx, task); err != nil {
log.Error("Unable to deliver webhook task[%d]: %v", task.ID, err)
}
}
return nil
}
func enqueueHookTask(taskID int64) error {
err := hookQueue.Push(taskID)
if err != nil && err != queue.ErrAlreadyInQueue {
return err
}
return nil
}
func checkBranch(w *webhook_model.Webhook, branch string) bool {
if w.BranchFilter == "" || w.BranchFilter == "*" {
return true
}
g, err := glob.Compile(w.BranchFilter)
if err != nil {
// should not really happen as BranchFilter is validated
log.Error("CheckBranch failed: %s", err)
return false
}
return g.Match(branch)
}
Store webhook event in database (#29145) Refactor the webhook logic, to have the type-dependent processing happen only in one place. --- 1. An event happens 2. It is pre-processed (depending on the webhook type) and its body is added to a task queue 3. When the task is processed, some more logic (depending on the webhook type as well) is applied to make an HTTP request This means that webhook-type dependant logic is needed in step 2 and 3. This is cumbersome and brittle to maintain. Updated webhook flow with this PR: 1. An event happens 2. It is stored as-is and added to a task queue 3. When the task is processed, the event is processed (depending on the webhook type) to make an HTTP request So the only webhook-type dependent logic happens in one place (step 3) which should be much more robust. - the raw event must be stored in the hooktask (until now, the pre-processed body was stored) - to ensure that previous hooktasks are correctly sent, a `payload_version` is added (version 1: the body has already been pre-process / version 2: the body is the raw event) So future webhook additions will only have to deal with creating an http.Request based on the raw event (no need to adjust the code in multiple places, like currently). Moreover since this processing happens when fetching from the task queue, it ensures that the queuing of new events (upon a `git push` for instance) does not get slowed down by a slow webhook. As a concrete example, the PR #19307 for custom webhooks, should be substantially smaller: - no need to change `services/webhook/deliver.go` - minimal change in `services/webhook/webhook.go` (add the new webhook to the map) - no need to change all the individual webhook files (since with this refactor the `*webhook_model.Webhook` is provided as argument) (cherry picked from commit 26653b196bd1d15c532af41f60351596dd4330bd) Conflicts: services/webhook/deliver_test.go trivial context conflict
2024-03-07 23:18:38 +01:00
// PrepareWebhook creates a hook task and enqueues it for processing.
// The payload is saved as-is. The adjustments depending on the webhook type happen
// right before delivery, in the [Deliver] method.
func PrepareWebhook(ctx context.Context, w *webhook_model.Webhook, event webhook_module.HookEventType, p api.Payloader) error {
// Skip sending if webhooks are disabled.
if setting.DisableWebhooks {
return nil
}
for _, e := range w.EventCheckers() {
if event == e.Type {
if !e.Has() {
return nil
}
2021-02-22 19:54:01 +01:00
break
}
}
// Avoid sending "0 new commits" to non-integration relevant webhooks (e.g. slack, discord, etc.).
// Integration webhooks (e.g. drone) still receive the required data.
if pushEvent, ok := p.(*api.PushPayload); ok &&
[BRANDING] add the forgejo webhook type & update webhook docs URLs templates/swagger/v1_json.tmpl updated with `make generate-swagger` (cherry picked from commit 88899c492efeedd138ba088a36b9c0bc733ead7b) (cherry picked from commit 7171bd9617c32c4911e3bdbc23c02a19e80d2465) (cherry picked from commit 1a742446c17aef9ca62fe75bfc0a388d40138154) (cherry picked from commit d7c189d7b2f9fea299a31adf068db969920ae39d) Conflicts: routers/web/web.go (cherry picked from commit cbdea868e41fb38ca491f8b449c3e525ec82d6b9) (cherry picked from commit 6cd150483b06e17aee023c0afd01a3f2460b3415) (cherry picked from commit 47246da8d3f50a02d11b77b3d402618b144aa720) (cherry picked from commit f2aa0e6b769d432e627798bcf294b04b7d253213) (cherry picked from commit 5a4fc69a16de8d6199ea24198299297ef7a3587b) (cherry picked from commit 48e444ca09c22f930514a01846b0c8bc3cef35ab) (cherry picked from commit 888e53781175d8d977f66d78991bd66563fcddfb) (cherry picked from commit 5121f493c99f19d8050aa09224ac3532b4100ec7) (cherry picked from commit 9394e55fdf80bf3d7bf8b2aba561ad44a84e3913) (cherry picked from commit 3a2ce51768de65892e3ec73596e3862354c9502e) (cherry picked from commit 719ead3a651f12afbb59c856914b0085e5cee157) (cherry picked from commit 83e6f82e2aea619a3cd502e133773d33c0e60133) (cherry picked from commit 494a429b21c6234be38b9e3db0f930fbb8118205) (cherry picked from commit 4d775db6b41f731e956cc6bb9217ef349b4a3635) (cherry picked from commit b68f777dc2822ec5c4e30186675cc82daec092a9) (cherry picked from commit 5b934023fa58820f27c349c26f2a1ce89aee6795) (cherry picked from commit 3b1ed8b16c73374cd5b6339f5315229dc82488da) (cherry picked from commit 6bc4a46c9fc6472e1c4bf0bb20dea6867f1b392f) (cherry picked from commit 8064bb24a3c752a86271f154ad4d0c4763e07295) Conflicts: templates/admin/hook_new.tmpl templates/org/settings/hook_new.tmpl templates/repo/settings/webhook/base_list.tmpl templates/repo/settings/webhook/new.tmpl templates/user/settings/hook_new.tmpl https://codeberg.org/forgejo/forgejo/pulls/1181 (cherry picked from commit 55f5588a9150d8912c0f8342495f858e4e1e2959) (cherry picked from commit a428bc06b353e7b678acc6a1ec2576608f1a5d9a) (cherry picked from commit d2186eceb960773d037756c991f50d177fd63954) (cherry picked from commit b4e126e9afd1c51b214e36a585bb454dac32862f) (cherry picked from commit e4c7a92c2d7142cac74cf05a2b1fe3f0bd0c7373) (cherry picked from commit a7165c8146a604b6ec5f8ee4db4c926fef180857) Conflicts: templates/admin/hook_new.tmpl templates/repo/settings/webhook/new.tmpl https://codeberg.org/forgejo/forgejo/pulls/1420 [BRANDING] add the forgejo webhook type & update webhook docs URLs (squash) gitea logo for gitea webhooks Refs: https://codeberg.org/forgejo/forgejo/issues/1367 (cherry picked from commit 2d8c1b93734acec918729e27c5613a8f9fb41ceb) (cherry picked from commit 9135a5e1dbcff07e9fff4c1001fcac8c05e0c82b) (cherry picked from commit 270f4020b0714ac553498fc986457b0c627d85a6) (cherry picked from commit 33e5e2f0d898639bae05225437aeb4afa2678360) (cherry picked from commit 7af8bcf4791ade3ca80d449629bb2eadd0ad25ba) Conflicts: tests/integration/links_test.go https://codeberg.org/forgejo/forgejo/pulls/1548 (cherry picked from commit ce2a434dcf9ad23337aaa0cc30dc069bbc79744c) (cherry picked from commit c2fa42b4fda37db8d5d4efc11834546d5de3d6ad) (cherry picked from commit 3858c8d4db2ceaf79bd5ebed54382cbdd6491a5b) (cherry picked from commit ebb0ae49208adeb5da4c6b442990db1b1dbe91d0) (cherry picked from commit 93d354c418e6da1971dd7f1749cb700627f1f6d2) (cherry picked from commit d5b08a2a4be2a48925dc089f5ec312188c60036c) (cherry picked from commit e5e5caffaa69c2f2be300cc97ec500a993ac6b3b) (cherry picked from commit 974d4057000f6989eb4b3440271c78dc05927312) (cherry picked from commit e654fbccb1ddfd9e494ce4560e5ac1620d85f7c6) (cherry picked from commit 85515977fb2fb6f6ae13f7b011bfe9f9c6d0425f) [BRANDING] add the forgejo webhook type & update webhook docs URLs (squash) ctx.Locale (cherry picked from commit e192647bc0f1d19b144e5ab1b286e11b2466eecd) (cherry picked from commit 95fb1b724201b4d0dc1ef0ffb2745b2977c73286) (cherry picked from commit ea7bd8022fc780b83de4a9ef5595cb0067e6e9d8) (cherry picked from commit bf8465ccd07a0be493d2faf21e1e2c62766513a4) (cherry picked from commit e1baa0a3007e2cc9a569da7d1e48785744d5cde6) (cherry picked from commit 01b4f4c51428fad992e95615e3d55be54b4f84f9) (cherry picked from commit da330b1e51916bb3b55167b5fefe86787a470e1f) (cherry picked from commit 7b26c27c209c7ee5dbb45b74bd253c1bc713f5b5) (cherry picked from commit 0c781e6f6d411335b4d22cd6a691b409379eba62) (cherry picked from commit 0f663a8414e248f67d18896863d34f03abd9738d) (cherry picked from commit 3297fd4f15ee7885d7b88c79e1ce70832cc60f78) (cherry picked from commit 2f4ef403cb9e92c0a0785b860747283bd754f826)
2023-01-14 10:07:01 +01:00
w.Type != webhook_module.FORGEJO && w.Type != webhook_module.GITEA && w.Type != webhook_module.GOGS &&
len(pushEvent.Commits) == 0 {
return nil
}
// If payload has no associated branch (e.g. it's a new tag, issue, etc.),
// branch filter has no effect.
if branch := getPayloadBranch(p); branch != "" {
if !checkBranch(w, branch) {
log.Info("Branch %q doesn't match branch filter %q, skipping", branch, w.BranchFilter)
return nil
}
}
Store webhook event in database (#29145) Refactor the webhook logic, to have the type-dependent processing happen only in one place. --- 1. An event happens 2. It is pre-processed (depending on the webhook type) and its body is added to a task queue 3. When the task is processed, some more logic (depending on the webhook type as well) is applied to make an HTTP request This means that webhook-type dependant logic is needed in step 2 and 3. This is cumbersome and brittle to maintain. Updated webhook flow with this PR: 1. An event happens 2. It is stored as-is and added to a task queue 3. When the task is processed, the event is processed (depending on the webhook type) to make an HTTP request So the only webhook-type dependent logic happens in one place (step 3) which should be much more robust. - the raw event must be stored in the hooktask (until now, the pre-processed body was stored) - to ensure that previous hooktasks are correctly sent, a `payload_version` is added (version 1: the body has already been pre-process / version 2: the body is the raw event) So future webhook additions will only have to deal with creating an http.Request based on the raw event (no need to adjust the code in multiple places, like currently). Moreover since this processing happens when fetching from the task queue, it ensures that the queuing of new events (upon a `git push` for instance) does not get slowed down by a slow webhook. As a concrete example, the PR #19307 for custom webhooks, should be substantially smaller: - no need to change `services/webhook/deliver.go` - minimal change in `services/webhook/webhook.go` (add the new webhook to the map) - no need to change all the individual webhook files (since with this refactor the `*webhook_model.Webhook` is provided as argument) (cherry picked from commit 26653b196bd1d15c532af41f60351596dd4330bd) Conflicts: services/webhook/deliver_test.go trivial context conflict
2024-03-07 23:18:38 +01:00
payload, err := p.JSONPayload()
if err != nil {
return fmt.Errorf("JSONPayload for %s: %w", event, err)
}
task, err := webhook_model.CreateHookTask(ctx, &webhook_model.HookTask{
Store webhook event in database (#29145) Refactor the webhook logic, to have the type-dependent processing happen only in one place. --- 1. An event happens 2. It is pre-processed (depending on the webhook type) and its body is added to a task queue 3. When the task is processed, some more logic (depending on the webhook type as well) is applied to make an HTTP request This means that webhook-type dependant logic is needed in step 2 and 3. This is cumbersome and brittle to maintain. Updated webhook flow with this PR: 1. An event happens 2. It is stored as-is and added to a task queue 3. When the task is processed, the event is processed (depending on the webhook type) to make an HTTP request So the only webhook-type dependent logic happens in one place (step 3) which should be much more robust. - the raw event must be stored in the hooktask (until now, the pre-processed body was stored) - to ensure that previous hooktasks are correctly sent, a `payload_version` is added (version 1: the body has already been pre-process / version 2: the body is the raw event) So future webhook additions will only have to deal with creating an http.Request based on the raw event (no need to adjust the code in multiple places, like currently). Moreover since this processing happens when fetching from the task queue, it ensures that the queuing of new events (upon a `git push` for instance) does not get slowed down by a slow webhook. As a concrete example, the PR #19307 for custom webhooks, should be substantially smaller: - no need to change `services/webhook/deliver.go` - minimal change in `services/webhook/webhook.go` (add the new webhook to the map) - no need to change all the individual webhook files (since with this refactor the `*webhook_model.Webhook` is provided as argument) (cherry picked from commit 26653b196bd1d15c532af41f60351596dd4330bd) Conflicts: services/webhook/deliver_test.go trivial context conflict
2024-03-07 23:18:38 +01:00
HookID: w.ID,
PayloadContent: string(payload),
EventType: event,
PayloadVersion: 2,
})
if err != nil {
Store webhook event in database (#29145) Refactor the webhook logic, to have the type-dependent processing happen only in one place. --- 1. An event happens 2. It is pre-processed (depending on the webhook type) and its body is added to a task queue 3. When the task is processed, some more logic (depending on the webhook type as well) is applied to make an HTTP request This means that webhook-type dependant logic is needed in step 2 and 3. This is cumbersome and brittle to maintain. Updated webhook flow with this PR: 1. An event happens 2. It is stored as-is and added to a task queue 3. When the task is processed, the event is processed (depending on the webhook type) to make an HTTP request So the only webhook-type dependent logic happens in one place (step 3) which should be much more robust. - the raw event must be stored in the hooktask (until now, the pre-processed body was stored) - to ensure that previous hooktasks are correctly sent, a `payload_version` is added (version 1: the body has already been pre-process / version 2: the body is the raw event) So future webhook additions will only have to deal with creating an http.Request based on the raw event (no need to adjust the code in multiple places, like currently). Moreover since this processing happens when fetching from the task queue, it ensures that the queuing of new events (upon a `git push` for instance) does not get slowed down by a slow webhook. As a concrete example, the PR #19307 for custom webhooks, should be substantially smaller: - no need to change `services/webhook/deliver.go` - minimal change in `services/webhook/webhook.go` (add the new webhook to the map) - no need to change all the individual webhook files (since with this refactor the `*webhook_model.Webhook` is provided as argument) (cherry picked from commit 26653b196bd1d15c532af41f60351596dd4330bd) Conflicts: services/webhook/deliver_test.go trivial context conflict
2024-03-07 23:18:38 +01:00
return fmt.Errorf("CreateHookTask for %s: %w", event, err)
}
return enqueueHookTask(task.ID)
}
// PrepareWebhooks adds new webhooks to task queue for given payload.
func PrepareWebhooks(ctx context.Context, source EventSource, event webhook_module.HookEventType, p api.Payloader) error {
owner := source.Owner
var ws []*webhook_model.Webhook
if source.Repository != nil {
repoHooks, err := db.Find[webhook_model.Webhook](ctx, webhook_model.ListWebhookOptions{
RepoID: source.Repository.ID,
IsActive: optional.Some(true),
})
if err != nil {
return fmt.Errorf("ListWebhooksByOpts: %w", err)
}
ws = append(ws, repoHooks...)
owner = source.Repository.MustOwner(ctx)
}
// append additional webhooks of a user or organization
if owner != nil {
ownerHooks, err := db.Find[webhook_model.Webhook](ctx, webhook_model.ListWebhookOptions{
OwnerID: owner.ID,
IsActive: optional.Some(true),
})
if err != nil {
return fmt.Errorf("ListWebhooksByOpts: %w", err)
}
ws = append(ws, ownerHooks...)
}
// Add any admin-defined system webhooks
systemHooks, err := webhook_model.GetSystemWebhooks(ctx, optional.Some(true))
if err != nil {
return fmt.Errorf("GetSystemWebhooks: %w", err)
}
ws = append(ws, systemHooks...)
if len(ws) == 0 {
return nil
}
for _, w := range ws {
if err := PrepareWebhook(ctx, w, event, p); err != nil {
return err
}
}
return nil
}
2022-01-05 22:00:20 +01:00
// ReplayHookTask replays a webhook task
func ReplayHookTask(ctx context.Context, w *webhook_model.Webhook, uuid string) error {
task, err := webhook_model.ReplayHookTask(ctx, w.ID, uuid)
2022-01-05 22:00:20 +01:00
if err != nil {
return err
}
return enqueueHookTask(task.ID)
2022-01-05 22:00:20 +01:00
}