Skip to content

Commit b4f62a1

Browse files
authored
fix: internal endpoints enabled in open source version of rudder-server (#7075)
🔒 Scanned for secrets using gitleaks 8.30.1 ## Description Fixing a security/misconfiguration issue where internal gateway and warehouse API endpoints were accessible in the open source version of `rudder-server`. Internal endpoints are now gated behind an enterprise token check and can only be registered when explicitly enabled. ### Changes - **Gate internal endpoints behind enterprise token** — added `WithInternalEndpointsEnabled(bool)` to the gateway and an equivalent `Warehouse.internalEndpointsEnabled` flag; both default to `true` in the enterprise version of rudder-server and `false` in the open source version. - **Move warehouse endpoints under `/internal`** — `/v1/warehouse/{pending-events,trigger-upload,jobs,jobs/status}` are now at `/internal/v1/warehouse/...`; legacy paths remain available behind a `legacyWarehouseEndpointsEnabled` flag (default `true`) for gradual rollout. - **Remove v1 job-status handler** — `NewV1Handler`, `GetFailedRecordsV1`, `Delete`, and all related tests removed; `/internal/v1/job-status` promoted to `/internal/v2/job-status`. ## Linear Ticket resolves RUD-2824 ## Security - [x] The code changed/added as part of this pull request won't create any security issues with how the software is being used.
1 parent 6b23a94 commit b4f62a1

20 files changed

Lines changed: 132 additions & 1071 deletions

app/apphandlers/embeddedAppHandler.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -412,11 +412,13 @@ func (a *embeddedApp) StartRudderCore(ctx context.Context, shutdownFn func(), op
412412
gw := gateway.Handle{}
413413
err = gw.Setup(ctx, config, logger.NewLogger().Child("gateway"), statsFactory, a.app, backendconfig.DefaultBackendConfig,
414414
gwWODB, rateLimiter, a.versionHandler, rsourcesService, transformerFeaturesService, sourceHandle,
415-
streamMsgValidator, gateway.WithInternalHttpHandlers(
415+
streamMsgValidator,
416+
gateway.WithInternalHttpHandlers(
416417
map[string]http.Handler{
417418
"/drain": drainConfigManager.DrainConfigHttpHandler(),
418419
},
419-
))
420+
),
421+
gateway.WithInternalEndpointsEnabled(config.GetBoolVar(options.EnterpriseToken != "", "Gateway.internalEndpointsEnabled")))
420422
if err != nil {
421423
return fmt.Errorf("could not setup gateway: %w", err)
422424
}

app/apphandlers/gatewayAppHandler.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -171,11 +171,13 @@ func (a *gatewayApp) StartRudderCore(ctx context.Context, _ func(), options *app
171171
streamMsgValidator := stream.NewMessageValidator()
172172
err = gw.Setup(ctx, config, logger.NewLogger().Child("gateway"), statsFactory, a.app, backendconfig.DefaultBackendConfig,
173173
gwWODB, rateLimiter, a.versionHandler, rsourcesService, transformerFeaturesService, sourceHandle,
174-
streamMsgValidator, gateway.WithInternalHttpHandlers(
174+
streamMsgValidator,
175+
gateway.WithInternalHttpHandlers(
175176
map[string]http.Handler{
176177
"/drain": drainConfigHttpHandler,
177178
},
178-
))
179+
),
180+
gateway.WithInternalEndpointsEnabled(config.GetBoolVar(options.EnterpriseToken != "", "Gateway.internalEndpointsEnabled")))
179181
if err != nil {
180182
return fmt.Errorf("failed to setup gateway: %w", err)
181183
}

gateway/gateway_test.go

Lines changed: 15 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -264,7 +264,6 @@ var _ = Describe("Gateway Enterprise", func() {
264264
conf = config.New()
265265
conf.Set("Gateway.enableRateLimit", false)
266266
conf.Set("Gateway.enableSuppressUserFeature", true)
267-
conf.Set("Gateway.enableEventSchemasFeature", false)
268267
})
269268

270269
AfterEach(func() {
@@ -402,7 +401,6 @@ var _ = Describe("Gateway", func() {
402401
BeforeEach(func() {
403402
conf = config.New()
404403
conf.Set("Gateway.enableRateLimit", false)
405-
conf.Set("Gateway.enableEventSchemasFeature", false)
406404
c = &testContext{}
407405
c.Setup()
408406
})
@@ -415,7 +413,7 @@ var _ = Describe("Gateway", func() {
415413
It("should wait for backend config", func() {
416414
c.initializeAppFeatures()
417415
gateway := &Handle{}
418-
err := gateway.Setup(context.Background(), conf, logger.NOP, stats.NOP, c.mockApp, c.mockBackendConfig, c.mockJobsDB, nil, c.mockVersionHandler, rsources.NewNoOpService(), transformer.NewNoOpService(), sourcedebugger.NewNoOpService(), nil)
416+
err := gateway.Setup(context.Background(), conf, logger.NOP, stats.NOP, c.mockApp, c.mockBackendConfig, c.mockJobsDB, nil, c.mockVersionHandler, rsources.NewNoOpService(), transformer.NewNoOpService(), sourcedebugger.NewNoOpService(), nil, WithInternalEndpointsEnabled(true))
419417
Expect(err).To(BeNil())
420418
waitForBackendConfigInit(gateway)
421419
err = gateway.Shutdown()
@@ -448,7 +446,7 @@ var _ = Describe("Gateway", func() {
448446
GinkgoT().Setenv("RSERVER_GATEWAY_WEB_PORT", strconv.Itoa(serverPort))
449447

450448
gateway = &Handle{}
451-
err = gateway.Setup(context.Background(), conf, logger.NOP, stats.NOP, c.mockApp, c.mockBackendConfig, c.mockJobsDB, nil, c.mockVersionHandler, rsources.NewNoOpService(), transformer.NewNoOpService(), sourcedebugger.NewNoOpService(), nil)
449+
err = gateway.Setup(context.Background(), conf, logger.NOP, stats.NOP, c.mockApp, c.mockBackendConfig, c.mockJobsDB, nil, c.mockVersionHandler, rsources.NewNoOpService(), transformer.NewNoOpService(), sourcedebugger.NewNoOpService(), nil, WithInternalEndpointsEnabled(true))
452450
Expect(err).To(BeNil())
453451
waitForBackendConfigInit(gateway)
454452
gateway.irh = mockRequestHandler{}
@@ -495,7 +493,7 @@ var _ = Describe("Gateway", func() {
495493
}
496494
resp, err := client.Do(req)
497495
Expect(err).To(BeNil())
498-
Expect(resp.StatusCode).To(SatisfyAny(Equal(http.StatusOK), Equal(http.StatusNoContent)), "endpoint: "+ep)
496+
Expect(resp.StatusCode).To(SatisfyAny(Equal(http.StatusOK), Equal(http.StatusNoContent)), "endpoint: "+ep, "method: "+method)
499497

500498
}
501499
}
@@ -543,7 +541,7 @@ var _ = Describe("Gateway", func() {
543541
Expect(err).To(BeNil())
544542

545543
gateway = &Handle{}
546-
err := gateway.Setup(context.Background(), conf, logger.NOP, statsStore, c.mockApp, c.mockBackendConfig, c.mockJobsDB, nil, c.mockVersionHandler, rsources.NewNoOpService(), transformer.NewNoOpService(), sourcedebugger.NewNoOpService(), nil)
544+
err := gateway.Setup(context.Background(), conf, logger.NOP, statsStore, c.mockApp, c.mockBackendConfig, c.mockJobsDB, nil, c.mockVersionHandler, rsources.NewNoOpService(), transformer.NewNoOpService(), sourcedebugger.NewNoOpService(), nil, WithInternalEndpointsEnabled(true))
547545
Expect(err).To(BeNil())
548546
waitForBackendConfigInit(gateway)
549547
})
@@ -1783,7 +1781,6 @@ var _ = Describe("Gateway", func() {
17831781
conf = config.New()
17841782
conf.Set("Gateway.enableRateLimit", false)
17851783
conf.Set("Gateway.enableSuppressUserFeature", true)
1786-
conf.Set("Gateway.enableEventSchemasFeature", false)
17871784

17881785
serverPort, err := kithelper.GetFreePort()
17891786
Expect(err).To(BeNil())
@@ -1799,7 +1796,7 @@ var _ = Describe("Gateway", func() {
17991796

18001797
gateway = &Handle{}
18011798
srcDebugger = mocksrcdebugger.NewMockSourceDebugger(c.mockCtrl)
1802-
err = gateway.Setup(context.Background(), conf, logger.NOP, statStore, c.mockApp, c.mockBackendConfig, c.mockJobsDB, nil, c.mockVersionHandler, rsources.NewNoOpService(), transformer.NewNoOpService(), srcDebugger, nil)
1799+
err = gateway.Setup(context.Background(), conf, logger.NOP, statStore, c.mockApp, c.mockBackendConfig, c.mockJobsDB, nil, c.mockVersionHandler, rsources.NewNoOpService(), transformer.NewNoOpService(), srcDebugger, nil, WithInternalEndpointsEnabled(true))
18031800
Expect(err).To(BeNil())
18041801
waitForBackendConfigInit(gateway)
18051802
c.mockBackendConfig.EXPECT().WaitForConfig(gomock.Any()).AnyTimes()
@@ -2138,7 +2135,7 @@ var _ = Describe("Gateway", func() {
21382135
BeforeEach(func() {
21392136
c.initializeAppFeatures()
21402137
gateway = &Handle{}
2141-
err := gateway.Setup(context.Background(), conf, logger.NOP, stats.NOP, c.mockApp, c.mockBackendConfig, c.mockJobsDB, nil, c.mockVersionHandler, rsources.NewNoOpService(), transformer.NewNoOpService(), sourcedebugger.NewNoOpService(), nil)
2138+
err := gateway.Setup(context.Background(), conf, logger.NOP, stats.NOP, c.mockApp, c.mockBackendConfig, c.mockJobsDB, nil, c.mockVersionHandler, rsources.NewNoOpService(), transformer.NewNoOpService(), sourcedebugger.NewNoOpService(), nil, WithInternalEndpointsEnabled(true))
21422139
Expect(err).To(BeNil())
21432140
waitForBackendConfigInit(gateway)
21442141
})
@@ -2410,12 +2407,10 @@ func endpointsToVerify() ([]string, []string, []string) {
24102407
"/pixel/v1/track",
24112408
"/pixel/v1/page",
24122409
"/v1/webhook",
2413-
"/v1/job-status/123",
2414-
"/v1/job-status/123/failed-records",
2415-
"/v1/warehouse/jobs/status",
2410+
"/internal/v1/warehouse/jobs/status",
24162411
"/internal/v1/warehouse/fetch-tables",
2417-
"/internal/v1/job-status/123",
2418-
"/internal/v1/job-status/123/failed-records",
2412+
"/internal/v2/job-status/123",
2413+
"/internal/v2/job-status/123/failed-records",
24192414
}
24202415

24212416
postEndpoints := []string{
@@ -2435,14 +2430,14 @@ func endpointsToVerify() ([]string, []string, []string) {
24352430
"/internal/v1/retl",
24362431
"/internal/v1/replay",
24372432
"/internal/v1/audiencelist",
2438-
"/v1/warehouse/pending-events",
2439-
"/v1/warehouse/trigger-upload",
2440-
"/v1/warehouse/jobs",
2441-
// "/internal/v1/batch", will be tested in new unit test
2433+
"/internal/v1/warehouse/pending-events",
2434+
"/internal/v1/warehouse/trigger-upload",
2435+
"/internal/v1/warehouse/jobs",
24422436
}
24432437

24442438
deleteEndpoints := []string{
2445-
"/v1/job-status/1234",
2439+
"/internal/v2/job-status/1234",
2440+
"/internal/v2/job-status/123/failed-records",
24462441
}
24472442
return getEndpoints, postEndpoints, deleteEndpoints
24482443
}
@@ -2622,7 +2617,7 @@ func createTestGatewayWithLeakyUploader(t *testing.T, endpoint, accessKeyID, sec
26222617
conf.Set("Gateway.leakyUploader.Storage.DisableSsl", true)
26232618
conf.Set("Gateway.leakyUploader.Storage.UseGlue", true)
26242619
conf.Set("Gateway.leakyUploader.Storage.S3ForcePathStyle", true)
2625-
err := gw.Setup(context.Background(), conf, logger.NewLogger().Withn(logger.NewStringField("component", "test")), stats.NOP, mockApp, mockBackendConfig, mockJobsDB, mockRateLimiter, mockVersionHandler, rsources.NewNoOpService(), transformer.NewNoOpService(), sourcedebugger.NewNoOpService(), nil)
2620+
err := gw.Setup(context.Background(), conf, logger.NewLogger().Withn(logger.NewStringField("component", "test")), stats.NOP, mockApp, mockBackendConfig, mockJobsDB, mockRateLimiter, mockVersionHandler, rsources.NewNoOpService(), transformer.NewNoOpService(), sourcedebugger.NewNoOpService(), nil, WithInternalEndpointsEnabled(true))
26262621
require.NoError(t, err)
26272622
require.Eventually(t, func() bool {
26282623
select {

gateway/handle.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,8 @@ type Handle struct {
126126
allowReqsWithoutUserIDAndAnonymousID config.ValueLoader[bool]
127127
gwAllowPartialWriteWithErrors config.ValueLoader[bool]
128128
webhookV2HandlerEnabled bool
129+
internalEndpointsEnabled bool
130+
legacyWarehouseEndpointsEnabled bool
129131
}
130132

131133
// additional internal http handlers

gateway/handle_lifecycle.go

Lines changed: 40 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,8 @@ func (gw *Handle) Setup(
117117
gw.conf.maxConcurrentRequests = config.GetIntVar(50000, 1, "Gateway.maxConcurrentRequests")
118118
// enable webhook v2 handler. disabled by default
119119
gw.conf.webhookV2HandlerEnabled = config.GetBoolVar(false, "Gateway.webhookV2HandlerEnabled")
120+
gw.conf.legacyWarehouseEndpointsEnabled = config.GetBoolVar(true, "Gateway.legacyWarehouseEndpointsEnabled") // TODO: remove legacy endpoints after next release
121+
120122
// Registering stats
121123
gw.batchSizeStat = gw.stats.NewStat("gateway.batch_size", stats.HistogramType)
122124
gw.requestSizeStat = gw.stats.NewStat("gateway.request_size", stats.HistogramType)
@@ -317,6 +319,12 @@ func WithNow(now func() time.Time) OptFunc {
317319
}
318320
}
319321

322+
func WithInternalEndpointsEnabled(enabled bool) OptFunc {
323+
return func(gw *Handle) {
324+
gw.conf.internalEndpointsEnabled = enabled
325+
}
326+
}
327+
320328
// initUserWebRequestWorkers initiates `maxUserWebRequestWorkerProcess` number of `webRequestWorkers` that listen on their `webRequestQ` for new WebRequests.
321329
func (gw *Handle) initUserWebRequestWorkers() {
322330
gw.userWebRequestWorkers = make([]*userWebRequestWorkerT, gw.conf.maxUserWebRequestWorkerProcess)
@@ -558,10 +566,6 @@ func (gw *Handle) StartWebHandler(ctx context.Context) error {
558566
component := "gateway"
559567
srvMux := chi.NewRouter()
560568
// rudder-sources new APIs
561-
rsourcesHandlerV1 := rsources_http.NewV1Handler(
562-
gw.rsourcesService,
563-
gw.logger.Child("rsources"),
564-
)
565569
rsourcesHandlerV2 := rsources_http.NewV2Handler(
566570
gw.rsourcesService,
567571
gw.logger.Child("rsources_failed_keys"),
@@ -578,25 +582,30 @@ func (gw *Handle) StartWebHandler(ctx context.Context) error {
578582
middleware.LimitConcurrentRequests(gw.conf.maxConcurrentRequests),
579583
middleware.UncompressMiddleware,
580584
)
581-
srvMux.Route("/internal", func(r chi.Router) {
582-
r.Post("/v1/extract", gw.webExtractHandler())
583-
r.Post("/v1/retl", gw.webRetlHandler())
584-
r.Get("/v1/warehouse/fetch-tables", gw.whProxy.ServeHTTP)
585-
r.Post("/v1/audiencelist", gw.webAudienceListHandler())
586-
r.Post("/v1/replay", gw.webReplayHandler())
587-
r.Post("/v1/batch", gw.internalBatchHandler())
588-
589-
// TODO: delete this handler once we are ready to remove support for the v1 api
590-
r.Mount("/v1/job-status", withContentType("application/json; charset=utf-8", rsourcesHandlerV1.ServeHTTP))
591-
592-
r.Mount("/v2/job-status", withContentType("application/json; charset=utf-8", rsourcesHandlerV2.ServeHTTP))
593-
for path, handler := range gw.internalHttpHandlers {
594-
r.Mount(path, withContentType("application/json; charset=utf-8", handler.ServeHTTP))
595-
}
596-
})
597-
598-
// TODO: delete this handler once we are ready to remove support for the v1 api
599-
srvMux.Mount("/v1/job-status", withContentType("application/json; charset=utf-8", rsourcesHandlerV1.ServeHTTP))
585+
if gw.conf.internalEndpointsEnabled {
586+
srvMux.Route("/internal", func(r chi.Router) {
587+
r.Route("/v1", func(r chi.Router) {
588+
r.Post("/extract", gw.webExtractHandler())
589+
r.Post("/retl", gw.webRetlHandler())
590+
r.Route("/warehouse", func(r chi.Router) {
591+
r.Get("/fetch-tables", gw.whProxy.ServeHTTP)
592+
r.Post("/pending-events", gw.whProxy.ServeHTTP)
593+
r.Post("/trigger-upload", gw.whProxy.ServeHTTP)
594+
r.Post("/jobs", gw.whProxy.ServeHTTP)
595+
r.Get("/jobs/status", gw.whProxy.ServeHTTP)
596+
})
597+
r.Post("/audiencelist", gw.webAudienceListHandler())
598+
r.Post("/replay", gw.webReplayHandler())
599+
r.Post("/batch", gw.internalBatchHandler())
600+
})
601+
r.Route("/v2", func(r chi.Router) {
602+
r.Mount("/job-status", withContentType("application/json; charset=utf-8", rsourcesHandlerV2.ServeHTTP))
603+
})
604+
for path, handler := range gw.internalHttpHandlers {
605+
r.Mount(path, withContentType("application/json; charset=utf-8", handler.ServeHTTP))
606+
}
607+
})
608+
}
600609

601610
srvMux.Route("/v1", func(r chi.Router) {
602611
r.Post("/alias", gw.webAliasHandler())
@@ -614,13 +623,14 @@ func (gw *Handle) StartWebHandler(ctx context.Context) error {
614623

615624
r.Get("/webhook", gw.webhookHandler())
616625

617-
r.Route("/warehouse", func(r chi.Router) {
618-
r.Post("/pending-events", gw.whProxy.ServeHTTP)
619-
r.Post("/trigger-upload", gw.whProxy.ServeHTTP)
620-
r.Post("/jobs", gw.whProxy.ServeHTTP)
621-
622-
r.Get("/jobs/status", gw.whProxy.ServeHTTP)
623-
})
626+
if gw.conf.internalEndpointsEnabled && gw.conf.legacyWarehouseEndpointsEnabled {
627+
r.Route("/warehouse", func(r chi.Router) {
628+
r.Post("/pending-events", gw.whProxy.ServeHTTP)
629+
r.Post("/trigger-upload", gw.whProxy.ServeHTTP)
630+
r.Post("/jobs", gw.whProxy.ServeHTTP)
631+
r.Get("/jobs/status", gw.whProxy.ServeHTTP)
632+
})
633+
}
624634
})
625635

626636
srvMux.Get("/health", withContentType("application/json; charset=utf-8", app.LivenessHandler(gw.jobsDB)))

gateway/handle_test.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,8 @@ func createTestGateway(t *testing.T, eventBlockingSettings backendconfig.EventBl
9191
allowReqsWithoutUserIDAndAnonymousID config.ValueLoader[bool]
9292
gwAllowPartialWriteWithErrors config.ValueLoader[bool]
9393
webhookV2HandlerEnabled bool
94+
internalEndpointsEnabled bool
95+
legacyWarehouseEndpointsEnabled bool
9496
}{
9597
webhookV2HandlerEnabled: false,
9698
},

gateway/integration_test.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,7 @@ func testGatewayByAppType(t *testing.T, appType, rsBinaryPath string) {
149149
fmt.Sprintf("RUDDER_TMPDIR=%s", rudderTmpDir),
150150
fmt.Sprintf("DEST_TRANSFORM_URL=%s", transformerContainer.TransformerURL),
151151
fmt.Sprintf("WORKSPACE_TOKEN=%s", workspaceToken),
152+
fmt.Sprintf("ENTERPRISE_TOKEN=%s", "token"),
152153
}
153154
if testing.Verbose() {
154155
envArr = append(envArr, "LOG_LEVEL=debug")

integration_test/docker_test/docker_test.go

Lines changed: 6 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -363,6 +363,8 @@ func setupMainFlow(svcCtx context.Context, cancel context.CancelFunc, t *testing
363363

364364
httpPort = strconv.Itoa(httpPortInt)
365365
t.Setenv("RSERVER_GATEWAY_WEB_PORT", httpPort)
366+
// we need to use the internal rETL endpoints in this test, even for open source
367+
t.Setenv("RSERVER_GATEWAY_INTERNAL_ENDPOINTS_ENABLED", "true")
366368

367369
t.Setenv("RSERVER_ENABLE_STATS", "false")
368370

@@ -760,30 +762,19 @@ func sendRETL(t *testing.T, payload *strings.Reader, sourceID, DestinationID str
760762
)
761763

762764
req, err := http.NewRequest(method, url, payload)
763-
if err != nil {
764-
t.Logf("sendEvent error: %v", err)
765-
return
766-
}
765+
require.NoError(t, err)
767766

768767
req.Header.Add("Content-Type", "application/json")
769768
req.Header.Add("X-Rudder-Source-Id", sourceID)
770769
req.Header.Add("X-Rudder-Destination-Id", DestinationID)
771770

772771
res, err := httpClient.Do(req)
773-
if err != nil {
774-
t.Logf("sendEvent error: %v", err)
775-
return
776-
}
772+
require.NoError(t, err)
777773
defer func() { httputil.CloseResponse(res) }()
778774

779775
body, err := io.ReadAll(res.Body)
780-
if err != nil {
781-
t.Logf("sendEvent error: %v", err)
782-
return
783-
}
784-
if res.Status != "200 OK" {
785-
return
786-
}
776+
require.NoError(t, err)
777+
require.Equal(t, 200, res.StatusCode)
787778

788779
t.Logf("Event Sent Successfully: (%s)", body)
789780
}

integration_test/retl_test/sut.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,7 @@ func (s *SUT) Start(t *testing.T) {
193193
t.Logf("--- Setup done (%s)", time.Since(setupStart))
194194

195195
go func() {
196-
r := runner.New(runner.ReleaseInfo{EnterpriseToken: os.Getenv("ENTERPRISE_TOKEN")})
196+
r := runner.New(runner.ReleaseInfo{EnterpriseToken: "token"})
197197
_ = r.Run(svcCtx, svcCancel, []string{"retl-test-rudder-server"})
198198
close(s.done)
199199
}()
@@ -335,7 +335,7 @@ func (s *SUT) SendRETL(t *testing.T, sourceID, destinationID string, payload bat
335335
func (s *SUT) JobStatus(t *testing.T, sourceID, jobRunID, jobTaskID string) (rsources.JobStatus, bool) {
336336
var (
337337
httpClient = &http.Client{}
338-
jobStatusURL = fmt.Sprintf("%s/v1/job-status/%s?%s", s.URL, jobRunID, url.Values{
338+
jobStatusURL = fmt.Sprintf("%s/internal/v2/job-status/%s?%s", s.URL, jobRunID, url.Values{
339339
"task_run_id": []string{jobTaskID},
340340
}.Encode()) // job_run_id
341341
)

0 commit comments

Comments
 (0)