feat: Add remotes from legacy catalog (#261)

* feat: Add remotes from legacy catalog

Add ability to represent the remote servers in our legacy catalog in the
new catalog format as well as in working sets.

- Add new `Type` 'remote' to `WorkingSet.Server and `Catalog.Server`
- Add new property `endpoint` to `Catalog.Server`

* fix: bug creating from working set

* Migration fix.

* Add remote endpoint to human readable print.

* Remove unnecessary if check.

---------

Co-authored-by: Cody Rigney <cody.rigney@docker.com>

Author: bobbyhouse

pkg/catalog_next/catalog.go

@@ -33,7 +33,7 @@ const (
 )
 
 type Server struct {
-	Type  workingset.ServerType `yaml:"type" json:"type" validate:"required,oneof=registry image"`
+	Type  workingset.ServerType `yaml:"type" json:"type" validate:"required,oneof=registry image remote"`
 	Tools []string              `yaml:"tools,omitempty" json:"tools,omitempty"`
 
 	// ServerTypeRegistry only
@@ -42,6 +42,9 @@ type Server struct {
 	// ServerTypeImage only
 	Image string `yaml:"image,omitempty" json:"image,omitempty" validate:"required_if=Type image"`
 
+	// ServerTypeRemote only
+	Endpoint string `yaml:"endpoint,omitempty" json:"endpoint,omitempty" validate:"required_if=Type remote"`
+
 	Snapshot *workingset.ServerSnapshot `yaml:"snapshot,omitempty" json:"snapshot,omitempty"`
 }
 
@@ -58,6 +61,9 @@ func NewFromDb(dbCatalog *db.Catalog) CatalogWithDigest {
 		if server.ServerType == "image" {
 			servers[i].Image = server.Image
 		}
+		if server.ServerType == "remote" {
+			servers[i].Endpoint = server.Endpoint
+		}
 		if server.Snapshot != nil {
 			servers[i].Snapshot = &workingset.ServerSnapshot{
 				Server: server.Snapshot.Server,
@@ -93,6 +99,9 @@ func (catalog Catalog) ToDb() (db.Catalog, error) {
 		if server.Type == workingset.ServerTypeImage {
 			dbServers[i].Image = server.Image
 		}
+		if server.Type == workingset.ServerTypeRemote {
+			dbServers[i].Endpoint = server.Endpoint
+		}
 		if server.Snapshot != nil {
 			dbServers[i].Snapshot = &db.ServerSnapshot{
 				Server: server.Snapshot.Server,

pkg/catalog_next/create.go

@@ -83,6 +83,7 @@ func createCatalogFromWorkingSet(ctx context.Context, dao db.DAO, workingSetID s
 			Tools:    server.Tools,
 			Source:   server.Source,
 			Image:    server.Image,
+			Endpoint: server.Endpoint,
 			Snapshot: server.Snapshot,
 		}
 	}
@@ -104,7 +105,6 @@ func createCatalogFromLegacyCatalog(ctx context.Context, legacyCatalogURL string
 
 	servers := make([]Server, 0, len(legacyCatalog.Servers))
 	for name, server := range legacyCatalog.Servers {
-		// TODO(cody): Add support for remote servers from the legacy catalog
 		if server.Type == "server" && server.Image != "" {
 			s := Server{
 				Type:  workingset.ServerTypeImage,
@@ -115,6 +115,16 @@ func createCatalogFromLegacyCatalog(ctx context.Context, legacyCatalogURL string
 			}
 			s.Snapshot.Server.Name = name
 			servers = append(servers, s)
+		} else if server.Type == "remote" {
+			s := Server{
+				Type:     workingset.ServerTypeRemote,
+				Endpoint: server.Remote.URL,
+				Snapshot: &workingset.ServerSnapshot{
+					Server: server,
+				},
+			}
+			s.Snapshot.Server.Name = name
+			servers = append(servers, s)
 		}
 	}
 

pkg/catalog_next/create_test.go

@@ -300,6 +300,11 @@ func TestCreateFromWorkingSetPreservesAllServerFields(t *testing.T) {
 				Image: "mycompany/myserver:v1.2.3",
 				Tools: []string{"deploy"},
 			},
+			{
+				Type:     string(workingset.ServerTypeRemote),
+				Endpoint: "https://remote.example.com/sse",
+				Tools:    []string{"remote-tool1", "remote-tool2"},
+			},
 		},
 		Secrets: db.SecretMap{},
 	}
@@ -319,7 +324,7 @@ func TestCreateFromWorkingSetPreservesAllServerFields(t *testing.T) {
 
 	assert.Equal(t, "Detailed Catalog", catalog.Title)
 	assert.Equal(t, "profile:detailed-ws", catalog.Source)
-	assert.Len(t, catalog.Servers, 2)
+	assert.Len(t, catalog.Servers, 3)
 
 	// Check registry server
 	assert.Equal(t, workingset.ServerTypeRegistry, catalog.Servers[0].Type)
@@ -330,6 +335,11 @@ func TestCreateFromWorkingSetPreservesAllServerFields(t *testing.T) {
 	assert.Equal(t, workingset.ServerTypeImage, catalog.Servers[1].Type)
 	assert.Equal(t, "mycompany/myserver:v1.2.3", catalog.Servers[1].Image)
 	assert.Equal(t, []string{"deploy"}, catalog.Servers[1].Tools)
+
+	// Check remote server
+	assert.Equal(t, workingset.ServerTypeRemote, catalog.Servers[2].Type)
+	assert.Equal(t, "https://remote.example.com/sse", catalog.Servers[2].Endpoint)
+	assert.Equal(t, []string{"remote-tool1", "remote-tool2"}, catalog.Servers[2].Tools)
 }
 
 func TestCreateFromLegacyCatalog(t *testing.T) {
@@ -501,3 +511,217 @@ registry:
 	assert.Equal(t, "Test Catalog", catalog.Title)
 	assert.Equal(t, "legacy-catalog:test-catalog", catalog.Source)
 }
+
+func TestCreateFromLegacyCatalogWithRemotes(t *testing.T) {
+	tests := []struct {
+		name           string
+		serverYAML     string
+		expectedType   workingset.ServerType
+		validateServer func(t *testing.T, server *catalog.Server)
+	}{
+		{
+			name: "basic remote with SSE transport",
+			serverYAML: `    title: "AIS Fleet"
+    type: remote
+    remote:
+      transport_type: sse
+      url: https://mcp.aisfleet.com/sse`,
+			expectedType: workingset.ServerTypeRemote,
+			validateServer: func(t *testing.T, server *catalog.Server) {
+				t.Helper()
+				assert.Equal(t, "sse", server.Remote.Transport)
+				assert.Equal(t, "https://mcp.aisfleet.com/sse", server.Remote.URL)
+				assert.Empty(t, server.Remote.Headers)
+				assert.Empty(t, server.Secrets)
+				assert.Nil(t, server.OAuth)
+			},
+		},
+		{
+			name: "remote with streamable-http and authorization header",
+			serverYAML: `    title: "Apify Remote"
+    type: remote
+    remote:
+      transport_type: streamable-http
+      url: https://mcp.apify.com
+      headers:
+        Authorization: "Bearer ${APIFY_API_KEY}"
+    secrets:
+      - name: apify.api_key
+        env: APIFY_API_KEY
+        example: <YOUR_API_KEY>`,
+			expectedType: workingset.ServerTypeRemote,
+			validateServer: func(t *testing.T, server *catalog.Server) {
+				t.Helper()
+				assert.Equal(t, "streamable-http", server.Remote.Transport)
+				assert.Equal(t, "https://mcp.apify.com", server.Remote.URL)
+				assert.Equal(t, "Bearer ${APIFY_API_KEY}", server.Remote.Headers["Authorization"])
+				assert.Len(t, server.Secrets, 1)
+				assert.Equal(t, "apify.api_key", server.Secrets[0].Name)
+				assert.Equal(t, "APIFY_API_KEY", server.Secrets[0].Env)
+			},
+		},
+		{
+			name: "remote with OAuth",
+			serverYAML: `    title: "Asana"
+    type: remote
+    remote:
+      transport_type: streamable-http
+      url: https://asana.com/api/mcp/v1/sse
+    oauth:
+      providers:
+        - provider: asana
+          secret: asana.personal_access_token
+          env: ASANA_PERSONAL_ACCESS_TOKEN`,
+			expectedType: workingset.ServerTypeRemote,
+			validateServer: func(t *testing.T, server *catalog.Server) {
+				t.Helper()
+				assert.Equal(t, "streamable-http", server.Remote.Transport)
+				assert.Equal(t, "https://asana.com/api/mcp/v1/sse", server.Remote.URL)
+				require.NotNil(t, server.OAuth)
+				assert.Len(t, server.OAuth.Providers, 1)
+				assert.Equal(t, "asana", server.OAuth.Providers[0].Provider)
+				assert.Equal(t, "asana.personal_access_token", server.OAuth.Providers[0].Secret)
+				assert.Equal(t, "ASANA_PERSONAL_ACCESS_TOKEN", server.OAuth.Providers[0].Env)
+			},
+		},
+		{
+			name: "remote with dynamic tools",
+			serverYAML: `    title: "Cloudflare Audit Logs"
+    type: remote
+    dynamic:
+      tools: true
+    remote:
+      transport_type: sse
+      url: https://auditlogs.mcp.cloudflare.com/sse
+    oauth:
+      providers:
+        - provider: cloudflare-audit-logs
+          secret: cloudflare-audit-logs.personal_access_token
+          env: CLOUDFLARE_PERSONAL_ACCESS_TOKEN`,
+			expectedType: workingset.ServerTypeRemote,
+			validateServer: func(t *testing.T, server *catalog.Server) {
+				t.Helper()
+				assert.Equal(t, "sse", server.Remote.Transport)
+				assert.Equal(t, "https://auditlogs.mcp.cloudflare.com/sse", server.Remote.URL)
+				require.NotNil(t, server.OAuth)
+				assert.Len(t, server.OAuth.Providers, 1)
+				assert.Equal(t, "cloudflare-audit-logs", server.OAuth.Providers[0].Provider)
+			},
+		},
+		{
+			name: "remote with static tools list (no headers/secrets)",
+			serverYAML: `    title: "GitMCP"
+    type: remote
+    remote:
+      transport_type: streamable-http
+      url: https://gitmcp.io/docs
+    tools:
+      - name: match_common_libs_owner_repo_mapping
+      - name: fetch_generic_documentation
+      - name: search_generic_documentation`,
+			expectedType: workingset.ServerTypeRemote,
+			validateServer: func(t *testing.T, server *catalog.Server) {
+				t.Helper()
+				assert.Equal(t, "streamable-http", server.Remote.Transport)
+				assert.Equal(t, "https://gitmcp.io/docs", server.Remote.URL)
+				assert.Empty(t, server.Remote.Headers)
+				assert.Empty(t, server.Secrets)
+				assert.Nil(t, server.OAuth)
+				assert.Len(t, server.Tools, 3)
+				assert.Equal(t, "match_common_libs_owner_repo_mapping", server.Tools[0].Name)
+			},
+		},
+		{
+			name: "remote with SSE, headers, and secrets",
+			serverYAML: `    title: "Dodo Payments"
+    type: remote
+    remote:
+      transport_type: sse
+      url: https://mcp.dodopayments.com/sse
+      headers:
+        Authorization: "Bearer ${DODO_PAYMENTS_API_KEY}"
+    secrets:
+      - name: dodo-payments.api_key
+        env: DODO_PAYMENTS_API_KEY
+        example: <YOUR_API_KEY>`,
+			expectedType: workingset.ServerTypeRemote,
+			validateServer: func(t *testing.T, server *catalog.Server) {
+				t.Helper()
+				assert.Equal(t, "sse", server.Remote.Transport)
+				assert.Equal(t, "https://mcp.dodopayments.com/sse", server.Remote.URL)
+				assert.Equal(t, "Bearer ${DODO_PAYMENTS_API_KEY}", server.Remote.Headers["Authorization"])
+				assert.Len(t, server.Secrets, 1)
+				assert.Equal(t, "dodo-payments.api_key", server.Secrets[0].Name)
+				assert.Equal(t, "DODO_PAYMENTS_API_KEY", server.Secrets[0].Env)
+			},
+		},
+		{
+			name: "remote documentation server (no auth)",
+			serverYAML: `    title: "Cloudflare Docs"
+    type: remote
+    remote:
+      transport_type: sse
+      url: https://docs.mcp.cloudflare.com/sse
+    tools:
+      - name: search_cloudflare_documentation
+      - name: migrate_pages_to_workers_guide`,
+			expectedType: workingset.ServerTypeRemote,
+			validateServer: func(t *testing.T, server *catalog.Server) {
+				t.Helper()
+				assert.Equal(t, "sse", server.Remote.Transport)
+				assert.Equal(t, "https://docs.mcp.cloudflare.com/sse", server.Remote.URL)
+				assert.Empty(t, server.Remote.Headers)
+				assert.Empty(t, server.Secrets)
+				assert.Nil(t, server.OAuth)
+				assert.Len(t, server.Tools, 2)
+				assert.Equal(t, "search_cloudflare_documentation", server.Tools[0].Name)
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			dao := setupTestDB(t)
+			ctx := t.Context()
+
+			// Create a temporary legacy catalog file
+			tempDir := t.TempDir()
+			catalogFile := filepath.Join(tempDir, "test-catalog.yaml")
+
+			legacyCatalogYAML := "name: test-catalog\nregistry:\n  test-server:\n" + tt.serverYAML + "\n"
+
+			err := os.WriteFile(catalogFile, []byte(legacyCatalogYAML), 0o644)
+			require.NoError(t, err)
+
+			// Create catalog from legacy catalog
+			output := captureStdout(t, func() {
+				err := Create(ctx, dao, "test/imported:latest", "", catalogFile, "Imported Catalog")
+				require.NoError(t, err)
+			})
+
+			assert.Contains(t, output, "Catalog test/imported:latest created")
+
+			// Verify the catalog was created
+			catalogs, err := dao.ListCatalogs(ctx)
+			require.NoError(t, err)
+			assert.Len(t, catalogs, 1)
+
+			catalog := NewFromDb(&catalogs[0])
+			assert.Equal(t, "Imported Catalog", catalog.Title)
+			assert.Equal(t, "legacy-catalog:test-catalog", catalog.Source)
+			require.Len(t, catalog.Servers, 1)
+
+			// Verify server basic properties
+			server := catalog.Servers[0]
+			assert.Equal(t, tt.expectedType, server.Type)
+			require.NotNil(t, server.Snapshot)
+			assert.Equal(t, "test-server", server.Snapshot.Server.Name)
+
+			assert.NotEmpty(t, server.Endpoint, "Endpoint should be set for remote servers")
+			assert.Equal(t, server.Snapshot.Server.Remote.URL, server.Endpoint, "Endpoint should match the Remote.URL from snapshot")
+
+			// Run custom validation
+			tt.validateServer(t, &server.Snapshot.Server)
+		})
+	}
+}

pkg/catalog_next/show.go

@@ -106,6 +106,8 @@ func printHumanReadable(catalog CatalogWithDigest) string {
 			servers += fmt.Sprintf("    Source: %s\n", server.Source)
 		case workingset.ServerTypeImage:
 			servers += fmt.Sprintf("    Image: %s\n", server.Image)
+		case workingset.ServerTypeRemote:
+			servers += fmt.Sprintf("    Endpoint: %s\n", server.Endpoint)
 		}
 	}
 	servers = strings.TrimSuffix(servers, "\n")

pkg/db/catalog.go

@@ -33,6 +33,7 @@ type CatalogServer struct {
 	Tools      ToolList `db:"tools" json:"tools"`
 	Source     string   `db:"source" json:"source"`
 	Image      string   `db:"image" json:"image"`
+	Endpoint   string   `db:"endpoint" json:"endpoint"`
 	CatalogRef string   `db:"catalog_ref" json:"catalog_ref"`
 
 	Snapshot *ServerSnapshot `db:"snapshot" json:"snapshot"`
@@ -63,7 +64,7 @@ func (d *dao) GetCatalog(ctx context.Context, ref string) (*Catalog, error) {
 		return nil, err
 	}
 
-	const serverQuery = `SELECT id, server_type, tools, source, image, catalog_ref, snapshot from catalog_server where catalog_ref = $1`
+	const serverQuery = `SELECT id, server_type, tools, source, image, endpoint, catalog_ref, snapshot from catalog_server where catalog_ref = $1`
 
 	var servers []CatalogServer
 	err = d.db.SelectContext(ctx, &servers, serverQuery, catalog.Ref)
@@ -103,8 +104,8 @@ func (d *dao) UpsertCatalog(ctx context.Context, catalog Catalog) error {
 
 	if len(catalog.Servers) > 0 {
 		const serverQuery = `INSERT INTO catalog_server (
-		server_type, tools, source, image, catalog_ref, snapshot
-	) VALUES (:server_type, :tools, :source, :image, :catalog_ref, :snapshot)`
+		server_type, tools, source, image, endpoint, catalog_ref, snapshot
+	) VALUES (:server_type, :tools, :source, :image, :endpoint, :catalog_ref, :snapshot)`
 
 		_, err = tx.NamedExecContext(ctx, serverQuery, catalog.Servers)
 		if err != nil {
@@ -137,7 +138,7 @@ func (d *dao) ListCatalogs(ctx context.Context) ([]Catalog, error) {
 
 	const query = `SELECT c.ref, c.digest, c.title, c.source, c.last_updated,
 	COALESCE(
-		json_group_array(json_object('id', s.id, 'server_type', s.server_type, 'tools', json(s.tools), 'source', s.source, 'image', s.image, 'snapshot', json(s.snapshot))),
+		json_group_array(json_object('id', s.id, 'server_type', s.server_type, 'tools', json(s.tools), 'source', s.source, 'image', s.image, 'endpoint', s.endpoint, 'snapshot', json(s.snapshot))),
 		'[]'
 	) AS server_json
 	FROM catalog c

pkg/db/migrations/005_add_remote_support.up.sql

@@ -0,0 +1,23 @@
+-- Add support for remote server type
+-- SQLite doesn't support modifying CHECK constraints, so we need to recreate the table
+
+CREATE TABLE catalog_server_new (
+  id integer primary key,
+  server_type text check(server_type in ('registry', 'image', 'remote')),
+  tools text CHECK (json_valid(tools)),
+  source text,
+  image text,
+  endpoint text,
+  snapshot text CHECK (json_valid(snapshot)),
+  catalog_ref text not null,
+  foreign key (catalog_ref) references catalog(ref) on delete cascade
+);
+
+-- Copy existing data
+INSERT INTO catalog_server_new (id, server_type, tools, source, image, endpoint, snapshot, catalog_ref)
+SELECT id, server_type, tools, source, image, "", snapshot, catalog_ref
+FROM catalog_server;
+
+DROP TABLE catalog_server;
+
+ALTER TABLE catalog_server_new RENAME TO catalog_server;