針對infra項目分析

2025-06-19

本嘗試是高度失敗的最終失敗作品

infra/
├── .github/
│ └── workflows/
├── .vscode/
├── packages/
│ ├── api (可以獨立編譯並啟動在docker中 )/
│ ├── buckets/
│ ├── client-proxy/
│ ├── cluster/
│ ├── cluster-disk-image/
│ ├── db (獨立啟動可灌入數據庫某種版本無效)/
│ ├── docker-reverse-proxy/
│ ├── envd (可以獨立編譯並啟動在docker中可編譯可獨立啟動)/
│ ├── fc-kernels (編譯VM vmlinux-6.1.102)/
│ ├── fc-versions (編譯firecracker v1.10.1_1fcdaec08)/
│ ├── init(未知)/
│ ├── nomad (目前分析是前端語言不懂大概是控制流程)/
│ ├── orchestrator (可以獨立啟動在docker中初始化templement)/
│ └── shared(共用檔)/
├── readme-assets(圖片資料夾)/
├── scripts/
├── spec/
├── terraform/
│ ├── github/
│ ├── grafana/
│ └── redis/
├── tests (測試區塊)/
├── .dockerignore
├── .env.template
├── .gitignore
├── .golangci.yml
├── DEV.md
├── LICENSE
├── Makefile (主要啟動代碼)
├── README.md
├── VERSION
├── go.work
├── self-host.md
├── main.tf
├── variables.tf
└── .terraform.lock.hcl

主項目Makefile 新增以下

#先導入數據庫才能啟動
RUN1:
    $(MAKE) -C packages/db migrate/up
#先啟動 orchestrator伺服器才能啟動api調用 
#需要提前啟動sudo modprobe nbd max_part=32 max_nbds=32
#需要提前安裝apt install golang -y
#需要提前安裝apt install make -y
#啟動有概率網路發生問題sudo ip route del default sudo ip route add default via 192.168.1.1 dev ens33
#make RUN2 > 2.txt 2>&1

RUN2:
    $(MAKE) -C packages/orchestrator run-debug
#啟動前端用的api調用
#需要提前到/kok/todo-api/main4/packages/shared/pkg/grpc/connection.go
#GetConnection()函數中新增
#if strings.TrimSpace(host) == "" {
#	// 當 host 空字串時，強制設定為 192.168.1.104:5008 (你的伺服器)
#	host = "192.168.1.104:5008"
#	fmt.Println("Host for gRPC not set, using default connection:", host)
#}
#make RUN3 > 3.txt 2>&1
RUN3:
    $(MAKE) -C packages/api run
#打開envd用途不明
RUN4:
    $(MAKE) -C packages/envd start-docker
#打開tests用途不明
RUN5:
    $(MAKE) -C tests test

#用途不明
RUN6:
    terraform init
    terraform plan
#編譯vmlinux-6.1.102起來
RUN7:
    $(MAKE) -C packages/fc-versions build
#編譯firecracker起來
RUN8:
    $(MAKE) -C packages/fc-kernels build

packages/orchestrator 資料夾的make需要進行微調為

ENV := $(shell cat ../../.last_used_env || echo "not-set")
-include ../../.env.${ENV}
IMAGE := e2b-orchestration/api


openapi := ../../spec/openapi.yml
codegen := go tool github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen
expectedMigration := $(shell ./../../scripts/get-latest-migration.sh)

.PHONY: generate
generate:
    $(codegen) -old-config-style -generate gin --package api $(openapi) > internal/api/api.gen.go
    $(codegen) -old-config-style -generate types --package api $(openapi) > internal/api/types.gen.go
    $(codegen) -old-config-style -generate spec --package api $(openapi) > internal/api/spec.gen.go


.PHONY: build
build:
    # Allow for passing commit sha directly for docker builds
    $(eval COMMIT_SHA ?= $(shell git rev-parse --short HEAD))
    $(eval EXPECTED_MIGRATION_TIMESTAMP ?= $(expectedMigration))
    CGO_ENABLED=0 go build -o bin/api -ldflags "-X=main.commitSHA=$(COMMIT_SHA) -X=main.expectedMigrationTimestamp=$(EXPECTED_MIGRATION_TIMESTAMP)" .

.PHONY: build-debug
build-debug:
    $(eval COMMIT_SHA ?= $(shell git rev-parse --short HEAD))
    $(eval EXPECTED_MIGRATION_TIMESTAMP ?= $(expectedMigration))
    CGO_ENABLED=1 go build -race -gcflags=all="-N -l" -o bin/api -ldflags "-X=main.commitSHA=$(COMMIT_SHA) -X=main.expectedMigrationTimestamp=$(EXPECTED_MIGRATION_TIMESTAMP)" .

.PHONY: run
run:
    make build-debug
    POSTGRES_CONNECTION_STRING=$(POSTGRES_CONNECTION_STRING) \
    SUPABASE_JWT_SECRETS=$(SUPABASE_JWT_SECRETS) \
    GOTRACEBACK=crash \
    GODEBUG=madvdontneed=1 \
    TEMPLATE_BUCKET_NAME=$(TEMPLATE_BUCKET_NAME) \
    SANDBOX_ACCESS_TOKEN_HASH_SEED=$(SANDBOX_ACCESS_TOKEN_HASH_SEED) \
    ENVIRONMENT=$(ENVIRONMENT) \
    ORCHESTRATOR_PORT=5008 \
    ./bin/api --port 3009 

# Run the API using air
.PHONY: dev
dev:
    go tool air

# You run the parametrized command like this:
# make metric=heap interval=90 profiler
.PHONY: profiler
profiler:
    go tool pprof -http :9991 http://localhost:3000/debug/pprof/$(metric)?seconds=$(interval)\&timeout=120

.PHONY: build-and-upload
build-and-upload:
    $(eval COMMIT_SHA := $(shell git rev-parse --short HEAD))
    $(eval EXPECTED_MIGRATION_TIMESTAMP := $(expectedMigration))
    @rm -rf .shared/ .db/
    @cp -r ../shared .shared/
    @cp -r ../db .db/
    @docker buildx install # sets up the buildx as default docker builder (otherwise the command below won't work)
    @docker build --platform linux/amd64 --tag "$(GCP_REGION)-docker.pkg.dev/$(GCP_PROJECT_ID)/$(IMAGE)" --push --build-arg COMMIT_SHA="$(COMMIT_SHA)" --build-arg EXPECTED_MIGRATION_TIMESTAMP="$(EXPECTED_MIGRATION_TIMESTAMP)" .
    @rm -rf .shared/
    @rm -rf .db/

.PHONY: test
test:
    go test -v ./...

packages/api 資料夾的make需要進行微調為

ENV := $(shell cat ../../.last_used_env || echo "not-set")
-include ../../.env.${ENV}
IMAGE := e2b-orchestration/api


openapi := ../../spec/openapi.yml
codegen := go tool github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen
expectedMigration := $(shell ./../../scripts/get-latest-migration.sh)

.PHONY: generate
generate:
    $(codegen) -old-config-style -generate gin --package api $(openapi) > internal/api/api.gen.go
    $(codegen) -old-config-style -generate types --package api $(openapi) > internal/api/types.gen.go
    $(codegen) -old-config-style -generate spec --package api $(openapi) > internal/api/spec.gen.go


.PHONY: build
build:
    # Allow for passing commit sha directly for docker builds
    $(eval COMMIT_SHA ?= $(shell git rev-parse --short HEAD))
    $(eval EXPECTED_MIGRATION_TIMESTAMP ?= $(expectedMigration))
    CGO_ENABLED=0 go build -o bin/api -ldflags "-X=main.commitSHA=$(COMMIT_SHA) -X=main.expectedMigrationTimestamp=$(EXPECTED_MIGRATION_TIMESTAMP)" .

.PHONY: build-debug
build-debug:
    $(eval COMMIT_SHA ?= $(shell git rev-parse --short HEAD))
    $(eval EXPECTED_MIGRATION_TIMESTAMP ?= $(expectedMigration))
    CGO_ENABLED=1 go build -race -gcflags=all="-N -l" -o bin/api -ldflags "-X=main.commitSHA=$(COMMIT_SHA) -X=main.expectedMigrationTimestamp=$(EXPECTED_MIGRATION_TIMESTAMP)" .

.PHONY: run
run:
    make build-debug
    POSTGRES_CONNECTION_STRING=$(POSTGRES_CONNECTION_STRING) \
    SUPABASE_JWT_SECRETS=$(SUPABASE_JWT_SECRETS) \
    GOTRACEBACK=crash \
    GODEBUG=madvdontneed=1 \
    TEMPLATE_BUCKET_NAME=$(TEMPLATE_BUCKET_NAME) \
    SANDBOX_ACCESS_TOKEN_HASH_SEED=$(SANDBOX_ACCESS_TOKEN_HASH_SEED) \
    ENVIRONMENT=$(ENVIRONMENT) \
    ORCHESTRATOR_PORT=5008 \
    ./bin/api --port 3009 

# Run the API using air
.PHONY: dev
dev:
    go tool air

# You run the parametrized command like this:
# make metric=heap interval=90 profiler
.PHONY: profiler
profiler:
    go tool pprof -http :9991 http://localhost:3000/debug/pprof/$(metric)?seconds=$(interval)\&timeout=120

.PHONY: build-and-upload
build-and-upload:
    $(eval COMMIT_SHA := $(shell git rev-parse --short HEAD))
    $(eval EXPECTED_MIGRATION_TIMESTAMP := $(expectedMigration))
    @rm -rf .shared/ .db/
    @cp -r ../shared .shared/
    @cp -r ../db .db/
    @docker buildx install # sets up the buildx as default docker builder (otherwise the command below won't work)
    @docker build --platform linux/amd64 --tag "$(GCP_REGION)-docker.pkg.dev/$(GCP_PROJECT_ID)/$(IMAGE)" --push --build-arg COMMIT_SHA="$(COMMIT_SHA)" --build-arg EXPECTED_MIGRATION_TIMESTAMP="$(EXPECTED_MIGRATION_TIMESTAMP)" .
    @rm -rf .shared/
    @rm -rf .db/

.PHONY: test
test:
    go test -v ./...

調用API使用 go run 1.go 計算出JWT
sub是數據庫auth中users表中的 id 欄位

package main

import (
    "encoding/json"
    "fmt"
    "time"

    "github.com/golang-jwt/jwt/v5"
)

// 定義密鑰結構
type JWTSecret struct {
    Type string `json:"type"`
    Key  string `json:"key"`
}

// 從 JSON 解析出一把密鑰
func parseSingleSecret(jsonStr string) (JWTSecret, error) {
    var secret JWTSecret
    err := json.Unmarshal([]byte(jsonStr), &secret)
    return secret, err
}

func main() {
    // 可改成從 os.Getenv("SUPABASE_JWT_SECRETS") 讀取
    supabaseJWTSecret := `{"type":"HS256", "key":"abcdefghijklmnop"}`

    // 解析密鑰
    secret, err := parseSingleSecret(supabaseJWTSecret)
    if err != nil {
        panic("解析失敗: " + err.Error())
    }

    // 用密鑰簽名 JWT
    signingKey := []byte(secret.Key)
    claims := jwt.MapClaims{
        "sub": "c91b0b27-017b-4308-84c0-073a93e1552b",
        "exp": time.Now().Add(time.Hour).Unix(),
        "iat": time.Now().Unix(),
    }
    token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)

    tokenStr, err := token.SignedString(signingKey)
    if err != nil {
        panic("簽名失敗: " + err.Error())
    }

    fmt.Println("🔐 JWT Token:", tokenStr)

    // 驗證
    parsed, err := jwt.Parse(tokenStr, func(t *jwt.Token) (interface{}, error) {
        if _, ok := t.Method.(*jwt.SigningMethodHMAC); !ok {
            return nil, fmt.Errorf("unexpected signing method: %v", t.Header["alg"])
        }
        return signingKey, nil
    })

    if err != nil {
        fmt.Println("❌ 驗證失敗:", err)
    } else if parsed.Valid {
        fmt.Println("✅ 驗證成功，用的 key:", secret.Key)
    } else {
        fmt.Println("❌ Token 無效")
    }
}

並使用 curl發送請求

curl -X POST http://192.168.1.104:3009/templates \
-H "Content-Type: application/json" \
-H "X-Supabase-Token: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3NTA1NTAzNjgsImlhdCI6MTc1MDU0Njc2OCwic3ViIjoiYzkxYjBiMjctMDE3Yi00MzA4LTg0YzAtMDczYTkzZTE1NTJiIn0.AD5aXE5eMV_eYGC0JwefsI0EeKMIZ7SVVbddMdM_TFM" \
-H "X-Supabase-Team: 7b878a9d-3dde-48a1-8913-31888d69dd78" \
-H "Authorization: Bearer sk_e2b_e12df59cc6206a3923a49a477a5db34b9685a98a" \
-d '{
    "dockerfile": "Dockerfile",
    "ReadyCmd":"ReadyCmd",
    "StartCmd":"/bin/true",
    "envd_version":"0.1.1"
}'

會獲取一個編號

{"aliases":null,"buildCount":0,
"buildID":"fdd3ae7b-1be9-4795-ac25-bd74cdf37951",
"cpuCount":0,"createdAt":"0001-01-01T00:00:00Z",
"createdBy":null,"lastSpawnedAt":"0001-01-01T00:00:00Z",
"memoryMB":0,"public":false,"spawnCount":0,
"templateID":"7s6n4ysowcq4jexcron3",
"updatedAt":"0001-01-01T00:00:00Z"}

伺服器需要先尋找一個docker資料夾進行

docker build -t di502spv70vn40ke7ysc:7df37656-7ddc-47f2-8979-21c40b285a4e .

客戶端才可以對伺服器進行第二個步驟

curl -X POST http://192.168.1.104:3009/templates/di502spv70vn40ke7ysc/builds/7df37656-7ddc-47f2-8979-21c40b285a4e \
-H "Content-Type: application/json" \
-H "X-Supabase-Token: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3NTA1NTAzNjgsImlhdCI6MTc1MDU0Njc2OCwic3ViIjoiYzkxYjBiMjctMDE3Yi00MzA4LTg0YzAtMDczYTkzZTE1NTJiIn0.AD5aXE5eMV_eYGC0JwefsI0EeKMIZ7SVVbddMdM_TFM" \
-H "X-Supabase-Team: 7b878a9d-3dde-48a1-8913-31888d69dd78" \
-H "Authorization: Bearer sk_e2b_e12df59cc6206a3923a49a477a5db34b9685a98a"

然後拉取了docker

步驟	說明
`Docker image size: 34 MB`	確認 image 體積
`Setting up system files`	開始準備 rootfs（root filesystem）
`executed provision script env`	執行初始化腳本，例如設定環境變數或掛載目錄
`set up filesystem`	建立 sandbox 或 microVM 的 filesystem
`Creating file system and pulling Docker image`	將 Docker image 寫入 rootfs（ext4）格式中
`closed rootfs file` → `created rootfs ext4 file`	把 rootfs 完整寫入磁碟映像檔，準備啟動
`tune2fs`, `resize2fs`	對 ext4 檔案系統進行調整，調整大小

下一步

| 步驟 | 命令                                                                                 | 說明                                       |
| -- | ---------------------------------------------------------------------------------- | ---------------------------------------- |
| 1  | `mount --make-rprivate /`                                                          | 隔離 mount namespace（避免 host 被影響）          |
| 2  | `mount -t tmpfs tmpfs ...`                                                         | 建立 tmpfs 掛載目錄，用於 sandbox 環境              |
| 3  | `ln -s /orchestrator/sandbox/rootfs-...link .../rootfs.ext4`                       | 建立 symlink，把 rootfs 指到對應檔案               |
| 4  | `ln -s /fc-kernels/vmlinux-6.1.102/vmlinux.bin /fc-vm/vmlinux-6.1.102/vmlinux.bin` | 建立 symlink 指向 kernel binary              |
| 5  | `ip netns exec ns-316 ... firecracker ...`                                         | 在特定 netns 中啟動 firecracker 並綁定 API socket |

然後發生問題

{"level":"info","timestamp":"2025-06-19T14:01:08.141766701Z","message":"2025-06-19T14:01:08.141732515 [anonymous-instance:fc_api] The API server received a Put request on \"/actions\" with body \"

{\\\"action_type\\\":\\\"InstanceStart\\\"}\\n\".","service":"template-manager","internal":true,"pid":179198,"logger":"template-manager","sandbox.id":"bb5rxi67qkgdxqmprh7xi","template.id":"7s6n4ysowcq4jexcron3","team.id":"","instanceID":"bb5rxi67qkgdxqmprh7xi","envID":"7s6n4ysowcq4jexcron3"}

{"level":"info","timestamp":"2025-06-19T14:01:08.147679108Z","message":"2025-06-19T14:01:08.147455261 [anonymous-instance:fc_api] Received Error. Status code: 400 Bad Request. Message: Start microvm error: Cannot load kernel due to invalid memory configuration or invalid kernel image: Kernel Loader: failed to load ELF kernel image: Kernel Loader: Unable to read kernel image","service":"template-manager","internal":true,"pid":179198,"logger":"template-manager","sandbox.id":"bb5rxi67qkgdxqmprh7xi","template.id":"7s6n4ysowcq4jexcron3","team.id":"","instanceID":"bb5rxi67qkgdxqmprh7xi","envID":"7s6n4ysowcq4jexcron3"}

終於發現問題所在

                    [ Nomad Server ]
                        |
        +---------------+---------------+
        |                               |
    [Nomad Client Node A]         [Nomad Client Node B]
        |                               |
    [Docker: API]                    [Docker: DB]
    [Docker: Worker]                [Docker: Proxy]

本質特點如下：

主控（Control Plane）+ 多台工作節點（Worker Nodes）
主控節點負責：
Job 排程
健康檢查
負載平衡
監控、指令發送
工作節點只跑：容器 + 監控 agent（例如 Nomad Client）
部署標準化： 開發者只需提供：
Dockerfile
Nomad job（或 Helm chart、Compose 等）
就能被部署到任意主機、任意區域
資源管理：
CPU/Memory 配額
network/port 分配
logs、metrics、volume 掛載
高度抽象：
Job 不管在哪台主機上跑
所有節點像是一個大主機池
管理員通過 Nomad / Kubernetes / Swarm 下發 job

本項目把我的docker弄得很亂

docker stop $(docker ps -aq)
docker rmi $(docker images -q)
docker images
docker rmi -f $(docker images -aq)

測試完畢清理介面卡

ip -o -4 addr show | grep '10\.12' | awk '{print $2}' | grep '^veth' | sort -u | while read iface; do
echo "🧹 刪除介面 $iface"
ip link delete "$iface"
done