diff --git a/clusters/tinycloud/platform/ai/inference/namespace.yaml b/clusters/tinycloud/platform/ai/inference/namespace.yaml deleted file mode 100644 index 52ab1d5..0000000 --- a/clusters/tinycloud/platform/ai/inference/namespace.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: vllm diff --git a/clusters/tinycloud/platform/ai/inference/reference-grant.yaml b/clusters/tinycloud/platform/ai/inference/reference-grant.yaml deleted file mode 100644 index a99806f..0000000 --- a/clusters/tinycloud/platform/ai/inference/reference-grant.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: gateway.networking.k8s.io/v1beta1 -kind: ReferenceGrant -metadata: - name: vllm-from-semantic-router - namespace: vllm -spec: - from: - - group: gateway.networking.k8s.io - kind: HTTPRoute - namespace: semantic-router - to: - - group: "" - kind: Service - name: vllm-qwen-instruct-engine-service diff --git a/clusters/tinycloud/platform/ai/kustomization.yaml b/clusters/tinycloud/platform/ai/kustomization.yaml deleted file mode 100644 index fccab96..0000000 --- a/clusters/tinycloud/platform/ai/kustomization.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - inference - - router -labels: - - pairs: - app.kubernetes.io/part-of: ai-platform - includeSelectors: false diff --git a/clusters/tinycloud/platform/ai/router/extension-policy.yaml b/clusters/tinycloud/platform/ai/router/extension-policy.yaml deleted file mode 100644 index 597c922..0000000 --- a/clusters/tinycloud/platform/ai/router/extension-policy.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: gateway.envoyproxy.io/v1alpha1 -kind: EnvoyExtensionPolicy -metadata: - name: semantic-router - namespace: semantic-router -spec: - targetRefs: - - group: gateway.networking.k8s.io - kind: HTTPRoute - name: semantic-router-api - extProc: - - backendRefs: - - group: "" - kind: Service - name: semantic-router - port: 50051 - failOpen: false - messageTimeout: 60s - processingMode: - allowModeOverride: true - request: - body: Buffered - response: - body: Buffered diff --git a/clusters/tinycloud/platform/ai/router/helm-release.yaml b/clusters/tinycloud/platform/ai/router/helm-release.yaml deleted file mode 100644 index 87bfd68..0000000 --- a/clusters/tinycloud/platform/ai/router/helm-release.yaml +++ /dev/null @@ -1,77 +0,0 @@ -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: semantic-router - namespace: semantic-router -spec: - interval: 30m - timeout: 15m - chartRef: - kind: OCIRepository - name: semantic-router - namespace: flux-system - dependsOn: - - name: vllm - namespace: vllm - install: - remediation: - retries: 3 - upgrade: - remediation: - retries: 3 - values: - persistence: - storageClassName: local-path - securityContext: - allowPrivilegeEscalation: false - image: - tag: v0.3.0 - pullPolicy: IfNotPresent - config: - listeners: - - name: http-8080 - address: 0.0.0.0 - port: 8080 - timeout: 300s - providers: - defaults: - default_model: qwen3-4b - models: - - name: qwen3-4b - provider_model_id: qwen3-4b - api_format: openai - backend_refs: - - name: primary - base_url: http://vllm-qwen-instruct-engine-service.vllm.svc.cluster.local:80/v1 - protocol: http - provider: openai - weight: 100 - routing: - modelCards: - - name: qwen3-4b - description: Local Qwen3 4B model served by vLLM. - capabilities: - - chat - - text - modality: text - decisions: - - name: qwen3-4b-default - description: Route automatic requests to the local Qwen3 4B backend. - priority: 1 - rules: - operator: AND - conditions: [] - modelRefs: - - model: qwen3-4b - use_reasoning: false - global: - router: - auto_model_name: auto - model_catalog: - embeddings: - semantic: - use_cpu: true - services: - response_api: - enabled: false - store_backend: memory diff --git a/clusters/tinycloud/platform/ai/router/kustomization.yaml b/clusters/tinycloud/platform/ai/router/kustomization.yaml deleted file mode 100644 index 3ed4d2a..0000000 --- a/clusters/tinycloud/platform/ai/router/kustomization.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - namespace.yaml - - oci-repository.yaml - - helm-release.yaml - - http-route.yaml - - extension-policy.yaml diff --git a/clusters/tinycloud/platform/ai/router/monitoring.yaml b/clusters/tinycloud/platform/ai/router/monitoring.yaml deleted file mode 100644 index 91108ee..0000000 --- a/clusters/tinycloud/platform/ai/router/monitoring.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: semantic-router - namespace: semantic-router - labels: - platform.tinycloud.dev/monitoring: prometheus -spec: - namespaceSelector: - matchNames: - - semantic-router - selector: - matchLabels: - app.kubernetes.io/name: semantic-router - app.kubernetes.io/instance: semantic-router - endpoints: - - port: metrics - path: /metrics - interval: 30s diff --git a/clusters/tinycloud/platform/ai/router/oci-repository.yaml b/clusters/tinycloud/platform/ai/router/oci-repository.yaml deleted file mode 100644 index f3ea60d..0000000 --- a/clusters/tinycloud/platform/ai/router/oci-repository.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: source.toolkit.fluxcd.io/v1 -kind: OCIRepository -metadata: - name: semantic-router - namespace: flux-system -spec: - interval: 1h - url: oci://ghcr.io/vllm-project/charts/semantic-router - ref: - tag: "0.3.0" - layerSelector: - mediaType: application/vnd.cncf.helm.chart.content.v1.tar+gzip - operation: copy diff --git a/clusters/tinycloud/platform/ai/inference/helm-release.yaml b/clusters/tinycloud/platform/inference/helm-release.yaml similarity index 82% rename from clusters/tinycloud/platform/ai/inference/helm-release.yaml rename to clusters/tinycloud/platform/inference/helm-release.yaml index 9509c79..99a1bc3 100644 --- a/clusters/tinycloud/platform/ai/inference/helm-release.yaml +++ b/clusters/tinycloud/platform/inference/helm-release.yaml @@ -27,32 +27,34 @@ spec: routerSpec: enableRouter: false servingEngineSpec: + strategy: + type: Recreate serviceMonitor: enabled: true additionalLabels: platform.tinycloud.dev/monitoring: prometheus modelSpec: - - name: qwen-instruct + - name: qwen3-8b repository: vllm/vllm-openai tag: v0.29.0 - modelURL: Qwen/Qwen3-4B-AWQ + modelURL: Qwen/Qwen3-8B-AWQ requestCPU: 2 - requestMemory: 4Gi + requestMemory: 6Gi requestGPU: 1 requestGPUType: nvidia.com/gpu limitCPU: "4" - limitMemory: 8Gi + limitMemory: 10Gi storageClass: local-path pvcStorage: 10Gi vllmConfig: tensorParallelSize: 1 maxModelLen: 8192 - dtype: float16 + dtype: bfloat16 quantization: awq - gpuMemoryUtilization: 0.8 + gpuMemoryUtilization: 0.9 extraArgs: - --served-model-name - - qwen3-4b + - qwen3-8b - --enable-auto-tool-choice - --tool-call-parser - hermes diff --git a/clusters/tinycloud/platform/ai/inference/helm-repository.yaml b/clusters/tinycloud/platform/inference/helm-repository.yaml similarity index 100% rename from clusters/tinycloud/platform/ai/inference/helm-repository.yaml rename to clusters/tinycloud/platform/inference/helm-repository.yaml diff --git a/clusters/tinycloud/platform/ai/router/http-route.yaml b/clusters/tinycloud/platform/inference/http-route.yaml similarity index 78% rename from clusters/tinycloud/platform/ai/router/http-route.yaml rename to clusters/tinycloud/platform/inference/http-route.yaml index b0459d4..542a757 100644 --- a/clusters/tinycloud/platform/ai/router/http-route.yaml +++ b/clusters/tinycloud/platform/inference/http-route.yaml @@ -1,8 +1,8 @@ apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: - name: semantic-router-api - namespace: semantic-router + name: vllm-openai + namespace: vllm spec: parentRefs: - group: gateway.networking.k8s.io @@ -22,6 +22,5 @@ spec: backendRefs: - group: "" kind: Service - name: vllm-qwen-instruct-engine-service - namespace: vllm + name: vllm-qwen3-8b-engine-service port: 80 diff --git a/clusters/tinycloud/platform/ai/inference/kustomization.yaml b/clusters/tinycloud/platform/inference/kustomization.yaml similarity index 56% rename from clusters/tinycloud/platform/ai/inference/kustomization.yaml rename to clusters/tinycloud/platform/inference/kustomization.yaml index 32ad03e..f431ecf 100644 --- a/clusters/tinycloud/platform/ai/inference/kustomization.yaml +++ b/clusters/tinycloud/platform/inference/kustomization.yaml @@ -4,4 +4,8 @@ resources: - namespace.yaml - helm-repository.yaml - helm-release.yaml - - reference-grant.yaml + - http-route.yaml +labels: + - pairs: + app.kubernetes.io/part-of: inference + includeSelectors: false diff --git a/clusters/tinycloud/platform/ai/router/namespace.yaml b/clusters/tinycloud/platform/inference/namespace.yaml similarity index 80% rename from clusters/tinycloud/platform/ai/router/namespace.yaml rename to clusters/tinycloud/platform/inference/namespace.yaml index b62535c..6b480c8 100644 --- a/clusters/tinycloud/platform/ai/router/namespace.yaml +++ b/clusters/tinycloud/platform/inference/namespace.yaml @@ -1,6 +1,6 @@ apiVersion: v1 kind: Namespace metadata: - name: semantic-router + name: vllm labels: platform.tinycloud.dev/gateway-access: envoy diff --git a/clusters/tinycloud/platform/kustomization.yaml b/clusters/tinycloud/platform/kustomization.yaml index 9158271..488e36e 100644 --- a/clusters/tinycloud/platform/kustomization.yaml +++ b/clusters/tinycloud/platform/kustomization.yaml @@ -1,7 +1,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - ai + - inference - gateway - observability labels: