Simulate scheduler with debuggable scheduler container (#370)

* move simulator entrypoint to simulator/cmd/simulator

* fix config position

* fix Dockerfile

* can restart pod from server

* fix docker-compose

* tweak

* tweak

* update extender.md

* delete StartScheduler

* tweak

* delete unused func

* add --master option

* rename WriteConfig to UpdateSchedulerConfig, and write func description RestartScheduler

* tweak errors

* refactor restartContainer

* delete unnecessary file copy

* scheduler config use default config

* extender example

* tweak

* tweak

* add comment to init-container

* add error log for restartContainer

* tweak

* fix extender document

* %w -> %v

* fix can't find simulator-scheduler case

* add comment

* return error when failed init

* update how-it-works.md

* change log text

* bug fix

Author: YamasouA

Makefile

@@ -74,4 +74,4 @@ docker_down:
 
 .PHONY: docker_down_local
 docker_down_local:
-	docker compose -f compose.yml -f compose.local.yml down
+	docker compose -f compose.yml -f compose.local.yml down --volumes

compose.local.yml

@@ -1,12 +1,25 @@
+# Note for maintainers: When making a change in this file,
+# you're likely to make the same change in simulator/docs/sample/plugin-extender/docker-compose.yaml.
+version: "3.7"
 services:
+  # This container copies the data,
+  # so any changes made to the configuration files within the Pod will not affect the original files.
+  init-container:
+      image: busybox
+      volumes:
+        - conf:/config
+        - ${PWD}/simulator/cmd/scheduler:/host-config:ro    
+      command: sh -c "cp -rf /host-config/* /config/"
   simulator-scheduler:
     image: simulator-scheduler
     container_name: simulator-scheduler
     environment:
-      - KUBE_APISERVER_URL=http://simulator-cluster:3131
-      - KUBECONFIG=/root/.kube/kubeconfig.yaml
+      - KUBECONFIG=/config/kubeconfig.yaml
     volumes:
-      - ./simulator/docs/sample/debuggable-scheduler/kubeconfig.yaml:/root/.kube/kubeconfig.yaml
+      - conf:/config
+    depends_on:
+      - init-container
+      - simulator-cluster
     restart: always
     tty: true
     networks:
@@ -15,6 +28,14 @@ services:
     image: simulator-server
     volumes:
       - ./simulator/kubeconfig.yaml:/kubeconfig.yaml
+      - /var/run/docker.sock:/var/run/docker.sock
+      - conf:/config
+    ports:
+      - "1212:1212"
+    restart: always
+    tty: true
+    networks:
+      - simulator-internal-network
     depends_on:
       fake-source-cluster:
         condition: "service_healthy"
@@ -39,3 +60,9 @@ services:
       - simulator-internal-network
     profiles:
       - externalImportEnabled
+networks:
+  simulator-internal-network:
+    driver: bridge
+volumes:
+  simulator-etcd-data:
+  conf:

simulator/cmd/scheduler/Dockerfile

@@ -24,4 +24,4 @@ FROM alpine:3.14.0
 COPY --from=build-env /go/src/simulator/bin/scheduler /scheduler
 RUN chmod a+x /scheduler
 
-CMD ["/scheduler"]
+CMD ["/scheduler", "--config", "/config/scheduler.yaml", "--master", "http://simulator-cluster:3131"]

simulator/cmd/scheduler/kubeconfig.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Config
+
+clusters:
+  - cluster:
+      server: http://simulator-cluster:3131
+    name: simulator
+
+contexts:
+  - context:
+      cluster: simulator
+    name: simulator
+
+current-context: simulator

simulator/cmd/scheduler/scheduler.yaml

@@ -0,0 +1,2 @@
+kind: KubeSchedulerConfiguration
+apiVersion: kubescheduler.config.k8s.io/v1

simulator/cmd/simulator/simulator.go

@@ -98,12 +98,6 @@ func startSimulator() error {
 	if err != nil {
 		return xerrors.Errorf("create di container: %w", err)
 	}
-	if !cfg.ExternalSchedulerEnabled {
-		if err := dic.SchedulerService().StartScheduler(cfg.InitialSchedulerCfg); err != nil {
-			return xerrors.Errorf("start scheduler: %w", err)
-		}
-		defer dic.SchedulerService().ShutdownScheduler()
-	}
 
 	// If ExternalImportEnabled is enabled, the simulator import resources
 	// from the target cluster that indicated by the `KUBECONFIG`.
@@ -116,6 +110,10 @@ func startSimulator() error {
 		}
 	}
 
+	if !cfg.ExternalSchedulerEnabled {
+		dic.SchedulerService().SetSchedulerConfig(cfg.InitialSchedulerCfg)
+	}
+
 	if cfg.ResourceSyncEnabled {
 		// Start the resource syncer to sync resources from the target cluster.
 		if err = dic.ResourceSyncer().Run(ctx); err != nil {

simulator/config/config.go

@@ -25,6 +25,9 @@ var ErrEmptyConfig = errors.New("config is required, but empty")
 // configYaml represents the value from the config file.
 var configYaml = &v1alpha1.SimulatorConfiguration{}
 
+// defaultSchedulerCfgPath is where we have the scheduler config in the container by default.
+const defaultSchedulerCfgPath = "/config/scheduler.yaml"
+
 // Config is configuration for simulator.
 type Config struct {
 	Port                  int
@@ -91,7 +94,7 @@ func NewConfig() (*Config, error) {
 		}
 	}
 
-	initialschedulerCfg, err := getSchedulerCfg()
+	initialschedulerCfg, err := GetSchedulerCfg()
 	if err != nil {
 		return nil, xerrors.Errorf("get SchedulerCfg: %w", err)
 	}
@@ -225,23 +228,25 @@ func parseStringListEnv(e string) []string {
 	return list
 }
 
-// getSchedulerCfg reads KUBE_SCHEDULER_CONFIG_PATH which means initial kube-scheduler configuration
+// GetSchedulerCfg reads KUBE_SCHEDULER_CONFIG_PATH which means initial kube-scheduler configuration
 // if empty from the config file.
 // and converts it into *configv1.KubeSchedulerConfiguration.
 // KUBE_SCHEDULER_CONFIG_PATH is not required.
 // If KUBE_SCHEDULER_CONFIG_PATH is not set, the default configuration of kube-scheduler will be used.
-func getSchedulerCfg() (*configv1.KubeSchedulerConfiguration, error) {
+func GetSchedulerCfg() (*configv1.KubeSchedulerConfiguration, error) {
 	kubeSchedulerConfigPath := os.Getenv("KUBE_SCHEDULER_CONFIG_PATH")
 	if kubeSchedulerConfigPath == "" {
 		kubeSchedulerConfigPath = configYaml.KubeSchedulerConfigPath
 		if kubeSchedulerConfigPath == "" {
+			config.SetKubeSchedulerCfgPath(defaultSchedulerCfgPath)
 			dsc, err := config.DefaultSchedulerConfig()
 			if err != nil {
 				return nil, xerrors.Errorf("create default scheduler config: %w", err)
 			}
 			return dsc, nil
 		}
 	}
+	config.SetKubeSchedulerCfgPath(kubeSchedulerConfigPath)
 	data, err := os.ReadFile(kubeSchedulerConfigPath)
 	if err != nil {
 		return nil, xerrors.Errorf("read scheduler config file: %w", err)

simulator/docs/extender.md

@@ -11,14 +11,12 @@ The simulator stores the results of each Extender in the annotation of a pod.
 Note: This feature is not available in [external scheduler](./external-scheduler.md).
 
 ## How to use
+In this example, we describe how you can run an extender with the simulator, using [k8s-scheduler-extender-example](https://github.com/everpeace/k8s-scheduler-extender-example).
 
-You need to configure your extender in KubeSchedulerConfig.
-(via [the simulator config](./simulator-server-config.md) or WebUI)
++ Create k8s-scheduler-extender-example's Image: Clone [k8s-scheduler-extender-example](https://github.com/everpeace/k8s-scheduler-extender-example) repository, and follow the step `1 build a docker image` on README.
 
-(No required special configuration is for the simulator to use this feature.)
-
-For example, if you run the server on `http://localhost:8080/scheduler/`,
-the configuration will look like this.
++ Set up your extender in KubeSchedulerConfiguration either through [`kubeSchedulerConfigPath`](./simulator-server-config.md) or the Web UI.
+For example, if you are running the server on http://kube-scheduler-simulator-extender-1:80/scheduler/, your configuration might look like the following:
 
 ```yaml
 apiVersion: kubescheduler.config.k8s.io/v1
@@ -28,7 +26,7 @@ leaderElection:
 profiles:
   - schedulerName: default-scheduler
 extenders:
-  - urlPrefix: "http://localhost:8080/scheduler/"
+  - urlPrefix: "http://kube-scheduler-simulator-extender-1:80/scheduler"
     filterVerb: "predicates/always_true"
     prioritizeVerb: "priorities/zero_score"
     preemptVerb: "preemption"
@@ -38,26 +36,31 @@ extenders:
     nodeCacheCapable: false
 ```
 
-After the above settings are made, when the simulator is started and the pod is scheduled, 
-you will see each Pod gets many results on the annotation like this:
++ Run Simulator:
+We have an example [`docker-compose.yaml`](./example/docker-compose.yaml); you can overwrite the [`docker-compose-local.yaml`](../../docker-compose-local.yml) file with this file, but make sure to update the extender's image name there.
+
+To run the simulator, use the following commands:
+```sh
+$ make docker_build docker_up_local
+```
+
++ Create a Pod and examine your Extender's Results:
+The simulator started with the above steps should have your extender(s) enabled. You can create Pod(s) in the simulator and see the result. 
+The result shows up in the Pod's annotations `scheduler-simulator/extender-xxx` like the following:
 
 ```yaml
 kind: Pod
 apiVersion: v1
 metadata:
   name: pod-2rsvz
 ...
-  annotations:
-    scheduler-simulator/bind-result: '{"DefaultBinder":"success"}'
-    scheduler-simulator/extender-bind-result: '{}'
-    scheduler-simulator/extender-filter-result: >-
-      {"http://localhost:8080/scheduler/":{"Nodes":{"metadata":{},"items":[{"metadata":{"name":"node-sc9ns","generateName":"node-","uid":"4b008c90-971e-4816-a0f4-dc1a3b6e856e","resourceVersion":"208","creationTimestamp":"2023-03-03T16:03:50Z","managedFields":[{"manager":"simulator","operation":"Update","apiVersion":"v1","time":"2023-03-03T16:03:50Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:generateName":{}}}}]},"spec":{},"status":{"capacity":{"cpu":"4","memory":"32Gi","pods":"110"},"allocatable":{"cpu":"4","memory":"32Gi","pods":"110"},"phase":"Running","conditions":[{"type":"Ready","status":"True","lastHeartbeatTime":null,"lastTransitionTime":null}],"daemonEndpoints":{"kubeletEndpoint":{"Port":0}},"nodeInfo":{"machineID":"","systemUUID":"","bootID":"","kernelVersion":"","osImage":"","containerRuntimeVersion":"","kubeletVersion":"","kubeProxyVersion":"","operatingSystem":"","architecture":""}}},{"metadata":{"name":"node-pwzdq","generateName":"node-","uid":"b24f918d-94ae-4c35-9e2c-2376998dbede","resourceVersion":"209","creationTimestamp":"2023-03-03T16:03:53Z","managedFields":[{"manager":"simulator","operation":"Update","apiVersion":"v1","time":"2023-03-03T16:03:53Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:generateName":{}}}}]},"spec":{},"status":{"capacity":{"cpu":"4","memory":"32Gi","pods":"110"},"allocatable":{"cpu":"4","memory":"32Gi","pods":"110"},"phase":"Running","conditions":[{"type":"Ready","status":"True","lastHeartbeatTime":null,"lastTransitionTime":null}],"daemonEndpoints":{"kubeletEndpoint":{"Port":0}},"nodeInfo":{"machineID":"","systemUUID":"","bootID":"","kernelVersion":"","osImage":"","containerRuntimeVersion":"","kubeletVersion":"","kubeProxyVersion":"","operatingSystem":"","architecture":""}}}]},"NodeNames":null,"FailedNodes":{},"FailedAndUnresolvableNodes":null,"Error":""}}
-    scheduler-simulator/extender-preempt-result: '{}'
-    scheduler-simulator/extender-prioritize-result: >-
-      {"http://localhost:8080/scheduler/":[{"Host":"node-sc9ns","Score":0},{"Host":"node-pwzdq","Score":0}]}
-    scheduler-simulator/score-result: >-
-      {"node-282x7":{"ImageLocality":"0","InterPodAffinity":"0","NodeAffinity":"0","NodeNumber":"0","NodeResourcesBalancedAllocation":"52","NodeResourcesFit":"47","PodTopologySpread":"0","TaintToleration":"0","VolumeBinding":"0"},"node-gp9t4":{"ImageLocality":"0","InterPodAffinity":"0","NodeAffinity":"0","NodeNumber":"0","NodeResourcesBalancedAllocation":"76","NodeResourcesFit":"73","PodTopologySpread":"0","TaintToleration":"0","VolumeBinding":"0"}}
-...
+annotations:
+      scheduler-simulator/extender-bind-result: '{}'
+      scheduler-simulator/extender-filter-result: '{"http://kube-scheduler-simulator-extender-1:80/scheduler":{"Nodes":{"metadata":{},"items":[{"metadata":{"name":"node-tzjll","generateName":"node-","uid":"a3e39211-2200-4dee-99a8-a27b2ac528b3","resourceVersion":"223","creationTimestamp":"2024-09-25T12:24:50Z","annotations":{"node.alpha.kubernetes.io/ttl":"0"},"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2024-09-25T12:24:50Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:node.alpha.kubernetes.io/ttl":{}}}}},{"manager":"simulator","operation":"Update","apiVersion":"v1","time":"2024-09-25T12:24:50Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:generateName":{}}}}]},"spec":{},"status":{"capacity":{"cpu":"4","memory":"32Gi","pods":"110"},"allocatable":{"cpu":"4","memory":"32Gi","pods":"110"},"phase":"Running","conditions":[{"type":"Ready","status":"True","lastHeartbeatTime":null,"lastTransitionTime":null}],"daemonEndpoints":{"kubeletEndpoint":{"Port":0}},"nodeInfo":{"machineID":"","systemUUID":"","bootID":"","kernelVersion":"","osImage":"","containerRuntimeVersion":"","kubeletVersion":"","kubeProxyVersion":"","operatingSystem":"","architecture":""}}}]},"NodeNames":null,"FailedNodes":{},"FailedAndUnresolvableNodes":null,"Error":""}}'
+      scheduler-simulator/extender-preempt-result: '{}'
+      scheduler-simulator/extender-prioritize-result: '{}'
+      ....
 ```
 
+You can also view the annotation results from the web UI. Simply select the Pod you created and scheduled, then check the Resource Definition section to see the annotations.
 

simulator/docs/how-it-works.md

@@ -4,11 +4,9 @@ This page describes how the simulator works.
 
 ### 0. starts the simulator.
 
-The simulator server works with the following:
-- scheduler
-- [HTTP server](api.md)
+The simulator server works with the [HTTP server](api.md).
 
-In advance, the simulator needs to launch etcd, controller-manager and kube-apiserver outside.
+In advance, the simulator needs to launch debuggable-scheduler, etcd, controller-manager and kube-apiserver outside.
 We recommend using [KWOK](https://github.com/kubernetes-sigs/kwok), see [compose.yml](../../compose.yml) to know how we wire things up.
 
 ### 1. users request creating resource.

simulator/docs/sample/plugin-extender/docker-compose.yaml

@@ -0,0 +1,97 @@
+version: "3.7"
+services:
+  extender:
+    image: test/test:test   # TODO(user): replace it with your extender's image
+    ports:
+      - "80:80"
+    networks:
+      - simulator-internal-network
+  init-container:
+      image: busybox
+      volumes:
+        - conf:/config
+        - ${PWD}/simulator/cmd/scheduler:/host-config:ro    
+      command: sh -c "cp -rf /host-config/* /config/"
+  simulator-scheduler:
+    image: simulator-scheduler
+    container_name: simulator-scheduler
+    environment:
+      - KUBECONFIG=/config/kubeconfig.yaml
+    volumes:
+      - conf:/config
+    depends_on:
+      - init-container
+      - simulator-cluster
+    restart: always
+    tty: true
+    networks:
+      - simulator-internal-network
+  simulator-server:
+    image: simulator-server
+    container_name: simulator-server
+    environment:
+      - PORT=1212
+      - KUBE_SCHEDULER_SIMULATOR_ETCD_URL=http://simulator-cluster:2379
+      - KUBE_APISERVER_URL=http://simulator-cluster:3131
+    volumes:
+      - ./simulator/config.yaml:/config.yaml
+      - ./simulator/kubeconfig.yaml:/kubeconfig.yaml
+      - /var/run/docker.sock:/var/run/docker.sock
+      - conf:/config
+    ports:
+      - "1212:1212"
+    restart: always
+    tty: true
+    networks:
+      - simulator-internal-network
+    depends_on:
+      fake-source-cluster:
+        condition: "service_healthy"
+        required: false
+  simulator-frontend:
+    image: simulator-frontend
+    restart: always
+    container_name: simulator-frontend
+    environment:
+      - HOST=0.0.0.0
+      - BASE_URL=http://${SIMULATOR_EXTERNAL_IP:-localhost}:1212
+      - KUBE_API_SERVER_URL=http://${SIMULATOR_EXTERNAL_IP:-localhost}:3131
+    ports:
+    - "3000:3000"
+    tty: true
+  simulator-cluster:
+    image: registry.k8s.io/kwok/cluster:v0.6.0-k8s.v1.30.2
+    container_name: simulator-cluster
+    restart: always
+    ports:
+      - "3131:3131"
+    volumes:
+      - simulator-etcd-data:/var/lib/etcd
+      - ./kwok.yaml:/root/.kwok/kwok.yaml
+    environment:
+      - KWOK_KUBE_APISERVER_PORT=3131
+    networks:
+      - simulator-internal-network
+  fake-source-cluster:
+    image: registry.k8s.io/kwok/cluster:v0.6.0-k8s.v1.30.2
+    container_name: fake-source-cluster
+    restart: always
+    healthcheck:
+      test: "kwokctl kubectl get nodes"
+      start_period: 60s
+      start_interval: 1s
+      interval: 600s
+    ports:
+      - "3132:3132"
+    environment:
+      - KWOK_KUBE_APISERVER_PORT=3132
+    networks:
+      - simulator-internal-network
+    profiles:
+      - externalImportEnabled
+networks:
+  simulator-internal-network:
+    driver: bridge
+volumes:
+  simulator-etcd-data:
+  conf: