Update guidellm

examples/guidellm_example.py

@@ -9,11 +9,12 @@
     GUIDELLM__MAX_CONCURRENCY=256,
     GUIDELLM__REQUEST_TIMEOUT=21600,
     target="http://localhost:8000/v1",
-    data_type="emulated",
     max_seconds=30,
-    data="prompt_tokens=512,generated_tokens=256",
+    #scenario = "benchmarking_32k",
+    data="prompt_tokens=128,output_tokens=128",
+    branch = "update_guidellm",
     vllm_kwargs={"enable-chunked-prefill": True}
 )
 
 task.execute_remotely("oneshot-a100x1")
-#task.execute_locally()
+#task.execute_locally()

examples/lmeval_example.py

@@ -6,8 +6,8 @@
     model_id="meta-llama/Llama-3.2-1B-Instruct",
     tasks="gsm8k",
     model_args="dtype=auto,max_model_len=8192",
-    batch_size="auto",    
+    batch_size="auto",
 )
 
 task.execute_remotely("oneshot-a100x1")
-#task.execute_locally()
+#task.execute_locally()

src/automation/configs.py

@@ -1,2 +1,4 @@
-DEFAULT_DOCKER_IMAGE = "498127099666.dkr.ecr.us-east-1.amazonaws.com/mlops/k8s-research-cuda12_5:latest"
-DEFAULT_OUTPUT_URI = "gs://neuralmagic-clearml"
+DEFAULT_DOCKER_IMAGE = "498127099666.dkr.ecr.us-east-1.amazonaws.com/mlops/k8s-research-cuda12_8:latest"
+DEFAULT_OUTPUT_URI = "gs://neuralmagic-clearml"
+DEFAULT_RESEARCH_BRANCH = "main"
+DEFAULT_GUIDELLM_SCENARIO = "chat"

src/automation/standards/benchmarking/benchmarking_128k.json

@@ -0,0 +1,13 @@
+{
+    "rate_type": "sweep",
+    "data": {
+        "prompt_tokens": 128000,
+        "prompt_tokens_stdev": 128,
+        "prompt_tokens_min": 1,
+        "prompt_tokens_max": 128000,
+        "output_tokens": 2048,
+        "output_tokens_stdev": 64,
+        "output_tokens_min": 1,
+        "output_tokens_max": 2048
+    }
+}

src/automation/standards/benchmarking/benchmarking_16k.json

@@ -0,0 +1,13 @@
+{
+    "rate_type": "sweep",
+    "data": {
+        "prompt_tokens": 16000,
+        "prompt_tokens_stdev": 128,
+        "prompt_tokens_min": 1,
+        "prompt_tokens_max": 16000,
+        "output_tokens": 2048,
+        "output_tokens_stdev": 64,
+        "output_tokens_min": 1,
+        "output_tokens_max": 2048
+    }
+}

src/automation/standards/benchmarking/benchmarking_32k.json

@@ -0,0 +1,13 @@
+{
+    "rate_type": "sweep",
+    "data": {
+        "prompt_tokens": 32000,
+        "prompt_tokens_stdev": 128,
+        "prompt_tokens_min": 1,
+        "prompt_tokens_max": 32000,
+        "output_tokens": 2048,
+        "output_tokens_stdev": 64,
+        "output_tokens_min": 1,
+        "output_tokens_max": 2048
+    }
+}

src/automation/standards/benchmarking/benchmarking_64k.json

@@ -0,0 +1,13 @@
+{
+    "rate_type": "sweep",
+    "data": {
+        "prompt_tokens": 64000,
+        "prompt_tokens_stdev": 128,
+        "prompt_tokens_min": 1,
+        "prompt_tokens_max": 64000,
+        "output_tokens": 2048,
+        "output_tokens_stdev": 64,
+        "output_tokens_min": 1,
+        "output_tokens_max": 2048
+    }
+}

src/automation/standards/benchmarking/benchmarking_chat.json

@@ -0,0 +1,13 @@
+{
+    "rate_type": "sweep",
+    "data": {
+        "prompt_tokens": 512,
+        "prompt_tokens_stdev": 128,
+        "prompt_tokens_min": 1,
+        "prompt_tokens_max": 512,
+        "output_tokens": 256,
+        "output_tokens_stdev": 64,
+        "output_tokens_min": 1,
+        "output_tokens_max": 256
+    }
+}

src/automation/standards/benchmarking/benchmarking_code_completion.json

@@ -0,0 +1,13 @@
+{
+    "rate_type": "sweep",
+    "data": {
+        "prompt_tokens": 256,
+        "prompt_tokens_stdev": 128,
+        "prompt_tokens_min": 1,
+        "prompt_tokens_max": 256,
+        "output_tokens": 1024,
+        "output_tokens_stdev": 64,
+        "output_tokens_min": 1,
+        "output_tokens_max": 1024
+    }
+}

src/automation/standards/benchmarking/benchmarking_code_fixing.json

@@ -0,0 +1,13 @@
+{
+    "rate_type": "sweep",
+    "data": {
+        "prompt_tokens": 1024,
+        "prompt_tokens_stdev": 128,
+        "prompt_tokens_min": 1,
+        "prompt_tokens_max": 1024,
+        "output_tokens": 1024,
+        "output_tokens_stdev": 64,
+        "output_tokens_min": 1,
+        "output_tokens_max": 1024
+    }
+}

src/automation/standards/benchmarking/benchmarking_docstring_generation.json

@@ -0,0 +1,13 @@
+{
+    "rate_type": "sweep",
+    "data": {
+        "prompt_tokens": 768,
+        "prompt_tokens_stdev": 128,
+        "prompt_tokens_min": 1,
+        "prompt_tokens_max": 768,
+        "output_tokens": 128,
+        "output_tokens_stdev": 64,
+        "output_tokens_min": 1,
+        "output_tokens_max": 128
+    }
+}

src/automation/standards/benchmarking/benchmarking_instruction.json

@@ -0,0 +1,13 @@
+{
+    "rate_type": "sweep",
+    "data": {
+        "prompt_tokens": 256,
+        "prompt_tokens_stdev": 128,
+        "prompt_tokens_min": 1,
+        "prompt_tokens_max": 256,
+        "output_tokens": 128,
+        "output_tokens_stdev": 64,
+        "output_tokens_min": 1,
+        "output_tokens_max": 128
+    }
+}

src/automation/standards/benchmarking/benchmarking_long_rag.json

@@ -0,0 +1,13 @@
+{
+    "rate_type": "sweep",
+    "data": {
+        "prompt_tokens": 10240,
+        "prompt_tokens_stdev": 128,
+        "prompt_tokens_min": 1,
+        "prompt_tokens_max": 10240,
+        "output_tokens": 1536,
+        "output_tokens_stdev": 64,
+        "output_tokens_min": 1,
+        "output_tokens_max": 1536
+    }
+}

src/automation/standards/benchmarking/benchmarking_rag.json

@@ -0,0 +1,13 @@
+{
+    "rate_type": "sweep",
+    "data": {
+        "prompt_tokens": 1024,
+        "prompt_tokens_stdev": 128,
+        "prompt_tokens_min": 1,
+        "prompt_tokens_max": 1024,
+        "output_tokens": 128,
+        "output_tokens_stdev": 64,
+        "output_tokens_min": 1,
+        "output_tokens_max": 128
+    }
+}

src/automation/standards/benchmarking/benchmarking_summarization.json

@@ -0,0 +1,13 @@
+{
+    "rate_type": "sweep",
+    "data": {
+        "prompt_tokens": 1024,
+        "prompt_tokens_stdev": 128,
+        "prompt_tokens_min": 1,
+        "prompt_tokens_max": 1024,
+        "output_tokens": 128,
+        "output_tokens_stdev": 64,
+        "output_tokens_min": 1,
+        "output_tokens_max": 128
+    }
+}

src/automation/standards/benchmarking/chat.json

@@ -0,0 +1,13 @@
+{
+    "rate_type": "sweep",
+    "data": {
+        "prompt_tokens": 512,
+        "prompt_tokens_stdev": 128,
+        "prompt_tokens_min": 1,
+        "prompt_tokens_max": 1024,
+        "output_tokens": 256,
+        "output_tokens_stdev": 64,
+        "output_tokens_min": 1,
+        "output_tokens_max": 1024
+    }
+}

src/automation/standards/benchmarking/rag.json

@@ -0,0 +1,13 @@
+{
+    "rate_type": "sweep",
+    "data": {
+        "prompt_tokens": 4096,
+        "prompt_tokens_stdev": 512,
+        "prompt_tokens_min": 2048,
+        "prompt_tokens_max": 6144,
+        "output_tokens": 512,
+        "output_tokens_stdev": 128,
+        "output_tokens_min": 1,
+        "output_tokens_max": 1024
+    }
+}

src/automation/tasks/base_task.py

@@ -1,34 +1,41 @@
 from clearml import Task
 from typing import Sequence, Optional
-from automation.configs import DEFAULT_OUTPUT_URI
+from automation.configs import DEFAULT_OUTPUT_URI, DEFAULT_RESEARCH_BRANCH
 from automation.standards import STANDARD_CONFIGS
 import yaml
 import os
 
 class BaseTask():
 
-    base_packages = ["git+https://github.com/neuralmagic/research.git"]
+    #base_packages = ["git+https://github.com/neuralmagic/research.git"]
+    #base_packages = ["git+https://github.com/neuralmagic/research.git@update_guidellm"]
 
     def __init__(
         self,
         project_name: str,
         task_name: str,
         docker_image: str,
+        branch: Optional[str] = DEFAULT_RESEARCH_BRANCH,
         packages: Optional[Sequence[str]]=None,
         task_type: str="training",
     ):
+        branch_name = branch or DEFAULT_RESEARCH_BRANCH
+        base_packages = [f"git+https://github.com/neuralmagic/research.git@{branch_name}"]
 
         if packages is not None:
-            packages = list(set(packages + self.base_packages))
+            packages = list(set(packages + base_packages))
         else:
-            packages = self.base_packages
+            packages = base_packages
+
+        print(packages)
 
         self.project_name = project_name
         self.task_name = task_name
         self.docker_image = docker_image
         self.packages = packages
         self.task_type = task_type
         self.task = None
+        self.branch= branch
         self.script_path = None
         self.callable_artifacts = None
 
@@ -50,8 +57,8 @@ def process_config(self, config):
             return yaml.safe_load(open(STANDARD_CONFIGS[config], "r"))
         elif os.path.exists(config):
             return yaml.safe_load(open(config, "r"))
-        elif os.path.exists(os.path.join("..", "standatrds", config)):
-            return yaml.safe_load(open(os.path.join("..", "standatrds", config)), "r")
+        elif os.path.exists(os.path.join("..", "standards", config)):
+            return yaml.safe_load(open(os.path.join("..", "standards", config)), "r")
         else:
             return yaml.safe_load(config)
 
@@ -91,7 +98,7 @@ def create_task(self):
             add_task_init_call=True,
             script=self.script_path,
             repo="https://github.com/neuralmagic/research.git",
-            branch="main",
+            branch=self.branch,
         )
         self.task.output_uri = DEFAULT_OUTPUT_URI
         self.set_arguments()

src/automation/tasks/guidellm.py

@@ -1,10 +1,10 @@
 from automation.tasks import BaseTask
-from automation.configs import DEFAULT_DOCKER_IMAGE
+from automation.configs import DEFAULT_DOCKER_IMAGE, DEFAULT_RESEARCH_BRANCH
 from typing import Optional, Sequence
 import os
 
 DEFAULT_SERVER_WAIT_TIME = 600 # 600 seconds = 10 minutes
-GUIDELLM_PACKAGE = "git+https://github.com/neuralmagic/guidellm.git@http_backend"
+GUIDELLM_PACKAGE = "git+https://github.com/neuralmagic/guidellm.git"
 
 class GuideLLMTask(BaseTask):
 
@@ -23,6 +23,7 @@ def __init__(
         docker_image: str=DEFAULT_DOCKER_IMAGE,
         packages: Optional[Sequence[str]]=None,
         clearml_model: bool=False,
+        branch: str= DEFAULT_RESEARCH_BRANCH,
         task_type: str="training",
         vllm_kwargs: dict={},
         target: str="http://localhost:8000/v1",
@@ -52,6 +53,7 @@ def __init__(
             docker_image=docker_image,
             packages=packages,
             task_type=task_type,
+            branch = branch,
         )
 
         # Check for conflicts in configs and constructor arguments