fix starcoder example (#1080)

Author: lvyufeng

llm/inference/starcoder/README.md

@@ -23,7 +23,9 @@ starcoder server for [llm-vscode](https://github.com/huggingface/llm-vscode) cus
 3. **Run application on port**
 
     ```shell
-    python main.py --pretrained bigcode/starcoder2-7b
+    python main.py --pretrained bigscience/starcoder2-7b
+    # use mirror if you cannot access huggingface
+    python main.py --pretrained AI-ModelScope/starcoder2-7b --mirror modelscope
     ```
 
     It may take a while to download the model checkpoint for the first time. Please change the mirror address by `--mirror` argument if it takes too long.
@@ -35,7 +37,7 @@ starcoder server for [llm-vscode](https://github.com/huggingface/llm-vscode) cus
     1) Select `Skip TLS verification for insecure connections`.
         ![llm-vscode-settings](./assets/llm-vscode-TLS-verification.png)
 
-    2) Change `Llm:ConfigTemplate` to `Custom`.
+    2) Change `Llm:Backend` to `tgi` and `Llm:ConfigTemplate` to `Custom`.
         ![llm-vscode-settings](./assets/llm-vscode-config-template.png)
 
     3) Select `Edit in settings.json` in `Llm:Url`.

llm/inference/starcoder/assets/llm-vscode-config-template.png

@@ -1,7 +1,5 @@
 from mindnlp.transformers import GenerationConfig
 from mindnlp.transformers import Pipeline, pipeline
-import mindspore
-
 
 class GeneratorBase:
     def generate(self, query: str, parameters: dict) -> str:
@@ -12,12 +10,12 @@ def __call__(self, query: str, parameters: dict = None) -> str:
 
 
 class StarCoder(GeneratorBase):
-    def __init__(self, pretrained: str, mirror: str = 'modelscope'):
+    def __init__(self, pretrained: str, mirror: str = 'huggingface'):
         self.pretrained: str = pretrained
         self.mirror: str = mirror
         self.pipe: Pipeline = pipeline(
             "text-generation", model=pretrained, mirror=mirror)
-        self.generation_config = GenerationConfig.from_pretrained(pretrained)
+        self.generation_config = GenerationConfig.from_pretrained(pretrained, mirror=mirror)
         self.generation_config.pad_token_id = self.pipe.tokenizer.eos_token_id
 
     def generate(self, query: str, parameters: dict) -> str:
@@ -27,4 +25,4 @@ def generate(self, query: str, parameters: dict) -> str:
         })
         json_response: dict = self.pipe(query, generation_config=config)[0]
         generated_text: str = json_response['generated_text']
-        return generated_text
+        return generated_text

llm/inference/starcoder/generators.py

@@ -18,9 +18,7 @@ async def api(request: Request):
     json_request: dict = await request.json()
     inputs: str = json_request['inputs']
     parameters: dict = json_request['parameters']
-    logger.info(f'{request.client.host}:{request.client.port} inputs = {json.dumps(inputs)}')
     generated_text: str = generator.generate(inputs, parameters)
-    logger.info(f'{request.client.host}:{request.client.port} generated_text = {json.dumps(generated_text)}')
     return {
         "generated_text": generated_text.replace(inputs, ""),
         "status": 200

llm/inference/starcoder/main.py

@@ -1,3 +1,3 @@
-uvicorn~=0.22.0
-fastapi~=0.95.1
-tokenizers~=0.15.0
+uvicorn
+fastapi
+mindnlp>=0.3.1

llm/inference/starcoder/requirements.txt

@@ -17,6 +17,6 @@ def get_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser()
     parser.add_argument('--port', type=int, default=8000)
     parser.add_argument('--host', type=str, default='0.0.0.0')
-    parser.add_argument('--mirror', type=str, default='modelscope')
+    parser.add_argument('--mirror', type=str, default='huggingface')
     parser.add_argument('--pretrained', type=str, required=True)
     return parser