better handling of tokenization (#708)

Author: justschen

src/platform/tokenizer/node/tokenizer.ts

@@ -145,8 +145,12 @@ class BPETokenizer extends Disposable implements ITokenizer {
 			case Raw.ChatCompletionContentPartKind.Opaque:
 				return text.tokenUsage || 0;
 			case Raw.ChatCompletionContentPartKind.Image:
-				if (text.imageUrl.url.startsWith('data')) {
-					return calculateImageTokenCost(text.imageUrl.url, text.imageUrl.detail);
+				if (text.imageUrl.url.startsWith('data:image/')) {
+					try {
+						return calculateImageTokenCost(text.imageUrl.url, text.imageUrl.detail);
+					} catch {
+						return this._textTokenLength(text.imageUrl.url);
+					}
 				}
 				return this._textTokenLength(text.imageUrl.url);
 			case Raw.ChatCompletionContentPartKind.CacheBreakpoint:
@@ -210,8 +214,12 @@ class BPETokenizer extends Disposable implements ITokenizer {
 				if (casted.type === 'text') {
 					numTokens += await this.tokenLength(casted.text);
 				} else if (casted.type === 'image_url' && casted.image_url) {
-					if (casted.image_url.url.startsWith('data')) {
-						numTokens += calculateImageTokenCost(casted.image_url.url, casted.image_url.detail);
+					if (casted.image_url.url.startsWith('data:image/')) {
+						try {
+							numTokens += calculateImageTokenCost(casted.image_url.url, casted.image_url.detail);
+						} catch {
+							numTokens += await this.tokenLength(casted.image_url.url);
+						}
 					} else {
 						numTokens += await this.tokenLength(casted.image_url.url);
 					}