11import * as child_process from 'child_process' ;
22import { ENV } from '../../env' ;
3- import { MetadataParseError } from './errors' ;
4- import { parseDataUrl , getFetchableUrl } from './metadata-helpers' ;
3+ import { MetadataParseError , MetadataTimeoutError , TooManyRequestsHttpError } from './errors' ;
4+ import { parseDataUrl , getFetchableDecentralizedStorageUrl } from './metadata-helpers' ;
55import { logger } from '@hirosystems/api-toolkit' ;
66import { PgStore } from '../../pg/pg-store' ;
7+ import { errors } from 'undici' ;
8+ import { RetryableJobError } from '../queue/errors' ;
79
810/**
9- * If an external image processor script is configured, then it will process the given image URL for
10- * the purpose of caching on a CDN (or whatever else it may be created to do). The script is
11- * expected to return a new URL for the image. If the script is not configured, then the original
12- * URL is returned immediately. If a data-uri is passed, it is also immediately returned without
13- * being passed to the script.
11+ * If an external image processor script is configured in the `METADATA_IMAGE_CACHE_PROCESSOR` ENV
12+ * var, this function will process the given image URL for the purpose of caching on a CDN (or
13+ * whatever else it may be created to do). The script is expected to return a new URL for the image
14+ * via `stdout`, with an optional 2nd line with another URL for a thumbnail version of the same
15+ * cached image. If the script is not configured, then the original URL is returned immediately. If
16+ * a data-uri is passed, it is also immediately returned without being passed to the script.
17+ *
18+ * The Image Cache script must return a status code of `0` to mark a successful cache. Other code
19+ * returns available are:
20+ * * `1`: A generic error occurred. Cache should not be retried.
21+ * * `2`: Image fetch timed out before caching was possible. Should be retried.
22+ * * `3`: Image fetch failed due to rate limits from the remote server. Should be retried.
1423 */
15- export async function processImageUrl (
24+ export async function processImageCache (
1625 imgUrl : string ,
1726 contractPrincipal : string ,
1827 tokenNumber : bigint
1928) : Promise < string [ ] > {
2029 const imageCacheProcessor = ENV . METADATA_IMAGE_CACHE_PROCESSOR ;
21- if ( ! imageCacheProcessor ) {
22- return [ imgUrl ] ;
23- }
24- if ( imgUrl . startsWith ( 'data:' ) ) {
25- return [ imgUrl ] ;
30+ if ( ! imageCacheProcessor || imgUrl . startsWith ( 'data:' ) ) return [ imgUrl ] ;
31+ logger . info ( `ImageCache processing token ${ contractPrincipal } ${ tokenNumber } ${ imgUrl } ) ;
32+ const { code, stdout, stderr } = await callImageCacheScript (
33+ imageCacheProcessor ,
34+ imgUrl ,
35+ contractPrincipal ,
36+ tokenNumber
37+ ) ;
38+ switch ( code ) {
39+ case 0 :
40+ try {
41+ const urls = stdout
42+ . trim ( )
43+ . split ( '\n' )
44+ . map ( r => new URL ( r ) . toString ( ) ) ;
45+ logger . info ( urls , `ImageCache processed token ${ contractPrincipal } ${ tokenNumber } ) ;
46+ return urls ;
47+ } catch ( error ) {
48+ // The script returned a code `0` but the results are invalid. This could happen because of
49+ // an unknown script error so we should mark it as retryable.
50+ throw new RetryableJobError (
51+ `ImageCache unknown error` ,
52+ new Error ( `Invalid cached url for ${ imgUrl } ${ stdout } ${ stderr } )
53+ ) ;
54+ }
55+ case 2 :
56+ throw new RetryableJobError ( `ImageCache fetch timed out` , new MetadataTimeoutError ( imgUrl ) ) ;
57+ case 3 :
58+ throw new RetryableJobError (
59+ `ImageCache fetch rate limited` ,
60+ new TooManyRequestsHttpError ( new URL ( imgUrl ) , new errors . ResponseStatusCodeError ( ) )
61+ ) ;
62+ default :
63+ throw new Error ( `ImageCache script error (code ${ code } ${ stderr } ) ;
2664 }
27- logger . info ( `ImageCache processing image for token ${ contractPrincipal } ${ tokenNumber } ) ;
65+ }
66+
67+ async function callImageCacheScript (
68+ imageCacheProcessor : string ,
69+ imgUrl : string ,
70+ contractPrincipal : string ,
71+ tokenNumber : bigint
72+ ) : Promise < {
73+ code : number ;
74+ stdout : string ;
75+ stderr : string ;
76+ } > {
2877 const repoDir = process . cwd ( ) ;
29- const { code , stdout , stderr } = await new Promise < {
78+ return await new Promise < {
3079 code : number ;
3180 stdout : string ;
3281 stderr : string ;
33- } > ( ( resolve , reject ) => {
82+ } > ( resolve => {
3483 const cp = child_process . spawn (
3584 imageCacheProcessor ,
3685 [ imgUrl , contractPrincipal , tokenNumber . toString ( ) ] ,
3786 { cwd : repoDir }
3887 ) ;
88+ let code = 0 ;
3989 let stdout = '' ;
4090 let stderr = '' ;
4191 cp . stdout . on ( 'data' , data => ( stdout += data ) ) ;
4292 cp . stderr . on ( 'data' , data => ( stderr += data ) ) ;
43- cp . on ( 'close' , code => resolve ( { code : code ?? 0 , stdout, stderr } ) ) ;
44- cp . on ( 'error' , error => reject ( error ) ) ;
93+ cp . on ( 'close' , _ => resolve ( { code, stdout, stderr } ) ) ;
94+ cp . on ( 'exit' , processCode => {
95+ code = processCode ?? 0 ;
96+ } ) ;
4597 } ) ;
46- if ( code !== 0 && stderr ) {
47- logger . warn ( stderr , `ImageCache error` ) ;
48- }
49- const result = stdout . trim ( ) . split ( '\n' ) ;
50- try {
51- return result . map ( r => new URL ( r ) . toString ( ) ) ;
52- } catch ( error ) {
53- throw new Error (
54- `Image processing script returned an invalid url for ${ imgUrl } ${ result } ${ stderr }
55- ) ;
56- }
5798}
5899
59- export function getImageUrl ( uri : string ) : string {
100+ /**
101+ * Converts a raw image URI from metadata into a fetchable URL.
102+ * @param uri - Original image URI
103+ * @returns Normalized URL string
104+ */
105+ export function normalizeImageUri ( uri : string ) : string {
60106 // Support images embedded in a Data URL
61107 if ( uri . startsWith ( 'data:' ) ) {
62108 const dataUrl = parseDataUrl ( uri ) ;
@@ -68,7 +114,7 @@ export function getImageUrl(uri: string): string {
68114 }
69115 return uri ;
70116 }
71- const fetchableUrl = getFetchableUrl ( uri ) ;
117+ const fetchableUrl = getFetchableDecentralizedStorageUrl ( uri ) ;
72118 return fetchableUrl . toString ( ) ;
73119}
74120
@@ -81,8 +127,8 @@ export async function reprocessTokenImageCache(
81127 const imageUris = await db . getTokenImageUris ( contractPrincipal , tokenIds ) ;
82128 for ( const token of imageUris ) {
83129 try {
84- const [ cached , thumbnail ] = await processImageUrl (
85- getFetchableUrl ( token . image ) . toString ( ) ,
130+ const [ cached , thumbnail ] = await processImageCache (
131+ getFetchableDecentralizedStorageUrl ( token . image ) . toString ( ) ,
86132 contractPrincipal ,
87133 BigInt ( token . token_number )
88134 ) ;
0 commit comments