Skip to content

Commit 2662ed3

Browse files
MForiMFori
authored
feat(input_schema): Enable secret objects (#515)
# Secret object/array inputs Based on the [issue](apify/apify-core#21369): > An ideal solution would be if we could encrypt objects This is proposal to enable `isSecret` boolean property for `object` and `array` fields in Input Schema. Currently this is available only for `string` fields with editors either `textfield` or `textarea`. ## Update based on the outcome of discussion in this PR: - Enable this also for `array` properties (not just `object`) - Enable "validation" properties like (`minProperties`, `maxProperties`, sub-schema in the future,..) even for secured fields - Encrypt the value to string and update the validation logic of JSON Schema (Ajv) validator to this form of string (encrypted object/array) as a value for `object`/`array` fields. - In the encrypted string capture the hash of the field schema (without unimportant fields like title, description, etc.) so we know that the stored encrypted value might no longer match the Actor's input schema. Approach suggested by @jancurn in the discussion below. The actual implementation stringify the object value and encrypts the string to final value in form of: ``` "ENCRYPTED_VALUE:FIELD_SCHEMA_HASH:ENCRYPTED_PASSWORD:ENCRYPTED_VALUE" // for strings "ENCRYPTED_JSON_VALUE:FIELD_SCHEMA_HASH:ENCRYPTED_PASSWORD:ENCRYPTED_VALUE" // for objects/arrays ``` Where the second group (`FIELD_SCHEMA_HASH`) is optional so all existing stored encrypted values are still matching and are backwards compatible. ----- ### Original description (❗outdated): >The `object` fields with `isSecret: true` won't be able to specify some properties that "normal" `object` fields can do, such as: `default`, `prefill`, `patternKey`, `patternValue`, `minProperties`, `maxProperties`. Editor can be only `json` or `hidden`. This restriction is basically the same as with the secret `string` property. > >In addition to change in the Input Schema's JSON schema it's also needed to change the encryption/decryption logic. If input is `string`, there is no change and the encrypted value is stored in `ENCRYPTED_VALUE:base64:base64` form. In case of `object` input, we need to keep the type of the encrypted value to be still `object` because of validation of input in other stages. > >This propose to store encrypted objects as object with structure: >``` >{ > "secret": "encrypted-stringified-json-of-original-object" >} >``` >where the `secret` key, contains same string value as normal encrypted string property. > >When decrypting an encrypted value the logic is exactly the same as with string. We would check if the object has `secret` field and if the value is string that match the encrypted string regex. > >- API, Console and Javascript SDK uses the `@apify/input_secrets` for encryption/decryption (it's part of this PR) >- Python SDK uses `apify._crypto` to decrypt secrets here is draft PR (I didn't test is yet), just to showcase what change >would be needed there: apify/apify-sdk-python#482 >- The last required change would be in console to handle secret inputs via input UI. Also draft, but this is tested and >works well: apify/apify-core#21454 > >I didn't find any other place where would this change causing issues.
1 parent e46c673 commit 2662ed3

File tree

11 files changed

+637
-84
lines changed

11 files changed

+637
-84
lines changed

‎package-lock.json

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/input_schema/package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
},
5050
"dependencies": {
5151
"@apify/consts": "^2.43.0",
52+
"@apify/input_secrets": "^1.1.76",
5253
"acorn-loose": "^8.4.0",
5354
"countries-list": "^3.0.0"
5455
},

‎packages/input_schema/src/intl.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ const intlStrings = {
3333
'Field schema.properties.{fieldKey} does not exist, but it is specified in schema.required. Either define the field or remove it from schema.required.',
3434
'inputSchema.validation.proxyGroupMustBeArrayOfStrings':
3535
'Field {rootName}.{fieldKey}.apifyProxyGroups must be an array of strings.',
36+
'inputSchema.validation.secretFieldSchemaChanged':
37+
'The field schema.properties.{fieldKey} is a secret field, but its schema has changed. Please update the value in the input editor.',
3638
};
3739

3840
/**

‎packages/input_schema/src/schema.json

Lines changed: 122 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,7 @@
152152
"type": { "enum": ["string"] },
153153
"title": { "type": "string" },
154154
"description": { "type": "string" },
155+
"prefill": { "type": "string" },
155156
"example": { "type": "string" },
156157
"nullable": { "type": "boolean" },
157158
"editor": { "enum": ["textfield", "textarea", "hidden"] },
@@ -166,59 +167,94 @@
166167
"type": "object",
167168
"properties": {
168169
"type": { "enum": ["array"] },
169-
"editor": { "enum": ["json", "requestListSources", "pseudoUrls", "globs", "keyValue", "stringList", "select", "hidden"] }
170+
"editor": { "enum": ["json", "requestListSources", "pseudoUrls", "globs", "keyValue", "stringList", "select", "hidden"] },
171+
"isSecret": { "type": "boolean" }
170172
},
171173
"additionalProperties": true,
172174
"required": ["type", "title", "description", "editor"],
173175
"if": {
174176
"properties": {
175-
"editor": { "const": "select" }
177+
"isSecret": {
178+
"not": {
179+
"const": true
180+
}
181+
}
176182
}
177183
},
178184
"then": {
179-
"additionalProperties": false,
180-
"required": ["items"],
181-
"properties": {
182-
"type": { "enum": ["array"] },
183-
"editor": { "enum": ["select"] },
184-
"title": { "type": "string" },
185-
"description": { "type": "string" },
186-
"default": { "type": "array" },
187-
"prefill": { "type": "array" },
188-
"example": { "type": "array" },
189-
"nullable": { "type": "boolean" },
190-
"minItems": { "type": "integer" },
191-
"maxItems": { "type": "integer" },
192-
"uniqueItems": { "type": "boolean" },
193-
"sectionCaption": { "type": "string" },
194-
"sectionDescription": { "type": "string" },
195-
"items": {
196-
"type": "object",
197-
"additionalProperties": false,
198-
"properties": {
199-
"type": { "enum": ["string"] },
200-
"enum": {
201-
"type": "array",
202-
"items": { "type": "string" },
203-
"uniqueItems": true
185+
"if": {
186+
"properties": {
187+
"editor": { "const": "select" }
188+
}
189+
},
190+
"then": {
191+
"additionalProperties": false,
192+
"required": ["items"],
193+
"properties": {
194+
"type": { "enum": ["array"] },
195+
"editor": { "enum": ["select"] },
196+
"title": { "type": "string" },
197+
"description": { "type": "string" },
198+
"default": { "type": "array" },
199+
"prefill": { "type": "array" },
200+
"example": { "type": "array" },
201+
"nullable": { "type": "boolean" },
202+
"minItems": { "type": "integer" },
203+
"maxItems": { "type": "integer" },
204+
"uniqueItems": { "type": "boolean" },
205+
"sectionCaption": { "type": "string" },
206+
"sectionDescription": { "type": "string" },
207+
"items": {
208+
"type": "object",
209+
"additionalProperties": false,
210+
"properties": {
211+
"type": { "enum": ["string"] },
212+
"enum": {
213+
"type": "array",
214+
"items": { "type": "string" },
215+
"uniqueItems": true
216+
},
217+
"enumTitles": {
218+
"type": "array",
219+
"items": { "type": "string" }
220+
}
204221
},
205-
"enumTitles": {
206-
"type": "array",
207-
"items": { "type": "string" }
208-
}
222+
"required": ["type", "enum"]
209223
},
210-
"required": ["type", "enum"]
224+
"isSecret": { "enum": [false] }
225+
}
226+
},
227+
"else": {
228+
"additionalProperties": false,
229+
"properties": {
230+
"type": { "enum": ["array"] },
231+
"editor": { "enum": ["json", "requestListSources", "pseudoUrls", "globs", "keyValue", "stringList", "hidden"] },
232+
"title": { "type": "string" },
233+
"description": { "type": "string" },
234+
"default": { "type": "array" },
235+
"prefill": { "type": "array" },
236+
"example": { "type": "array" },
237+
"nullable": { "type": "boolean" },
238+
"minItems": { "type": "integer" },
239+
"maxItems": { "type": "integer" },
240+
"uniqueItems": { "type": "boolean" },
241+
"sectionCaption": { "type": "string" },
242+
"sectionDescription": { "type": "string" },
243+
"placeholderKey": { "type": "string" },
244+
"placeholderValue": { "type": "string" },
245+
"patternKey": { "type": "string" },
246+
"patternValue": { "type": "string" },
247+
"isSecret": { "enum": [false] }
211248
}
212249
}
213250
},
214251
"else": {
215252
"additionalProperties": false,
216253
"properties": {
217254
"type": { "enum": ["array"] },
218-
"editor": { "enum": ["json", "requestListSources", "pseudoUrls", "globs", "keyValue", "stringList", "hidden"] },
255+
"editor": { "enum": ["json", "hidden"] },
219256
"title": { "type": "string" },
220257
"description": { "type": "string" },
221-
"default": { "type": "array" },
222258
"prefill": { "type": "array" },
223259
"example": { "type": "array" },
224260
"nullable": { "type": "boolean" },
@@ -227,35 +263,70 @@
227263
"uniqueItems": { "type": "boolean" },
228264
"sectionCaption": { "type": "string" },
229265
"sectionDescription": { "type": "string" },
230-
"placeholderKey": { "type": "string" },
231-
"placeholderValue": { "type": "string" },
232-
"patternKey": { "type": "string" },
233-
"patternValue": { "type": "string" }
266+
"isSecret": { "enum": [true] }
234267
}
235268
}
236269
},
237270
"objectProperty": {
238271
"title": "Object property",
239272
"type": "object",
240-
"additionalProperties": false,
273+
"additionalProperties": true,
241274
"properties": {
242275
"type": { "enum": ["object"] },
243276
"title": { "type": "string" },
244277
"description": { "type": "string" },
245-
"default": { "type": "object" },
246-
"prefill": { "type": "object" },
247-
"example": { "type": "object" },
248-
"patternKey": { "type": "string" },
249-
"patternValue": { "type": "string" },
250-
"nullable": { "type": "boolean" },
251-
"minProperties": { "type": "integer" },
252-
"maxProperties": { "type": "integer" },
253-
254278
"editor": { "enum": ["json", "proxy", "hidden"] },
255-
"sectionCaption": { "type": "string" },
256-
"sectionDescription": { "type": "string" }
279+
"isSecret": { "type": "boolean" }
257280
},
258-
"required": ["type", "title", "description", "editor"]
281+
"required": ["type", "title", "description", "editor"],
282+
"if": {
283+
"properties": {
284+
"isSecret": {
285+
"not": {
286+
"const": true
287+
}
288+
}
289+
}
290+
},
291+
"then": {
292+
"additionalProperties": false,
293+
"properties": {
294+
"type": { "enum": ["object"] },
295+
"title": { "type": "string" },
296+
"description": { "type": "string" },
297+
"default": { "type": "object" },
298+
"prefill": { "type": "object" },
299+
"example": { "type": "object" },
300+
"patternKey": { "type": "string" },
301+
"patternValue": { "type": "string" },
302+
"nullable": { "type": "boolean" },
303+
"minProperties": { "type": "integer" },
304+
"maxProperties": { "type": "integer" },
305+
"editor": { "enum": ["json", "proxy", "hidden"] },
306+
"sectionCaption": { "type": "string" },
307+
"sectionDescription": { "type": "string" },
308+
"isSecret": { "enum": [false] }
309+
}
310+
},
311+
"else": {
312+
"additionalProperties": false,
313+
"properties": {
314+
"type": { "enum": ["object"] },
315+
"title": { "type": "string" },
316+
"description": { "type": "string" },
317+
"prefill": { "type": "object" },
318+
"example": { "type": "object" },
319+
"patternKey": { "type": "string" },
320+
"patternValue": { "type": "string" },
321+
"nullable": { "type": "boolean" },
322+
"minProperties": { "type": "integer" },
323+
"maxProperties": { "type": "integer" },
324+
"editor": { "enum": ["json", "hidden"] },
325+
"sectionCaption": { "type": "string" },
326+
"sectionDescription": { "type": "string" },
327+
"isSecret": { "enum": [true] }
328+
}
329+
}
259330
},
260331
"integerProperty": {
261332
"title": "Integer property",

‎packages/input_schema/src/utilities.ts

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import type { ValidateFunction } from 'ajv';
33
import { countries } from 'countries-list';
44

55
import { PROXY_URL_REGEX, URL_REGEX } from '@apify/consts';
6+
import { isEncryptedValueForFieldSchema, isEncryptedValueForFieldType } from '@apify/input_secrets';
67

78
import { parseAjvError } from './input_schema';
89
import { m } from './intl';
@@ -133,13 +134,34 @@ export function validateInputUsingValidator(
133134
// Process AJV validation errors
134135
if (!isValid) {
135136
errors = validator.errors!
137+
.filter((error) => {
138+
// We are storing encrypted objects/arrays as strings, so AJV will throw type the error here.
139+
// So we need to skip these errors.
140+
if (error.keyword === 'type' && error.instancePath) {
141+
const path = error.instancePath.replace(/^\//, '').split('/')[0];
142+
const propSchema = inputSchema.properties?.[path];
143+
const value = input[path];
144+
145+
// Check if the property is a secret and if the value is an encrypted value.
146+
// We do additional validation of the field schema in the later part of this function
147+
if (
148+
propSchema?.isSecret
149+
&& typeof value === 'string'
150+
&& (propSchema.type === 'object' || propSchema.type === 'array')
151+
&& isEncryptedValueForFieldType(value, propSchema.type)
152+
) {
153+
return false;
154+
}
155+
}
156+
return true;
157+
})
136158
.map((error) => parseAjvError(error, 'input', properties, input))
137159
.filter((error) => !!error) as any[];
138160
}
139161

140162
Object.keys(properties).forEach((property) => {
141163
const value = input[property];
142-
const { type, editor, patternKey, patternValue } = properties[property];
164+
const { type, editor, patternKey, patternValue, isSecret } = properties[property];
143165
const fieldErrors = [];
144166
// Check that proxy is required, if yes, valides that it's correctly setup
145167
if (type === 'object' && editor === 'proxy') {
@@ -215,7 +237,7 @@ export function validateInputUsingValidator(
215237
}
216238
}
217239
// Check that object items fit patternKey and patternValue
218-
if (type === 'object' && value) {
240+
if (type === 'object' && value && typeof value === 'object') {
219241
if (patternKey) {
220242
const check = new RegExp(patternKey);
221243
const invalidKeys: any[] = [];
@@ -249,6 +271,16 @@ export function validateInputUsingValidator(
249271
}
250272
}
251273

274+
// Additional validation for secret fields
275+
if (isSecret && value && typeof value === 'string') {
276+
// If the value is a valid encrypted string for the field type,
277+
// we check if the field schema is likely to be still valid (is unchanged from the time of encryption).
278+
if (isEncryptedValueForFieldType(value, type) && !isEncryptedValueForFieldSchema(value, properties[property])) {
279+
// If not, we add an error message to the field errors and user needs to update the value in the input editor.
280+
fieldErrors.push(m('inputSchema.validation.secretFieldSchemaChanged', { fieldKey: property }));
281+
}
282+
}
283+
252284
if (fieldErrors.length > 0) {
253285
const message = fieldErrors.join(', ');
254286
errors.push({ fieldKey: property, message });

‎packages/input_secrets/src/field_schema_utils.ts

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
import crypto from 'node:crypto';
2+
3+
/**
4+
* These keys are omitted from the field schema normalization process
5+
* because they are not relevant for validation of values against the schema.
6+
*/
7+
const OMIT_KEYS = new Set(['title', 'description', 'sectionCaption', 'sectionDescription', 'nullable', 'example', 'prefill', 'editor']);
8+
9+
/**
10+
* Normalizes the field schema by removing irrelevant keys and sorting the remaining keys.
11+
*/
12+
function normalizeFieldSchema(value: any): any {
13+
if (Array.isArray(value)) {
14+
return value.map(normalizeFieldSchema);
15+
}
16+
17+
if (value && typeof value === 'object') {
18+
const result: Record<string, any> = {};
19+
Object.keys(value)
20+
.filter((key) => !OMIT_KEYS.has(key))
21+
.sort()
22+
.forEach((key) => {
23+
result[key] = normalizeFieldSchema(value[key]);
24+
});
25+
return result;
26+
}
27+
28+
return value;
29+
}
30+
31+
/**
32+
* Generates a stable hash for the field schema.
33+
* @param fieldSchema
34+
*/
35+
export function getFieldSchemaHash(fieldSchema: Record<string, any>): string {
36+
try {
37+
const stringifiedSchema = JSON.stringify(normalizeFieldSchema(fieldSchema));
38+
// Create a SHA-256 hash of the stringified schema and return the first 10 characters in hex.
39+
return crypto.createHash('sha256').update(stringifiedSchema).digest('hex').slice(0, 10);
40+
} catch (err) {
41+
throw new Error(`The field schema could not be stringified for hash: ${err}`);
42+
}
43+
}

0 commit comments

Comments
 (0)