Skip to content

Commit 2af5407

Browse files
VinciGit00VinciGit00
authored
Merge pull request #57 from ScrapeGraphAI/mock-info
Mock info
2 parents cafda2c + 2a1e71b commit 2af5407

File tree

6 files changed

+139
-14
lines changed

6 files changed

+139
-14
lines changed

‎scrapegraph-js/examples/mock_mode_example.js‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ import {
3333
agenticScraper,
3434
getAgenticScraperRequest,
3535
getCredits,
36-
submitFeedback
36+
sendFeedback
3737
} from '../index.js';
3838

3939
import {
@@ -77,9 +77,9 @@ async function basicMockUsage() {
7777
const credits = await getCredits(API_KEY);
7878
console.log('Credits:', credits);
7979

80-
// Test submitFeedback endpoint
81-
console.log('\n-- Testing submitFeedback endpoint --');
82-
const feedback = await submitFeedback(API_KEY, 'mock-request-id', 5, 'Great service!');
80+
// Test sendFeedback endpoint
81+
console.log('\n-- Testing sendFeedback endpoint --');
82+
const feedback = await sendFeedback(API_KEY, 'mock-request-id', 5, 'Great service!');
8383
console.log('Feedback result:', feedback);
8484

8585
} catch (error) {
@@ -242,8 +242,8 @@ async function testAllEndpoints() {
242242
const credits = await getCredits(API_KEY);
243243
console.log('GetCredits:', credits.remaining_credits ? '✅' : '❌');
244244

245-
const feedback = await submitFeedback(API_KEY, 'mock-id', 5, 'Great!');
246-
console.log('SubmitFeedback:', feedback.status ? '✅' : '❌');
245+
const feedback = await sendFeedback(API_KEY, 'mock-id', 5, 'Great!');
246+
console.log('SendFeedback:', feedback.status ? '✅' : '❌');
247247

248248
} catch (error) {
249249
console.error('Error testing endpoints:', error.message);

‎scrapegraph-js/src/agenticScraper.js‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
import axios from 'axios';
22
import handleError from './utils/handleError.js';
3+
import { isMockEnabled, getMockConfig } from './utils/mockConfig.js';
4+
import { getMockResponse } from './utils/mockResponse.js';
35

46
/**
57
* Perform automated browser actions on a webpage using AI-powered agentic scraping.
@@ -11,6 +13,8 @@ import handleError from './utils/handleError.js';
1113
* @param {string} [userPrompt=null] - Prompt for AI extraction (required when aiExtraction=true)
1214
* @param {Object} [outputSchema=null] - Schema for structured data extraction (optional, used with aiExtraction=true)
1315
* @param {boolean} [aiExtraction=false] - Whether to use AI for data extraction from the scraped content
16+
* @param {Object} options - Optional configuration options
17+
* @param {boolean} options.mock - Override mock mode for this request
1418
* @returns {Promise<Object>} Response from the API containing request_id and initial status
1519
* @throws {Error} Will throw an error in case of an HTTP failure or invalid parameters.
1620
*
@@ -60,7 +64,19 @@ import handleError from './utils/handleError.js';
6064
* console.error('Error:', error.message);
6165
* }
6266
*/
63-
export async function agenticScraper(apiKey, url, steps, useSession = true, userPrompt = null, outputSchema = null, aiExtraction = false) {
67+
export async function agenticScraper(apiKey, url, steps, useSession = true, userPrompt = null, outputSchema = null, aiExtraction = false, options = {}) {
68+
const { mock = null } = options;
69+
70+
// Check if mock mode is enabled
71+
const useMock = mock !== null ? mock : isMockEnabled();
72+
73+
if (useMock) {
74+
console.log('🧪 Mock mode active. Returning stub for agenticScraper request');
75+
const mockConfig = getMockConfig();
76+
const mockData = getMockResponse('POST', 'https://api.scrapegraphai.com/v1/agentic-scrapper', mockConfig.customResponses, mockConfig.customHandler);
77+
return mockData;
78+
}
79+
6480
const endpoint = 'https://api.scrapegraphai.com/v1/agentic-scrapper';
6581
const headers = {
6682
'accept': 'application/json',
@@ -166,7 +182,19 @@ export async function agenticScraper(apiKey, url, steps, useSession = true, user
166182
* allowing for complex interactions like form filling, clicking buttons,
167183
* and navigating through multi-step workflows with session management.
168184
*/
169-
export async function getAgenticScraperRequest(apiKey, requestId) {
185+
export async function getAgenticScraperRequest(apiKey, requestId, options = {}) {
186+
const { mock = null } = options;
187+
188+
// Check if mock mode is enabled
189+
const useMock = mock !== null ? mock : isMockEnabled();
190+
191+
if (useMock) {
192+
console.log('🧪 Mock mode active. Returning stub for getAgenticScraperRequest');
193+
const mockConfig = getMockConfig();
194+
const mockData = getMockResponse('GET', `https://api.scrapegraphai.com/v1/agentic-scrapper/${requestId}`, mockConfig.customResponses, mockConfig.customHandler);
195+
return mockData;
196+
}
197+
170198
const endpoint = 'https://api.scrapegraphai.com/v1/agentic-scrapper/' + requestId;
171199
const headers = {
172200
'accept': 'application/json',

‎scrapegraph-js/src/crawl.js‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ import axios from 'axios';
22
import handleError from './utils/handleError.js';
33
import { ZodType } from 'zod';
44
import { zodToJsonSchema } from 'zod-to-json-schema';
5+
import { isMockEnabled, getMockConfig } from './utils/mockConfig.js';
6+
import { getMockResponse } from './utils/mockResponse.js';
57

68
/**
79
* Start a crawl job using the ScrapeGraphAI API.
@@ -18,6 +20,7 @@ import { zodToJsonSchema } from 'zod-to-json-schema';
1820
* @param {boolean} [options.sameDomainOnly=true] - Whether to only crawl pages from the same domain
1921
* @param {boolean} [options.sitemap] - Whether to use sitemap for better page discovery
2022
* @param {number} [options.batchSize=1] - Batch size for processing pages (1-10)
23+
* @param {boolean} [options.mock] - Override mock mode for this request
2124
* @returns {Promise<Object>} The crawl job response
2225
* @throws {Error} Throws an error if the HTTP request fails
2326
*/
@@ -28,6 +31,17 @@ export async function crawl(
2831
schema,
2932
options = {}
3033
) {
34+
const { mock = null } = options;
35+
36+
// Check if mock mode is enabled
37+
const useMock = mock !== null ? mock : isMockEnabled();
38+
39+
if (useMock) {
40+
console.log('🧪 Mock mode active. Returning stub for crawl request');
41+
const mockConfig = getMockConfig();
42+
const mockData = getMockResponse('POST', 'https://api.scrapegraphai.com/v1/crawl', mockConfig.customResponses, mockConfig.customHandler);
43+
return mockData;
44+
}
3145
const endpoint = 'https://api.scrapegraphai.com/v1/crawl';
3246
const headers = {
3347
'accept': 'application/json',
@@ -81,7 +95,19 @@ export async function crawl(
8195
* @returns {Promise<Object>} The crawl result
8296
* @throws {Error} Throws an error if the HTTP request fails
8397
*/
84-
export async function getCrawlRequest(apiKey, crawlId) {
98+
export async function getCrawlRequest(apiKey, crawlId, options = {}) {
99+
const { mock = null } = options;
100+
101+
// Check if mock mode is enabled
102+
const useMock = mock !== null ? mock : isMockEnabled();
103+
104+
if (useMock) {
105+
console.log('🧪 Mock mode active. Returning stub for getCrawlRequest');
106+
const mockConfig = getMockConfig();
107+
const mockData = getMockResponse('GET', `https://api.scrapegraphai.com/v1/crawl/${crawlId}`, mockConfig.customResponses, mockConfig.customHandler);
108+
return mockData;
109+
}
110+
85111
const endpoint = `https://api.scrapegraphai.com/v1/crawl/${crawlId}`;
86112
const headers = {
87113
'accept': 'application/json',

‎scrapegraph-js/src/feedback.js‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
import axios from 'axios';
22
import handleError from './utils/handleError.js';
3+
import { isMockEnabled, getMockConfig } from './utils/mockConfig.js';
4+
import { getMockResponse } from './utils/mockResponse.js';
35

46
/**
57
* Send feedback to the API.
@@ -8,9 +10,23 @@ import handleError from './utils/handleError.js';
810
* @param {string} requestId - The request ID associated with the feedback
911
* @param {number} rating - The rating score
1012
* @param {string} feedbackText - Optional feedback message to send
13+
* @param {Object} options - Optional configuration options
14+
* @param {boolean} options.mock - Override mock mode for this request
1115
* @returns {Promise<string>} Response from the API in JSON format
1216
*/
13-
export async function sendFeedback(apiKey, requestId, rating, feedbackText = null) {
17+
export async function sendFeedback(apiKey, requestId, rating, feedbackText = null, options = {}) {
18+
const { mock = null } = options;
19+
20+
// Check if mock mode is enabled
21+
const useMock = mock !== null ? mock : isMockEnabled();
22+
23+
if (useMock) {
24+
console.log('🧪 Mock mode active. Returning stub for sendFeedback request');
25+
const mockConfig = getMockConfig();
26+
const mockData = getMockResponse('POST', 'https://api.scrapegraphai.com/v1/feedback', mockConfig.customResponses, mockConfig.customHandler);
27+
return mockData;
28+
}
29+
1430
const endpoint = 'https://api.scrapegraphai.com/v1/feedback';
1531
const headers = {
1632
'accept': 'application/json',

‎scrapegraph-js/src/markdownify.js‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,31 @@
11
import axios from 'axios';
22
import handleError from './utils/handleError.js';
3+
import { isMockEnabled, getMockConfig } from './utils/mockConfig.js';
4+
import { getMockResponse } from './utils/mockResponse.js';
35

46
/**
57
* Converts a webpage into clean, well-structured markdown format.
68
*
79
* @param {string} apiKey - Your ScrapeGraph AI API key.
810
* @param {string} url - The URL of the webpage to be converted.
11+
* @param {Object} options - Optional configuration options.
12+
* @param {boolean} options.mock - Override mock mode for this request
913
* @returns {Promise<string>} A promise that resolves to the markdown representation of the webpage.
1014
* @throws {Error} Throws an error if the HTTP request fails.
1115
*/
12-
export async function markdownify(apiKey, url) {
16+
export async function markdownify(apiKey, url, options = {}) {
17+
const { mock = null } = options;
18+
19+
// Check if mock mode is enabled
20+
const useMock = mock !== null ? mock : isMockEnabled();
21+
22+
if (useMock) {
23+
console.log('🧪 Mock mode active. Returning stub for markdownify request');
24+
const mockConfig = getMockConfig();
25+
const mockData = getMockResponse('POST', 'https://api.scrapegraphai.com/v1/markdownify', mockConfig.customResponses, mockConfig.customHandler);
26+
return mockData;
27+
}
28+
1329
const endpoint = 'https://api.scrapegraphai.com/v1/markdownify';
1430
const headers = {
1531
'accept': 'application/json',
@@ -66,7 +82,19 @@ export async function markdownify(apiKey, url) {
6682
* - Links and images
6783
* - Text formatting (bold, italic, etc.)
6884
*/
69-
export async function getMarkdownifyRequest(apiKey, requestId) {
85+
export async function getMarkdownifyRequest(apiKey, requestId, options = {}) {
86+
const { mock = null } = options;
87+
88+
// Check if mock mode is enabled
89+
const useMock = mock !== null ? mock : isMockEnabled();
90+
91+
if (useMock) {
92+
console.log('🧪 Mock mode active. Returning stub for getMarkdownifyRequest');
93+
const mockConfig = getMockConfig();
94+
const mockData = getMockResponse('GET', `https://api.scrapegraphai.com/v1/markdownify/${requestId}`, mockConfig.customResponses, mockConfig.customHandler);
95+
return mockData;
96+
}
97+
7098
const endpoint = 'https://api.scrapegraphai.com/v1/markdownify/' + requestId;
7199
const headers = {
72100
'accept': 'application/json',

‎scrapegraph-js/src/searchScraper.js‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ import axios from 'axios';
22
import handleError from './utils/handleError.js';
33
import { ZodType } from 'zod';
44
import { zodToJsonSchema } from 'zod-to-json-schema';
5+
import { isMockEnabled, getMockConfig } from './utils/mockConfig.js';
6+
import { getMockResponse } from './utils/mockResponse.js';
57

68
/**
79
* Search and extract information from multiple web sources using AI.
@@ -13,10 +15,23 @@ import { zodToJsonSchema } from 'zod-to-json-schema';
1315
* Credit calculation: 30 base + 10 per additional website beyond 3.
1416
* @param {Object} [schema] - Optional schema object defining the output structure
1517
* @param {String} userAgent - the user agent like "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
18+
* @param {Object} options - Optional configuration options
19+
* @param {boolean} options.mock - Override mock mode for this request
1620
* @returns {Promise<string>} Extracted data in JSON format matching the provided schema
1721
* @throws - Will throw an error in case of an HTTP failure.
1822
*/
19-
export async function searchScraper(apiKey, prompt, numResults = 3, schema = null, userAgent = null) {
23+
export async function searchScraper(apiKey, prompt, numResults = 3, schema = null, userAgent = null, options = {}) {
24+
const { mock = null } = options;
25+
26+
// Check if mock mode is enabled
27+
const useMock = mock !== null ? mock : isMockEnabled();
28+
29+
if (useMock) {
30+
console.log('🧪 Mock mode active. Returning stub for searchScraper request');
31+
const mockConfig = getMockConfig();
32+
const mockData = getMockResponse('POST', 'https://api.scrapegraphai.com/v1/searchscraper', mockConfig.customResponses, mockConfig.customHandler);
33+
return mockData;
34+
}
2035
const endpoint = 'https://api.scrapegraphai.com/v1/searchscraper';
2136
const headers = {
2237
'accept': 'application/json',
@@ -92,7 +107,19 @@ export async function searchScraper(apiKey, prompt, numResults = 3, schema = nul
92107
* information based on the original search query. The results are structured according to
93108
* the schema provided in the original searchScraper call, if any.
94109
*/
95-
export async function getSearchScraperRequest(apiKey, requestId) {
110+
export async function getSearchScraperRequest(apiKey, requestId, options = {}) {
111+
const { mock = null } = options;
112+
113+
// Check if mock mode is enabled
114+
const useMock = mock !== null ? mock : isMockEnabled();
115+
116+
if (useMock) {
117+
console.log('🧪 Mock mode active. Returning stub for getSearchScraperRequest');
118+
const mockConfig = getMockConfig();
119+
const mockData = getMockResponse('GET', `https://api.scrapegraphai.com/v1/searchscraper/${requestId}`, mockConfig.customResponses, mockConfig.customHandler);
120+
return mockData;
121+
}
122+
96123
const endpoint = 'https://api.scrapegraphai.com/v1/searchscraper/' + requestId;
97124
const headers = {
98125
'accept': 'application/json',

0 commit comments

Comments
 (0)