Skip to content

Commit b30bd7a

Browse files
VinciGit00VinciGit00
authored
Merge pull request #56 from ScrapeGraphAI/mock-info
add mock
2 parents c853dfe + e056508 commit b30bd7a

File tree

13 files changed

+1554
-9
lines changed

13 files changed

+1554
-9
lines changed

‎scrapegraph-js/examples/mock_mode_example.js‎

Lines changed: 297 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,297 @@
1+
/**
2+
* Example demonstrating how to use the ScrapeGraph AI SDK in mock mode.
3+
*
4+
* This example shows how to:
5+
* 1. Enable mock mode globally or per-request
6+
* 2. Use custom mock responses
7+
* 3. Use custom mock handlers
8+
* 4. Test different endpoints in mock mode
9+
* 5. Demonstrate environment variable activation
10+
*
11+
* Requirements:
12+
* - Node.js 16+
13+
* - scrapegraph-js
14+
*
15+
* Usage:
16+
* node mock_mode_example.js
17+
*
18+
* Or with environment variable:
19+
* SGAI_MOCK=1 node mock_mode_example.js
20+
*/
21+
22+
import {
23+
scrape,
24+
getScrapeRequest,
25+
smartScraper,
26+
getSmartScraperRequest,
27+
searchScraper,
28+
getSearchScraperRequest,
29+
markdownify,
30+
getMarkdownifyRequest,
31+
crawl,
32+
getCrawlRequest,
33+
agenticScraper,
34+
getAgenticScraperRequest,
35+
getCredits,
36+
submitFeedback
37+
} from '../index.js';
38+
39+
import {
40+
initMockConfig,
41+
enableMock,
42+
disableMock,
43+
setMockResponses,
44+
setMockHandler
45+
} from '../src/utils/mockConfig.js';
46+
47+
// Configuration
48+
const API_KEY = process.env.SGAI_API_KEY || 'sgai-00000000-0000-0000-0000-000000000000';
49+
50+
/**
51+
* Basic mock mode usage demonstration
52+
*/
53+
async function basicMockUsage() {
54+
console.log('\n=== Basic Mock Usage ===');
55+
56+
// Enable mock mode globally
57+
enableMock();
58+
59+
try {
60+
// Test scrape endpoint
61+
console.log('\n-- Testing scrape endpoint --');
62+
const scrapeResult = await scrape(API_KEY, 'https://example.com', { renderHeavyJs: true });
63+
console.log('Scrape result:', scrapeResult);
64+
65+
// Test getScrapeRequest endpoint
66+
console.log('\n-- Testing getScrapeRequest endpoint --');
67+
const scrapeStatus = await getScrapeRequest(API_KEY, 'mock-request-id');
68+
console.log('Scrape status:', scrapeStatus);
69+
70+
// Test smartScraper endpoint
71+
console.log('\n-- Testing smartScraper endpoint --');
72+
const smartResult = await smartScraper(API_KEY, 'https://example.com', 'Extract the title');
73+
console.log('SmartScraper result:', smartResult);
74+
75+
// Test getCredits endpoint
76+
console.log('\n-- Testing getCredits endpoint --');
77+
const credits = await getCredits(API_KEY);
78+
console.log('Credits:', credits);
79+
80+
// Test submitFeedback endpoint
81+
console.log('\n-- Testing submitFeedback endpoint --');
82+
const feedback = await submitFeedback(API_KEY, 'mock-request-id', 5, 'Great service!');
83+
console.log('Feedback result:', feedback);
84+
85+
} catch (error) {
86+
console.error('Error in basic mock usage:', error.message);
87+
}
88+
}
89+
90+
/**
91+
* Mock mode with custom responses
92+
*/
93+
async function mockWithCustomResponses() {
94+
console.log('\n=== Mock Mode with Custom Responses ===');
95+
96+
// Set custom responses for specific endpoints
97+
setMockResponses({
98+
'/v1/credits': {
99+
remaining_credits: 42,
100+
total_credits_used: 58,
101+
custom_field: 'This is a custom response'
102+
},
103+
'/v1/smartscraper': () => ({
104+
request_id: 'custom-mock-request-id',
105+
custom_data: 'Generated by custom function'
106+
})
107+
});
108+
109+
try {
110+
// Test credits with custom response
111+
console.log('\n-- Testing credits with custom response --');
112+
const credits = await getCredits(API_KEY);
113+
console.log('Custom credits:', credits);
114+
115+
// Test smartScraper with custom response
116+
console.log('\n-- Testing smartScraper with custom response --');
117+
const smartResult = await smartScraper(API_KEY, 'https://example.com', 'Extract data');
118+
console.log('Custom smartScraper result:', smartResult);
119+
120+
} catch (error) {
121+
console.error('Error in custom responses:', error.message);
122+
}
123+
}
124+
125+
/**
126+
* Mock mode with custom handler
127+
*/
128+
async function mockWithCustomHandler() {
129+
console.log('\n=== Mock Mode with Custom Handler ===');
130+
131+
// Set a custom handler that overrides all responses
132+
setMockHandler((method, url) => {
133+
return {
134+
custom_handler: true,
135+
method: method,
136+
url: url,
137+
timestamp: new Date().toISOString(),
138+
message: 'This response was generated by a custom handler'
139+
};
140+
});
141+
142+
try {
143+
// Test various endpoints with custom handler
144+
console.log('\n-- Testing with custom handler --');
145+
146+
const scrapeResult = await scrape(API_KEY, 'https://example.com');
147+
console.log('Scrape with custom handler:', scrapeResult);
148+
149+
const smartResult = await smartScraper(API_KEY, 'https://example.com', 'Test prompt');
150+
console.log('SmartScraper with custom handler:', smartResult);
151+
152+
const credits = await getCredits(API_KEY);
153+
console.log('Credits with custom handler:', credits);
154+
155+
} catch (error) {
156+
console.error('Error in custom handler:', error.message);
157+
}
158+
}
159+
160+
/**
161+
* Per-request mock mode (without global enable)
162+
*/
163+
async function perRequestMockMode() {
164+
console.log('\n=== Per-Request Mock Mode ===');
165+
166+
// Disable global mock mode
167+
disableMock();
168+
169+
try {
170+
// Test individual requests with mock enabled
171+
console.log('\n-- Testing per-request mock mode --');
172+
173+
const scrapeResult = await scrape(API_KEY, 'https://example.com', { mock: true });
174+
console.log('Per-request mock scrape:', scrapeResult);
175+
176+
const smartResult = await smartScraper(API_KEY, 'https://example.com', 'Test', null, null, null, null, { mock: true });
177+
console.log('Per-request mock smartScraper:', smartResult);
178+
179+
const scrapeStatus = await getScrapeRequest(API_KEY, 'test-id', { mock: true });
180+
console.log('Per-request mock getScrapeRequest:', scrapeStatus);
181+
182+
} catch (error) {
183+
console.error('Error in per-request mock mode:', error.message);
184+
}
185+
}
186+
187+
/**
188+
* Test all available endpoints in mock mode
189+
*/
190+
async function testAllEndpoints() {
191+
console.log('\n=== Testing All Endpoints in Mock Mode ===');
192+
193+
enableMock();
194+
195+
try {
196+
// Test all available endpoints
197+
console.log('\n-- Testing all endpoints --');
198+
199+
// Scrape endpoints
200+
const scrapeResult = await scrape(API_KEY, 'https://example.com');
201+
console.log('Scrape:', scrapeResult.request_id ? '✅' : '❌');
202+
203+
const scrapeStatus = await getScrapeRequest(API_KEY, 'mock-id');
204+
console.log('GetScrapeRequest:', scrapeStatus.status ? '✅' : '❌');
205+
206+
// SmartScraper endpoints
207+
const smartResult = await smartScraper(API_KEY, 'https://example.com', 'Extract title');
208+
console.log('SmartScraper:', smartResult.request_id ? '✅' : '❌');
209+
210+
const smartStatus = await getSmartScraperRequest(API_KEY, 'mock-id');
211+
console.log('GetSmartScraperRequest:', smartStatus.status ? '✅' : '❌');
212+
213+
// SearchScraper endpoints
214+
const searchResult = await searchScraper(API_KEY, 'Search for information');
215+
console.log('SearchScraper:', searchResult.request_id ? '✅' : '❌');
216+
217+
const searchStatus = await getSearchScraperRequest(API_KEY, 'mock-id');
218+
console.log('GetSearchScraperRequest:', searchStatus.status ? '✅' : '❌');
219+
220+
// Markdownify endpoints
221+
const markdownResult = await markdownify(API_KEY, 'https://example.com');
222+
console.log('Markdownify:', markdownResult.request_id ? '✅' : '❌');
223+
224+
const markdownStatus = await getMarkdownifyRequest(API_KEY, 'mock-id');
225+
console.log('GetMarkdownifyRequest:', markdownStatus.status ? '✅' : '❌');
226+
227+
// Crawl endpoints
228+
const crawlResult = await crawl(API_KEY, 'https://example.com');
229+
console.log('Crawl:', crawlResult.crawl_id ? '✅' : '❌');
230+
231+
const crawlStatus = await getCrawlRequest(API_KEY, 'mock-id');
232+
console.log('GetCrawlRequest:', crawlStatus.status ? '✅' : '❌');
233+
234+
// AgenticScraper endpoints
235+
const agenticResult = await agenticScraper(API_KEY, 'https://example.com', ['click button']);
236+
console.log('AgenticScraper:', agenticResult.request_id ? '✅' : '❌');
237+
238+
const agenticStatus = await getAgenticScraperRequest(API_KEY, 'mock-id');
239+
console.log('GetAgenticScraperRequest:', agenticStatus.status ? '✅' : '❌');
240+
241+
// Utility endpoints
242+
const credits = await getCredits(API_KEY);
243+
console.log('GetCredits:', credits.remaining_credits ? '✅' : '❌');
244+
245+
const feedback = await submitFeedback(API_KEY, 'mock-id', 5, 'Great!');
246+
console.log('SubmitFeedback:', feedback.status ? '✅' : '❌');
247+
248+
} catch (error) {
249+
console.error('Error testing endpoints:', error.message);
250+
}
251+
}
252+
253+
/**
254+
* Environment variable activation test
255+
*/
256+
async function testEnvironmentActivation() {
257+
console.log('\n=== Environment Variable Activation Test ===');
258+
259+
console.log('Current SGAI_MOCK value:', process.env.SGAI_MOCK || 'not set');
260+
261+
// Reinitialize mock config to check environment
262+
initMockConfig();
263+
264+
try {
265+
const credits = await getCredits(API_KEY);
266+
console.log('Credits with env check:', credits);
267+
} catch (error) {
268+
console.error('Error in environment test:', error.message);
269+
}
270+
}
271+
272+
/**
273+
* Main function to run all examples
274+
*/
275+
async function main() {
276+
console.log('🧪 ScrapeGraph AI SDK - Mock Mode Examples');
277+
console.log('==========================================');
278+
279+
try {
280+
await basicMockUsage();
281+
await mockWithCustomResponses();
282+
await mockWithCustomHandler();
283+
await perRequestMockMode();
284+
await testAllEndpoints();
285+
await testEnvironmentActivation();
286+
287+
console.log('\n✅ All mock mode examples completed successfully!');
288+
289+
} catch (error) {
290+
console.error('\n❌ Error running examples:', error.message);
291+
}
292+
}
293+
294+
// Run the examples
295+
if (import.meta.url === `file://${process.argv[1]}`) {
296+
main();
297+
}

‎scrapegraph-js/index.js‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,14 @@ export { searchScraper, getSearchScraperRequest } from './src/searchScraper.js';
66
export { getCredits } from './src/credits.js';
77
export { sendFeedback } from './src/feedback.js';
88
export { crawl, getCrawlRequest } from './src/crawl.js';
9+
10+
// Mock utilities
11+
export {
12+
initMockConfig,
13+
enableMock,
14+
disableMock,
15+
setMockResponses,
16+
setMockHandler,
17+
getMockConfig,
18+
isMockEnabled
19+
} from './src/utils/mockConfig.js';

‎scrapegraph-js/src/credits.js‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,27 @@
11
import axios from 'axios';
22
import handleError from './utils/handleError.js';
3+
import { isMockEnabled, getMockConfig } from './utils/mockConfig.js';
4+
import { getMockResponse, createMockAxiosResponse } from './utils/mockResponse.js';
35

46
/**
57
* Retrieve credits from the API.
68
*
79
* @param {string} apiKey - Your ScrapeGraph AI API key
810
* @returns {Promise<string>} Response from the API in JSON format
911
*/
10-
export async function getCredits(apiKey) {
12+
export async function getCredits(apiKey, options = {}) {
13+
const { mock = null } = options;
14+
15+
// Check if mock mode is enabled
16+
const useMock = mock !== null ? mock : isMockEnabled();
17+
18+
if (useMock) {
19+
console.log('🧪 Mock mode active. Returning stub for getCredits');
20+
const mockConfig = getMockConfig();
21+
const mockData = getMockResponse('GET', 'https://api.scrapegraphai.com/v1/credits', mockConfig.customResponses, mockConfig.customHandler);
22+
return mockData;
23+
}
24+
1125
const endpoint = 'https://api.scrapegraphai.com/v1/credits';
1226
const headers = {
1327
'accept': 'application/json',

‎scrapegraph-js/src/scrape.js‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
import axios from 'axios';
22
import handleError from './utils/handleError.js';
3+
import { isMockEnabled, getMockConfig } from './utils/mockConfig.js';
4+
import { getMockResponse, createMockAxiosResponse } from './utils/mockResponse.js';
35

46
/**
57
* Converts a webpage into HTML format with optional JavaScript rendering.
@@ -44,9 +46,20 @@ import handleError from './utils/handleError.js';
4446
export async function scrape(apiKey, url, options = {}) {
4547
const {
4648
renderHeavyJs = false,
47-
headers: customHeaders = {}
49+
headers: customHeaders = {},
50+
mock = null
4851
} = options;
4952

53+
// Check if mock mode is enabled
54+
const useMock = mock !== null ? mock : isMockEnabled();
55+
56+
if (useMock) {
57+
console.log('🧪 Mock mode active. Returning stub for scrape request');
58+
const mockConfig = getMockConfig();
59+
const mockData = getMockResponse('POST', 'https://api.scrapegraphai.com/v1/scrape', mockConfig.customResponses, mockConfig.customHandler);
60+
return mockData;
61+
}
62+
5063
const endpoint = 'https://api.scrapegraphai.com/v1/scrape';
5164
const headers = {
5265
'accept': 'application/json',
@@ -114,7 +127,19 @@ export async function scrape(apiKey, url, options = {}) {
114127
* - CSS styles and formatting
115128
* - Images, links, and other media elements
116129
*/
117-
export async function getScrapeRequest(apiKey, requestId) {
130+
export async function getScrapeRequest(apiKey, requestId, options = {}) {
131+
const { mock = null } = options;
132+
133+
// Check if mock mode is enabled
134+
const useMock = mock !== null ? mock : isMockEnabled();
135+
136+
if (useMock) {
137+
console.log('🧪 Mock mode active. Returning stub for getScrapeRequest');
138+
const mockConfig = getMockConfig();
139+
const mockData = getMockResponse('GET', `https://api.scrapegraphai.com/v1/scrape/${requestId}`, mockConfig.customResponses, mockConfig.customHandler);
140+
return mockData;
141+
}
142+
118143
const endpoint = 'https://api.scrapegraphai.com/v1/scrape/' + requestId;
119144
const headers = {
120145
'accept': 'application/json',

0 commit comments

Comments
 (0)