1
- const fs = require ( 'fs' ) ;
2
- const path = require ( ' path' ) ;
1
+ const fs = require ( "fs" ) ;
2
+ const path = require ( " path" ) ;
3
3
4
4
function getAllHtmlFiles ( dir , fileList = [ ] ) {
5
5
const files = fs . readdirSync ( dir ) ;
6
-
7
- files . forEach ( file => {
6
+
7
+ files . forEach ( ( file ) => {
8
8
const filePath = path . join ( dir , file ) ;
9
9
const stat = fs . statSync ( filePath ) ;
10
-
10
+
11
11
if ( stat . isDirectory ( ) ) {
12
12
getAllHtmlFiles ( filePath , fileList ) ;
13
- } else if ( file === ' index.html' ) {
13
+ } else if ( file === " index.html" ) {
14
14
fileList . push ( filePath ) ;
15
15
}
16
16
} ) ;
17
-
17
+
18
18
return fileList ;
19
19
}
20
20
21
21
module . exports = function customLLMsTxtPlugin ( context , options ) {
22
22
return {
23
- name : ' custom-llms-txt' ,
23
+ name : " custom-llms-txt" ,
24
24
async postBuild ( { siteConfig, outDir } ) {
25
25
// Group docs by their sidebar category
26
26
const sections = {
27
- ' Getting Started' : [ ] ,
28
- ' Features' : [ ] ,
29
- ' Guides' : [ ] ,
30
- ' Customization' : [ ] ,
31
- ' Advanced' : [ ] ,
32
- ' Hub' : [ ]
27
+ " Getting Started" : [ ] ,
28
+ Features : [ ] ,
29
+ Guides : [ ] ,
30
+ Customization : [ ] ,
31
+ Advanced : [ ] ,
32
+ Hub : [ ] ,
33
33
} ;
34
-
34
+
35
35
// Get all HTML files
36
36
const htmlFiles = getAllHtmlFiles ( outDir ) ;
37
-
37
+
38
38
for ( const htmlPath of htmlFiles ) {
39
39
try {
40
- const htmlContent = fs . readFileSync ( htmlPath , ' utf8' ) ;
41
-
40
+ const htmlContent = fs . readFileSync ( htmlPath , " utf8" ) ;
41
+
42
42
// Extract title from HTML (handle both formats)
43
43
const titleMatch = htmlContent . match ( / < t i t l e [ ^ > ] * > ( [ ^ < ] + ) < \/ t i t l e > / ) ;
44
- const title = titleMatch ? titleMatch [ 1 ] . replace ( ' | Continue' , '' ) . trim ( ) : 'Untitled' ;
45
-
44
+ const title = titleMatch
45
+ ? titleMatch [ 1 ] . replace ( " | Continue" , "" ) . trim ( )
46
+ : "Untitled" ;
47
+
46
48
// Skip if title is just "Continue" (likely homepage)
47
- if ( title === ' Continue' ) continue ;
48
-
49
+ if ( title === " Continue" ) continue ;
50
+
49
51
// Extract description from meta tag
50
- const descMatch = htmlContent . match ( / < m e t a \s + n a m e = " d e s c r i p t i o n " \s + c o n t e n t = " ( [ ^ " ] + ) " / ) ;
51
- const description = descMatch ? descMatch [ 1 ] : '' ;
52
-
52
+ const descMatch = htmlContent . match (
53
+ / < m e t a \s + n a m e = " d e s c r i p t i o n " \s + c o n t e n t = " ( [ ^ " ] + ) " / ,
54
+ ) ;
55
+ const description = descMatch ? descMatch [ 1 ] : "" ;
56
+
53
57
// Get relative path from build directory
54
58
const relativePath = path . relative ( outDir , htmlPath ) ;
55
59
// Convert to URL path
56
- const cleanPath = relativePath . replace ( / i n d e x \. h t m l $ / , '' ) . replace ( / \/ $ / , '' ) ;
60
+ const cleanPath = relativePath
61
+ . replace ( / i n d e x \. h t m l $ / , "" )
62
+ . replace ( / \/ $ / , "" ) ;
57
63
const url = `https://docs.continue.dev/${ cleanPath } ` ;
58
-
64
+
59
65
const docInfo = { title, url, description, path : cleanPath } ;
60
-
66
+
61
67
// Determine section based on path
62
- if ( cleanPath . includes ( ' getting-started' ) ) {
63
- sections [ ' Getting Started' ] . push ( docInfo ) ;
64
- } else if ( cleanPath . includes ( ' features' ) ) {
65
- sections [ ' Features' ] . push ( docInfo ) ;
66
- } else if ( cleanPath . includes ( ' guides' ) ) {
67
- sections [ ' Guides' ] . push ( docInfo ) ;
68
- } else if ( cleanPath . includes ( ' customization' ) ) {
69
- sections [ ' Customization' ] . push ( docInfo ) ;
70
- } else if ( cleanPath . includes ( ' advanced' ) ) {
71
- sections [ ' Advanced' ] . push ( docInfo ) ;
72
- } else if ( cleanPath . includes ( ' hub' ) ) {
73
- sections [ ' Hub' ] . push ( docInfo ) ;
74
- } else if ( cleanPath === '' || cleanPath === '/' ) {
68
+ if ( cleanPath . includes ( " getting-started" ) ) {
69
+ sections [ " Getting Started" ] . push ( docInfo ) ;
70
+ } else if ( cleanPath . includes ( " features" ) ) {
71
+ sections [ " Features" ] . push ( docInfo ) ;
72
+ } else if ( cleanPath . includes ( " guides" ) ) {
73
+ sections [ " Guides" ] . push ( docInfo ) ;
74
+ } else if ( cleanPath . includes ( " customization" ) ) {
75
+ sections [ " Customization" ] . push ( docInfo ) ;
76
+ } else if ( cleanPath . includes ( " advanced" ) ) {
77
+ sections [ " Advanced" ] . push ( docInfo ) ;
78
+ } else if ( cleanPath . includes ( " hub" ) ) {
79
+ sections [ " Hub" ] . push ( docInfo ) ;
80
+ } else if ( cleanPath === "" || cleanPath === "/" ) {
75
81
// Root page
76
- sections [ ' Getting Started' ] . unshift ( docInfo ) ;
82
+ sections [ " Getting Started" ] . unshift ( docInfo ) ;
77
83
}
78
84
} catch ( error ) {
79
85
console . warn ( `Failed to process ${ htmlPath } :` , error . message ) ;
80
86
}
81
87
}
82
-
88
+
83
89
// Generate the structured llms.txt content
84
- let content = '# Continue Documentation\n\n' ;
85
- content += 'Documentation for Continue - the open-source AI code assistant for developers\n\n' ;
86
-
90
+ let content = "# Continue Documentation\n\n" ;
91
+ content +=
92
+ "Documentation for Continue - the open-source AI code assistant for developers\n\n" ;
93
+
87
94
// Add each section
88
95
Object . entries ( sections ) . forEach ( ( [ sectionName , docs ] ) => {
89
96
if ( docs . length > 0 ) {
90
97
content += `## ${ sectionName } \n\n` ;
91
-
98
+
92
99
// Sort docs within each section
93
100
docs . sort ( ( a , b ) => {
94
101
// Sort by path depth first (shorter paths first)
95
102
const depthA = ( a . path . match ( / \/ / g) || [ ] ) . length ;
96
103
const depthB = ( b . path . match ( / \/ / g) || [ ] ) . length ;
97
104
if ( depthA !== depthB ) return depthA - depthB ;
98
-
105
+
99
106
// Then alphabetically
100
107
return a . path . localeCompare ( b . path ) ;
101
108
} ) ;
102
-
103
- docs . forEach ( doc => {
109
+
110
+ docs . forEach ( ( doc ) => {
104
111
content += `- [${ doc . title } ](${ doc . url } )` ;
105
112
if ( doc . description ) {
106
113
content += `: ${ doc . description } ` ;
107
114
}
108
- content += '\n' ;
115
+ content += "\n" ;
109
116
} ) ;
110
- content += '\n' ;
117
+ content += "\n" ;
111
118
}
112
119
} ) ;
113
-
120
+
114
121
// Write to file
115
- const outputPath = path . join ( outDir , 'llms.txt' ) ;
116
- fs . writeFileSync ( outputPath , content , 'utf8' ) ;
117
-
118
- console . log ( `Generated structured llms.txt with ${ Object . values ( sections ) . flat ( ) . length } pages` ) ;
119
- }
122
+ const outputPath = path . join ( outDir , "llms.txt" ) ;
123
+ fs . writeFileSync ( outputPath , content , "utf8" ) ;
124
+
125
+ console . log (
126
+ `Generated structured llms.txt with ${ Object . values ( sections ) . flat ( ) . length } pages` ,
127
+ ) ;
128
+ } ,
120
129
} ;
121
- } ;
130
+ } ;
0 commit comments