Skip to content

Commit b551e1e

Browse files
ToMESSKaToMESSKa
committed
build: create llms.txt for testing
1 parent e66ad32 commit b551e1e

File tree

4 files changed

+886
-4
lines changed

4 files changed

+886
-4
lines changed

‎packages/__docs__/buildScripts/build-docs.mts‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,14 @@ import { theme as canvasTheme } from '@instructure/canvas-theme'
3131
import { theme as canvasHighContrastTheme } from '@instructure/canvas-high-contrast-theme'
3232
import type {
3333
LibraryOptions,
34-
MainDocsData, ProcessedFile
34+
MainDocsData,
35+
ProcessedFile
3536
} from './DataTypes.mjs'
3637
import { getFrontMatter } from './utils/getFrontMatter.mjs'
37-
import { createRequire } from "module"
38+
import { createRequire } from 'module'
3839
import { fileURLToPath, pathToFileURL } from 'url'
40+
import { generateIndividualMarkdownFiles } from './generate-markdowns.mjs'
41+
import { generateLLMSTxt } from './generate-llms.mjs'
3942

4043
const __filename = fileURLToPath(import.meta.url)
4144
const __dirname = path.dirname(__filename)
@@ -101,7 +104,7 @@ const pathsToIgnore = [
101104
// deprecated packages and modules:
102105
'**/InputModeListener.ts',
103106
// regression testing app:
104-
'**/regression-test/**',
107+
'**/regression-test/**'
105108
]
106109

107110
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
@@ -146,9 +149,24 @@ function buildDocs() {
146149
markdownsAndSources
147150
)
148151

152+
generateIndividualMarkdownFiles(
153+
buildDir + 'docs/',
154+
buildDir + 'markdowns/'
155+
)
156+
const parentOfDocs = path.dirname(buildDir + 'docs/')
157+
generateLLMSTxt(
158+
buildDir + 'markdown-and-sources-data.json',
159+
path.join(parentOfDocs, 'llms.txt'),
160+
path.join(__dirname, '../buildScripts/summaries.json')
161+
162+
)
163+
149164
// eslint-disable-next-line no-console
150165
console.log('Copying icons data...')
151-
fs.copyFileSync(projectRoot + '/packages/ui-icons/__build__/icons-data.json', buildDir + 'icons-data.json')
166+
fs.copyFileSync(
167+
projectRoot + '/packages/ui-icons/__build__/icons-data.json',
168+
buildDir + 'icons-data.json'
169+
)
152170

153171
// eslint-disable-next-line no-console
154172
console.log('Finished building documentation data')

‎packages/__docs__/buildScripts/generate-llms.mts‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
import { writeFileSync, readFileSync } from 'fs'
2+
3+
// Define the section interface
4+
interface SectionData {
5+
docs?: string[]
6+
sections?: string[]
7+
title?: string
8+
level?: number
9+
}
10+
11+
function generateDocsMarkdown(
12+
filePath: string,
13+
summariesFilePath?: string
14+
): string {
15+
const fileContent = readFileSync(filePath, 'utf-8')
16+
const jsonData = JSON.parse(fileContent)
17+
const { sections } = jsonData
18+
19+
// Load summaries from the provided summaries file
20+
let summaries: Record<string, string> = {}
21+
if (summariesFilePath) {
22+
try {
23+
const summariesContent = readFileSync(summariesFilePath, 'utf-8')
24+
const summariesData = JSON.parse(summariesContent)
25+
summariesData.forEach(
26+
(item: { title: string; description?: string; summary?: string }) => {
27+
const normalizedKey = item.title
28+
.replace(/\s+/g, '')
29+
.replace(/[^a-zA-Z0-9]/g, '')
30+
summaries[normalizedKey] = item.description || item.summary || ''
31+
summaries[item.title] = item.description || item.summary || ''
32+
}
33+
)
34+
} catch (error) {
35+
console.warn('Could not load summaries file:', error)
36+
}
37+
}
38+
39+
const filepath = 'https://instructure.design/pr-preview/pr-2109/markdowns/'
40+
41+
let markdownContent = `# Instructure UI (InstUI) - React Component Library\n\n - version 10.24.1 \n\n `
42+
markdownContent += `- Instructure UI (InstUI) is a comprehensive React component library.\n\n`
43+
44+
// Main Documentation section
45+
markdownContent += `## Documentation\n\n`
46+
47+
// Add User Guides wrapper
48+
markdownContent += `### User Guides\n`
49+
50+
// Process all non-component, non-utility sections
51+
Object.entries(sections as Record<string, SectionData>).forEach(([sectionKey, sectionData]) => {
52+
if (
53+
sectionKey === '__uncategorized' ||
54+
sectionKey === 'components' ||
55+
sectionKey.toLowerCase().includes('utilities') ||
56+
(sectionData.level || 0) > 0 // Fixed: Handle undefined level
57+
) {
58+
return
59+
}
60+
61+
const sectionTitle = sectionData.title || sectionKey
62+
markdownContent += `#### ${sectionTitle}\n\n`
63+
64+
if (sectionData.docs && sectionData.docs.length > 0) {
65+
const uniqueDocs = [...new Set(sectionData.docs)]
66+
uniqueDocs.forEach((doc) => {
67+
const normalizedDoc = doc
68+
.replace(/\s+/g, '')
69+
.replace(/[^a-zA-Z0-9]/g, '')
70+
const summary = summaries[doc] || summaries[normalizedDoc] || ''
71+
markdownContent += `- [${doc}](${filepath}${doc}.md)${
72+
summary ? `: ${summary}` : ''
73+
}\n`
74+
})
75+
}
76+
markdownContent += '\n'
77+
})
78+
79+
// Add Components section (sibling of User Guides)
80+
markdownContent += `### Components\n\n`
81+
82+
const componentsSection = sections['components'] as SectionData | undefined
83+
if (componentsSection) {
84+
const allComponents = [...new Set(componentsSection.docs || [])]
85+
allComponents.forEach((component) => {
86+
const normalizedComponent = component
87+
.replace(/\s+/g, '')
88+
.replace(/[^a-zA-Z0-9]/g, '')
89+
const summary =
90+
summaries[component] || summaries[normalizedComponent] || ''
91+
markdownContent += `- [${component}](${filepath}${component}.md)${
92+
summary ? `: ${summary}` : ''
93+
}\n`
94+
})
95+
}
96+
97+
// Process component subsections (keep AI Components + utilities, skip deprecated)
98+
if (componentsSection?.sections && componentsSection.sections.length > 0) {
99+
const subsections = componentsSection.sections
100+
subsections.forEach((subsectionPath: string) => {
101+
if (subsectionPath.toLowerCase().includes('deprecated')) return
102+
103+
const subsection = sections[subsectionPath] as SectionData | undefined
104+
if (subsection && subsection.docs && subsection.docs.length > 0) {
105+
const subsectionTitle =
106+
subsection.title || subsectionPath.split('/').pop() || subsectionPath
107+
markdownContent += `\n#### ${subsectionTitle}\n\n`
108+
const uniqueSubDocs = [...new Set(subsection.docs)]
109+
uniqueSubDocs.forEach((doc) => {
110+
const normalizedDoc = doc
111+
.replace(/\s+/g, '')
112+
.replace(/[^a-zA-Z0-9]/g, '')
113+
const summary = summaries[doc] || summaries[normalizedDoc] || ''
114+
markdownContent += `- [${doc}](${filepath}${doc}.md)${
115+
summary ? `: ${summary}` : ''
116+
}\n`
117+
})
118+
}
119+
})
120+
}
121+
122+
return markdownContent
123+
}
124+
125+
function generateLLMSTxt(
126+
inputFilePath: string,
127+
outputFilePath: string,
128+
summariesFilePath?: string
129+
) {
130+
const markdownContent = generateDocsMarkdown(inputFilePath, summariesFilePath)
131+
writeFileSync(outputFilePath, markdownContent)
132+
}
133+
134+
export { generateLLMSTxt }

0 commit comments

Comments
 (0)