The code is a JavaScript part of a larger project that manages and exports notebooks and functions, with the exportAll
function being the main entry point. It generates an index page and a search page, but several TODO comments indicate areas that need to be implemented or improved.
npm run import -- "export documentation"
const fs = require('fs')
const path = require('path')
const { functionCache } = importer.import("cache rpc functions with llm descriptions")
const { storeAllLlamaFunctions } = importer.import("store all notebook llm functions")
const { listInProject } = importer.import("list project files");
const { askLlamaAboutCode } = importer.import("ask llm about code")
const { askLlamaToSummerize, askLlamaToGeneralize } = importer.import("ask llm to summerize")
const { cacheCells } = importer.import("cache notebook")
const { safeurl } = importer.import("domain cache tools")
const { Remarkable } = require('remarkable');
const md = new Remarkable({ html: true, xhtmlOut: true, breaks: true });
const PROJECT_PATH = path.resolve(path.join(path.dirname(__dirname), 'docs'))
const INTERPRET = {}
async function exportAll(query) {
// await storeAllLlamaFunctions()
if (!fs.existsSync(PROJECT_PATH)) {
fs.mkdirSync(PROJECT_PATH)
}
// TODO: get a list of all notebooks
let notebooks = listInProject(path.resolve(__dirname, '../'), '{,*,*/,*/*/*,*/*/*/*}*.ipynb')
let languageTemplate = importer.interpret('highlight js')
let javadocTemplate = importer.interpret('javadoc template')
// TODO: generate an index page that describes the index.js and __init__.py loaders
let results = {}
const search = path.join(PROJECT_PATH, 'search.html')
results[search] = {
output: '',
indexOutput: '',
title: 'Search',
parent: '<a href="./index.html">Home</a>',
next: '',
prev: ''
}
results[search].index = search
results[search].output += '<h1>Search</h1>\n\n'
const intro = path.join(PROJECT_PATH, 'index.html')
results[intro] = {
output: '',
indexOutput: '',
title: 'Categories',
parent: '',
next: '',
prev: ''
}
results[intro].output += '<h1>Categories</h1>\n\n'
let prevCategory = ''
let prevIndex = ''
let namespaces = {}
for (let i = 0; i < notebooks.length; i++) {
// TODO: output folders and pages for every namespace and notebook
const relative = path.relative(path.resolve(__dirname, '../'), notebooks[i])
let category = relative.split(path.sep)[0]
if (category.endsWith('.ipynb')) {
category = 'Introduction'
}
if (!fs.existsSync(path.join(PROJECT_PATH, category))) {
fs.mkdirSync(path.join(PROJECT_PATH, category))
}
const catpath = path.join(PROJECT_PATH, category, 'index.html')
if (typeof results[catpath] == 'undefined') {
prevIndex = '' // start fresh with nav
results[intro].output += '<a href="./' + category + '/index.html">' + category + '</a>' + '\n<br /><br />\n'
results[intro].indexOutput += '<a href="../' + category + '/index.html">' + category + '</a>' + '\n<br /><br />\n'
results[search].indexOutput += '<h3><a href="./' + category + '/index.html">' + category + '</a></h3>'
results[catpath] = {
output: '',
indexOutput: '',
title: category,
next: '',
prev: ''
}
results[catpath].index = intro
results[catpath].parent = '<a href="../index.html">Categories</a>'
results[catpath].output += '<h1>' + category + '</h1>\n\n'
results[search].output += '<h2>' + category + '</h2>\n\n'
results[catpath].link = '<a href="../' + category + '/index.html">' + category + '</a>'
if (i > 0 && prevCategory) {
results[prevCategory].next = results[catpath].link
results[catpath].prev = results[prevCategory].link
}
prevCategory = catpath
}
const namespace = path.basename(notebooks[i]).replace('.ipynb', '')
const fullpath = path.join(PROJECT_PATH, category, safeurl(namespace))
if (!fs.existsSync(fullpath)) {
fs.mkdirSync(fullpath)
}
results[catpath].output += '<a href="./' + safeurl(namespace) + '/index.html">' + namespace + '</a>' + '\n<br /><br />\n'
// used by the next level down
results[catpath].indexOutput += '<a href="../' + safeurl(namespace) + '/index.html">' + namespace + '</a>' + '\n<br /><br />\n'
const index = path.join(fullpath, 'index.html')
results[index] = {
output: '',
indexOutput: '',
title: namespace,
next: '',
prev: ''
}
results[index].index = catpath
results[index].parent = '<a href="../../' + category + '/index.html">' + category + '</a>'
results[index].output += '<h1>' + namespace + '</h1>\n\n'
results[search].output += '<h3>' + namespace + '</h3>\n\n'
results[search].indexOutput += '<h4><a href="./' + category + '/' + safeurl(namespace) + '/index.html">' + namespace + '</a></h4>'
results[index].link = '<a href="../' + safeurl(namespace) + '/index.html">' + namespace + '</a>'
if (i > 0 && prevIndex) {
results[prevIndex].next = results[index].link
results[index].prev = results[prevIndex].link
}
prevIndex = index
namespaces[relative] = []
const newCache = cacheCells(notebooks[i])
let prevKey = ''
for (let j = 0; j < newCache.length; j++) {
let cellname = newCache[j].questions[0]
if (!cellname) {
cellname = 'cell_' + j
} else {
cellname = safeurl(cellname)
}
const key = path.join(fullpath, cellname + '.html')
results[key] = {
output: '\n\n<pre class="' + newCache[j].language + '"><code>' + newCache[j].code.replaceAll('&', '&').replaceAll('<', '<').replaceAll('>', '>').replaceAll(/(import_notebook|importNotebook|import)\([\\\n\r\[\]\s'"]*([\s\S]*?)[\\\n\r\]\[\s'"]*\)/gi, (match, ...args) => {
let searches = args[1].split(/[\\\n\r\s'"]*,[\\\n\r\s'"]*/gi)
let links = []
for(let k = 0; k < searches.length; k++) {
try {
if(!searches[k].trim()) continue
let cell
if(typeof INTERPRET[searches[k]] != 'undefined') {
cell = INTERPRET[searches[k]]
} else {
console.log('Searching: ', searches[k])
cell = importer.interpret(searches[k])
INTERPRET[searches[k]] = cell
}
if(!cell) continue
const relative = path.relative(path.resolve(__dirname, '../'), cell.filename)
links.push('"<a href="../../' + relative.split(path.sep)[0]
+ '/' + safeurl(path.basename(cell.filename).replace('.ipynb', ''))
+ '/' + safeurl(cell.questions[0]) + '.html">' + searches[k] + '</a>"')
} catch(e) {}
}
return args[0] + '(' + links.join(',\n') + ')'
}) + '</code></pre>\n\n'
}
results[key].output += languageTemplate.code.replaceAll('${LANGUAGE}', newCache[j].language)
let title = newCache[j].questions[0]
if (!title) {
title = 'Cell ' + j
}
results[key].output = '<h1>' + title + '</h1>\n\n' + results[key].output
if (newCache[j].questions[0])
results[key].output = '<h2>Run example</h2>\n\n<pre language="bash"><code>npm run import -- "' + newCache[j].questions[0] + '"</code></pre>' + results[key].output
results[key].index = index
results[key].title = title
results[key].link = '<a href="./' + cellname + '.html">' + title + '</a>'
if (j > 0) {
results[prevKey].next = results[key].link
results[key].prev = results[prevKey].link
} else {
results[key].prev = ''
}
if (j == newCache.length - 1) {
results[key].next = ''
}
results[key].parent = '<a href="../' + safeurl(namespace) + '/index.html">' + namespace + '</a>'
results[search].output += '<a href="./' + category + '/' + safeurl(namespace) + '/' + cellname + '.html">' + title + '</a>' + '\n<br /><br />\n'
results[search].indexOutput += '<a href="./' + category + '/' + safeurl(namespace) + '/' + cellname + '.html">' + title + '</a>' + '\n<br /><br />\n'
results[index].output += '<a href="./' + cellname + '.html">' + title + '</a>' + '\n<br /><br />\n'
results[index].indexOutput += '<a href="./' + cellname + '.html">' + title + '</a>' + '\n<br /><br />\n'
prevKey = key
const description = functionCache[notebooks[i] + '[' + j + ']']
if (!description) {
continue
}
if(cellname.match('ask_llm_for_a_shorter_list_of_categories')) {
debugger
}
results[search].output += md.render(description.summary.replaceAll(/^.*?summary.*?:/gmi, ''))
results[index].output += md.render(description.summary.replaceAll(/^.*?summary.*?:/gmi, ''))
if(description.amazing) {
let codeMatch = description.amazing.replace(/^```(markdown)\n*|\n```$/gi, '')
codeMatch = (/```(javascript|python|bash|[a-z0-9]+)\n*([\s\S]*?)\n```/gi).exec(codeMatch)
if(codeMatch) {
results[key].output += '<div class="gold"><h2>What the code could have been:</h2>\n<pre class="' + codeMatch[1] + '"><code>' + codeMatch[2] + '</code></pre></div>'
} else {
results[key].output += '<div class="gold"><h2>What the code could have been:</h2>\n<pre class="' + newCache[j].language + '"><code>' + description.amazing.replaceAll('&', '&').replaceAll('<', '<').replaceAll('>', '>') + '</code></pre></div>'
}
}
results[key].output = md.render(description.summary.replaceAll(/^.*?summary.*?:/gmi, '')) + results[key].output
results[key].output += md.render(description.description)
}
// TODO: generate parameter, return type descriptions
}
// TODO: export all the markdown
const paths = Object.keys(results)
for (let i = 0; i < paths.length; i++) {
// TODO: add navigation and full template
let output = javadocTemplate.code
output = output.replaceAll('${TITLE}', results[paths[i]].title)
output = output.replaceAll('${PARENT}', results[paths[i]].parent)
output = output.replaceAll('${NEXT}', results[paths[i]].next)
output = output.replaceAll('${PREV}', results[paths[i]].prev)
const relative = path.relative(path.resolve(__dirname, '../docs'), paths[i])
const searchRelative = path.relative(path.dirname(relative), 'search.html')
const pluginRelative = path.relative(path.dirname(relative), 'mergehtml.js')
output = output.replaceAll('${SEARCH}', '<a href="' + searchRelative + '">Search</a>')
if (results[paths[i]].index)
output = output.replaceAll('${CLASSES}', results[results[paths[i]].index].indexOutput)
else
output = output.replaceAll('${CLASSES}', '')
// do this last because it should already the necessary replacements
output = output.replaceAll('${OUTPUT}', results[paths[i]].output)
output = output.replaceAll('../../mergehtml.js', pluginRelative)
fs.writeFileSync(paths[i], output)
}
}
module.exports = exportAll
// Import required modules
const fs = require('fs');
const path = require('path');
const importer = require('./importer'); // assuming importer module is in the same directory
const Remarkable = require('remarkable');
const md = new Remarkable({ html: true, xhtmlOut: true, breaks: true });
// Define constants
const PROJECT_PATH = path.resolve(path.join(path.dirname(__dirname), 'docs'));
const TEMPLATE_PATH = path.resolve(__dirname, 'templates');
const LANGUAGE_TEMPLATE_FILE = 'highlight.js';
const JAVADOC_TEMPLATE_FILE = 'javadoc-template.js';
const CACHE_FUNCTIONS_FILE = 'cache-rpc-functions-with-llm-descriptions.js';
// Define template variables
const languageTemplate = importer.interpret(LANGUAGE_TEMPLATE_FILE);
const javadocTemplate = importer.interpret(JAVADOC_TEMPLATE_FILE);
// Define cache variables
const cacheFunctions = importer.import(CACHE_FUNCTIONS_FILE);
// Define function to export all notebooks
async function exportAll() {
// Create project directory if it doesn't exist
if (!fs.existsSync(PROJECT_PATH)) {
fs.mkdirSync(PROJECT_PATH);
}
// Get list of all notebooks
const notebooks = await listInProject(path.resolve(__dirname, '../'), '{,*,*/,*/*/*,*/*/*/*}*.ipynb');
// Initialize variables to store results
const results = {};
// Loop through each notebook
for (const notebook of notebooks) {
// Get namespace and category from notebook path
const relative = path.relative(path.resolve(__dirname, '../'), notebook);
const category = relative.split(path.sep)[0];
const namespace = path.basename(notebook).replace('.ipynb', '');
// Create category directory if it doesn't exist
if (!fs.existsSync(path.join(PROJECT_PATH, category))) {
fs.mkdirSync(path.join(PROJECT_PATH, category));
}
// Create index page for category
const categoryIndex = path.join(PROJECT_PATH, category, 'index.html');
results[categoryIndex] = {
output: '',
indexOutput: '',
title: category,
next: '',
prev: '',
};
results[categoryIndex].index = categoryIndex;
results[categoryIndex].output += '' + category + '
\n\n';
results[categoryIndex].link = '' + category + '';
// Loop through each cell in notebook
for (const cell of cacheCells(notebook)) {
// Create cell index page
const cellIndex = path.join(path.join(PROJECT_PATH, category, namespace), 'index.html');
results[cellIndex] = {
output: '',
indexOutput: '',
title: cell.questions[0],
next: '',
prev: '',
};
results[cellIndex].index = cellIndex;
results[cellIndex].parent = '' + namespace + '';
results[cellIndex].output += '' + cell.questions[0] + '
\n\n';
// Add cell to category index
results[categoryIndex].output += '' + cell.questions[0] + '\n\n';
// Create cell HTML page
const cellHtml = path.join(path.join(PROJECT_PATH, category, namespace), safeurl(cell.questions[0]) + '.html');
results[cellHtml] = {
output: '',
indexOutput: '',
title: cell.questions[0],
next: '',
prev: '',
};
results[cellHtml].index = cellIndex;
results[cellHtml].parent = '' + namespace + '';
results[cellHtml].output += '' + cell.questions[0] + '
\n\n';
// Add cell to search results
results[categoryIndex].output += '' + cell.questions[0] + '\n\n';
// Render HTML page
results[cellHtml].output = languageTemplate.code.replaceAll('${LANGUAGE}', cell.language) + '\n\n';
if (cell.questions[0]) {
results[cellHtml].output = 'Run example
\n\nnpm run import -- "' + cell.questions[0] + '"
' + results[cellHtml].output;
}
results[cellHtml].output += md.render(cell.description);
fs.writeFileSync(cellHtml, results[cellHtml].output);
}
}
// Export all HTML pages
const pathList = Object.keys(results);
for (const path of pathList) {
const relative = path.relative(path.resolve(__dirname, '../docs'), path);
const template = importer.interpret('mergehtml.js');
const output = template.code
.replaceAll('${TITLE}', results[path].title)
.replaceAll('${PARENT}', results[path].parent)
.replaceAll('${NEXT}', results[path].next)
.replaceAll('${PREV}', results[path].prev)
.replaceAll('${OUTPUT}', results[path].output)
.replaceAll('${SEARCH}', 'Search');
fs.writeFileSync(path, output);
}
}
// Call exportAll function
exportAll();
Breakdown of Code Structure
The code is written in JavaScript and appears to be a part of a larger project that manages and exports notebooks and functions.
The code starts with a series of import statements that bring in various functions and modules from other parts of the project. These imports include:
fs
(File System) for interacting with the file system.path
for working with file paths.importer
module, which seems to be a custom module for importing and managing functions from other parts of the project. These functions include:
functionCache
storeAllLlamaFunctions
listInProject
askLlamaAboutCode
askLlamaToSummerize
and askLlamaToGeneralize
cacheCells
safeurl
Remarkable
for working with Markdown documents.The code defines several constants and variables:
PROJECT_PATH
: the path to the project directory.INTERPRET
: an empty object that is not used anywhere in the code.md
: an instance of the Remarkable
class for working with Markdown documents.exportAll
FunctionThe exportAll
function is an asynchronous function that appears to be the main entry point of the code. It takes a query
parameter that is not used anywhere in the function.
The function does the following:
listInProject
function.The code contains several TODO comments that indicate areas that need to be implemented or improved. These include:
The code appears to be a part of a larger project that manages and exports notebooks and functions. The exportAll
function is the main entry point, and it sets up some variables and generates an index page and a search page. The code also contains several TODO comments that indicate areas that need to be implemented or improved.