documentation | discover well-known apis | highlight js | Search

The code is a JavaScript part of a larger project that manages and exports notebooks and functions, with the exportAll function being the main entry point. It generates an index page and a search page, but several TODO comments indicate areas that need to be implemented or improved.

Run example

npm run import -- "export documentation"

export documentation

const fs = require('fs')
const path = require('path')
const { functionCache } = importer.import("cache rpc functions with llm descriptions")
const { storeAllLlamaFunctions } = importer.import("store all notebook llm functions")
const { listInProject } = importer.import("list project files");
const { askLlamaAboutCode } = importer.import("ask llm about code")
const { askLlamaToSummerize, askLlamaToGeneralize } = importer.import("ask llm to summerize")
const { cacheCells } = importer.import("cache notebook")
const { safeurl } = importer.import("domain cache tools")
const { Remarkable } = require('remarkable');
const md = new Remarkable({ html: true, xhtmlOut: true, breaks: true });

const PROJECT_PATH = path.resolve(path.join(path.dirname(__dirname), 'docs'))
const INTERPRET = {}

async function exportAll(query) {
  // await storeAllLlamaFunctions()
  if (!fs.existsSync(PROJECT_PATH)) {
    fs.mkdirSync(PROJECT_PATH)
  }

  // TODO: get a list of all notebooks
  let notebooks = listInProject(path.resolve(__dirname, '../'), '{,*,*/,*/*/*,*/*/*/*}*.ipynb')

  let languageTemplate = importer.interpret('highlight js')
  let javadocTemplate = importer.interpret('javadoc template')

  // TODO: generate an index page that describes the index.js and __init__.py loaders

  let results = {}
  const search = path.join(PROJECT_PATH, 'search.html')
  results[search] = {
    output: '',
    indexOutput: '',
    title: 'Search',
    parent: '<a href="./index.html">Home</a>',
    next: '',
    prev: ''
  }
  results[search].index = search
  results[search].output += '<h1>Search</h1>\n\n'
  const intro = path.join(PROJECT_PATH, 'index.html')
  results[intro] = {
    output: '',
    indexOutput: '',
    title: 'Categories',
    parent: '',
    next: '',
    prev: ''
  }
  results[intro].output += '<h1>Categories</h1>\n\n'

  let prevCategory = ''
  let prevIndex = ''
  let namespaces = {}
  for (let i = 0; i < notebooks.length; i++) {
    // TODO: output folders and pages for every namespace and notebook
    const relative = path.relative(path.resolve(__dirname, '../'), notebooks[i])
    let category = relative.split(path.sep)[0]
    if (category.endsWith('.ipynb')) {
      category = 'Introduction'
    }

    if (!fs.existsSync(path.join(PROJECT_PATH, category))) {
      fs.mkdirSync(path.join(PROJECT_PATH, category))
    }

    const catpath = path.join(PROJECT_PATH, category, 'index.html')
    if (typeof results[catpath] == 'undefined') {
      prevIndex = '' // start fresh with nav
      results[intro].output += '<a href="./' + category + '/index.html">' + category + '</a>' + '\n<br /><br />\n'
      results[intro].indexOutput += '<a href="../' + category + '/index.html">' + category + '</a>' + '\n<br /><br />\n'
      results[search].indexOutput += '<h3><a href="./' + category + '/index.html">' + category + '</a></h3>'
      results[catpath] = {
        output: '',
        indexOutput: '',
        title: category,
        next: '',
        prev: ''
      }
      results[catpath].index = intro
      results[catpath].parent = '<a href="../index.html">Categories</a>'
      results[catpath].output += '<h1>' + category + '</h1>\n\n'
      results[search].output += '<h2>' + category + '</h2>\n\n'
      results[catpath].link = '<a href="../' + category + '/index.html">' + category + '</a>'
      if (i > 0 && prevCategory) {
        results[prevCategory].next = results[catpath].link
        results[catpath].prev = results[prevCategory].link
      }
      prevCategory = catpath
    }


    const namespace = path.basename(notebooks[i]).replace('.ipynb', '')
    const fullpath = path.join(PROJECT_PATH, category, safeurl(namespace))
    if (!fs.existsSync(fullpath)) {
      fs.mkdirSync(fullpath)
    }

    results[catpath].output += '<a href="./' + safeurl(namespace) + '/index.html">' + namespace + '</a>' + '\n<br /><br />\n'
    // used by the next level down
    results[catpath].indexOutput += '<a href="../' + safeurl(namespace) + '/index.html">' + namespace + '</a>' + '\n<br /><br />\n'
    const index = path.join(fullpath, 'index.html')
    results[index] = {
      output: '',
      indexOutput: '',
      title: namespace,
      next: '',
      prev: ''
    }
    results[index].index = catpath
    results[index].parent = '<a href="../../' + category + '/index.html">' + category + '</a>'
    results[index].output += '<h1>' + namespace + '</h1>\n\n'
    results[search].output += '<h3>' + namespace + '</h3>\n\n'
    results[search].indexOutput += '<h4><a href="./' + category + '/' + safeurl(namespace) + '/index.html">' + namespace + '</a></h4>'
    results[index].link = '<a href="../' + safeurl(namespace) + '/index.html">' + namespace + '</a>'
    if (i > 0 && prevIndex) {
      results[prevIndex].next = results[index].link
      results[index].prev = results[prevIndex].link
    }
    prevIndex = index

    namespaces[relative] = []

    const newCache = cacheCells(notebooks[i])


    let prevKey = ''
    for (let j = 0; j < newCache.length; j++) {
      let cellname = newCache[j].questions[0]
      if (!cellname) {
        cellname = 'cell_' + j
      } else {
        cellname = safeurl(cellname)
      }
      const key = path.join(fullpath, cellname + '.html')
      results[key] = {
        output: '\n\n<pre class="' + newCache[j].language + '"><code>' + newCache[j].code.replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;').replaceAll(/(import_notebook|importNotebook|import)\([\\\n\r\[\]\s'"]*([\s\S]*?)[\\\n\r\]\[\s'"]*\)/gi, (match, ...args) => {
          let searches = args[1].split(/[\\\n\r\s'"]*,[\\\n\r\s'"]*/gi)
          let links = []
          for(let k = 0; k < searches.length; k++) {
            try {
              if(!searches[k].trim()) continue
              let cell
              if(typeof INTERPRET[searches[k]] != 'undefined') {
                cell = INTERPRET[searches[k]]
              } else {
                console.log('Searching: ', searches[k])
                cell = importer.interpret(searches[k])
                INTERPRET[searches[k]] = cell
              }
              if(!cell) continue
              const relative = path.relative(path.resolve(__dirname, '../'), cell.filename)
              links.push('"<a href="../../' + relative.split(path.sep)[0] 
                + '/' + safeurl(path.basename(cell.filename).replace('.ipynb', '')) 
                + '/' + safeurl(cell.questions[0]) + '.html">' + searches[k] + '</a>"')
            } catch(e) {}
          }
          return args[0] + '(' + links.join(',\n') + ')'
        }) + '</code></pre>\n\n'
      }
      results[key].output += languageTemplate.code.replaceAll('${LANGUAGE}', newCache[j].language)

      let title = newCache[j].questions[0]
      if (!title) {
        title = 'Cell ' + j
      }
      results[key].output = '<h1>' + title + '</h1>\n\n' + results[key].output
      if (newCache[j].questions[0])
        results[key].output = '<h2>Run example</h2>\n\n<pre language="bash"><code>npm run import -- "' + newCache[j].questions[0] + '"</code></pre>' + results[key].output
      
      results[key].index = index
      results[key].title = title
      results[key].link = '<a href="./' + cellname + '.html">' + title + '</a>'
      if (j > 0) {
        results[prevKey].next = results[key].link
        results[key].prev = results[prevKey].link
      } else {
        results[key].prev = ''
      }
      if (j == newCache.length - 1) {
        results[key].next = ''
      }

      results[key].parent = '<a href="../' + safeurl(namespace) + '/index.html">' + namespace + '</a>'
      results[search].output += '<a href="./' + category + '/' + safeurl(namespace) + '/' + cellname + '.html">' + title + '</a>' + '\n<br /><br />\n'
      results[search].indexOutput += '<a href="./' + category + '/' + safeurl(namespace) + '/' + cellname + '.html">' + title + '</a>' + '\n<br /><br />\n'
      results[index].output += '<a href="./' + cellname + '.html">' + title + '</a>' + '\n<br /><br />\n'
      results[index].indexOutput += '<a href="./' + cellname + '.html">' + title + '</a>' + '\n<br /><br />\n'
      prevKey = key

      const description = functionCache[notebooks[i] + '[' + j + ']']
      if (!description) {
        continue
      }

if(cellname.match('ask_llm_for_a_shorter_list_of_categories')) {
  debugger
}

      results[search].output += md.render(description.summary.replaceAll(/^.*?summary.*?:/gmi, ''))
      results[index].output += md.render(description.summary.replaceAll(/^.*?summary.*?:/gmi, ''))
      if(description.amazing) {
        let codeMatch = description.amazing.replace(/^```(markdown)\n*|\n```$/gi, '')
        codeMatch = (/```(javascript|python|bash|[a-z0-9]+)\n*([\s\S]*?)\n```/gi).exec(codeMatch)
        if(codeMatch) {
          results[key].output += '<div class="gold"><h2>What the code could have been:</h2>\n<pre class="' + codeMatch[1] + '"><code>' + codeMatch[2] + '</code></pre></div>'
        } else {
          results[key].output += '<div class="gold"><h2>What the code could have been:</h2>\n<pre class="' + newCache[j].language + '"><code>' + description.amazing.replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;') + '</code></pre></div>'
        }
      }
      results[key].output = md.render(description.summary.replaceAll(/^.*?summary.*?:/gmi, '')) + results[key].output
      results[key].output += md.render(description.description)
    }

    // TODO: generate parameter, return type descriptions


  }


  // TODO: export all the markdown

  const paths = Object.keys(results)
  for (let i = 0; i < paths.length; i++) {
    // TODO: add navigation and full template
    let output = javadocTemplate.code
    output = output.replaceAll('${TITLE}', results[paths[i]].title)
    output = output.replaceAll('${PARENT}', results[paths[i]].parent)
    output = output.replaceAll('${NEXT}', results[paths[i]].next)
    output = output.replaceAll('${PREV}', results[paths[i]].prev)
    const relative = path.relative(path.resolve(__dirname, '../docs'), paths[i])
    const searchRelative = path.relative(path.dirname(relative), 'search.html')
    const pluginRelative = path.relative(path.dirname(relative), 'mergehtml.js')
    output = output.replaceAll('${SEARCH}', '<a href="' + searchRelative + '">Search</a>')
    if (results[paths[i]].index)
      output = output.replaceAll('${CLASSES}', results[results[paths[i]].index].indexOutput)
    else
      output = output.replaceAll('${CLASSES}', '')
    // do this last because it should already the necessary replacements
    output = output.replaceAll('${OUTPUT}', results[paths[i]].output)
    output = output.replaceAll('../../mergehtml.js', pluginRelative)
    fs.writeFileSync(paths[i], output)
  }

}

module.exports = exportAll

What the code could have been:

// Import required modules
const fs = require('fs');
const path = require('path');
const importer = require('./importer'); // assuming importer module is in the same directory
const Remarkable = require('remarkable');
const md = new Remarkable({ html: true, xhtmlOut: true, breaks: true });

// Define constants
const PROJECT_PATH = path.resolve(path.join(path.dirname(__dirname), 'docs'));
const TEMPLATE_PATH = path.resolve(__dirname, 'templates');
const LANGUAGE_TEMPLATE_FILE = 'highlight.js';
const JAVADOC_TEMPLATE_FILE = 'javadoc-template.js';
const CACHE_FUNCTIONS_FILE = 'cache-rpc-functions-with-llm-descriptions.js';

// Define template variables
const languageTemplate = importer.interpret(LANGUAGE_TEMPLATE_FILE);
const javadocTemplate = importer.interpret(JAVADOC_TEMPLATE_FILE);

// Define cache variables
const cacheFunctions = importer.import(CACHE_FUNCTIONS_FILE);

// Define function to export all notebooks
async function exportAll() {
  // Create project directory if it doesn't exist
  if (!fs.existsSync(PROJECT_PATH)) {
    fs.mkdirSync(PROJECT_PATH);
  }

  // Get list of all notebooks
  const notebooks = await listInProject(path.resolve(__dirname, '../'), '{,*,*/,*/*/*,*/*/*/*}*.ipynb');

  // Initialize variables to store results
  const results = {};

  // Loop through each notebook
  for (const notebook of notebooks) {
    // Get namespace and category from notebook path
    const relative = path.relative(path.resolve(__dirname, '../'), notebook);
    const category = relative.split(path.sep)[0];
    const namespace = path.basename(notebook).replace('.ipynb', '');

    // Create category directory if it doesn't exist
    if (!fs.existsSync(path.join(PROJECT_PATH, category))) {
      fs.mkdirSync(path.join(PROJECT_PATH, category));
    }

    // Create index page for category
    const categoryIndex = path.join(PROJECT_PATH, category, 'index.html');
    results[categoryIndex] = {
      output: '',
      indexOutput: '',
      title: category,
      next: '',
      prev: '',
    };
    results[categoryIndex].index = categoryIndex;
    results[categoryIndex].output += '

' + category + '

\n\n'; results[categoryIndex].link = '' + category + ''; // Loop through each cell in notebook for (const cell of cacheCells(notebook)) { // Create cell index page const cellIndex = path.join(path.join(PROJECT_PATH, category, namespace), 'index.html'); results[cellIndex] = { output: '', indexOutput: '', title: cell.questions[0], next: '', prev: '', }; results[cellIndex].index = cellIndex; results[cellIndex].parent = '' + namespace + ''; results[cellIndex].output += '

' + cell.questions[0] + '

\n\n'; // Add cell to category index results[categoryIndex].output += '' + cell.questions[0] + '\n\n'; // Create cell HTML page const cellHtml = path.join(path.join(PROJECT_PATH, category, namespace), safeurl(cell.questions[0]) + '.html'); results[cellHtml] = { output: '', indexOutput: '', title: cell.questions[0], next: '', prev: '', }; results[cellHtml].index = cellIndex; results[cellHtml].parent = '' + namespace + ''; results[cellHtml].output += '

' + cell.questions[0] + '

\n\n'; // Add cell to search results results[categoryIndex].output += '' + cell.questions[0] + '\n\n'; // Render HTML page results[cellHtml].output = languageTemplate.code.replaceAll('${LANGUAGE}', cell.language) + '\n\n'; if (cell.questions[0]) { results[cellHtml].output = '

Run example

\n\n
npm run import -- "' + cell.questions[0] + '"
' + results[cellHtml].output; } results[cellHtml].output += md.render(cell.description); fs.writeFileSync(cellHtml, results[cellHtml].output); } } // Export all HTML pages const pathList = Object.keys(results); for (const path of pathList) { const relative = path.relative(path.resolve(__dirname, '../docs'), path); const template = importer.interpret('mergehtml.js'); const output = template.code .replaceAll('${TITLE}', results[path].title) .replaceAll('${PARENT}', results[path].parent) .replaceAll('${NEXT}', results[path].next) .replaceAll('${PREV}', results[path].prev) .replaceAll('${OUTPUT}', results[path].output) .replaceAll('${SEARCH}', 'Search'); fs.writeFileSync(path, output); } } // Call exportAll function exportAll();

Breakdown of Code Structure

The code is written in JavaScript and appears to be a part of a larger project that manages and exports notebooks and functions.

Import Statements

The code starts with a series of import statements that bring in various functions and modules from other parts of the project. These imports include:

Constants and Variables

The code defines several constants and variables:

exportAll Function

The exportAll function is an asynchronous function that appears to be the main entry point of the code. It takes a query parameter that is not used anywhere in the function.

The function does the following:

  1. Checks if the project directory exists and creates it if it doesn't.
  2. Lists all notebooks in the project directory using the listInProject function.
  3. Sets up some variables for generating an index page and a search page.
  4. Iterates over the list of notebooks and generates an entry for each one.

TODO Comments

The code contains several TODO comments that indicate areas that need to be implemented or improved. These include:

Overall Structure

The code appears to be a part of a larger project that manages and exports notebooks and functions. The exportAll function is the main entry point, and it sets up some variables and generates an index page and a search page. The code also contains several TODO comments that indicate areas that need to be implemented or improved.