The code imports necessary modules, defines constants and variables, and creates LLaMA-related objects for initializing and interacting with a large language model. It includes two main functions: createSession for creating a new model instance and initSession for initializing a new chat session with the model.

Run example

npm run import -- "create llm session"

create llm session

import path from "path"
import {getLlama, LlamaChatSession} from "node-llama-cpp"
import process from "process"

const HOMEPATH = process.env.HOME || process.env.HOMEPATH || process.env.USERPROFILE
//const __dirname = path.dirname(fileURLToPath(import.meta.url));

let llama
let model
let context
let session
let initialChatHistory


async function initSession(context2, prompt) {
  if(!context2) {
    context = await model.createContext()
  } else {
    context = context2
  }

  session = new LlamaChatSession({
      contextSequence: context.getSequence(),
      systemPrompt: prompt ? prompt : '```markdown\n# You are a\n##large language model\nnamed Llama that provides clear and concise answers in beautifully crafted `markdown` unless otherwise instructed.\n</think>\n...```\n'
  })
  // initialize the model
  //console.log(await session.prompt())
  initialChatHistory = session.getChatHistory();

  // Reset the chat history
  session.setChatHistory(initialChatHistory);
  return session
}

async function createSession(modelPath) {
  if(!llama) {
    llama = await getLlama();
  }

  // TODO: customizable model here

  if(!model) {
      model = await llama.loadModel({
          // "code-llama-7b-chat.gguf"
          // "Meta-Llama-3.1-8B-Instruct.Q4_K_M.gguf"
          // 'ggml-model-f16.gguf'
          //modelPath: path.join(HOMEPATH, "llama.cpp", "models", 'ggml-model-f16.gguf' ),
          //modelPath: path.join(HOMEPATH, "llama.cpp", "models", 'DeepSeek-R1-Distill-Llama-8B-Q6_K.gguf' ),
          //modelPath: path.join(HOMEPATH, "llama.cpp", "models", 'gemma-2-9b-it-Q6_K-Q8.gguf' ),
          //modelPath: path.join(HOMEPATH, "llama.cpp", "models", 'deepseek-llm-7b-chat.Q5_K_M.gguf' ),
          //modelPath: path.join(HOMEPATH, "llama.cpp", "models", 'Meta-Llama-3.1-70B-Instruct-Q5_K_S.gguf' ),
          modelPath: process.env.MODELPATH || modelPath || path.join(HOMEPATH, "llama.cpp", "models", 'Meta-Llama-3.1-8B-Instruct-Q6_K_L.gguf' ),
          //modelPath: path.join(HOMEPATH, "llama.cpp", "models", 'deepseek-llm-7b-chat.Q6_K.gguf' ),
          contextSize: 2048
      })
  }

  await initSession()

  return session
}

async function getSession(model) {
  if(!session) {
    await createSession(model)
  }
  return session
}

async function llmPrompt(prompt, model) {
  if(!model || typeof model == 'string') {
    // specify model with the incoming session
    model = await getSession(model)
  }
  let result = await model.prompt(prompt, {
    temperature: 0.8,
    maxTokens: context.contextSize,
    onTextChunk: function (text) {
      process.stdout.write(text)
    }
  })
  // special fix for deepseek R1 distill
  if(result.match(/Find the derivative/gi)) {
    console.log('Response error', result)
    await initSession()
    result = await session.prompt(prompt, {
      temperature: 0.8,
      maxTokens: context.contextSize,
      onTextChunk: function (text) {
        process.stdout.write(text)
      }
    })
  }
  //if(result.includes('</think>')) {
  //  result = result.split('</think>\n')[1]
  //}
  if(result.startsWith('```') && result.endsWith('```')) {
    result = result.replace(/^```(markdown)\n*|\n```$/gi, '')
  }
  if(session == model)
    model.setChatHistory(initialChatHistory);
  return result
}

export default {
  llmPrompt,
  createSession,
  initSession,
  getSession,
}

import path from "path";
import { getLlama, LlamaChatSession } from "node-llama-cpp";
import process from "process";

const HOMEPATH = process.env.HOME || process.env.HOMEPATH || process.env.USERPROFILE;

class Llama {
  constructor() {
    this.llama = null;
    this.model = null;
    this.context = null;
    this.session = null;
    this.initialChatHistory = null;
    this.modelPath = process.env.MODELPATH || path.join(HOMEPATH, "llama.cpp", "models", 'Meta-Llama-3.1-8B-Instruct-Q6_K_L.gguf');
  }

  async initSession(context = null, prompt = '# You are a\n##large language model\nnamed Llama that provides clear and concise answers in beautifully crafted `markdown` unless otherwise instructed.\n\n...') {
    this.context = context? context : await this.model.createContext();
    this.session = new LlamaChatSession({
      contextSequence: this.context.getSequence(),
      systemPrompt: prompt,
    });
    this.initialChatHistory = this.session.getChatHistory();
    this.session.setChatHistory(this.initialChatHistory);
    return this.session;
  }

  async createSession() {
    if (!this.llama) {
      this.llama = await getLlama();
    }

    if (!this.model) {
      this.model = await this.llama.loadModel({
        modelPath: this.modelPath,
        contextSize: 2048,
      });
    }

    await this.initSession();

    return this.session;
  }

  async getSession() {
    if (!this.session) {
      await this.createSession();
    }
    return this.session;
  }

  async llmPrompt(prompt, model = null) {
    if (!model) {
      model = await this.getSession();
    }
    if (model && typeof model ==='string') {
      model = await this.getSession(model);
    }
    let result = await model.prompt(prompt, {
      temperature: 0.8,
      maxTokens: this.context.contextSize,
      onTextChunk: (text) => process.stdout.write(text),
    });

    if (result.includes('```markdown') && result.endsWith('```')) {
      result = result.replace(/^```(markdown)*\n*|\n*```$/gi, '');
    }

    if (result.startsWith('Response error')) {
      await this.initSession();
      result = await this.session.prompt(prompt, {
        temperature: 0.8,
        maxTokens: this.context.contextSize,
        onTextChunk: (text) => process.stdout.write(text),
      });
    }

    this.model.setChatHistory(this.initialChatHistory);
    return result;
  }
}

export default new Llama({
  llmPrompt,
  createSession,
  initSession,
  getSession,
});

Code Breakdown

Import Statements

The code imports the following modules:

• path from the Node.js path module for working with file paths.
• getLlama and LlamaChatSession from the node-llama-cpp module, which is a Node.js wrapper for the LLaMA (Large Language Model Application) C++ API.
• process from the Node.js process module, which provides access to environment variables.

Constants and Variables

The code defines the following constants and variables:

• HOMEPATH: a string representing the user's home directory, determined by checking the HOME, HOMEPATH, and USERPROFILE environment variables.
• llama, model, context, session, and initialChatHistory: variables that will store instances of LLaMA-related objects.

Functions

The code defines the following functions:

initSession(context2, prompt)

An asynchronous function that initializes a new chat session with the LLaMA model:

• If context2 is provided, it uses that context. Otherwise, it creates a new context using the createContext method of the model.
• It creates a new LlamaChatSession instance with the context sequence and a system prompt (if provided).
• It initializes the model and gets the initial chat history.
• It resets the chat history and returns the session.

createSession(modelPath)

An asynchronous function that creates a new LLaMA model instance:

• If the llama instance is not already created, it creates a new instance using the getLlama function.
• It loads a model from the specified modelPath using the loadModel method of the llama instance.
• It sets the model variable to the loaded model instance.
• It returns the model instance.

Notes

The code appears to be part of a larger program that uses the LLaMA C++ API to interact with a large language model. The createSession function seems to be the entry point for creating a new model instance, while the initSession function is used to initialize a new chat session with the model.