The code imports necessary modules and functions, defines constants for file system paths and regular expression patterns, and defines a function cellNeedsTidying to check if a cell in a chat log needs tidying. The cellNeedsTidying function checks five conditions, including the existence of certain properties like emotions, date, and dat, to determine if the cell needs tidying.

Run example

npm run import -- "scan chat logs"

scan chat logs

//import {fileURLToPath} from "url";
const fs = require('fs')
const path = require('path')
const {askLlamaAboutEmotions} = importer.import("ask llm about emotions")
const {storeChatHistory} = importer.import("how to cache chat logs")
//const { chatCache } = importer.import("cache chat history with llm descriptions")
const { askLlamaAboutConversation, askLlamaAboutCategory } = importer.import("ask llm about chat conversations")

const hashCode = s => s.split('').reduce((a,b) => (((a << 5) - a) + b.charCodeAt(0))|0, 0)

const CHAT_DIRECTORIES = [
  '/Volumes/External/Personal/Collections/conversations',
  '/Volumes/External/Personal/Collections/conversations/Trillian/logs/AIM/Query',
  '/Volumes/External/Personal/Collections/conversations/Trillian/logs/MSN/Query',
  '/Volumes/External/Personal/Collections/conversations/Trillian/logs/old logs/AIM',
  '/Volumes/External/Personal/Collections/conversations/Trillian/logs/old logs/MSN',
  '/Volumes/External/Personal/Collections/conversations/Trillian/logs/older logs/AIM',
  '/Volumes/External/Personal/Collections/conversations/Trillian/logs/older logs/MSN',
]

const CHAT_DATES = [
  /(January|February|March|April|May|June|July|August|September|October|November|December) (\d{1,2}), \d{4}/gi,
  /(0[1-9]|[12]\d|3[01])\.(0[1-9]|1[0-2])\.\d{4}/gi,
  /(0[1-9]|1[0-2])\/(0[1-9]|[12]\d|3[01])\/\d{4}/gi,
  /\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])/gi,
  /(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) (\d{1,2}), \d{4}/gi
]

const CHAT_TIMES = [
  /(0[1-9]|1[0-2]):([0-5]\d):([0-5]\d)\s?(AM|PM)/gi,
  /([01]\d|2[0-3]):([0-5]\d):([0-5]\d)/gi,
  /(0[1-9]|1[0-2]):([0-5]\d)\s?(AM|PM)/gi,
  /([01]\d|2[0-3]):([0-5]\d)/gi
]

const CHAT_PATH = path.join(__dirname, '..', 'Resources', 'Projects', 'conversations')

function cellNeedsTidying(cellId, chatCache) {
  return typeof chatCache[cellId] == 'undefined'
  || typeof chatCache[cellId].emotions == 'undefined'
  || typeof chatCache[cellId].category != 'string'
  || typeof chatCache[cellId].date == 'undefined'
  || !chatCache[cellId].date
  || chatCache[cellId].summary.match(/Find the derivative/gi)
  || chatCache[cellId].category == '<h1>'
  || typeof chatCache[cellId].from == 'undefined' || typeof chatCache[cellId].to == 'undefined'
}

function convertMessagesToNoDates(currentMessages) {
  let messageNoTimes = currentMessages
  if(typeof currentMessages != 'string') {
    messageNoTimes = messageNoTimes.join('\n').trim()
  }
  for(let i = 0; i < CHAT_DATES.length; i++) {
    messageNoTimes = messageNoTimes.replace(CHAT_DATES[i], '')
  }
  for(let i = 0; i < CHAT_TIMES.length; i++) {
    messageNoTimes = messageNoTimes.replace(CHAT_TIMES[i], '')
  }
  messageNoTimes = messageNoTimes.replace(/\[\s*\]\s*/gi, '')
  return messageNoTimes.split('\n')
}

async function askLlamaAboutChatLog(chatLog) {

  if(!chatLog) {
    chatLog = 'fuji897@hotmail.com'
  }

  let chatHistory = []
  let chatPath
  for(let i = 0; i < CHAT_DIRECTORIES.length; i++) {
    chatPath = path.join(CHAT_DIRECTORIES[i], chatLog + '.log')
    if(fs.existsSync(chatPath)) {
      chatHistory = fs.readFileSync(chatPath).toLocaleString('utf-8').split('\n')
      break
    } else {
      chatPath = void 0
    }
  }
  if(!chatPath) return

  let chatCacheFile = path.join(CHAT_PATH, path.basename(chatPath).replace('.log', '') + '.json')
  let chatCache = {}
  if(fs.existsSync(chatCacheFile))
    chatCache = JSON.parse(fs.readFileSync(chatCacheFile).toString('utf-8'))

  let from = 0
  let mtime = fs.statSync(chatPath).mtime.getTime()
  let previousTime = mtime
  let cellCounter = 0
  let currentMessages = []
  let currentTotal = 0
  let cellId = chatPath + '[' + cellCounter + ']'
  for(let i = 0; i < chatHistory.length; i++) {
    if(currentMessages.length > 0 && currentTotal + chatHistory[i].length > 2048 || currentMessages.length == 20) {
      // process now
      // TODO: store result
      let messageBlock = currentMessages.join('\n')
      let messageNoTimes = convertMessagesToNoDates(currentMessages)
      let date = CHAT_DATES.map(d => (messageBlock.match(d) || [])[0]).filter(d=>d)[0]
      let time = CHAT_TIMES.map(d => (messageBlock.match(d) || [])[0]).filter(d=>d)[0]
      let now = new Date(date + ' ' + time).getTime() || previousTime
      if(date && time)
        previousTime = now
      let hash = hashCode(messageBlock)
      if(cellNeedsTidying(cellId, chatCache) || chatCache[cellId].hash != hash) {
        let summary = await askLlamaAboutConversation(messageNoTimes)
        let emotions = await askLlamaAboutEmotions(messageNoTimes)
        let category = await askLlamaAboutCategory(messageNoTimes)
        storeChatHistory(cellId, mtime, summary, emotions, category, hash, now, from, i - 1)
      }
      currentMessages = []
      currentTotal = 0
      from = i
      cellCounter++
      cellId = chatPath + '[' + cellCounter + ']'
    }

    if(chatHistory[i].trim().length == 0) continue

    currentMessages[currentMessages.length] = chatHistory[i]
    currentTotal += chatHistory[i].length
  }
  if(currentMessages.length > 0) {
    // process now
    let messageBlock = currentMessages.join('\n')
    let messageNoTimes = convertMessagesToNoDates(currentMessages)
    let date = CHAT_DATES.map(d => (messageBlock.match(d) || [])[0]).filter(d=>d)[0]
    let time = CHAT_TIMES.map(d => (messageBlock.match(d) || [])[0]).filter(d=>d)[0]
    let now = new Date(date + ' ' + time).getTime()
    let hash = hashCode(messageBlock)
    if(cellNeedsTidying(cellId, chatCache) || chatCache[cellId].hash != hash) {
      let summary = await askLlamaAboutConversation(messageNoTimes)
      let emotions = await askLlamaAboutEmotions(messageNoTimes)
      let category = await askLlamaAboutCategory(messageNoTimes)
      storeChatHistory(cellId, mtime, summary, emotions, category, hash, now, from, chatHistory.length - 1)
    }
    // TODO: store result
  }

  
}

module.exports = {
  askLlamaAboutChatLog,
  askLlamaAboutConversation,
  convertMessagesToNoDates,
  CHAT_DIRECTORIES
}

// Import required modules
const fs = require('fs');
const path = require('path');
const importer = require('./importer');
const { askLlamaAboutEmotions, askLlamaAboutConversation, askLlamaAboutCategory } = importer.import([
  'ask llm about emotions',
  'ask llm about chat conversations',
  'ask llm about chat categories',
]);
const { storeChatHistory } = importer.import('store chat history');

// Define regular expressions for chat dates and times
const CHAT_DATES = [
  /(\w+) (\d{1,2}), \d{4}/gi,
  /\d{2}(\.|\/)(\d{2})\.(?:\d{2}|\d{4})/gi,
  /(\d{2})\/(\d{2})\/\d{4}/gi,
  /\d{4}-(\d{2})-(\d{2})/gi,
  /(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) (\d{1,2}), \d{4}/gi,
];

const CHAT_TIMES = [
  /(0[1-9]|1[0-2]):([0-5]\d):([0-5]\d)\s?(AM|PM)/gi,
  /([01]\d|2[0-3]):([0-5]\d):([0-5]\d)/gi,
  /(0[1-9]|1[0-2]):([0-5]\d)\s?(AM|PM)/gi,
  /([01]\d|2[0-3]):([0-5]\d)/gi,
];

// Define chat directories and resource path
const CHAT_DIRECTORIES = [
  '/Volumes/External/Personal/Collections/conversations',
  '/Volumes/External/Personal/Collections/conversations/Trillian/logs/AIM/Query',
  '/Volumes/External/Personal/Collections/conversations/Trillian/logs/MSN/Query',
  '/Volumes/External/Personal/Collections/conversations/Trillian/logs/old logs/AIM',
  '/Volumes/External/Personal/Collections/conversations/Trillian/logs/old logs/MSN',
  '/Volumes/External/Personal/Collections/conversations/Trillian/logs/older logs/AIM',
  '/Volumes/External/Personal/Collections/conversations/Trillian/logs/older logs/MSN',
];

const CHAT_RESOURCE_PATH = path.join(__dirname, '..', 'Resources', 'Projects', 'conversations');

// Define a function to check if a cell needs tidying
function cellNeedsTidying(cellId, chatCache) {
  return (
   !chatCache[cellId] ||
   !chatCache[cellId].emotions ||
    typeof chatCache[cellId].category!=='string' ||
   !chatCache[cellId].date ||
    chatCache[cellId].summary.match(/Find the derivative/gi) ||
    chatCache[cellId].category === '
' ||
   !chatCache[cellId].from ||!chatCache[cellId].to
  );
}

// Define a function to convert messages to no dates
function convertMessagesToNoDates(messageBlock) {
  const messageNoTimes = messageBlock
   .toString()
   .replace(/(\d{2}\/\d{2}\/\d{4}|\d{4}\/\d{2}\/\d{2}|\d{2}\/\d{2}\/\d{4}|\d{4}-\d{2}-\d{2})/gi, '')
   .replace(/\[\s*\]\s*/gi, '')
   .split('\n')
   .filter((item) => item.trim().length > 0);
  return messageNoTimes;
}

// Define a function to get chat log information
async function getChatLogInformation(chatLog) {
  if (!chatLog) {
    chatLog = 'fuji897@hotmail.com';
  }

  const chatPath = CHAT_DIRECTORIES.find((directory) => {
    const fullPath = path.join(directory, chatLog + '.log');
    return fs.existsSync(fullPath);
  });

  if (!chatPath) {
    return null;
  }

  const chatHistory = fs.readFileSync(path.join(chatPath, chatLog + '.log'), 'utf8').split('\n');
  const chatCacheFile = path.join(CHAT_RESOURCE_PATH, path.basename(chatPath).replace('.log', '') + '.json');
  const chatCache = fs.existsSync(chatCacheFile)? JSON.parse(fs.readFileSync(chatCacheFile, 'utf8')) : {};

  return { chatHistory, chatCache, chatPath };
}

// Define the main function to ask LLaMA about a chat log
async function askLlamaAboutChatLog(chatLog) {
  const { chatHistory, chatCache, chatPath } = await getChatLogInformation(chatLog);

  if (!chatHistory) {
    return;
  }

  const from = 0;
  let mtime = fs.statSync(path.join(chatPath, chatLog + '.log')).mtime.getTime();
  const previousTime = mtime;
  const cellCounter = 0;
  const currentMessages = [];
  const currentTotal = 0;
  let cellId = chatPath + '[' + cellCounter + ']';

  for (let i = 0; i < chatHistory.length; i++) {
    if (
      currentMessages.length > 0 &&
      currentTotal + chatHistory[i].length > 2048 ||
      currentMessages.length === 20
    ) {
      const messageBlock = currentMessages.join('\n');
      const messageNoTimes = convertMessagesToNoDates(messageBlock);
      const date = CHAT_DATES
       .map((d) => (messageBlock.match(d) || [])[0])
       .filter((d) => d)[0];
      const time = CHAT_TIMES
       .map((d) => (messageBlock.match(d) || [])[0])
       .filter((d) => d)[0];

      const now = new Date(date +'' + time).getTime() || previousTime;
      if (date && time) {
        previousTime = now;
      }

      const hash = hashCode(messageBlock);

      if (cellNeedsTidying(cellId, chatCache) || chatCache[cellId].hash!== hash) {
        const summary = await askLlamaAboutConversation(messageNoTimes);
        const emotions = await askLlamaAboutEmotions(messageNoTimes);
        const category = await askLlamaAboutCategory(messageNoTimes);
        storeChatHistory(
          cellId,
          mtime,
          summary,
          emotions,
          category,
          hash,
          now,
          from,
          i - 1
        );
      }

      currentMessages = [];
      currentTotal = 0;
      from = i;
      cellCounter++;
      cellId = chatPath + '[' + cellCounter + ']';
    }

    if (chatHistory[i].trim().length === 0) {
      continue;
    }

    currentMessages[currentMessages.length] = chatHistory[i];
    currentTotal += chatHistory[i].length;
  }

  if (currentMessages.length > 0) {
    const messageBlock = currentMessages.join('\n');
    const messageNoTimes = convertMessagesToNoDates(messageBlock);
    const date = CHAT_DATES
     .map((d) => (messageBlock.match(d) || [])[0])
     .filter((d) => d)[0];
    const time = CHAT_TIMES
     .map((d) => (messageBlock.match(d) || [])[0])
     .filter((d) => d)[0];

    const now = new Date(date +'' + time).getTime();
    const hash = hashCode(messageBlock);

    if (cellNeedsTidying(cellId, chatCache) || chatCache[cellId].hash!== hash) {
      const summary = await askLlamaAboutConversation(messageNoTimes);
      const emotions = await askLlamaAboutEmotions(messageNoTimes);
      const category = await askLlamaAboutCategory(messageNoTimes);
      storeChatHistory(
        cellId,
        mtime,
        summary,
        emotions,
        category,
        hash,
        now,
        from,
        chatHistory.length - 1
      );
    }
  }
}

// Define the hash code function
function hashCode(s) {
  return s
   .toString()
   .split('')
   .reduce((a, b) => ((a << 5) - a) + b.charCodeAt(0) | 0, 0);
}

module.exports = {
  askLlamaAboutChatLog,
  askLlamaAboutConversation,
  convertMessagesToNoDates,
  CHAT_DIRECTORIES,
  getChatLogInformation,
};

Breakdown of the Code

Importing Modules and Functions

• The code begins by importing necessary modules and functions:
  • fs and path from the Node.js standard library for file system operations.
  • Functions from the importer module:
    • askLlamaAboutEmotions
    • storeChatHistory
    • askLlamaAboutConversation
    • askLlamaAboutCategory
• The hashCode function is defined to generate a hash code from a string.

Defining Constants

• CHAT_DIRECTORIES: an array of file system paths to directories containing chat logs.
• CHAT_DATES: an array of regular expression patterns to match dates in different formats.
• CHAT_TIMES: an array of regular expression patterns to match times in different formats.
• CHAT_PATH: a file system path to the root directory of the conversations project.

Defining the cellNeedsTidying Function

• This function takes two arguments: cellId and chatCache.
• It returns a boolean value indicating whether the cell with the given cellId needs tidying in the chatCache object.
• The function checks the following conditions:
  • The cell is not present in the chatCache object or its value is undefined.
  • The cell's emotions property is undefined.
  • The cell's category property is not a string.
  • The cell's date property is undefined.
  • The cell's dat property is undefined (note: this condition is not documented, but it appears to be checking the existence of a dat property).