llama vision | mask image | image 2 image | Search

The doInpaintMask function performs inpainting on an image using a provided mask and text prompt, sending a POST request to a local stable diffusion API endpoint.

Run example

npm run import -- "inpaint mask"

inpaint mask

const fs = require('fs')
const path = require('path')
const {request} = require('gaxios')

const OUTPUT_PATH = path.join(process.env.HOME || process.env.HOMEPATH || process.env.USERPROFILE, 'stable-diffusion-webui/outputs')

async function doInpaintMask(image, mask, prompt) {
  let width = 1024
  
  if(!prompt) {
    // TODO: image 2 image with ollama vision?
    return
  }

  if(prompt.includes('View360')) {
    width = 2048
  }

  let base64_image
  if(typeof image == 'string') {
    if(image.startsWith('data:image/'))
      image = image.replace(/^data:image\/.*?;base64,/gi, '')
  
    if(image.includes('://')) {
      let result = await request({
        url: image,
        method: 'GET',
      })
      base64_image = Buffer.from(await result.data.arrayBuffer()).toString('base64')
    } else if (!fs.existsSync(image)) {
      base64_image = Buffer.from(image, 'base64').toString('base64')
    } else {
      base64_image = fs.readFileSync(image).toString('base64')
    }  
  } else {
    base64_image = image.toString('base64')
  }

  let base64_mask
  if(typeof mask == 'string') {
    if(mask.startsWith('data:image/'))
      mask = mask.replace(/^data:image\/.*?;base64,/gi, '')
  
    if(mask.includes('://')) {
      let result = await request({
        url: mask,
        method: 'GET',
      })
      base64_mask = Buffer.from(await result.data.arrayBuffer()).toString('base64')
    } else if (!fs.existsSync(mask)) {
      base64_mask = Buffer.from(mask, 'base64').toString('base64')
    } else {
      base64_mask = fs.readFileSync(mask).toString('base64')
    }  
  } else {
    base64_mask = mask.toString('base64')
  }

  try {
    let result = await request({
      url: 'http://127.0.0.1:7860/sdapi/v1/img2img',
      method: 'POST',
      headers: {
        'Content-Type': 'application/json'
      },
      data: JSON.stringify({
        prompt: prompt,
        negative_prompt: 'bad hands, bad feet, bad faces, bad eyes, bad anatomy, extra limbs, missing limbs, tattoo, statue, picture frame, anime, cartoon, signature, abstract',
        save_images: true,
        "width": width,
        "height": 1024,
        "steps": 30,
        tiling: false,
        init_images: [base64_image],
        //denoising_strength: 0.35,
        mask: base64_mask,
        mask_blur: 4,
        inpainting_fill: 1,
        inpaint_full_res: false,
        inpaint_full_res_padding: 32,
        inpainting_mask_invert: 1,
      })
    })
    
    let seed = JSON.parse(result.data.info).seed
    let buff = Buffer.from(result.data.images[0], 'base64');
    let now = new Date()
    let folderName = now.getFullYear() + '-' + String(now.getMonth() + 1).padStart(2, '0') + '-' + String(now.getDate()).padStart(2, '0')
    let stablePath = path.join(OUTPUT_PATH, 'img2img-images', folderName)
    let imagePath
    if(fs.existsSync(stablePath)) {
      let images = fs.readdirSync(stablePath)
      for(let i = 0; i < images.length; i++) {
        if(images[i].match('-' + seed + '-')) {
          imagePath = path.join('img2img-images', folderName, images[i])
          break
        }
      }
    }
    return {seed, image: buff, imagePath, prompt}

  } catch (e) {
    console.error(e)
  }
}

module.exports = {
  doInpaintMask,
  OUTPUT_PATH
}

What the code could have been:

const fs = require('fs');
const path = require('path');
const axios = require('axios');

const OUTPUT_PATH = path.join(process.env.HOME || process.env.HOMEPATH || process.env.USERPROFILE,'stable-diffusion-webui/outputs');

async function doInpaintMask(image, mask, prompt) {
  const WIDTH = 1024;

  if (!prompt) {
    throw new Error('Prompt is required for image processing');
  }

  if (prompt.includes('View360')) {
    WIDTH = 2048;
  }

  async function loadImageFromUrl(url) {
    const { data } = await axios.get(url);
    return Buffer.from(data.arrayBuffer()).toString('base64');
  }

  async function loadImageFromPath(path) {
    if (!fs.existsSync(path)) {
      throw new Error('Image file not found');
    }
    return fs.readFileSync(path).toString('base64');
  }

  const base64Image = await getBase64Image(image);
  const base64Mask = await getBase64Mask(mask);

  try {
    const result = await axios.post('http://127.0.0.1:7860/sdapi/v1/img2img', {
      prompt,
      negative_prompt: 'bad hands, bad feet, bad faces, bad eyes, bad anatomy, extra limbs, missing limbs, tattoo, statue, picture frame, anime, cartoon, signature, abstract',
      save_images: true,
      width: WIDTH,
      height: 1024,
      steps: 30,
      tiling: false,
      init_images: [base64Image],
      mask: base64Mask,
      mask_blur: 4,
      inpainting_fill: 1,
      inpaint_full_res: false,
      inpaint_full_res_padding: 32,
      inpainting_mask_invert: 1,
    }, {
      headers: {
        'Content-Type': 'application/json',
      },
    });

    const seed = JSON.parse(result.data.info).seed;
    const buff = Buffer.from(result.data.images[0], 'base64');
    const now = new Date();
    const folderName = now.getFullYear() + '-' + String(now.getMonth() + 1).padStart(2, '0') + '-' + String(now.getDate()).padStart(2, '0');
    const stablePath = path.join(OUTPUT_PATH, 'img2img-images', folderName);

    if (!fs.existsSync(stablePath)) {
      fs.mkdirSync(stablePath);
    }

    let imagePath;
    const images = fs.readdirSync(stablePath);
    for (let i = 0; i < images.length; i++) {
      if (images[i].match('-' + seed + '-')) {
        imagePath = path.join('img2img-images', folderName, images[i]);
        break;
      }
    }

    return { seed, image: buff, imagePath, prompt };
  } catch (error) {
    console.error(error);
  }
}

async function getBase64Image(image) {
  if (typeof image ==='string' && image.startsWith('data:image/')) {
    return image.replace(/^data:image\/.*?;base64,/gi, '');
  } else if (typeof image ==='string' && image.includes('://')) {
    return loadImageFromUrl(image);
  } else if (typeof image ==='string') {
    return loadImageFromPath(image);
  } else {
    return image.toString('base64');
  }
}

async function getBase64Mask(mask) {
  if (typeof mask ==='string' && mask.startsWith('data:image/')) {
    return mask.replace(/^data:image\/.*?;base64,/gi, '');
  } else if (typeof mask ==='string' && mask.includes('://')) {
    return loadImageFromUrl(mask);
  } else if (typeof mask ==='string') {
    return loadImageFromPath(mask);
  } else {
    return mask.toString('base64');
  }
}

module.exports = {
  doInpaintMask,
  OUTPUT_PATH,
};

Function Breakdown: doInpaintMask

This is an asynchronous function named doInpaintMask that takes three parameters:

  1. image: The input image, which can be a string or a buffer.
  2. mask: The inpainting mask, which can be a string or a buffer.
  3. prompt: The text prompt used for the inpainting process.

Functionality

The function performs the following steps:

  1. Checks if the prompt parameter is provided. If not, it returns without performing any actions.
  2. If the prompt includes the string "View360", it sets the width variable to 2048. Otherwise, it sets it to 1024.
  3. Converts the image and mask parameters to base64 strings if they are not already in that format.
  4. If the image and mask are URLs, it downloads the image data using the gaxios library. Otherwise, it reads the image data from the local file system using the fs library.
  5. Sends a POST request to the stable diffusion API (http://127.0.0.1:7860/sdapi/v1/img2img) with the prompt, image, and mask data.
  6. Returns the result of the API request.

Notes