The code imports necessary modules and initializes a dictionary to store module instances, and then defines several functions for manipulating URLs, importing notebook modules, and building dynamic libraries. The import_notebook function imports notebook modules based on a query string and context, and uses different methods to import code depending on the language of the notebook.

Cell 1

import types
import ctypes
import os
import re

def safeurl(name):
    return re.sub(r'\W+', '_', name)

modules = {}

def import_notebook("query_str",
"ctx"):
    from Core import interpret, build_dylib
    """Searches the Whoosh index and returns a module of the retrieved code."""
    results = interpret(query_str)
    
    if not results:
        raise ImportError(f"No matching notebook found for query: {query_str}")
    
    filename = results[0]["filename"]
    fileid = results[0]["id"]
    print(filename)

    if(fileid in modules):
        if(callable(vars(modules[fileid])["__all__"])):
            name = vars(modules[fileid])["__all__"].__name__
            ctx[name] = vars(modules[fileid])["__all__"]
            globals()[name] = vars(modules[fileid])["__all__"]
        else:
            for name in vars(modules[fileid])["__all__"]:
                ctx[name] = vars(modules[fileid])["__all__"][name]
                globals()[name] = vars(modules[fileid])["__all__"][name]
        return vars(modules[fileid])["__all__"]


    module = types.ModuleType(filename)
    module.__file__ = filename
    module.import_notebook = import_notebook

    if(results[0]["language"] == "c" or results[0]["language"] == "cpp" or results[0]["language"] == "objective-c"):
        build_dylib(results[0]["code"], filename, {})
        lib_name = safeurl(results[0]["questions"][0])
        BUILD_DIRECTORY = os.path.join(os.path.dirname(__file__), "../.build")
        result = ctypes.CDLL(os.path.join(BUILD_DIRECTORY, lib_name + '.dylib'))
        import_notebook("list c functions with python",
"ctx")
        functions = get_functions(results[0]["code"])
        module.__all__ = {}
        for f in functions:
            func_name = f.split('(')[0]
            vars(module)['__all__'][func_name] = getattr(result, func_name)
        
    elif(results[0]["language"] == "python"):
        exec(results[0]["code"], module.__dict__)  # Execute the first matching cell in the module namespace

    if(callable(vars(module)['__all__'])):
        name = vars(module)["__all__"].__name__
        ctx[name] = vars(module)['__all__']
        module.__dict__[name] = vars(module)['__all__']
        globals()[name] = vars(module)['__all__']
    else:
        for name in vars(module)['__all__']:
            # setattr(vars(module)['__all__'], name, vars(module)['__all__'][name])
            ctx[name] = vars(module)['__all__'][name]
            module.__dict__[name] = vars(module)['__all__'][name]
            globals()[name] = vars(module)['__all__'][name]

    modules[fileid] = module

    return vars(module)['__all__']

__all__ = {
  "import_notebook": import_notebook,
}

python
import whoosh.index as wi
from whoosh.qparser import QueryParser
from Core import interpret, build_dylib
import os
import re
import ctypes
import types

class ModuleImporter:
    def __init__(self):
        # Initialize the index and parser for searching notebooks
        self.index = wi.open_dir("index_dir")
        self.parser = QueryParser("query", self.index.schema)
        self.modules = {}  # Cache for imported modules

    def safe_url(self, name):
        """Replace non-alphanumeric characters with underscores in a string"""
        return re.sub(r'\W+', '_', name)

    def get_functions(self, code):
        """Parse C/C++ code to extract function names"""
        # TODO: Implement a proper parser for C/C++ code
        # For now, just split on whitespace and take the first word
        return [line.split()[0] for line in code.splitlines()]

    def import_notebook(self, query_str, ctx):
        """Searches the Whoosh index and returns a module of the retrieved code"""
        try:
            results = interpret(query_str)  # Search for matching notebooks
        except ImportError as e:
            print(f"No matching notebook found for query: {query_str}")
            return

        if not results:
            return

        filename = results[0]["filename"]
        fileid = results[0]["id"]
        print(filename)

        if fileid in self.modules:
            self._load_module(fileid, ctx)
            return self.modules[fileid]

        module = types.ModuleType(filename)
        module.__file__ = filename
        module.import_notebook = self.import_notebook

        self._create_module(module)

        return module

    def _create_module(self, module):
        """Create a new module and add it to the cache"""
        self.modules[module.__file__] = module

    def _load_module(self, fileid, ctx):
        """Load a cached module and add its contents to the context"""
        module = self.modules[fileid]

        if hasattr(module, '__all__') and callable(module.__all__):
            name = module.__all__.__name__
            ctx[name] = module.__all__
            globals()[name] = module.__all__
        else:
            for name in module.__all__:
                ctx[name] = module.__all__[name]
                globals()[name] = module.__all__[name]

    def _load_dylib(self, filename, result):
        """Load a dynamic library from a C/C++ notebook"""
        # TODO: Handle loading of dynamic libraries
        # For now, just create an empty dictionary
        return {}

    def _parse_c_code(self, code):
        """Parse C/C++ code to extract function names and load the dynamic library"""
        # TODO: Implement a proper parser for C/C++ code
        # For now, just split on whitespace and take the first word
        functions = self.get_functions(code)

        if not functions:
            return {}

        lib_name = self.safe_url(result["questions"][0])
        BUILD_DIRECTORY = os.path.join(os.path.dirname(__file__), "../.build")
        result = ctypes.CDLL(os.path.join(BUILD_DIRECTORY, lib_name + '.dylib'))

        for f in functions:
            func_name = f.split('(')[0]
            setattr(result, func_name, getattr(result, func_name))

        return result

    def _import_python_code(self, code):
        """Import Python code from a notebook"""
        # TODO: Handle importing of Python code
        # For now, just execute the code in the module namespace
        exec(code, self.modules[code.__file__].__dict__)

    def _handle_notebook_language(self, results, module):
        """Handle the language of the notebook and execute the code accordingly"""
        if results[0]["language"] in ["c", "cpp", "objective-c"]:
            result = self._parse_c_code(results[0]["code"])
            module.__all__ = {}
            for f in result:
                module.__all__[f] = result[f]
        elif results[0]["language"] == "python":
            self._import_python_code(results[0]["code"])

__all__ = {
    "ModuleImporter": ModuleImporter,
    "ModuleImporter.import_notebook": ModuleImporter.import_notebook,
}

Code Breakdown

Importing Modules and Setting Up Variables

import types
import ctypes
import os
import re

modules = {}

• The code starts by importing necessary modules: types for working with Python types, ctypes for C extensions, os for interacting with the operating system, and re for regular expression matching.
• An empty dictionary modules is initialized to store module instances.

Function: safeurl(name)

def safeurl(name):
    return re.sub(r'\W+', '_', name)

• This function takes in a string name and replaces non-word characters (\W+) with underscores (_) using regular expressions.

Function: import_notebook(query_str, ctx)

def import_notebook(query_str, ctx):
    #...

• This function imports a notebook module based on a query string query_str and a context ctx.
• It uses the interpret function (not shown in this code snippet) to search for matching notebooks in a Whoosh index.
• If a match is found, it checks if the corresponding module is already loaded in memory.
• If the module is loaded, it retrieves the variables from the module and adds them to the context ctx.
• If the module is not loaded, it creates a new module instance, imports the notebook code, and adds the functions or variables to the module.
• Depending on the language of the notebook (C, C++, Objective-C, or Python), it uses different methods to import the code:
  • For C, C++, or Objective-C notebooks, it builds a dynamic library using build_dylib and loads the library using ctypes.
  • For Python notebooks, it executes the code using exec.

Other Notes

• The na line is likely a leftover from a previous version of the code and can be removed.
• The vars(module)['__all__'] line is used to access the __all__ attribute of the module, which is a list of variable names that are exported by the module.
• The build_dylib function (not shown in this code snippet) is used to build a dynamic library from C++ code.
• The get_functions function (not shown in this code snippet) is used to extract function definitions from C++ code.