diff --git a/c/src/metta.rs b/c/src/metta.rs index 35c8c3aec..a77d649ef 100644 --- a/c/src/metta.rs +++ b/c/src/metta.rs @@ -822,22 +822,20 @@ pub extern "C" fn metta_new_with_space_environment_and_stdlib(space: *mut space_ /// @brief Creates a new core MeTTa Interpreter, with no loaded stdlib nor initialization /// @ingroup interpreter_group /// @param[in] space A pointer to a handle for the Space for use by the Interpreter -/// @param[in] tokenizer A pointer to a handle for the Tokenizer for use by the Interpreter /// @param[in] environment An `env_builder_t` handle to configure the environment to use /// @return A `metta_t` handle to the newly created Interpreter /// @note The caller must take ownership responsibility for the returned `metta_t`, and free it with `metta_free()` /// @note This function does not load any stdlib, nor does it run the `init.metta` file from the environment /// #[no_mangle] -pub extern "C" fn metta_new_core(space: *mut space_t, tokenizer: *mut tokenizer_t, env_builder: env_builder_t) -> metta_t { +pub extern "C" fn metta_new_core(space: *mut space_t, env_builder: env_builder_t) -> metta_t { let dyn_space = unsafe{ &*space }.borrow(); - let tokenizer = unsafe{ &*tokenizer }.clone_handle(); let env_builder = if env_builder.is_default() { None } else { Some(env_builder.into_inner()) }; - let metta = Metta::new_core(dyn_space.clone(), tokenizer, env_builder); + let metta = Metta::new_core(dyn_space.clone(), env_builder); metta.into() } diff --git a/lib/src/metta/runner/mod.rs b/lib/src/metta/runner/mod.rs index c4da04c9c..224fac2a9 100644 --- a/lib/src/metta/runner/mod.rs +++ b/lib/src/metta/runner/mod.rs @@ -1,3 +1,59 @@ +//! +//! # MeTTa Runner Implementation +//! +//! This documentation addresses the different objects involved with the MeTTa runner, and how they fit together. +//! +//! ## [Environment] +//! [Environment] is the gateway to the outside world. It creates a platform-abstraction layer for MeTTa, and +//! is responsible for managing configuration and implementing a security model with permissions. In a typical +//! situation, there will only be one [Environment] needed. +//! +//! ## [Metta] +//! [Metta] is the runner object. It is owned by the caller and hosts all state associated with MeTTa execution, +//! including loaded [MettaMod] modules. A [Metta] has one top-level module, (known as the runner's `&self`) and +//! all other modules are loaded as dependents (or transitive dependents) of the top-level module. [Metta] is a +//! long-lived object, and it may be sufficient to create one [Metta] runner that lasts for the duration of the +//! host program. +//! +//! ## [RunnerState] +//! A [RunnerState] object encapsulates one conceptual "thread" of MeTTa execution (although it may be +//! parallelized in its implementation) A [RunnerState] is short-lived; it is created to evaluate some +//! MeTTa code, and can be run until it finishes or encounters an error. Multiple [RunnerState] objects may +//! be executing within the same [Metta] at the same time. +//! UPDATE: I think I will be removing the [RunnerState] shortly, in favor of a delegate interface that allows +//! a function to interact with the runner in specific ways for the implementation of a debugger. +//! +//! ## [ModuleDescriptor] +//! A self-contained data-structure that uniquely identifies a specific version of a specific module. Two +//! modules that have the same ModuleDescriptor are considered to be the same module from the perspective of +//! the implementation. +//! +//! ## [MettaMod] +//! A [MettaMod] represents a loaded module. A module is fundamentally a [Space] of atoms, although it also +//! contains an associated [Tokenizer] to help with the conversion from text to [Atom]s and a set of sub-modules +//! upon which it depends. Modules can be implemented in pure MeTTa as well as through extensions in other host +//! languages, namely Rust, C, and Python. +//! +//! ## [RunContext] +//! A [RunContext] objects encapsulates the interface accessible to code running inside a [RunnerState]. It +//! provides access to the currently loaded module and any other shared state required for the atoms executing +//! within the MeTTa interpreter. A [RunContext] is created inside the runner, and it is not possible for +//! code outside the MeTTa core library to own a [RunContext]. +//! +//! Metta (Runner) +//! ┌─────────────────────────────────────────────────────────────────┐ +//! │ MettaMods (Modules) │ +//! │ ┌────────────────────────────────────────────────────┐ │ +//! │ │ Space Tokenizer ├─┐ │ +//! │ │ ┌─────────────────┐ ┌─────────────────────┐ │ ├─┐ │ +//! │ │ │ │ │ │ │ │ │ │ +//! │ │ └─────────────────┘ └─────────────────────┘ │ │ │ │ +//! │ └─┬──────────────────────────────────────────────────┘ │ │ │ +//! │ └─┬──────────────────────────────────────────────────┘ │ │ +//! │ └────────────────────────────────────────────────────┘ │ +//! └─────────────────────────────────────────────────────────────────┘ +//! + use crate::*; use crate::common::shared::Shared; @@ -6,10 +62,13 @@ use super::space::*; use super::text::{Tokenizer, SExprParser}; use super::types::validate_atom; +mod modules; +use modules::*; + use std::rc::Rc; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::collections::HashMap; -use std::sync::Arc; +use std::sync::{Arc, Mutex, RwLock}; mod environment; pub use environment::{Environment, EnvBuilder}; @@ -31,6 +90,11 @@ mod arithmetics; const EXEC_SYMBOL : Atom = sym!("!"); +// *-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-* +// Metta & related objects +// *-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-* + +/// A Metta object encapsulates everything needed to execute MeTTa code #[derive(Clone, Debug)] pub struct Metta(Rc); @@ -42,37 +106,27 @@ impl PartialEq for Metta { #[derive(Debug)] pub struct MettaContents { - space: DynSpace, - tokenizer: Shared, + /// All the runner's loaded modules + modules: RwLock>, + /// An index, to find a loaded module from a ModuleDescriptor + module_descriptors: Mutex>, + /// A clone of the top module's Space, so we don't need to do any locking to access it, + /// to support the metta.space() public function + top_mod_space: DynSpace, + /// A clone of the top module's Tokenizer + top_mod_tokenizer: Shared, + /// The runner's pragmas, affecting runner-wide behavior settings: Shared>, - modules: Shared>, - working_dir: Option, + /// The runner's Environment environment: Arc, } -#[derive(Debug, PartialEq, Eq)] -enum MettaRunnerMode { - ADD, - INTERPRET, - TERMINATE, -} +/// A reference to a [MettaMod] that is loaded into a [Metta] runner +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ModId(usize); -pub struct RunnerState<'m, 'i> { - mode: MettaRunnerMode, - metta: &'m Metta, - parser: Option>, - atoms: Option<&'i [Atom]>, - interpreter_state: Option>, - results: Vec>, -} - -impl std::fmt::Debug for RunnerState<'_, '_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("RunnerState") - .field("mode", &self.mode) - .field("interpreter_state", &self.interpreter_state) - .finish() - } +impl ModId { + pub const TOP_MOD: ModId = ModId(0); } impl Metta { @@ -97,18 +151,34 @@ impl Metta { }; //Create the raw MeTTa runner - let metta = Metta::new_core(space, Shared::new(Tokenizer::new()), env_builder); + let metta = Metta::new_core(space, env_builder); + //LP-TODO-NEXT, revisit this below, once all tests are passing // TODO: Reverse the loading order between the Rust stdlib and user-supplied stdlib loader, // because user-supplied stdlibs might need to build on top of the Rust stdlib. // Currently this is problematic because https://github.com/trueagi-io/hyperon-experimental/issues/408, // and the right fix is value-bridging (https://github.com/trueagi-io/hyperon-experimental/issues/351) + //LP-TODO-NEXT, need to get the Python stdlib loading via hyperonpy //Load the custom stdlib loader(&metta); - //Load the Rust stdlib - metta.load_module(PathBuf::from("stdlib")).expect("Could not load stdlib"); + //Load the Rust stdlib into the runner + let rust_stdlib_desc = ModuleDescriptor::new("stdlib".to_string(), None); + let rust_stdlib_id = metta.get_or_init_module(rust_stdlib_desc, |context, descriptor| { + init_rust_stdlib(context, descriptor) + }).expect("Could not load stdlib"); + + //Add the libs + let mut runner_state = RunnerState::new(&metta); + runner_state.run_in_context(|context| { + + //Import the Rust stdlib into &self + context.module().import_dependency_legacy(&metta, rust_stdlib_id).unwrap(); + + Ok(()) + }).unwrap(); + drop(runner_state); //Run the `init.metta` file if let Some(init_meta_file_path) = metta.0.environment.initialization_metta_file_path() { @@ -126,107 +196,82 @@ impl Metta { /// /// NOTE: If `env_builder` is `None`, the common environment will be used /// NOTE: This function does not load any stdlib atoms, nor run the [Environment]'s 'init.metta' - pub fn new_core(space: DynSpace, tokenizer: Shared, env_builder: Option) -> Self { + pub fn new_core(space: DynSpace, env_builder: Option) -> Self { let settings = Shared::new(HashMap::new()); - let modules = Shared::new(HashMap::new()); let environment = match env_builder { Some(env_builder) => Arc::new(env_builder.build()), None => Environment::common_env_arc() }; + let top_mod_working_dir = environment.working_dir().map(|path| path.into()); + let top_mod_tokenizer = Shared::new(Tokenizer::new()); let contents = MettaContents{ - space, - tokenizer, + modules: RwLock::new(vec![]), + module_descriptors: Mutex::new(HashMap::new()), + top_mod_space: space.clone(), + top_mod_tokenizer: top_mod_tokenizer.clone(), settings, - modules, - working_dir: environment.working_dir().map(|path| path.into()), - environment, + environment }; let metta = Self(Rc::new(contents)); - register_runner_tokens(&metta); - register_common_tokens(&metta); - metta - } - - /// Returns a new MeTTa interpreter intended for use loading MeTTa modules during import - fn new_loading_runner(metta: &Metta, path: &Path) -> Self { - let space = DynSpace::new(GroundingSpace::new()); - let tokenizer = metta.tokenizer().clone_inner(); - let environment = metta.0.environment.clone(); - let settings = metta.0.settings.clone(); - let modules = metta.0.modules.clone(); - //Start search for sub-modules in the parent directory of the module we're loading - let working_dir = path.parent().map(|path| path.into()); + let top_mod = MettaMod::new_with_tokenizer(&metta, ModuleDescriptor::top(), space, top_mod_tokenizer, top_mod_working_dir); + assert_eq!(metta.add_module(top_mod), ModId::TOP_MOD); - let metta = Self(Rc::new(MettaContents { space, tokenizer, settings, modules, environment, working_dir })); - register_runner_tokens(&metta); metta } - pub fn load_module_space(&self, path: PathBuf) -> Result { - log::debug!("Metta::load_module_space: load module space {}", path.display()); - let loaded_module = self.0.modules.borrow().get(&path).cloned(); + /// Returns the ModId of a module, or None if it isn't loaded + pub fn get_module(&self, descriptor: &ModuleDescriptor) -> Option { + let descriptors = self.0.module_descriptors.lock().unwrap(); + descriptors.get(descriptor).cloned() + } - // Loading the module only once - // TODO: force_reload? - match loaded_module { - Some(module_space) => { - log::debug!("Metta::load_module_space: module is already loaded {}", path.display()); - Ok(module_space) - }, - None => { - // TODO: This is a hack. We need a way to register tokens at module-load-time, for any module - if path.to_str().unwrap() == "stdlib" { - register_rust_tokens(self); - } + /// Returns the ModId of a module, initializing it with the provided function if it isn't already loaded + /// + /// The init function will then call `context.init_self_module()` along with any other initialization code + pub fn get_or_init_module Result<(), String>>(&self, descriptor: ModuleDescriptor, f: F) -> Result { - // Load the module to the new space - let runner = Metta::new_loading_runner(self, &path); - let program = match path.to_str() { - Some("stdlib") => METTA_CODE.to_string(), - _ => std::fs::read_to_string(&path).map_err( - |err| format!("Could not read file, path: {}, error: {}", path.display(), err))?, - }; - // Make the imported module be immediately available to itself - // to mitigate circular imports - self.0.modules.borrow_mut().insert(path.clone(), runner.space().clone()); - runner.run(SExprParser::new(program.as_str())) - .map_err(|err| format!("Cannot import module, path: {}, error: {}", path.display(), err))?; - - Ok(runner.space().clone()) - } + //If we already have the module loaded, then return it + if let Some(mod_id) = self.get_module(&descriptor) { + return Ok(mod_id); + } + + //Create a new RunnerState in order to initialize the new module, and push the init function + // to run within the new RunnerState. The init function will then call `context.init_self_module()` + let mut runner_state = RunnerState::new_internal(&self); + runner_state.run_in_context(|context| { + context.push_func(|context| f(context, descriptor)); + Ok(()) + })?; + + //Finish the execution, and add the newly initialized module to the Runner + while !runner_state.is_complete() { + runner_state.run_step()?; } + let module = runner_state.into_module(); + Ok(self.add_module(module)) } - pub fn load_module(&self, path: PathBuf) -> Result<(), String> { - // Load module to &self - // TODO: Should we register the module name? - // self.tokenizer.borrow_mut().register_token(stdlib::regex(name), move |_| { space_atom.clone() }); - // TODO: check if it is already there (if the module is newly loaded) - let module_space = self.load_module_space(path)?; - let space_atom = Atom::gnd(module_space); - self.0.space.borrow_mut().add(space_atom); - Ok(()) + /// Internal function to add a loaded module to the runner + pub(crate) fn add_module(&self, module: MettaMod) -> ModId { + let mut vec_ref = self.0.modules.write().unwrap(); + let new_id = ModId(vec_ref.len()); + let mut descriptors = self.0.module_descriptors.lock().unwrap(); + if descriptors.insert(module.descriptor().clone(), new_id).is_some() { + panic!(); //We should never try and add the same module twice. + } + vec_ref.push(module); + new_id } + /// Returns a reference to the Space associated with the runner's top module pub fn space(&self) -> &DynSpace { - &self.0.space + &self.0.top_mod_space } + /// Returns a reference to the Tokenizer associated with the runner's top module pub fn tokenizer(&self) -> &Shared { - &self.0.tokenizer - } - - /// Returns the search paths to explore for MeTTa modules, in search priority order - /// - /// The runner's working_dir is always returned first - pub fn search_paths<'a>(&'a self) -> impl Iterator + 'a { - [&self.0.working_dir].into_iter().filter_map(|opt| opt.as_deref()) - .chain(self.0.environment.extra_include_paths()) - } - - pub fn modules(&self) -> &Shared> { - &self.0.modules + &self.0.top_mod_tokenizer } pub fn settings(&self) -> &Shared> { @@ -250,64 +295,116 @@ impl Metta { state.run_to_completion() } + pub fn run_in_module(&self, mod_id: ModId, parser: SExprParser) -> Result>, String> { + let mut state = RunnerState::new_with_module(self, mod_id); + state.i_wrapper.input_src.push_parser(parser); + state.run_to_completion() + } + // TODO: this method is deprecated and should be removed after switching // to the minimal MeTTa pub fn evaluate_atom(&self, atom: Atom) -> Result, String> { #[cfg(feature = "minimal")] - let atom = wrap_atom_by_metta_interpreter(self, atom); - match self.type_check(atom) { - Err(atom) => Ok(vec![atom]), - Ok(atom) => interpret(self.space(), &atom), + let atom = wrap_atom_by_metta_interpreter(self.0.top_mod_space.clone(), atom); + if self.type_check_is_enabled() && !validate_atom(self.0.top_mod_space.borrow().as_space(), &atom) { + Ok(vec![Atom::expr([ERROR_SYMBOL, atom, BAD_TYPE_SYMBOL])]) + } else { + interpret(self.space(), &atom) } } - fn add_atom(&self, atom: Atom) -> Result<(), Atom>{ - let atom = self.type_check(atom)?; - self.0.space.borrow_mut().add(atom); - Ok(()) + fn type_check_is_enabled(&self) -> bool { + self.get_setting_string("type-check").map_or(false, |val| val == "auto") } - fn type_check(&self, atom: Atom) -> Result { - let is_type_check_enabled = self.get_setting_string("type-check").map_or(false, |val| val == "auto"); - if is_type_check_enabled && !validate_atom(self.0.space.borrow().as_space(), &atom) { - Err(Atom::expr([ERROR_SYMBOL, atom, BAD_TYPE_SYMBOL])) - } else { - Ok(atom) - } - } +} +// *-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-* +// RunnerState & related objects +// *-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-* + +//TODO: Proposed API change to eliminate RunnerState from public API +// After a lot of experimentation with a design that is capable of moving execution work across threads, +// I think it makes sense to reverse course on the idea to expose "RunnerState" public API object. +// +//In essence, I am running into all exactly the same set of challenges that async Rust went through, +// but their solution is very involved. For example: https://rust-lang.github.io/async-book/04_pinning/01_chapter.html +// We could end up using the same approaches (and possibly utilizing the same mechanims (like pinning)), but +// I feel like that is overkill for what we require from Rust-language interoperability. +// +//Instead, I would like to simplify the Runner API to include a delegate interface that the runner will call +// with periodic events and status updates. This delegate interface would then be used to implement a +// debugger and any other code that needs to influence execution from outside the MeTTa runner. +// +//Multi-threading inside a single Runner is tricky no matter which design we choose, but my thinking is that +// the delegate would be an FnMut closure that is called by the receiver on a mpsc channel, so delegate +// callbacks will always happen on the same thread, and the callback won't need to be Send nor Sync. +// +//So, the API change will be: +// - `run_step` goes away, and is replaced by a delegate that is called periodically, and has the ability to: +// * receive incremental new results +// * interrupt (terminate) execution early +// * access additional debug info (specifics TBD) +// - New runner APIs including: `Metta::run_from_parser`, `Metta::run_atoms`, etc. will replace existing +// RunnerState APIs like `RunnerState::new_with_parser`, etc. +// + +/// A RunnerState encapsulates a single in-flight process, executing code within a [Metta] runner +pub struct RunnerState<'m, 'i> { + metta: &'m Metta, + module: StateMod, + i_wrapper: InterpreterWrapper<'m, 'i>, } -#[cfg(feature = "minimal")] -fn wrap_atom_by_metta_interpreter(runner: &Metta, atom: Atom) -> Atom { - let space = Atom::gnd(runner.space().clone()); - let interpret = Atom::expr([Atom::sym("interpret"), atom, ATOM_TYPE_UNDEFINED, space]); - let eval = Atom::expr([EVAL_SYMBOL, interpret]); - eval +impl std::fmt::Debug for RunnerState<'_, '_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RunnerState") + .field("mode", &self.i_wrapper.mode) + .field("interpreter_state", &self.i_wrapper.interpreter_state) + .finish() + } +} + +/// Internal, refers to the MeTTa module used by a RunnerState +enum StateMod { + None, + UseLoaded(ModId), + Initializing(MettaMod), } -impl<'m, 'i> RunnerState<'m, 'i> { - fn new(metta: &'m Metta) -> Self { +impl<'m, 'input> RunnerState<'m, 'input> { + + fn new_internal(metta: &'m Metta) -> Self { Self { metta, - mode: MettaRunnerMode::ADD, - interpreter_state: None, - parser: None, - atoms: None, - results: vec![], + module: StateMod::None, + i_wrapper: InterpreterWrapper::default() } } + + /// Returns a new RunnerState to execute code in the context of a MeTTa runner's top module + pub fn new(metta: &'m Metta) -> Self { + Self::new_with_module(metta, ModId::TOP_MOD) + } + + /// Returns a new RunnerState to execute code in the context of any loaded module + pub(crate) fn new_with_module(metta: &'m Metta, mod_id: ModId) -> Self { + let mut state = Self::new_internal(metta); + state.module = StateMod::UseLoaded(mod_id); + state + } + /// Returns a new RunnerState, for running code from the [SExprParser] with the specified [Metta] runner - pub fn new_with_parser(metta: &'m Metta, parser: SExprParser<'i>) -> Self { + pub fn new_with_parser(metta: &'m Metta, parser: SExprParser<'input>) -> Self { let mut state = Self::new(metta); - state.parser = Some(parser); + state.i_wrapper.input_src.push_parser(parser); state } /// Returns a new RunnerState, for running code encoded as a slice of [Atom]s with the specified [Metta] runner - pub fn new_with_atoms(metta: &'m Metta, atoms: &'i[Atom]) -> Self { + pub fn new_with_atoms(metta: &'m Metta, atoms: &'input[Atom]) -> Self { let mut state = Self::new(metta); - state.atoms = Some(atoms); + state.i_wrapper.input_src.push_atoms(atoms); state } @@ -321,102 +418,338 @@ impl<'m, 'i> RunnerState<'m, 'i> { /// Runs one step of the interpreter pub fn run_step(&mut self) -> Result<(), String> { + self.run_in_context(|context| context.step()) + } + + /// Returns `true` if the RunnerState has completed execution of all input or has encountered a + /// fatal error, otherwise returns `false` + pub fn is_complete(&self) -> bool { + self.i_wrapper.mode == MettaRunnerMode::TERMINATE + } + + /// Returns a reference to the current in-progress results within the RunnerState + pub fn current_results(&self) -> &Vec> { + &self.i_wrapper.results + } + + /// Consumes the RunnerState and returns the final results + pub fn into_results(self) -> Vec> { + self.i_wrapper.results + } + + /// Private method. Creates the Runner's context, and executes an arbitrary function within that context + //TODO: When we eliminate the RunnerState, this method should become a private method of Metta, + // and an argument of type `Option` should be added. When this function is used to initialize + // modules, the module type can be returned from this function + fn run_in_context) -> Result>(&mut self, f: F) -> Result { + + // Construct the RunContext + let mod_ref; + let module = match &mut self.module { + StateMod::UseLoaded(mod_id) => { + mod_ref = self.metta.0.modules.read().unwrap(); + ModRef::Borrowed(mod_ref.get(mod_id.0).unwrap()) + }, + StateMod::Initializing(_) | + StateMod::None => ModRef::Local(&mut self.module) + }; + let mut context = RunContext { + metta: &self.metta, + i_wrapper: &mut self.i_wrapper, + module, + }; + + // Call our function + f(&mut context) + } + + /// Internal method to return the MettaMod for a RunnerState that just initialized the module + pub(crate) fn into_module(self) -> MettaMod { + match self.module { + StateMod::Initializing(module) => module, + _ => unreachable!() + } + } +} + +// *-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-* +// RunContext & related objects +// *-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-* + +/// Runtime data that is available to Grounded Atom execution +// TODO: I think we may be able to remove the `'interpreter`` lifetime after the minimal MeTTa migration +// because the lifetime is separated on account of the inability of the compiler to shorten a lifetime +// used as a generic parameter on a trait. In this case, the `Plan` trait. +pub struct RunContext<'a, 'interpreter, 'input> { + metta: &'a Metta, + module: ModRef<'a>, + i_wrapper: &'a mut InterpreterWrapper<'interpreter, 'input> +} + +enum ModRef<'a> { + Local(&'a mut StateMod), + Borrowed(&'a MettaMod) +} + +impl ModRef<'_> { + fn try_borrow(&self) -> Option<&MettaMod> { + match &self { + ModRef::Borrowed(module) => Some(*module), + ModRef::Local(state_mod) => { + match state_mod { + StateMod::Initializing(module) => Some(module), + _ => None + } + } + } + } + pub fn try_borrow_mut(&mut self) -> Option<&mut MettaMod> { + match self { + ModRef::Borrowed(_) => None, + ModRef::Local(state_mod) => { + match state_mod { + StateMod::Initializing(module) => Some(module), + _ => None + } + } + } + } +} + +impl<'input> RunContext<'_, '_, 'input> { + /// Returns access to the context's current module + pub fn module(&self) -> &MettaMod { + self.module.try_borrow().unwrap_or_else(|| panic!("No module available")) + } + + /// Returns mutable access the context's current module, if possible + pub fn module_mut(&mut self) -> Option<&mut MettaMod> { + self.module.try_borrow_mut() + } + + /// Pushes the parser as a source of operations to subsequently execute + pub fn push_parser(&mut self, parser: SExprParser<'input>) { + self.i_wrapper.input_src.push_parser(parser); + } + + /// Pushes the atoms as a source of operations to subsequently execute + pub fn push_atoms(&mut self, atoms: &'input[Atom]) { + self.i_wrapper.input_src.push_atoms(atoms); + } + + /// Pushes an executable function as an operation to be executed + pub fn push_func Result<(), String> + 'input>(&mut self, f: F) { + self.i_wrapper.input_src.push_func(f); + } + + /// Initializes the context's module. Used in the implementation of a module loader function + pub fn init_self_module(&mut self, descriptor: ModuleDescriptor, space: DynSpace, working_dir: Option) { + match &mut self.module { + ModRef::Borrowed(_) => panic!("Module already initialized"), + ModRef::Local(ref mut state_mod_ref) => { + if matches!(state_mod_ref, StateMod::Initializing(_)) { + panic!("Module already initialized"); + } + let tokenizer = Shared::new(Tokenizer::new()); + **state_mod_ref = StateMod::Initializing(MettaMod::new_with_tokenizer(self.metta, descriptor, space, tokenizer, working_dir)); + } + } + } + + /// Private method to advance the context forward one step + fn step(&mut self) -> Result<(), String> { // If we're in the middle of interpreting an atom... - if let Some(interpreter_state) = core::mem::take(&mut self.interpreter_state) { + if let Some(interpreter_state) = core::mem::take(&mut self.i_wrapper.interpreter_state) { if interpreter_state.has_next() { //Take a step with the interpreter, and put it back for next time - self.interpreter_state = Some(interpret_step(interpreter_state)) + self.i_wrapper.interpreter_state = Some(interpret_step(interpreter_state)) } else { //This interpreter is finished, process the results let result = interpreter_state.into_result().unwrap(); let error = result.iter().any(|atom| atom_is_error(atom)); - self.results.push(result); + self.i_wrapper.results.push(result); if error { - self.mode = MettaRunnerMode::TERMINATE; + self.i_wrapper.mode = MettaRunnerMode::TERMINATE; return Ok(()); } } + Ok(()) } else { - // Get the next atom, and start a new intperpreter - let next_atom = if let Some(parser) = self.parser.as_mut() { - match parser.parse(&self.metta.0.tokenizer.borrow()) { - Ok(atom) => atom, - Err(err) => { - self.mode = MettaRunnerMode::TERMINATE; - return Err(err); - } + // Get the next operation + let tokenizer_option = self.module.try_borrow().map(|module| module.tokenizer().borrow()); + let tokenizer = tokenizer_option.as_ref().map(|tok| &**tok as &Tokenizer); + let next_op = match self.i_wrapper.input_src.next_op(tokenizer) { + Ok(atom) => atom, + Err(err) => { + self.i_wrapper.mode = MettaRunnerMode::TERMINATE; + return Err(err); } - } else { - if let Some(atoms) = self.atoms.as_mut() { - if let Some((atom, rest)) = atoms.split_first() { - *atoms = rest; - Some(atom.clone()) - } else { - None + }; + drop(tokenizer_option); + + // Start execution of the operation + match next_op { + Some(Executable::Func(func)) => { + func(self) + }, + // If the next operation is an atom, start a new intperpreter + Some(Executable::Atom(atom)) => { + if atom == EXEC_SYMBOL { + self.i_wrapper.mode = MettaRunnerMode::INTERPRET; + return Ok(()); } + match self.i_wrapper.mode { + MettaRunnerMode::ADD => { + if let Err(atom) = self.module().add_atom(atom, self.metta.type_check_is_enabled()) { + self.i_wrapper.results.push(vec![atom]); + self.i_wrapper.mode = MettaRunnerMode::TERMINATE; + return Ok(()); + } + }, + MettaRunnerMode::INTERPRET => { + + if self.metta.type_check_is_enabled() && !validate_atom(self.module().space().borrow().as_space(), &atom) { + let type_err_exp = Atom::expr([ERROR_SYMBOL, atom, BAD_TYPE_SYMBOL]); + self.i_wrapper.interpreter_state = Some(InterpreterState::new_finished(self.module().space().clone(), vec![type_err_exp])); + } else { + #[cfg(feature = "minimal")] + let atom = wrap_atom_by_metta_interpreter(self.module().space().clone(), atom); + self.i_wrapper.interpreter_state = Some(interpret_init(self.module().space().clone(), &atom)); + } + }, + MettaRunnerMode::TERMINATE => { + return Ok(()); + }, + } + self.i_wrapper.mode = MettaRunnerMode::ADD; + Ok(()) + }, + None => { + self.i_wrapper.mode = MettaRunnerMode::TERMINATE; + Ok(()) + } + } + } + } + +} + +// *-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-* +// InterpreterWrapper & related objects +// *-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-* + +/// Private structure to contain everything associated with an InterpreterState. +/// This is basically the part of RunContext that lasts across calls to run_step +#[derive(Default)] +struct InterpreterWrapper<'interpreter, 'i> { + mode: MettaRunnerMode, + input_src: InputStream<'i>, + interpreter_state: Option>, + results: Vec>, +} + +#[derive(Debug, Default, PartialEq, Eq)] +enum MettaRunnerMode { + #[default] + ADD, + INTERPRET, + TERMINATE, +} + +/// Private type representing a source for operations for the runner +enum InputSource<'i> { + Parser(SExprParser<'i>), + Slice(&'i [Atom]), + //TODO: Func should be deleted in favor of just an Atom. See comment on Executable::Func below + Func(Box Result<(), String> + 'i>) +} + +impl InputSource<'_> { + fn next_atom(&mut self, tokenizer: Option<&Tokenizer>) -> Result, String> { + let atom = match self { + InputSource::Parser(parser) => { + parser.parse(tokenizer.as_ref().unwrap_or_else(|| panic!("Module must be initialized to parse MeTTa code")))? + }, + InputSource::Slice(atoms) => { + if let Some((atom, rest)) = atoms.split_first() { + *atoms = rest; + Some(atom.clone()) } else { None } - }; + }, + InputSource::Func(_) => unreachable!() //If we got here, there is a bug in the calling function + }; + Ok(atom) + } +} - if let Some(atom) = next_atom { - if atom == EXEC_SYMBOL { - self.mode = MettaRunnerMode::INTERPRET; - return Ok(()); - } - match self.mode { - MettaRunnerMode::ADD => { - if let Err(atom) = self.metta.add_atom(atom) { - self.results.push(vec![atom]); - self.mode = MettaRunnerMode::TERMINATE; - return Ok(()); - } - }, - MettaRunnerMode::INTERPRET => { +/// Private type representing an operation for the runner +/// TODO: Delete this type, once I can wrap any function in an Atom, and have access to the RunContext +enum Executable<'i> { + Atom(Atom), + Func(Box Result<(), String> + 'i>) +} - self.interpreter_state = Some(match self.metta.type_check(atom) { - Err(atom) => { - InterpreterState::new_finished(self.metta.space().clone(), vec![atom]) - }, - Ok(atom) => { - #[cfg(feature = "minimal")] - let atom = wrap_atom_by_metta_interpreter(&self.metta, atom); - interpret_init(self.metta.space().clone(), &atom) - }, - }); - }, - MettaRunnerMode::TERMINATE => { - return Ok(()); +/// A private structure representing a heterogeneous source of operations for the runner to execute +#[derive(Default)] +struct InputStream<'a>(Vec>); + +impl<'i> InputStream<'i> { + fn push_parser(&mut self, parser: SExprParser<'i>) { + self.0.push(InputSource::Parser(parser)) + } + fn push_atoms(&mut self, atoms: &'i[Atom]) { + self.0.push(InputSource::Slice(atoms)); + } + fn push_func Result<(), String> + 'i>(&mut self, f: F) { + self.0.push(InputSource::Func(Box::new(f))) + } + /// Returns the next operation in the InputStream, and removes it from the stream. Returns None if the + /// InputStream is empty. + fn next_op(&mut self, tokenizer: Option<&Tokenizer>) -> Result>, String> { + match self.0.get_mut(0) { + None => Ok(None), + Some(src) => { + match src { + InputSource::Func(_) => match self.0.remove(0) { + InputSource::Func(f) => Ok(Some(Executable::Func(f))), + _ => unreachable!() }, + InputSource::Parser(_) | + InputSource::Slice(_) => { + match src.next_atom(tokenizer)? { + Some(atom) => Ok(Some(Executable::Atom(atom))), + None => { + self.0.remove(0); + self.next_op(tokenizer) + } + } + } } - self.mode = MettaRunnerMode::ADD; - } else { - self.mode = MettaRunnerMode::TERMINATE; } } - - Ok(()) } +} - pub fn is_complete(&self) -> bool { - self.mode == MettaRunnerMode::TERMINATE - } - /// Returns a reference to the current in-progress results within the RunnerState - pub fn current_results(&self) -> &Vec> { - &self.results - } - /// Consumes the RunnerState and returns the final results - pub fn into_results(self) -> Vec> { - self.results - } +#[cfg(feature = "minimal")] +fn wrap_atom_by_metta_interpreter(space: DynSpace, atom: Atom) -> Atom { + let space = Atom::gnd(space); + let interpret = Atom::expr([Atom::sym("interpret"), atom, ATOM_TYPE_UNDEFINED, space]); + let eval = Atom::expr([EVAL_SYMBOL, interpret]); + eval } +// *-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-* +// Tests +// *-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-*-=-* + #[cfg(test)] mod tests { use super::*; @@ -447,7 +780,7 @@ mod tests { (foo b) "; - let metta = Metta::new_core(DynSpace::new(GroundingSpace::new()), Shared::new(Tokenizer::new()), Some(EnvBuilder::test_env())); + let metta = Metta::new_core(DynSpace::new(GroundingSpace::new()), Some(EnvBuilder::test_env())); metta.set_setting("type-check".into(), sym!("auto")); let result = metta.run(SExprParser::new(program)); assert_eq!(result, Ok(vec![vec![expr!("Error" ("foo" "b") "BadType")]])); @@ -461,7 +794,7 @@ mod tests { !(foo b) "; - let metta = Metta::new_core(DynSpace::new(GroundingSpace::new()), Shared::new(Tokenizer::new()), Some(EnvBuilder::test_env())); + let metta = Metta::new_core(DynSpace::new(GroundingSpace::new()), Some(EnvBuilder::test_env())); metta.set_setting("type-check".into(), sym!("auto")); let result = metta.run(SExprParser::new(program)); assert_eq!(result, Ok(vec![vec![expr!("Error" ("foo" "b") "BadType")]])); @@ -516,7 +849,7 @@ mod tests { !(foo a) "; - let metta = Metta::new_core(DynSpace::new(GroundingSpace::new()), Shared::new(Tokenizer::new()), Some(EnvBuilder::test_env())); + let metta = Metta::new_core(DynSpace::new(GroundingSpace::new()), Some(EnvBuilder::test_env())); metta.set_setting("type-check".into(), sym!("auto")); let result = metta.run(SExprParser::new(program)); assert_eq!(result, Ok(vec![vec![expr!("Error" ("foo" "b") "BadType")]])); diff --git a/lib/src/metta/runner/modules/mod.rs b/lib/src/metta/runner/modules/mod.rs new file mode 100644 index 000000000..b1b1c0d44 --- /dev/null +++ b/lib/src/metta/runner/modules/mod.rs @@ -0,0 +1,274 @@ + +use std::path::{Path, PathBuf}; +use std::collections::HashMap; +use std::sync::Mutex; + +use crate::*; +use crate::space::DynSpace; +use crate::metta::*; +use crate::metta::runner::{Metta, ModId, RunnerState}; +use crate::metta::types::validate_atom; +use crate::metta::text::Tokenizer; +use crate::common::shared::Shared; + +use regex::Regex; + +#[cfg(not(feature = "minimal"))] +use super::interpreter::interpret; +#[cfg(not(feature = "minimal"))] +use super::stdlib::*; + +#[cfg(feature = "minimal")] +use super::interpreter2::interpret; +#[cfg(feature = "minimal")] +use super::stdlib2::*; + +/// A data structure that uniquely identifies an exact version of a module with a particular provenance +/// +/// If two modules have the same ModuleDescriptor, they are considered to be the same module +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct ModuleDescriptor { + name: String, + path: Option, + //TODO: version, checksum? +} + +impl ModuleDescriptor { + /// Internal method to create a ModuleDescriptor for a runner's "top" module + pub(crate) fn top() -> Self { + Self::new("top".to_string(), None) + } + /// Create a new ModuleDescriptor + pub fn new(name: String, path: Option<&Path>) -> Self { + Self { + name, + path: path.map(|path| path.into()) + } + } +} + +/// Contains state associated with a loaded MeTTa module +#[derive(Debug)] +pub struct MettaMod { + descriptor: ModuleDescriptor, + working_dir: Option, + space: DynSpace, + tokenizer: Shared, + deps: Mutex>, +} + +impl Clone for MettaMod { + fn clone(&self) -> Self { + Self { + descriptor: self.descriptor.clone(), + space: self.space.clone(), + tokenizer: self.tokenizer.clone(), + working_dir: self.working_dir.clone(), + deps: Mutex::new(self.deps.lock().unwrap().clone()) + } + } +} + +impl MettaMod { + + /// Internal method to initialize an empty MettaMod + // TODO: may be able to remove the metta argument when the dust settles + pub(crate) fn new_with_tokenizer(metta: &Metta, descriptor: ModuleDescriptor, space: DynSpace, tokenizer: Shared, working_dir: Option) -> Self { + let new_mod = Self { + descriptor, + space, + tokenizer, + working_dir, + deps: Mutex::new(HashMap::new()), + }; + + // Load the base tokens for the module's new Tokenizer + register_runner_tokens(&mut *new_mod.tokenizer().borrow_mut(), new_mod.tokenizer().clone(), metta); + register_common_tokens(&mut *new_mod.tokenizer().borrow_mut(), new_mod.tokenizer().clone(), metta); + + new_mod + } + + /// Adds a loaded module as a dependency of the `&self` [MettaMod], and adds the dependent module's Space + /// as an atom in the Space of &self called "&name" + pub fn import_dependency_as(&self, metta: &Metta, mod_id: ModId, name: Option) -> Result<(), String> { + + // Get the space and name associated with the dependent module + let mut runner_state = RunnerState::new_with_module(metta, mod_id); + let (dep_space, name) = runner_state.run_in_context(|context| { + let dep_space = context.module().space().clone(); + let name = match name { + Some(name) => name, + None => context.module().descriptor().name.clone() + }; + Ok((dep_space, name)) + })?; + + // Add a new atom to the &self space, so we can access the dependent module + let new_token = format!("&{name}"); + let dep_space_atom = Atom::gnd(dep_space); + self.add_atom(dep_space_atom.clone(), false).map_err(|a| a.to_string())?; + self.tokenizer.borrow_mut().register_token_with_regex_str(&new_token, move |_| { dep_space_atom.clone() }); + + // Include the dependent module's mod_id in our deps + let mut deps = self.deps.lock().unwrap(); + deps.insert(name, mod_id); + + Ok(()) + } + + /// Adds a specific atom and/or Tokenizer entry from a dependency module to the &self module + /// + /// Behavior: + /// * If the `from_name` argument exactly matches a [Tokenizer] entry in the source module, + /// then that entry will be imported, and the `name` argument will be ignored + /// * If an exact [Tokenizer] entry was not found, this method will attempt to resolve `from_name` + /// into an atom, using the [Tokenizer] and [Space] associated with the dependent module, and + /// the resolved atom will be imported into the `&self` [Space] + /// * If `name` is provided, then if the resolved atom not a Symbol or if the resolved atom is a + /// symbol that doesn't perfectly match `name`, a new [Tokenizer] entry will be created to + /// access the atom + /// + // QUESTION: This behavior of exactly matching a regex makes importing a Tokenizer pattern pretty + // unfriendly. Does it make sense to require Tokenizers entries to be associated with atoms, for + // example "Type Atoms"? For example, we could have an "Number" type that is tied to all the + // Tokenizer regex patters used to parse different types of numbers? Then a user could + // "!(import! Number from Arithmetic)" or whatever, and get all the Tokenizer patters that parse + // numbers? + // + // Another alternative behavior might be to allow any match for a Tokenizer entry to cause the + // import of that Tokenizer entry instead of a specific atom. For example, importing "3.14" + // would import the RegEx entry that parsed that first. However that strikes me as not great + // for 2 reasons. 1.) A string matched by a Tokenizer entry is definitely not the same thing as + // a Tokenizer entry, so it's a bad way to refer to one and 2.) multiple RegEx Tokenizer entries + // might be used to compose one conceptual type. + pub fn import_item_from_dependency_as(&self, metta: &Metta, from_name: &str, mod_id: ModId, name: Option<&str>) -> Result<(), String> { + + // Get the space and tokenizer associated with the dependent module + let mut runner_state = RunnerState::new_with_module(metta, mod_id); + let (dep_space, dep_tokenizer, src_mod_name) = runner_state.run_in_context(|context| { + let dep_space = context.module().space().clone(); + let dep_tokenizer = context.module().tokenizer().clone(); + let src_mod_name = context.module().descriptor().name.clone(); + Ok((dep_space, dep_tokenizer, src_mod_name)) + })?; + + //See if there is a match in the dependent module's Tokenizer + if let Some(found_constructor) = dep_tokenizer.borrow().find_exact(from_name) { + + // If so, this method just transplants the Tokenizer entry + self.tokenizer.borrow_mut().register_token_with_func_ptr(Regex::new(from_name).unwrap(), found_constructor); + } else { + //Otherwise we will try and transplant an atom + + // Get and reduce the atom we are importing, from the dependent module + let mut parser = SExprParser::new(from_name); + let import_sym = parser.parse(&dep_tokenizer.borrow())?.ok_or_else(|| format!("Import failed to resolve \"{from_name}\""))?; + if let Some(extra_atom) = parser.parse(&dep_tokenizer.borrow())? { return Err(format!("Extraneous token in import \"{extra_atom}\""));} + let src_atom_vec = interpret(dep_space, &import_sym)?; + match src_atom_vec.len() { + 0 => return Err(format!("Failed to resolve import \"{from_name}\" in module \"{src_mod_name}\"")), + 1 => {}, + _ => return Err(format!("Ambiguous import \"{from_name}\" in module \"{src_mod_name}\"")), + } + let src_atom = src_atom_vec.into_iter().next().unwrap(); + + // Add the atom to our module's space + self.add_atom(src_atom.clone(), false).map_err(|a| a.to_string())?; + + // Finally, Add a Tokenizer entry to access this atom, if one is needed + let name = match name { + Some(name) => name, + None => from_name + }; + //We will add Tokenizer entries for all non-symbols, and symbol atoms that don't match name + let should_add_tok = match &src_atom { + Atom::Symbol(s) => s.name() != name, + _ => true, + }; + if should_add_tok { + self.tokenizer.borrow_mut().register_token_with_regex_str(&name, move |_| { src_atom.clone() }); + } + } + + Ok(()) + } + + /// Adds all atom and tokens in a dependency module to the &self module + /// + /// Currently this function effectively merges the spaces & tokenizers. + // + //QUESTION: How do we prevent duplication of transitive imports? For example, consider: + // ModA imports ModOne and ModTwo. ModOne imports ModB. ModTwo also imports ModB. How do we + // make sure ModA doesn't end up with duplicate definitions for everything in ModB? + // + //This is usually solved in other languages using some combination of strategies. + // 1.) Keep loaded modules contained within their own name-spaces. And allow aliasing between a parent's + // space and a child module's space. This is not really sufficient for us because we also want to + // import Tokenizer entries and their associated functionality, such as the ability to parse integers + // 2.) Allow a conditional "import-unique" brute-force import which is tantamount to inlining the dependency + // at the point of the import statement. e.g. `#include` in C., but with the "ifdef" guards to make sure + // the same header isn't inlined multiple times. We may want to offer this functionality, but I assume + // we'll want to offer this in addition to a more hygenic module system structure, given that most languages + // (Python, JavaScript, even Perl!) that started off with brute-force imports have retrofitted more + // sophisticated dependency management as they have matured + // 3.) Force a module to be explicit about what can be exported from the module. e.g. the `pub` visibility + // qualifiers in Rust, etc. + // + //Personally, I feel like 3. (being explicit about exported items) is the cleanest solution and provides + // the most flexibility into the future (such as the ability to unload modules, etc.) + // + pub fn import_all_from_dependency(&self, _metta: &Metta, _mod_id: ModId) -> Result<(), String> { + //TODO: Unimplemented until we decide what "import *" actually means + unimplemented!(); + } + + /// Merges all Tokenizer entries in a dependency module into &self + pub(crate) fn import_all_tokens_from_dependency(&self, metta: &Metta, mod_id: ModId) -> Result<(), String> { + + // Get the tokenizer associated with the dependent module + let mut runner_state = RunnerState::new_with_module(metta, mod_id); + let dep_tokenizer = runner_state.run_in_context(|context| Ok(context.module().tokenizer().clone()))?; + + //Import all the Tokenizer entries from the dependency + let mut dep_tok_clone = dep_tokenizer.borrow().clone(); + self.tokenizer.borrow_mut().move_front(&mut dep_tok_clone); + + Ok(()) + } + + /// Implements the prior import behavior of importing all atoms into a sub-space, loaded into the + /// &self Space, and merging all tokens + //TODO: This method is a stop-gap, until we figure out all the details of the import behavior we want + pub(crate) fn import_dependency_legacy(&self, metta: &Metta, mod_id: ModId) -> Result<(), String> { + self.import_dependency_as(metta, mod_id, None)?; + self.import_all_tokens_from_dependency(metta, mod_id) + } + + pub fn descriptor(&self) -> &ModuleDescriptor { + &self.descriptor + } + + pub fn space(&self) -> &DynSpace { + &self.space + } + + pub fn tokenizer(&self) -> &Shared { + &self.tokenizer + } + + pub fn working_dir(&self) -> Option<&Path> { + self.working_dir.as_ref().map(|p| p.as_ref()) + } + + /// A convenience to add an an atom to a module's Space, if it passes type-checking + pub(crate) fn add_atom(&self, atom: Atom, type_check: bool) -> Result<(), Atom> { + if type_check && !validate_atom(self.space.borrow().as_space(), &atom) { + return Err(Atom::expr([ERROR_SYMBOL, atom, BAD_TYPE_SYMBOL])); + } + self.space.borrow_mut().add(atom); + Ok(()) + } + +} + diff --git a/lib/src/metta/runner/stdlib.rs b/lib/src/metta/runner/stdlib.rs index d23ec750d..090d4ea27 100644 --- a/lib/src/metta/runner/stdlib.rs +++ b/lib/src/metta/runner/stdlib.rs @@ -4,7 +4,7 @@ use crate::space::*; use crate::metta::*; use crate::metta::text::Tokenizer; use crate::metta::interpreter::interpret; -use crate::metta::runner::Metta; +use crate::metta::runner::{Metta, RunContext, ModuleDescriptor}; use crate::metta::types::get_atom_types; use crate::common::shared::Shared; use crate::common::assert::vec_eq_no_order; @@ -16,7 +16,6 @@ use std::cell::RefCell; use std::fmt::Display; use std::collections::HashMap; use std::iter::FromIterator; -use std::path::PathBuf; use regex::Regex; use super::arithmetics::*; @@ -38,101 +37,102 @@ fn interpret_no_error(space: DynSpace, expr: &Atom) -> Result, String> } } -#[derive(Clone, Debug)] -pub struct ImportOp { - metta: Metta, -} - -impl PartialEq for ImportOp { - fn eq(&self, _other: &Self) -> bool { true } -} - -impl ImportOp { - pub fn new(metta: Metta) -> Self { - Self{ metta } - } -} - -impl Display for ImportOp { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "import!") - } -} - -impl Grounded for ImportOp { - fn type_(&self) -> Atom { - Atom::expr([ARROW_SYMBOL, ATOM_TYPE_ATOM, ATOM_TYPE_ATOM, UNIT_TYPE()]) - } - - fn execute(&self, args: &[Atom]) -> Result, ExecError> { - let arg_error = || ExecError::from("import! expects two arguments: space and file path"); - let space = args.get(0).ok_or_else(arg_error)?; - let file = args.get(1).ok_or_else(arg_error)?; - let mut module_path = None; - - // TODO: replace Symbol by grounded String? - if let Atom::Symbol(file) = file { - - //Check each include directory in order, until we find the module we're looking for - for include_dir in self.metta.search_paths() { - let mut path: PathBuf = include_dir.into(); - path.push(file.name()); - path = path.canonicalize().unwrap_or(path); - if path.exists() { - module_path = Some(path); - break; - } - } - } else { - return Err("import! expects a file path as a second argument".into()) - } - let module_space = match module_path { - Some(path) => { - log::debug!("import! load file, full path: {}", path.display()); - self.metta.load_module_space(path)? - } - None => return Err(format!("Failed to load module {file:?}; could not locate file").into()) - }; - - match space { - // If the module is to be associated with a new space, - // we register it in the tokenizer - works as "import as" - Atom::Symbol(space) => { - let name = space.name(); - let space_atom = Atom::gnd(module_space); - let regex = Regex::new(name) - .map_err(|err| format!("Could not convert space name {} into regex: {}", name, err))?; - self.metta.tokenizer().borrow_mut() - .register_token(regex, move |_| { space_atom.clone() }); - }, - // If the reference space exists, the module space atom is inserted into it - // (but the token is not added) - works as "import to" - Atom::Grounded(_) => { - let space = Atom::as_gnd::(space) - .ok_or("import! expects a space as a first argument")?; - // Moving space atoms from children to parent - let modules = self.metta.modules().borrow(); - for (_path, mspace) in modules.iter() { - let aspace = Atom::gnd(mspace.clone()); - if module_space.borrow_mut().remove(&aspace) { - self.metta.space().borrow_mut().remove(&aspace); - self.metta.space().borrow_mut().add(aspace); - } - } - let module_space_atom = Atom::gnd(module_space); - if space.borrow_mut().query(&module_space_atom).is_empty() { - space.borrow_mut().add(module_space_atom); - } - }, - _ => return Err("import! expects space as a first argument".into()), - }; - unit_result() - } - - fn match_(&self, other: &Atom) -> MatchResultIter { - match_by_equality(self, other) - } -} +//LP-TODO-NEXT, come back and fix this +// #[derive(Clone, Debug)] +// pub struct ImportOp { +// metta: Metta, +// } + +// impl PartialEq for ImportOp { +// fn eq(&self, _other: &Self) -> bool { true } +// } + +// impl ImportOp { +// pub fn new(metta: Metta) -> Self { +// Self{ metta } +// } +// } + +// impl Display for ImportOp { +// fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +// write!(f, "import!") +// } +// } + +// impl Grounded for ImportOp { +// fn type_(&self) -> Atom { +// Atom::expr([ARROW_SYMBOL, ATOM_TYPE_ATOM, ATOM_TYPE_ATOM, UNIT_TYPE()]) +// } + +// fn execute(&self, args: &[Atom]) -> Result, ExecError> { +// let arg_error = || ExecError::from("import! expects two arguments: space and file path"); +// let space = args.get(0).ok_or_else(arg_error)?; +// let file = args.get(1).ok_or_else(arg_error)?; +// let mut module_path = None; + +// // TODO: replace Symbol by grounded String? +// if let Atom::Symbol(file) = file { + +// //Check each include directory in order, until we find the module we're looking for +// for include_dir in self.metta.search_paths() { +// let mut path: PathBuf = include_dir.into(); +// path.push(file.name()); +// path = path.canonicalize().unwrap_or(path); +// if path.exists() { +// module_path = Some(path); +// break; +// } +// } +// } else { +// return Err("import! expects a file path as a second argument".into()) +// } +// let module_space = match module_path { +// Some(path) => { +// log::debug!("import! load file, full path: {}", path.display()); +// self.metta.load_module_space(path)? +// } +// None => return Err(format!("Failed to load module {file:?}; could not locate file").into()) +// }; + +// match space { +// // If the module is to be associated with a new space, +// // we register it in the tokenizer - works as "import as" +// Atom::Symbol(space) => { +// let name = space.name(); +// let space_atom = Atom::gnd(module_space); +// let regex = Regex::new(name) +// .map_err(|err| format!("Could not convert space name {} into regex: {}", name, err))?; +// self.metta.tokenizer().borrow_mut() +// .register_token(regex, move |_| { space_atom.clone() }); +// }, +// // If the reference space exists, the module space atom is inserted into it +// // (but the token is not added) - works as "import to" +// Atom::Grounded(_) => { +// let space = Atom::as_gnd::(space) +// .ok_or("import! expects a space as a first argument")?; +// // Moving space atoms from children to parent +// let modules = self.metta.modules().borrow(); +// for (_path, mspace) in modules.iter() { +// let aspace = Atom::gnd(mspace.clone()); +// if module_space.borrow_mut().remove(&aspace) { +// self.metta.space().borrow_mut().remove(&aspace); +// self.metta.space().borrow_mut().add(aspace); +// } +// } +// let module_space_atom = Atom::gnd(module_space); +// if space.borrow_mut().query(&module_space_atom).is_empty() { +// space.borrow_mut().add(module_space_atom); +// } +// }, +// _ => return Err("import! expects space as a first argument".into()), +// }; +// unit_result() +// } + +// fn match_(&self, other: &Atom) -> MatchResultIter { +// match_by_equality(self, other) +// } +// } #[derive(Clone, PartialEq, Debug)] pub struct MatchOp {} @@ -1088,13 +1088,13 @@ fn regex(regex: &str) -> Regex { Regex::new(regex).unwrap() } -pub fn register_common_tokens(metta: &Metta) { - let tokenizer = metta.tokenizer(); - let mut tref = tokenizer.borrow_mut(); +//TODO: The additional arguments are a temporary hack on account of the way the operation atoms store references +// to the runner & module state. https://github.com/trueagi-io/hyperon-experimental/issues/410 +pub fn register_common_tokens(tref: &mut Tokenizer, tokenizer: Shared, _metta: &Metta) { let match_op = Atom::gnd(MatchOp{}); tref.register_token(regex(r"match"), move |_| { match_op.clone() }); - let bind_op = Atom::gnd(BindOp::new(tokenizer.clone())); + let bind_op = Atom::gnd(BindOp::new(tokenizer)); tref.register_token(regex(r"bind!"), move |_| { bind_op.clone() }); let new_space_op = Atom::gnd(NewSpaceOp{}); tref.register_token(regex(r"new-space"), move |_| { new_space_op.clone() }); @@ -1128,11 +1128,10 @@ pub fn register_common_tokens(metta: &Metta) { tref.register_token(regex(r"get-state"), move |_| { get_state_op.clone() }); } -pub fn register_runner_tokens(metta: &Metta) { +//TODO: The metta argument is a temporary hack on account of the way the operation atoms store references +// to the runner & module state. https://github.com/trueagi-io/hyperon-experimental/issues/410 +pub fn register_runner_tokens(tref: &mut Tokenizer, _tokenizer: Shared, metta: &Metta) { let space = metta.space(); - let tokenizer = metta.tokenizer(); - - let mut tref = tokenizer.borrow_mut(); let case_op = Atom::gnd(CaseOp::new(space.clone())); tref.register_token(regex(r"case"), move |_| { case_op.clone() }); @@ -1146,8 +1145,9 @@ pub fn register_runner_tokens(metta: &Metta) { tref.register_token(regex(r"superpose"), move |_| { superpose_op.clone() }); let get_type_op = Atom::gnd(GetTypeOp::new(space.clone())); tref.register_token(regex(r"get-type"), move |_| { get_type_op.clone() }); - let import_op = Atom::gnd(ImportOp::new(metta.clone())); - tref.register_token(regex(r"import!"), move |_| { import_op.clone() }); +//LP-TODO-NEXT, come back and fix this + // let import_op = Atom::gnd(ImportOp::new(metta.clone())); + // tref.register_token(regex(r"import!"), move |_| { import_op.clone() }); let pragma_op = Atom::gnd(PragmaOp::new(metta.settings().clone())); tref.register_token(regex(r"pragma!"), move |_| { pragma_op.clone() }); @@ -1162,7 +1162,7 @@ pub fn register_runner_tokens(metta: &Metta) { tref.register_token(regex(r"&self"), move |_| { self_atom.clone() }); } -pub fn register_rust_tokens(metta: &Metta) { +pub fn register_rust_stdlib_tokens(target: &mut Tokenizer) { let mut rust_tokens = Tokenizer::new(); let tref = &mut rust_tokens; @@ -1183,7 +1183,7 @@ pub fn register_rust_tokens(metta: &Metta) { let mod_op = Atom::gnd(ModOp{}); tref.register_token(regex(r"%"), move |_| { mod_op.clone() }); - metta.tokenizer().borrow_mut().move_front(&mut rust_tokens); + target.move_front(&mut rust_tokens); } pub static METTA_CODE: &'static str = " @@ -1195,6 +1195,20 @@ pub static METTA_CODE: &'static str = " (: Error (-> Atom Atom ErrorType)) "; +/// Initializes the Rust stdlib module +pub(crate) fn init_rust_stdlib(context: &mut RunContext, descriptor: ModuleDescriptor) -> Result<(), String> { + + let space = DynSpace::new(GroundingSpace::new()); + context.init_self_module(descriptor, space, None); + + register_rust_stdlib_tokens(&mut *context.module().tokenizer().borrow_mut()); + + let parser = SExprParser::new(METTA_CODE); + context.push_parser(parser); + + Ok(()) +} + #[cfg(all(test, not(feature = "minimal")))] mod tests { use super::*; diff --git a/lib/src/metta/runner/stdlib2.rs b/lib/src/metta/runner/stdlib2.rs index 8e5db2455..e90e2691d 100644 --- a/lib/src/metta/runner/stdlib2.rs +++ b/lib/src/metta/runner/stdlib2.rs @@ -3,9 +3,10 @@ use crate::matcher::MatchResultIter; use crate::space::*; use crate::metta::*; use crate::metta::text::Tokenizer; -use crate::metta::runner::Metta; +use crate::metta::runner::{Metta, RunContext, ModuleDescriptor}; use crate::metta::types::{get_atom_types, get_meta_type}; use crate::common::assert::vec_eq_no_order; +use crate::common::shared::Shared; use crate::metta::runner::stdlib; use std::fmt::Display; @@ -408,9 +409,9 @@ fn regex(regex: &str) -> Regex { Regex::new(regex).unwrap() } -pub fn register_common_tokens(metta: &Metta) { - let tokenizer = metta.tokenizer(); - let mut tref = tokenizer.borrow_mut(); +//TODO: The additional arguments are a temporary hack on account of the way the operation atoms store references +// to the runner & module state. https://github.com/trueagi-io/hyperon-experimental/issues/410 +pub fn register_common_tokens(tref: &mut Tokenizer, tokenizer: Shared, metta: &Metta) { let space = metta.space(); let get_type_op = Atom::gnd(GetTypeOp::new(space.clone())); @@ -437,11 +438,10 @@ pub fn register_common_tokens(metta: &Metta) { tref.register_token(regex(r"get-state"), move |_| { get_state_op.clone() }); } -pub fn register_runner_tokens(metta: &Metta) { +//TODO: The additional arguments are a temporary hack on account of the way the operation atoms store references +// to the runner & module state. https://github.com/trueagi-io/hyperon-experimental/issues/410 +pub fn register_runner_tokens(tref: &mut Tokenizer, tokenizer: Shared, metta: &Metta) { let space = metta.space(); - let tokenizer = metta.tokenizer(); - - let mut tref = tokenizer.borrow_mut(); let assert_equal_op = Atom::gnd(AssertEqualOp::new(space.clone())); tref.register_token(regex(r"assertEqual"), move |_| { assert_equal_op.clone() }); @@ -455,8 +455,9 @@ pub fn register_runner_tokens(metta: &Metta) { tref.register_token(regex(r"case"), move |_| { case_op.clone() }); let pragma_op = Atom::gnd(stdlib::PragmaOp::new(metta.settings().clone())); tref.register_token(regex(r"pragma!"), move |_| { pragma_op.clone() }); - let import_op = Atom::gnd(stdlib::ImportOp::new(metta.clone())); - tref.register_token(regex(r"import!"), move |_| { import_op.clone() }); + //LP-TODO-NEXT + // let import_op = Atom::gnd(stdlib::ImportOp::new(metta.clone())); + // tref.register_token(regex(r"import!"), move |_| { import_op.clone() }); let bind_op = Atom::gnd(stdlib::BindOp::new(tokenizer.clone())); tref.register_token(regex(r"bind!"), move |_| { bind_op.clone() }); // &self should be updated @@ -470,7 +471,7 @@ pub fn register_runner_tokens(metta: &Metta) { tref.register_token(regex(r"&self"), move |_| { self_atom.clone() }); } -pub fn register_rust_tokens(metta: &Metta) { +pub fn register_rust_stdlib_tokens(target: &mut Tokenizer) { let mut rust_tokens = Tokenizer::new(); let tref = &mut rust_tokens; @@ -491,11 +492,25 @@ pub fn register_rust_tokens(metta: &Metta) { let mod_op = Atom::gnd(ModOp{}); tref.register_token(regex(r"%"), move |_| { mod_op.clone() }); - metta.tokenizer().borrow_mut().move_front(&mut rust_tokens); + target.move_front(&mut rust_tokens); } pub static METTA_CODE: &'static str = include_str!("stdlib.metta"); +/// Initializes the Rust stdlib module +pub(crate) fn init_rust_stdlib(context: &mut RunContext, descriptor: ModuleDescriptor) -> Result<(), String> { + + let space = DynSpace::new(GroundingSpace::new()); + context.init_self_module(descriptor, space, None); + + register_rust_stdlib_tokens(&mut *context.module().tokenizer().borrow_mut()); + + let parser = SExprParser::new(METTA_CODE); + context.push_parser(parser); + + Ok(()) +} + #[cfg(test)] mod tests { use super::*; diff --git a/lib/src/metta/text.rs b/lib/src/metta/text.rs index 6c0f1d007..ec2977af2 100644 --- a/lib/src/metta/text.rs +++ b/lib/src/metta/text.rs @@ -34,7 +34,7 @@ impl Tokenizer { } pub fn register_token Atom>(&mut self, regex: Regex, constr: C) { - self.tokens.push(TokenDescr{ regex, constr: Rc::new(constr) }) + self.register_token_with_func_ptr(regex, Rc::new(constr)) } pub fn register_token_with_regex_str Atom>(&mut self, regex: &str, constr: C) { @@ -60,6 +60,19 @@ impl Tokenizer { }).map(|descr| &*(descr.constr)) } + /// Registers the regex-function pair, for a function that's already wrapped in an RC pointer + pub(crate) fn register_token_with_func_ptr(&mut self, regex: Regex, constr: Rc) { + self.tokens.push(TokenDescr{ regex, constr: constr }) + } + + /// Returns the constructor function associated with an exact regex string, or None if the Tokenizer + /// does not contain the specified regex + pub(crate) fn find_exact(&self, regex_str: &str) -> Option> { + self.tokens.iter().rfind(|descr| { + descr.regex.as_str() == regex_str + }).map(|descr| descr.constr.clone()) + } + } /// The meaning of a parsed syntactic element, generated from a substring in the input text diff --git a/repl/src/metta_shim.rs b/repl/src/metta_shim.rs index 5aa80e0e2..edb41bd8a 100644 --- a/repl/src/metta_shim.rs +++ b/repl/src/metta_shim.rs @@ -369,7 +369,8 @@ pub mod metta_interface_mod { } pub fn load_metta_module(&mut self, module: PathBuf) { - self.metta.load_module(module).unwrap(); + //LP-TODO-NEXT, put this back when it's working again + //self.metta.load_module(module).unwrap(); } pub fn parse_and_unroll_syntax_tree(&self, line: &str) -> Vec<(SyntaxNodeType, std::ops::Range)> {