Change parser to build syntax tree, which is used for syntax highlighting, add config.metta to repl

c/src/metta.rs

@@ -1,10 +1,8 @@
 use hyperon::common::shared::Shared;
 use hyperon::space::DynSpace;
-use hyperon::Atom;
 use hyperon::metta::text::*;
 use hyperon::metta::interpreter;
-use hyperon::metta::interpreter::InterpretedAtom;
-use hyperon::common::plan::StepResult;
+use hyperon::metta::interpreter::InterpreterState;
 use hyperon::metta::runner::Metta;
 use hyperon::rust_type_atom;
 
@@ -345,19 +343,19 @@ pub struct step_result_t {
     result: *mut RustStepResult,
 }
 
-struct RustStepResult(StepResult<'static, Vec<InterpretedAtom>, (Atom, Atom)>);
+struct RustStepResult(InterpreterState<'static, DynSpace>);
 
-impl From<StepResult<'static, Vec<InterpretedAtom>, (Atom, Atom)>> for step_result_t {
-    fn from(result: StepResult<'static, Vec<InterpretedAtom>, (Atom, Atom)>) -> Self {
-        Self{ result: Box::into_raw(Box::new(RustStepResult(result))) }
+impl From<InterpreterState<'static, DynSpace>> for step_result_t {
+    fn from(state: InterpreterState<'static, DynSpace>) -> Self {
+        Self{ result: Box::into_raw(Box::new(RustStepResult(state))) }
     }
 }
 
 impl step_result_t {
-    fn into_inner(self) -> StepResult<'static, Vec<InterpretedAtom>, (Atom, Atom)> {
+    fn into_inner(self) -> InterpreterState<'static, DynSpace> {
         unsafe{ Box::from_raw(self.result).0 }
     }
-    fn borrow(&self) -> &StepResult<'static, Vec<InterpretedAtom>, (Atom, Atom)> {
+    fn borrow(&self) -> &InterpreterState<'static, DynSpace> {
         &unsafe{ &*(&*self).result }.0
     }
 }
@@ -427,13 +425,9 @@ pub extern "C" fn step_has_next(step: *const step_result_t) -> bool {
 pub extern "C" fn step_get_result(step: step_result_t,
         callback: c_atom_vec_callback_t, context: *mut c_void) {
     let step = step.into_inner();
-    match step {
-        StepResult::Return(mut res) => {
-            let res = res.drain(0..).map(|res| res.into_tuple().0).collect();
-            return_atoms(&res, callback, context);
-        },
-        StepResult::Error(_) => return_atoms(&vec![], callback, context),
-        _ => panic!("Not expected step result: {:?}", step),
+    match step.into_result() {
+        Ok(res) => return_atoms(&res, callback, context),
+        Err(_) => return_atoms(&vec![], callback, context),
     }
 }
 

lib/src/metta/interpreter.rs

@@ -79,6 +79,43 @@ use std::ops::Deref;
 use std::rc::Rc;
 use std::fmt::{Debug, Display, Formatter};
 
+/// Wrapper, So the old interpreter can present the same public interface as the new intperpreter
+pub struct InterpreterState<'a, T: SpaceRef<'a>> {
+    step_result: StepResult<'a, Results, InterpreterError>,
+    phantom: core::marker::PhantomData<T>
+}
+
+impl<'a, T: SpaceRef<'a>> InterpreterState<'a, T> {
+
+    /// INTERNAL USE ONLY. Create an InterpreterState that is ready to yield results
+    pub(crate) fn new_finished(_space: T, results: Vec<Atom>) -> Self {
+        Self {
+            step_result: StepResult::Return(results.into_iter().map(|atom| InterpretedAtom(atom, Bindings::new())).collect()),
+            phantom: <_>::default(),
+        }
+    }
+
+    pub fn has_next(&self) -> bool {
+        self.step_result.has_next()
+    }
+    pub fn into_result(self) -> Result<Vec<Atom>, String> {
+        match self.step_result {
+            StepResult::Return(mut res) => {
+                let res = res.drain(0..).map(|res| res.into_tuple().0).collect();
+                Ok(res)
+            },
+            StepResult::Error((atom, err)) => Ok(vec![Atom::expr([ERROR_SYMBOL, atom, err])]),
+            StepResult::Execute(_) => Err("Evaluation is not finished".into())
+        }
+    }
+}
+
+impl<'a, T: SpaceRef<'a>> Debug for InterpreterState<'a, T> {
+    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
+        Debug::fmt(&self.step_result, f)
+    }
+}
+
 /// Result of atom interpretation plus variable bindings found
 #[derive(Clone, PartialEq)]
 pub struct InterpretedAtom(Atom, Bindings);
@@ -132,7 +169,12 @@ type NoInputPlan<'a> = Box<dyn Plan<'a, (), Results, InterpreterError> + 'a>;
 /// # Arguments
 /// * `space` - atomspace to query for interpretation
 /// * `expr` - atom to interpret
-pub fn interpret_init<'a, T: Space + 'a>(space: T, expr: &Atom) -> StepResult<'a, Results, InterpreterError> {
+pub fn interpret_init<'a, T: Space + 'a>(space: T, expr: &Atom) -> InterpreterState<'a, T> {
+    let step_result = interpret_init_internal(space, expr);
+    InterpreterState { step_result: step_result, phantom: <_>::default() }
+}
+
+fn interpret_init_internal<'a, T: Space + 'a>(space: T, expr: &Atom) -> StepResult<'a, Results, InterpreterError> {
     let context = InterpreterContextRef::new(space);
     interpret_as_type_plan(context,
         InterpretedAtom(expr.clone(), Bindings::new()),
@@ -145,10 +187,10 @@ pub fn interpret_init<'a, T: Space + 'a>(space: T, expr: &Atom) -> StepResult<'a
 ///
 /// # Arguments
 /// * `step` - [StepResult::Execute] result from the previous step.
-pub fn interpret_step<'a>(step: StepResult<'a, Results, InterpreterError>) -> StepResult<'a, Results, InterpreterError> {
+pub fn interpret_step<'a, T: Space + 'a>(step: InterpreterState<'a, T>) -> InterpreterState<'a, T> {
     log::debug!("current plan:\n{:?}", step);
-    match step {
-        StepResult::Execute(plan) => plan.step(()),
+    match step.step_result {
+        StepResult::Execute(plan) => InterpreterState { step_result: plan.step(()), phantom: <_>::default() },
         StepResult::Return(_) => panic!("Plan execution is finished already"),
         StepResult::Error(_) => panic!("Plan execution is finished with error"),
     }
@@ -162,10 +204,10 @@ pub fn interpret_step<'a>(step: StepResult<'a, Results, InterpreterError>) -> St
 /// * `expr` - atom to interpret
 pub fn interpret<T: Space>(space: T, expr: &Atom) -> Result<Vec<Atom>, String> {
     let mut step = interpret_init(space, expr);
-    while step.has_next() {
+    while step.step_result.has_next() {
         step = interpret_step(step);
     }
-    match step {
+    match step.step_result {
         StepResult::Return(mut result) => Ok(result.drain(0..)
             .map(|InterpretedAtom(atom, _)| atom).collect()),
         // TODO: return (Error atom err) expression
@@ -226,7 +268,7 @@ impl SpaceObserver for InterpreterCache {
 
 use std::marker::PhantomData;
 
-trait SpaceRef<'a> : Space + 'a {}
+pub trait SpaceRef<'a> : Space + 'a {}
 impl<'a, T: Space + 'a> SpaceRef<'a> for T {}
 
 struct InterpreterContext<'a, T: SpaceRef<'a>> {

lib/src/metta/interpreter2.rs

@@ -91,11 +91,21 @@ fn atom_into_array<const N: usize>(atom: Atom) -> Option<[Atom; N]> {
 
 impl<'a, T: SpaceRef<'a>> InterpreterState<'a, T> {
 
-    fn has_next(&self) -> bool {
+    /// INTERNAL USE ONLY. Create an InterpreterState that is ready to yield results
+    #[allow(dead_code)] //TODO: only silence the warning until interpreter2 replaces interpreter
+    pub(crate) fn new_finished(space: T, results: Vec<Atom>) -> Self {
+        Self {
+            plan: vec![],
+            finished: results,
+            context: InterpreterContextRef::new(space),
+        }
+    }
+
+    pub fn has_next(&self) -> bool {
         !self.plan.is_empty()
     }
 
-    fn into_result(self) -> Result<Vec<Atom>, String> {
+    pub fn into_result(self) -> Result<Vec<Atom>, String> {
         if self.has_next() {
             Err("Evaluation is not finished".into())
         } else {
@@ -142,6 +152,7 @@ pub fn interpret_init<'a, T: Space + 'a>(space: T, expr: &Atom) -> InterpreterSt
     }
 }
 
+//TODO: These docs are out of date for the new interpreter
 /// Perform next step of the interpretation plan and return the result. Panics
 /// when [StepResult::Return] or [StepResult::Error] are passed as input.
 /// See [crate::metta::interpreter] for algorithm explanation.

lib/src/metta/runner/mod.rs

@@ -11,13 +11,13 @@ use std::path::PathBuf;
 use std::collections::HashMap;
 
 pub mod stdlib;
-use super::interpreter::interpret;
+use super::interpreter::{interpret, interpret_init, interpret_step, InterpreterState};
 use stdlib::*;
 
 // Uncomment three lines below and comment three lines above to
 // switch to the minimal MeTTa version
 //pub mod stdlib2;
-//use super::interpreter2::interpret;
+//use super::interpreter2::{interpret, interpret_init, interpret_step, InterpreterState};
 //use stdlib2::*;
 
 mod arithmetics;
@@ -36,9 +36,17 @@ pub struct MettaContents {
     search_paths: Vec<PathBuf>,
 }
 
-enum Mode {
+#[derive(Debug, PartialEq, Eq)]
+enum MettaRunnerMode {
     ADD,
     INTERPRET,
+    TERMINATE,
+}
+
+pub struct RunnerState<'a> {
+    mode: MettaRunnerMode,
+    interpreter_state: Option<InterpreterState<'a, DynSpace>>,
+    results: Vec<Vec<Atom>>,
 }
 
 impl Metta {
@@ -124,7 +132,7 @@ impl Metta {
         &self.0.search_paths
     }
 
-    pub(crate) fn modules(&self) -> &Shared<HashMap<PathBuf, DynSpace>> {
+    pub fn modules(&self) -> &Shared<HashMap<PathBuf, DynSpace>> {
         &self.0.modules
     }
 
@@ -142,54 +150,91 @@ impl Metta {
     }
 
     pub fn run(&self, parser: &mut SExprParser) -> Result<Vec<Vec<Atom>>, String> {
-        let mut mode = Mode::ADD;
-        let mut results: Vec<Vec<Atom>> = Vec::new();
+        let mut state = self.start_run();
+
+        while !state.is_complete() {
+            self.run_step(parser, &mut state)?;
+        }
+        Ok(state.into_results())
+    }
+
+    pub fn start_run(&self) -> RunnerState {
+        RunnerState::new()
+    }
+
+    pub fn run_step(&self, parser: &mut SExprParser, state: &mut RunnerState) -> Result<(), String> {
+
+        // If we're in the middle of interpreting an atom...
+        if let Some(interpreter_state) = core::mem::replace(&mut state.interpreter_state, None) {
 
-        loop {
-            let atom = parser.parse(&self.0.tokenizer.borrow())?;
+            if interpreter_state.has_next() {
+
+                //Take a step with the interpreter, and put it back for next time
+                state.interpreter_state = Some(interpret_step(interpreter_state))
+            } else {
+
+                //This interpreter is finished, process the results
+                match interpreter_state.into_result() {
+                    Err(msg) => return Err(msg),
+                    Ok(result) => {
+                        fn is_error(atom: &Atom) -> bool {
+                            match atom {
+                                Atom::Expression(expr) => {
+                                    expr.children().len() > 0 && expr.children()[0] == ERROR_SYMBOL
+                                },
+                                _ => false,
+                            }
+                        }
+
+                        let error = result.iter().any(|atom| is_error(atom));
+                        state.results.push(result);
+                        if error {
+                            state.mode = MettaRunnerMode::TERMINATE;
+                            return Ok(());
+                        }
+                    }
+                }
+            }
 
-            if let Some(atom) = atom {
+        } else {
+
+            // We'll parse the next atom, and start a new intperpreter
+            if let Some(atom) = parser.parse(&self.0.tokenizer.borrow())? {
                 if atom == EXEC_SYMBOL {
-                    mode = Mode::INTERPRET;
-                    continue;
+                    state.mode = MettaRunnerMode::INTERPRET;
+                    return Ok(());
                 }
-                match mode {
-                    Mode::ADD => {
+                match state.mode {
+                    MettaRunnerMode::ADD => {
                         if let Err(atom) = self.add_atom(atom) {
-                            results.push(vec![atom]);
-                            break;
+                            state.results.push(vec![atom]);
+                            state.mode = MettaRunnerMode::TERMINATE;
+                            return Ok(());
                         }
                     },
-                    Mode::INTERPRET => {
-                        match self.evaluate_atom(atom) {
-                            Err(msg) => return Err(msg),
-                            Ok(result) => {
-                                fn is_error(atom: &Atom) -> bool {
-                                    match atom {
-                                        Atom::Expression(expr) => {
-                                            expr.children().len() > 0 && expr.children()[0] == ERROR_SYMBOL
-                                        },
-                                        _ => false,
-                                    }
-                                }
-                                let error = result.iter().any(|atom| is_error(atom));
-                                results.push(result);
-                                if error {
-                                    break;
-                                }
-                            }
-                        }
+                    MettaRunnerMode::INTERPRET => {
+
+                        state.interpreter_state = Some(match self.type_check(atom) {
+                            Err(atom) => {
+                                InterpreterState::new_finished(self.space().clone(), vec![atom])
+                            },
+                            Ok(atom) => {
+                                interpret_init(self.space().clone(), &atom)
+                            },
+                        });
+                    },
+                    MettaRunnerMode::TERMINATE => {
+                        return Ok(());
                     },
                 }
-                mode = Mode::ADD;
-            } else {
-                break;
+                state.mode = MettaRunnerMode::ADD;
+            }  else {
+                state.mode = MettaRunnerMode::TERMINATE;
             }
         }
-        Ok(results)
-    }
-
 
+        Ok(())
+    }
 
     pub fn evaluate_atom(&self, atom: Atom) -> Result<Vec<Atom>, String> {
         match self.type_check(atom) {
@@ -215,6 +260,25 @@ impl Metta {
 
 }
 
+impl<'a> RunnerState<'a> {
+    fn new() -> Self {
+        Self {
+            mode: MettaRunnerMode::ADD,
+            interpreter_state: None,
+            results: vec![],
+        }
+    }
+    pub fn is_complete(&self) -> bool {
+        self.mode == MettaRunnerMode::TERMINATE
+    }
+    pub fn intermediate_results(&self) -> &Vec<Vec<Atom>> {
+        &self.results
+    }
+    pub fn into_results(self) -> Vec<Vec<Atom>> {
+        self.results
+    }
+}
+
 pub fn new_metta_rust() -> Metta {
     let metta = Metta::new(DynSpace::new(GroundingSpace::new()),
         Shared::new(Tokenizer::new()));

lib/src/metta/runner/stdlib2.rs

@@ -7,7 +7,6 @@ use crate::metta::runner::Metta;
 use crate::metta::types::{get_atom_types, get_meta_type};
 
 use std::fmt::Display;
-use std::path::PathBuf;
 use regex::Regex;
 
 use super::arithmetics::*;
@@ -115,7 +114,7 @@ pub fn register_common_tokens(metta: &Metta) {
     tref.register_token(regex(r"if-equal"), move |_| { is_equivalent.clone() });
 }
 
-pub fn register_runner_tokens(metta: &Metta, _cwd: PathBuf) {
+pub fn register_runner_tokens(metta: &Metta) {
     let _space = metta.space();
     let tokenizer = metta.tokenizer();