Update example code and add tests and parsee support for blocks (#406)

* Update example code to match parser sytax and add test tags

* Parser changes to support planned features

* Add tests for example code using test-each

examples/contexts.tk

@@ -1,27 +1,28 @@
 #!/usr/bin/env tako
+// test: all
 
 test() = {
   allocator = import("allocator")
-  ptr = allocator.allocate(sizeof(Char[100]))
-  if (ptr == nullptr) {
+  ptr = allocator.allocate(sizeof(Char)*100)
+  if(ptr == nullptr, {
     print("Success")
     exit(0)
-  } else {
+  }, {
     print("Failure")
-  }
+  })
 }
 
 main() = {
   allocator = import("allocator")
   logger = import("logger")
 
   test_logger = logger.new_logger()
-  with {
+  with(
     allocator.allocate = allocator.report_oom,
     system.out = test_logger,
     system.error = test_logger,
     run = test
-  }
+  )
 
   print(test_logger)
 }

examples/enums.tk

@@ -1,4 +1,5 @@
 #!/usr/bin/env tako
+// test: all
 
 boolean = Enum.new(
   true=1,
@@ -9,7 +10,7 @@ boolean.import_literals()
 
 color = Enum
   .based_on(
-    byte[3]
+    [byte; 3]
   )
   .open(
     red=#ff0000,

examples/fib.tk

@@ -1,4 +1,5 @@
 #!/usr/bin/env tako
+// test: todo
 
 fib(n: T, {T: Nat}): T=if(n<=1, 1, fib(n-1)+fib(n-2))
 

examples/fib_acc.tk

@@ -1,4 +1,5 @@
 #!/usr/bin/env tako
+// test: todo
 
 fib_helper(n: T, a: T, b: T, {T: Nat}): T=if(n==0, a, fib_helper(n-1, b, a+b))
 

examples/generic_abstract_data_types.tk

@@ -1,17 +1,18 @@
 #!/usr/bin/env tako
+// test: todo
 
 // A simple, untyped  implementation of S expressions.
 SExpr: Type = struct.new(
-  func: (args: Value[n]),
-  args: SExpr[n],
+  func: (args: [Value; n]),
+  args: [SExpr; n],
   {n: Int},
 )
 
 // A well typed, but little, language using GADTs.
 Expr(
   SubExpr(R: AllowedType): Reference(Expr(SubExpr, R))=Expr,
   R: AllowedType,
-  with {AllowedType = Integer|Boolean}
+  {AllowedType = Integer|Boolean}
 ) = gadt.new(
 
   // Control flow:

examples/instances.tk

@@ -1,4 +1,5 @@
 #!/usr/bin/env tako
+// test: todo parse error
 
 Bounded = Enum
   .open()

examples/simple_sum.tk

@@ -1,4 +1,6 @@
 #!/usr/bin/env tako
+// test: all
+
 1/*one*/+2/*two*/+3/*three*/
 
 /* Comment

examples/vector_transpose.tk

@@ -1,4 +1,5 @@
 #!/usr/bin/env tako
+// test: todo parse
 
 main = {
   x3by2 = Vector.new([

examples/vector_transpose_failing.tk

@@ -1,4 +1,5 @@
 #!/usr/bin/env tako
+// test: todo type error fix parse error
 
 main() = {
   x3by2 = Vector.new([

takolib/Cargo.toml

@@ -52,6 +52,7 @@ melior = { version = "0.17.0", optional = true }
 smallvec = { version = "2.0.0-alpha.5" }
 llamada = { path = "../llamada", features = [ ] }
 better-std = { path = "../better-std", features = [ ] }
+test-each = "0.2.1"
 
 [dev-dependencies]
 strum = "0.26.2"

takolib/src/parser/mod.rs

@@ -5,7 +5,7 @@ use crate::ast::string_interner::Identifier;
 use crate::ast::{Ast, Atom, Call, Contains, Definition, NodeData, NodeId, Op};
 use crate::error::TError;
 use better_std::include_strs;
-use log::trace;
+use log::{debug, trace};
 use semantics::BindingMode;
 use semantics::Literal;
 use smallvec::smallvec;
@@ -313,15 +313,20 @@ impl<'src, 'toks, T: Iterator<Item = &'toks Token>> ParseState<'src, 'toks, T> {
         Ok(args)
     }
 
-    fn call_or_definition(&mut self, name: Token, binding: Symbol) -> Result<NodeId, TError> {
+    fn call_definition_or_repeated(
+        &mut self,
+        name: Token,
+        binding: Symbol,
+    ) -> Result<NodeId, TError> {
         let name_id = self.name(name);
         trace!(
-            "Call or definition: {name:?}: {:?}",
+            "Call, definition or repeated: {name:?}: {:?}",
             self.ast.string_interner.get_str(name_id)
         );
         let location = name.location();
         let mut bindings = vec![];
         let mut has_args = false;
+        let mut is_a_repeated = false;
         let mut has_non_bind_args = false; // i.e. this should be a definition...
         if self.operator_is(Symbol::OpenParen).is_ok() {
             trace!("has arguments");
@@ -334,6 +339,17 @@ impl<'src, 'toks, T: Iterator<Item = &'toks Token>> ParseState<'src, 'toks, T> {
                     break;
                 }
             }
+        } else if self.operator_is(Symbol::OpenBracket).is_ok() {
+            has_args = true;
+            is_a_repeated = true;
+            // Read args...
+            while self.operator_is(Symbol::CloseBracket).is_err() {
+                bindings.push(self.binding_or_arg(&mut has_non_bind_args)?);
+                if self.require(TokenType::Op(Symbol::Comma)).is_err() {
+                    self.require(TokenType::Op(Symbol::CloseBracket))?;
+                    break;
+                }
+            }
         }
         let ty = if self.has_type().is_ok() {
             trace!("HasType started");
@@ -420,6 +436,9 @@ impl<'src, 'toks, T: Iterator<Item = &'toks Token>> ParseState<'src, 'toks, T> {
             // TODO: USE bindings
             trace!("Add call");
             let args = self.handle_bindings(bindings)?.into();
+            if is_a_repeated {
+                todo!("Handle lists and arrays");
+            }
             let call = self.ast.add_call(Call { inner, args }, location);
             if let Some(ty) = ty {
                 return Ok(self.ast.add_annotation(call, ty));
@@ -434,6 +453,21 @@ impl<'src, 'toks, T: Iterator<Item = &'toks Token>> ParseState<'src, 'toks, T> {
         Ok(ident)
     }
 
+    fn file(&mut self) -> Result<NodeId, TError> {
+        let mut left = self.expr(Symbol::OpenParen)?;
+        while let Ok(token) = self.peek().copied() {
+            let right = self.expr(Symbol::OpenParen)?;
+            left = self.ast.add_op(
+                Op {
+                    op: Symbol::Sequence,
+                    args: smallvec![left, right],
+                },
+                token.location(),
+            );
+        }
+        Ok(left)
+    }
+
     fn expr(&mut self, binding: Symbol) -> Result<NodeId, TError> {
         let Ok(mut token) = self.peek().copied() else {
             return Err(ParseError::UnexpectedEof.into());
@@ -450,8 +484,30 @@ impl<'src, 'toks, T: Iterator<Item = &'toks Token>> ParseState<'src, 'toks, T> {
         } else if self.operator_is(Symbol::OpenCurly).is_ok() {
             // TODO: Support sequence&dictionary syntax.
             // Tuple, parenthesized expr... etc.
-            let left = self.expr(Symbol::OpenParen)?;
-            self.require(TokenType::Op(Symbol::CloseCurly))?;
+            let mut left = self.expr(Symbol::OpenParen)?;
+            let mut joiner = Symbol::Sequence;
+            while self.operator_is(Symbol::CloseCurly).is_err() {
+                // TODO: Clean up sequence generation.
+                // TODO: Check that a sequence / args is of the right structure?
+                let next = self.expr(Symbol::OpenParen)?;
+                left = self.ast.add_op(
+                    Op {
+                        op: joiner,
+                        args: smallvec![left, next],
+                    },
+                    location,
+                );
+                if self.require(TokenType::Op(Symbol::Comma)).is_ok() {
+                    // TODO: Set next joiner
+                    joiner = Symbol::Comma;
+                } else if self.require(TokenType::Op(Symbol::Sequence)).is_ok() {
+                    // TODO: Set next joiner
+                    joiner = Symbol::Sequence;
+                } else {
+                    // Allow it? Maybe...
+                    joiner = Symbol::Sequence;
+                }
+            }
             left
         } else if self.operator_is(Symbol::OpenParen).is_ok() {
             // TODO: Support list syntax.
@@ -478,22 +534,27 @@ impl<'src, 'toks, T: Iterator<Item = &'toks Token>> ParseState<'src, 'toks, T> {
                         )
                     }
                     _ => {
+                        let st = location.start.into();
+                        let end = std::cmp::min(st + 50, self.contents.len());
+                        debug!("ERROR AT:\n{}", &self.contents[st..end]);
                         return Err(ParseError::MissingLeftHandSideOfOperator {
                             op: symbol,
                             location,
                         }
-                        .into())
+                        .into());
                     }
                 }
             }
         } else if let Ok(token) = self.ident() {
-            self.call_or_definition(token, binding)?
+            self.call_definition_or_repeated(token, binding)?
         } else if let Ok(token) = self.token_of_type(TokenType::Atom) {
             self.atom(token, location)
         } else if let Ok(token) = self.token_of_type(TokenType::NumberLit) {
             self.number_literal(token, location)
         } else if let Ok(token) = self.token_of_type(TokenType::StringLit) {
             self.string_literal(token, location)
+        } else if let Ok(token) = self.token_of_type(TokenType::ColorLit) {
+            self.color_literal(token, location)
         } else {
             return Err(ParseError::UnexpectedTokenTypeInExpression {
                 got: token.kind,
@@ -589,6 +650,16 @@ impl<'src, 'toks, T: Iterator<Item = &'toks Token>> ParseState<'src, 'toks, T> {
         self.ast.add_literal(Literal::Text, location)
     }
 
+    fn color_literal(&mut self, res: Token, location: Location) -> NodeId {
+        assert!(res.kind == TokenType::ColorLit);
+        trace!("Saving literal: {res:?}");
+        let _id = self
+            .ast
+            .string_interner
+            .register_str_by_loc(res.get_src(self.contents), location.start);
+        self.ast.add_literal(Literal::Color, location)
+    }
+
     fn number_literal(&mut self, res: Token, location: Location) -> NodeId {
         assert!(res.kind == TokenType::NumberLit);
         trace!("Saving literal: {res:?}");
@@ -609,7 +680,7 @@ pub fn parse(filepath: &Path, contents: &str, tokens: &[Token]) -> Result<Ast, T
     };
     if !tokens.is_empty() {
         // Support empty files!
-        let root = state.expr(Symbol::OpenParen)?;
+        let root = state.file()?;
         state.ast.set_root(root);
     }
     // TODO(testing): REMOVE THIS (it's just to test the threading model)
@@ -1002,10 +1073,15 @@ pub mod tests {
         setup("a << b + 1").expect("Disallowed syntax");
     }
 
+    #[test]
+    fn parse_atom_atom_in_file() {
+        setup("$a $b").expect("Top level allowed syntax");
+    }
+
     #[test]
     #[should_panic] // TODO(errors): Implement!
-    fn parse_atom_atom() {
-        setup("$a $b").expect("Disallowed syntax");
+    fn parse_atom_atom_in_expr() {
+        setup("($a $b)").expect("Disallowed syntax");
     }
 
     #[test]