diff --git a/README.md b/README.md index ba7e806..263c1c7 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,16 @@ The CONTROL-CORE framework consists of the below projects. _concore_ enables composing studies from programs developed in different languages. Currently supported languages are, Python, Matlab/Octave, Verilog, and C++. The studies are designed through the visual _concore_ Editor (DHGWorkflow) and interpreted into _concore_ through its parser. Neural control systems consist of loops (dicycles). Therefore, they cannot be represented by classic workflow standards (such as CWL or WDL). Therefore, _concore_ addresses a significant research gap to model closed-loop neuromodulation control systems. The _concore_ protocol shares data between the programs through file sharing, with no centralized entity (a broker or an orchestrator) to arbitrate communications between the programs. (In the distributed executions, the CONTROL-CORE Mediator enables connecting the disjoint pieces of the study through REST APIs). +## Wire Format + +Concore payloads follow Python literal syntax compatible with `ast.literal_eval()`. All language implementations (Python, C++, Java, MATLAB) parse this shared format. Supported value types include: + +* **Numbers** — integers and floats, including scientific notation (e.g., `1e3`, `-2.5`) +* **Booleans** — `True` / `False` (converted to `1.0` / `0.0` in numeric contexts) +* **Strings** — single- or double-quoted (e.g., `"start"`, `'label'`) +* **Nested arrays** — `[1, [2, 3]]` +* **Tuples** — `(1.0, 2.0)` (treated identically to arrays) + # Installation and Getting Started Guide diff --git a/TestLiteralEvalCpp.cpp b/TestLiteralEvalCpp.cpp new file mode 100644 index 0000000..5746837 --- /dev/null +++ b/TestLiteralEvalCpp.cpp @@ -0,0 +1,307 @@ +/** + * TestLiteralEvalCpp.cpp + * + * Test suite for the C++ Python-literal-compatible parser in concore_base.hpp. + * Validates Issue #389 fix: C++ parser must accept all valid concore payloads + * that Python's ast.literal_eval() accepts. + * + * Compile: g++ -std=c++11 -o TestLiteralEvalCpp TestLiteralEvalCpp.cpp + * Run: ./TestLiteralEvalCpp (Linux/macOS) + * TestLiteralEvalCpp.exe (Windows) + */ + +#include +#include +#include +#include +#include +#include + +#include "concore_base.hpp" + +using namespace concore_base; + +static int passed = 0; +static int failed = 0; + +// ------------- helpers ------------------------------------------------- + +static void check(const std::string& testName, bool condition) { + if (condition) { + std::cout << "PASS: " << testName << std::endl; + ++passed; + } else { + std::cout << "FAIL: " << testName << std::endl; + ++failed; + } +} + +static bool approx(double a, double b, double eps = 1e-9) { + return std::fabs(a - b) < eps; +} + +// ------------- backward-compatibility tests ---------------------------- + +static void test_flat_numeric_list() { + std::vector v = parselist_double("[10.0, 0.5, 2.3]"); + check("flat_numeric size==3", v.size() == 3); + check("flat_numeric[0]==10.0", approx(v[0], 10.0)); + check("flat_numeric[1]==0.5", approx(v[1], 0.5)); + check("flat_numeric[2]==2.3", approx(v[2], 2.3)); +} + +static void test_empty_list() { + std::vector v = parselist_double("[]"); + check("empty_list size==0", v.size() == 0); +} + +static void test_single_element() { + std::vector v = parselist_double("[42.0]"); + check("single_element size==1", v.size() == 1); + check("single_element[0]==42", approx(v[0], 42.0)); +} + +static void test_negative_numbers() { + std::vector v = parselist_double("[-1.5, -3.0, 2.0]"); + check("negative size==3", v.size() == 3); + check("negative[0]==-1.5", approx(v[0], -1.5)); + check("negative[1]==-3.0", approx(v[1], -3.0)); +} + +static void test_scientific_notation() { + std::vector v = parselist_double("[1e3, 2.5E-2, -1.0e+1]"); + check("sci size==3", v.size() == 3); + check("sci[0]==1000", approx(v[0], 1000.0)); + check("sci[1]==0.025", approx(v[1], 0.025)); + check("sci[2]==-10", approx(v[2], -10.0)); +} + +static void test_integer_values() { + std::vector v = parselist_double("[1, 2, 3]"); + check("int size==3", v.size() == 3); + check("int[0]==1", approx(v[0], 1.0)); + check("int[2]==3", approx(v[2], 3.0)); +} + +// ------------- mixed-type payload tests (Issue #389 core) -------------- + +static void test_string_element() { + // [10.0, "start", 0.5] – string should be skipped in numeric flatten + std::vector v = parselist_double("[10.0, \"start\", 0.5]"); + check("string_elem size==2", v.size() == 2); + check("string_elem[0]==10.0", approx(v[0], 10.0)); + check("string_elem[1]==0.5", approx(v[1], 0.5)); +} + +static void test_boolean_element() { + // [10.0, True, 0.5] + std::vector v = parselist_double("[10.0, True, 0.5]"); + check("bool_elem size==3", v.size() == 3); + check("bool_elem[0]==10.0", approx(v[0], 10.0)); + check("bool_elem[1]==1.0 (True)", approx(v[1], 1.0)); + check("bool_elem[2]==0.5", approx(v[2], 0.5)); +} + +static void test_bool_false() { + std::vector v = parselist_double("[False, 5.0]"); + check("bool_false size==2", v.size() == 2); + check("bool_false[0]==0.0", approx(v[0], 0.0)); +} + +static void test_nested_list() { + // [10.0, [0.5, 0.3], 0.1] – nested list flattened to [10.0, 0.5, 0.3, 0.1] + std::vector v = parselist_double("[10.0, [0.5, 0.3], 0.1]"); + check("nested size==4", v.size() == 4); + check("nested[0]==10.0", approx(v[0], 10.0)); + check("nested[1]==0.5", approx(v[1], 0.5)); + check("nested[2]==0.3", approx(v[2], 0.3)); + check("nested[3]==0.1", approx(v[3], 0.1)); +} + +static void test_tuple_payload() { + // (10.0, 0.3) – tuple treated as array + std::vector v = parselist_double("(10.0, 0.3)"); + check("tuple size==2", v.size() == 2); + check("tuple[0]==10.0", approx(v[0], 10.0)); + check("tuple[1]==0.3", approx(v[1], 0.3)); +} + +static void test_nested_tuple() { + // [10.0, (0.5, 0.3)] + std::vector v = parselist_double("[10.0, (0.5, 0.3)]"); + check("nested_tuple size==3", v.size() == 3); + check("nested_tuple[0]==10.0", approx(v[0], 10.0)); + check("nested_tuple[1]==0.5", approx(v[1], 0.5)); + check("nested_tuple[2]==0.3", approx(v[2], 0.3)); +} + +static void test_mixed_types() { + // [10.0, "label", True, [1, 2], (3,), False, "end"] + std::vector v = parselist_double("[10.0, \"label\", True, [1, 2], (3,), False, \"end\"]"); + // numeric values: 10.0, 1.0(True), 1, 2, 3, 0.0(False) = 6 values + check("mixed size==6", v.size() == 6); + check("mixed[0]==10.0", approx(v[0], 10.0)); + check("mixed[1]==1.0", approx(v[1], 1.0)); // True + check("mixed[2]==1.0", approx(v[2], 1.0)); // nested [1,...] + check("mixed[3]==2.0", approx(v[3], 2.0)); // nested [...,2] + check("mixed[4]==3.0", approx(v[4], 3.0)); // tuple (3,) + check("mixed[5]==0.0", approx(v[5], 0.0)); // False +} + +// ------------- full ConcoreValue parse tests --------------------------- + +static void test_parse_literal_string() { + ConcoreValue v = parse_literal("[10.0, \"start\", 0.5]"); + check("literal_string is ARRAY", v.type == ConcoreValueType::ARRAY); + check("literal_string len==3", v.array.size() == 3); + check("literal_string[0] NUMBER", v.array[0].type == ConcoreValueType::NUMBER); + check("literal_string[1] STRING", v.array[1].type == ConcoreValueType::STRING); + check("literal_string[1]==\"start\"", v.array[1].str == "start"); + check("literal_string[2] NUMBER", v.array[2].type == ConcoreValueType::NUMBER); +} + +static void test_parse_literal_bool() { + ConcoreValue v = parse_literal("[True, False]"); + check("literal_bool is ARRAY", v.type == ConcoreValueType::ARRAY); + check("literal_bool[0] BOOL", v.array[0].type == ConcoreValueType::BOOL); + check("literal_bool[0]==true", v.array[0].boolean == true); + check("literal_bool[1]==false", v.array[1].boolean == false); +} + +static void test_parse_literal_nested() { + ConcoreValue v = parse_literal("[1, [2, [3]]]"); + check("literal_nested outer ARRAY", v.type == ConcoreValueType::ARRAY); + check("literal_nested[1] ARRAY", v.array[1].type == ConcoreValueType::ARRAY); + check("literal_nested[1][1] ARRAY", v.array[1].array[1].type == ConcoreValueType::ARRAY); + check("literal_nested[1][1][0]==3", approx(v.array[1].array[1].array[0].number, 3.0)); +} + +static void test_parse_single_quoted_string() { + ConcoreValue v = parse_literal("['hello']"); + check("single_quote ARRAY", v.type == ConcoreValueType::ARRAY); + check("single_quote[0] STRING", v.array[0].type == ConcoreValueType::STRING); + check("single_quote[0]=='hello'", v.array[0].str == "hello"); +} + +static void test_parse_escape_sequences() { + ConcoreValue v = parse_literal("[\"line\\none\"]"); + check("escape STRING", v.array[0].type == ConcoreValueType::STRING); + check("escape has newline", v.array[0].str == "line\none"); +} + +static void test_parse_none() { + ConcoreValue v = parse_literal("[None, 1]"); + check("none[0] STRING", v.array[0].type == ConcoreValueType::STRING); + check("none[0]==\"None\"", v.array[0].str == "None"); +} + +static void test_trailing_comma() { + // Python allows trailing comma: [1, 2,] + std::vector v = parselist_double("[1, 2,]"); + check("trailing_comma size==2", v.size() == 2); + check("trailing_comma[1]==2", approx(v[1], 2.0)); +} + +// ------------- error / failure case tests ------------------------------ + +static void test_malformed_bracket() { + bool caught = false; + try { + parse_literal("[1, 2"); + } catch (const std::runtime_error&) { + caught = true; + } + check("malformed_bracket throws", caught); +} + +static void test_malformed_string() { + bool caught = false; + try { + parse_literal("[\"unterminated]"); + } catch (const std::runtime_error&) { + caught = true; + } + check("malformed_string throws", caught); +} + +static void test_unsupported_object() { + bool caught = false; + try { + parse_literal("{1: 2}"); + } catch (const std::runtime_error&) { + caught = true; + } + check("unsupported_object throws", caught); +} + +static void test_empty_string_input() { + std::vector v = parselist_double(""); + check("empty_input size==0", v.size() == 0); +} + +// ------------- cross-language round-trip tests ------------------------- + +static void test_python_write_cpp_read_flat() { + // Simulate Python write: "[5.0, 1.0, 2.0]" + std::vector v = parselist_double("[5.0, 1.0, 2.0]"); + check("py2cpp_flat size==3", v.size() == 3); + check("py2cpp_flat[0]==5.0", approx(v[0], 5.0)); +} + +static void test_python_write_cpp_read_mixed() { + // Simulate Python write: "[5.0, 'sensor_a', True, [0.1, 0.2]]" + std::vector v = parselist_double("[5.0, 'sensor_a', True, [0.1, 0.2]]"); + // numeric: 5.0, 1.0(True), 0.1, 0.2 = 4 + check("py2cpp_mixed size==4", v.size() == 4); + check("py2cpp_mixed[0]==5.0", approx(v[0], 5.0)); + check("py2cpp_mixed[1]==1.0", approx(v[1], 1.0)); + check("py2cpp_mixed[2]==0.1", approx(v[2], 0.1)); + check("py2cpp_mixed[3]==0.2", approx(v[3], 0.2)); +} + +// ------------- main ---------------------------------------------------- + +int main() { + std::cout << "===== C++ Literal Parser Tests (Issue #389) =====\n\n"; + + // Backward compatibility + test_flat_numeric_list(); + test_empty_list(); + test_single_element(); + test_negative_numbers(); + test_scientific_notation(); + test_integer_values(); + + // Mixed-type payloads (core of Issue #389) + test_string_element(); + test_boolean_element(); + test_bool_false(); + test_nested_list(); + test_tuple_payload(); + test_nested_tuple(); + test_mixed_types(); + + // Full ConcoreValue structure tests + test_parse_literal_string(); + test_parse_literal_bool(); + test_parse_literal_nested(); + test_parse_single_quoted_string(); + test_parse_escape_sequences(); + test_parse_none(); + test_trailing_comma(); + + // Error / failure cases + test_malformed_bracket(); + test_malformed_string(); + test_unsupported_object(); + test_empty_string_input(); + + // Cross-language round-trip + test_python_write_cpp_read_flat(); + test_python_write_cpp_read_mixed(); + + std::cout << "\n=== Results: " << passed << " passed, " << failed + << " failed out of " << (passed + failed) << " tests ===\n"; + + return (failed > 0) ? 1 : 0; +} diff --git a/concore.hpp b/concore.hpp index da2c792..02a72f5 100644 --- a/concore.hpp +++ b/concore.hpp @@ -256,6 +256,14 @@ class Concore{ return concore_base::parselist_double(f); } + concore_base::ConcoreValue parse_literal(string f){ + return concore_base::parse_literal(f); + } + + vector flatten_numeric(const concore_base::ConcoreValue& v){ + return concore_base::flatten_numeric(v); + } + /** * @brief deviate the read to either the SM (Shared Memory) or FM (File Method) communication protocol based on iport and oport. * @param port The port number. diff --git a/concore_base.hpp b/concore_base.hpp index 6479942..e5a7b34 100644 --- a/concore_base.hpp +++ b/concore_base.hpp @@ -81,17 +81,210 @@ inline std::vector parselist(const std::string& str) { return result; } -/** - * Parses a double-valued list like "[0.0, 1.5, 2.3]" into a vector. - * Used by concore.hpp's read/write which work with numeric data. - */ +inline std::vector parselist_double(const std::string& str); + +enum class ConcoreValueType { NUMBER, BOOL, STRING, ARRAY }; + +struct ConcoreValue { + ConcoreValueType type; + double number; + bool boolean; + std::string str; + std::vector array; + + ConcoreValue() : type(ConcoreValueType::NUMBER), number(0.0), boolean(false) {} + + static ConcoreValue make_number(double v) { + ConcoreValue cv; + cv.type = ConcoreValueType::NUMBER; + cv.number = v; + return cv; + } + static ConcoreValue make_bool(bool v) { + ConcoreValue cv; + cv.type = ConcoreValueType::BOOL; + cv.boolean = v; + cv.number = v ? 1.0 : 0.0; + return cv; + } + static ConcoreValue make_string(const std::string& v) { + ConcoreValue cv; + cv.type = ConcoreValueType::STRING; + cv.str = v; + return cv; + } + static ConcoreValue make_array(const std::vector& v) { + ConcoreValue cv; + cv.type = ConcoreValueType::ARRAY; + cv.array = v; + return cv; + } +}; + +inline void skip_ws(const std::string& s, size_t& pos) { + while (pos < s.size() && std::isspace(static_cast(s[pos]))) + ++pos; +} + +inline ConcoreValue parse_literal_value(const std::string& s, size_t& pos); + +inline ConcoreValue parse_literal_string(const std::string& s, size_t& pos) { + char quote = s[pos]; + ++pos; + std::string result; + while (pos < s.size() && s[pos] != quote) { + if (s[pos] == '\\' && pos + 1 < s.size()) { + ++pos; + switch (s[pos]) { + case 'n': result += '\n'; break; + case 't': result += '\t'; break; + case '\\': result += '\\'; break; + case '\'': result += '\''; break; + case '"': result += '"'; break; + default: result += '\\'; result += s[pos]; break; + } + } else { + result += s[pos]; + } + ++pos; + } + if (pos >= s.size()) + throw std::runtime_error("Invalid concore payload: unterminated string"); + ++pos; + return ConcoreValue::make_string(result); +} + +inline ConcoreValue parse_literal_array(const std::string& s, size_t& pos) { + char open = s[pos]; + char close = (open == '[') ? ']' : ')'; + ++pos; + std::vector elements; + skip_ws(s, pos); + if (pos < s.size() && s[pos] == close) { ++pos; return ConcoreValue::make_array(elements); } + while (pos < s.size()) { + elements.push_back(parse_literal_value(s, pos)); + skip_ws(s, pos); + if (pos < s.size() && s[pos] == ',') { ++pos; skip_ws(s, pos); } + if (pos < s.size() && s[pos] == close) { ++pos; return ConcoreValue::make_array(elements); } + } + throw std::runtime_error("Invalid concore payload: unterminated array/tuple"); +} + +inline ConcoreValue parse_literal_value(const std::string& s, size_t& pos) { + skip_ws(s, pos); + if (pos >= s.size()) + throw std::runtime_error("Invalid concore payload: unexpected end of input"); + + char c = s[pos]; + + if (c == '[' || c == '(') + return parse_literal_array(s, pos); + + if (c == '\'' || c == '"') + return parse_literal_string(s, pos); + + if (s.compare(pos, 4, "True") == 0 && + (pos + 4 >= s.size() || !std::isalnum(static_cast(s[pos + 4])))) { + pos += 4; + return ConcoreValue::make_bool(true); + } + if (s.compare(pos, 5, "False") == 0 && + (pos + 5 >= s.size() || !std::isalnum(static_cast(s[pos + 5])))) { + pos += 5; + return ConcoreValue::make_bool(false); + } + if (s.compare(pos, 4, "None") == 0 && + (pos + 4 >= s.size() || !std::isalnum(static_cast(s[pos + 4])))) { + pos += 4; + return ConcoreValue::make_string("None"); + } + + { + size_t start = pos; + if (pos < s.size() && (s[pos] == '+' || s[pos] == '-')) ++pos; + bool has_digits = false; + while (pos < s.size() && std::isdigit(static_cast(s[pos]))) { + ++pos; has_digits = true; + } + if (pos < s.size() && s[pos] == '.') { + ++pos; + while (pos < s.size() && std::isdigit(static_cast(s[pos]))) { + ++pos; has_digits = true; + } + } + if (has_digits && pos < s.size() && (s[pos] == 'e' || s[pos] == 'E')) { + ++pos; + if (pos < s.size() && (s[pos] == '+' || s[pos] == '-')) ++pos; + while (pos < s.size() && std::isdigit(static_cast(s[pos]))) ++pos; + } + if (has_digits && pos > start) { + std::string numstr = s.substr(start, pos - start); + try { + double val = std::stod(numstr); + return ConcoreValue::make_number(val); + } catch (...) { + throw std::runtime_error( + "Invalid concore payload: bad number '" + numstr + "'"); + } + } + pos = start; + } + + throw std::runtime_error( + std::string("Invalid concore payload: unsupported literal at position ") + + std::to_string(pos)); +} + +inline ConcoreValue parse_literal(const std::string& s) { + size_t pos = 0; + ConcoreValue v = parse_literal_value(s, pos); + skip_ws(s, pos); + if (pos != s.size()) + throw std::runtime_error( + "Invalid concore payload: unexpected trailing content"); + return v; +} + +inline void flatten_numeric_impl(const ConcoreValue& v, std::vector& out) { + switch (v.type) { + case ConcoreValueType::NUMBER: + out.push_back(v.number); + break; + case ConcoreValueType::BOOL: + out.push_back(v.boolean ? 1.0 : 0.0); + break; + case ConcoreValueType::STRING: + break; + case ConcoreValueType::ARRAY: + for (const auto& elem : v.array) + flatten_numeric_impl(elem, out); + break; + } +} + +inline std::vector flatten_numeric(const ConcoreValue& v) { + std::vector out; + flatten_numeric_impl(v, out); + return out; +} + inline std::vector parselist_double(const std::string& str) { - std::vector result; - std::vector tokens = parselist(str); - for (const auto& tok : tokens) { - result.push_back(std::stod(tok)); + std::string trimmed = stripstr(str); + if (trimmed.empty()) return {}; + try { + ConcoreValue v = parse_literal(trimmed); + return flatten_numeric(v); + } catch (...) { + std::vector result; + if (trimmed.size() < 2) return result; + if (trimmed.front() == '[' || trimmed.front() == '(') { + std::vector tokens = parselist(trimmed); + for (const auto& tok : tokens) { + try { result.push_back(std::stod(tok)); } catch (...) {} + } + } + return result; } - return result; } /**