1 // Written in the D programming language. 2 /** 3 Haystack zinc decode. 4 5 Copyright: Copyright (c) 2017, Radu Racariu <radu.racariu@gmail.com> 6 License: $(LINK2 www.boost.org/LICENSE_1_0.txt, Boost License 1.0) 7 Authors: Radu Racariu 8 **/ 9 module haystack.zinc.decode; 10 11 import std.algorithm : move; 12 import std.conv : to; 13 import std.traits : isSomeChar; 14 import std.range.primitives : empty, isInputRange, ElementEncodingType; 15 16 import haystack.tag; 17 import haystack.zinc.lexer; 18 import haystack.util.misc : Own; 19 20 /////////////////////////////////////////////////////////////////// 21 // 22 // Tag Decoding from Zinc 23 // 24 /////////////////////////////////////////////////////////////////// 25 26 /* 27 Parses a Zinc encoded $(Input Range) 28 */ 29 struct ZincParser(Range) 30 if (isInputRange!Range && isSomeChar!(ElementEncodingType!Range)) 31 { 32 alias Parser = ZincParser!Range; 33 alias Lexer = ZincLexer!Range; 34 35 this(Range r) 36 { 37 this.lexer = Lexer(r); 38 if (!r.empty) 39 popFront(); 40 else 41 state = ParserState.fault; 42 } 43 @disable this(); 44 @disable this(this); 45 46 /// True until parsing error or parsing complete 47 @property bool empty() 48 { 49 return (state == ParserState.ok || state == ParserState.fault); 50 } 51 52 /// The last parsed $(D Element) 53 @property ref Element front() 54 { 55 assert(!empty, "Attempting to access front of an empty Grid."); 56 return element; 57 } 58 59 /// Parse next $(D Element) 60 void popFront() 61 { 62 assert(!empty, "Attempting to pop from an empty Grid."); 63 // grid parsing state machine 64 loop: 65 for (; !empty; lexer.popFront()) 66 { 67 final switch (state) 68 { 69 case ParserState.header: 70 if (element.type == Element.Type.header) 71 { 72 if (element.header.consume() && isNewLine) 73 { 74 lexer.popFront(); 75 state = ParserState.colums; 76 } 77 else 78 { 79 state = ParserState.fault; 80 } 81 } 82 else 83 { 84 element.header = Header(this); 85 } 86 return; 87 88 case ParserState.colums: 89 if (element.type == Element.Type.columns) 90 { 91 if (element.columns.consume() && isNewLine) 92 { 93 lexer.popFront(); 94 state = ParserState.rows; 95 } 96 else 97 { 98 state = ParserState.fault; 99 } 100 } 101 else 102 { 103 element.columns = Columns(this); 104 } 105 return; 106 107 case ParserState.rows: 108 if (element.type == Element.Type.rows) 109 { 110 if (element.rows.consume()) 111 { 112 if (isNewLine) 113 { 114 lexer.popFront(); 115 if (isNewLine || lexer.empty) 116 state = ParserState.ok; 117 else 118 element.rows = Rows(this); 119 } 120 else if (lexer.empty || hasChr('>')) 121 { 122 state = ParserState.ok; 123 } 124 else 125 { 126 state = ParserState.fault; 127 } 128 } 129 else 130 { 131 state = ParserState.fault; 132 } 133 } 134 else 135 { 136 if (isSpace) 137 continue; 138 if (isNewLine) 139 state = ParserState.ok; 140 else 141 element.rows = Rows(this); 142 } 143 return; 144 145 case ParserState.ok: 146 case ParserState.fault: 147 assert(false, "Invalid parser state."); 148 } 149 } 150 } 151 152 /* 153 The atomic parts of a Grid. 154 It holds type and value information about header, columns, rows 155 */ 156 struct Element 157 { 158 /// Each Element type 159 enum Type { header, columns, rows, none = uint.max } 160 Type type = Type.none; 161 162 @property void header(Header header) 163 { 164 type = Type.header; 165 move(header, _value.header); 166 } 167 168 @property ref Header header() 169 { 170 assert(type == Type.header); 171 return _value.header; 172 } 173 174 @property void columns(Columns columns) 175 { 176 type = Type.columns; 177 move(columns, _value.columns); 178 } 179 180 @property ref Columns columns() 181 { 182 assert(type == Type.columns); 183 return _value.columns; 184 } 185 186 @property void rows(Rows rows) 187 { 188 type = Type.rows; 189 move(rows, _value.rows); 190 } 191 192 @property ref Rows rows() 193 { 194 assert(type == Type.rows); 195 return _value.rows; 196 } 197 198 string toString() 199 { 200 import std.format : format; 201 return format("%s", type); 202 } 203 204 private: 205 union Value 206 { 207 Header header = void; 208 Columns columns = void; 209 Rows rows = void; 210 } 211 Value _value; 212 @disable this(this); 213 } 214 215 /** 216 Parses the zinc `InputRange` as an `InputRange` of `Dict`s 217 This allows lazy row based parsing. 218 */ 219 auto asRows() scope return 220 { 221 import std.array : appender; 222 auto colList = appender!(Grid.Col[])(); 223 224 // Wraps the row parser into a `InputRange` 225 struct Result 226 { 227 this (scope Rows* rows) 228 { 229 this.rows = rows; 230 if (rows !is null && !rows.empty) 231 popFront(); 232 } 233 234 @property bool empty() scope { return _empty; } 235 236 @property Dict front() { return row; } 237 238 void popFront() 239 { 240 row = Dict.init; 241 for (size_t i = 0; !rows.empty; rows.popFront(), i++) 242 { 243 row[colList.data[i].dis] = rows.front.asTag; 244 } 245 _empty = rows.parser.empty; 246 if (!_empty) 247 rows.parser.popFront(); 248 } 249 250 private: 251 Dict row; 252 bool _empty = true; 253 Rows* rows; 254 } 255 // addvance the parser till the rows portion 256 for (; !empty; popFront) 257 { 258 auto el = &front(); 259 if (el.type == Parser.Element.Type.header) 260 { 261 for (; !el.header.empty; el.header.popFront) 262 { 263 auto h = &el.header.front(); 264 if (h.type == Parser.Header.Type.tags) 265 h.tags.consume(); 266 } 267 } 268 if (el.type == Parser.Element.Type.columns) 269 { 270 for (; !el.columns.empty; el.columns.popFront) 271 { 272 scope col = &el.columns.front(); 273 if (col.type == Parser.Columns.Type.name) 274 colList.put(Grid.Col(col.name)); 275 else if (col.type == Parser.Columns.Type.tags) 276 col.tags.consume(); 277 } 278 } 279 if (el.type == Parser.Element.Type.rows) 280 { 281 return Result(&el.rows()); 282 } 283 } 284 return Result(null); 285 } 286 287 /** 288 Parses the $(D InputRange) and constructs an in-memory $(D Grid) 289 */ 290 immutable(Grid) asGrid() 291 { 292 string ver; 293 Dict gridMeta; 294 import std.array : appender; 295 auto colList = appender!(Grid.Col[])(); 296 auto rowsList = appender!(Dict[])(); 297 for (; !empty; popFront) 298 { 299 auto el = &front(); 300 if (el.type == Parser.Element.Type.header) 301 { 302 for (; !el.header.empty; el.header.popFront) 303 { 304 auto h = &el.header.front(); 305 if (h.type == Parser.Header.Type.ver) 306 { 307 ver = h.ver; 308 } 309 else if (h.type == Parser.Header.Type.tags) 310 { 311 for (; !h.tags.empty; h.tags.popFront) 312 { 313 h.tags.front.asDict(gridMeta); 314 } 315 } 316 } 317 } 318 if (el.type == Parser.Element.Type.columns) 319 { 320 for (; !el.columns.empty; el.columns.popFront) 321 { 322 auto col = &el.columns.front(); 323 if (col.type == Parser.Columns.Type.name) 324 { 325 colList.put(Grid.Col(col.name)); 326 } 327 else if (col.type == Parser.Columns.Type.tags) 328 { 329 scope column = &colList.data[$ - 1]; 330 for (; !col.tags.empty; col.tags.popFront) 331 { 332 col.tags.front.asDict(column.meta); 333 } 334 } 335 } 336 } 337 if (el.type == Parser.Element.Type.rows) 338 { 339 Dict row; 340 for (size_t i = 0; !el.rows.empty; el.rows.popFront(), i++) 341 { 342 row[colList.data[i].dis] = el.rows.front.asTag; 343 } 344 if (row !is null) 345 rowsList.put(row); 346 } 347 } 348 return cast(immutable) Grid(rowsList.data, colList.data, gridMeta, ver); 349 } 350 351 Tag asTag() 352 { 353 return Tag(cast(Grid) asGrid); 354 } 355 356 /// Definitions of parser states 357 enum ParserState { header, colums, rows, ok, fault }; 358 /// Current parser state 359 ParserState state; 360 361 /// The $(D InputRange) the parser uses 362 @property ref Range range() 363 { 364 return lexer.range; 365 } 366 @property void range(ref Range range) 367 { 368 lexer.range = range; 369 } 370 371 /////////////////////////////////////////////////////////////////////// 372 // 373 // Header parsing 374 // 375 /////////////////////////////////////////////////////////////////////// 376 377 /// Parser the header portion of a $(D Grid) 378 struct Header 379 { 380 enum Type { ver, tags } 381 382 struct Value 383 { 384 Type type; 385 union 386 { 387 string ver = void; 388 NameValueList tags; 389 } 390 } 391 392 @property bool empty() 393 { 394 return state == HeaderState.ok 395 || state == HeaderState.fault; 396 } 397 398 @property ref Value front() 399 { 400 return value; 401 } 402 403 void popFront() 404 { 405 for (; !empty; parser.lexer.popFront()) 406 { 407 switch (state) 408 { 409 case HeaderState.ver: 410 if (!parseVer()) 411 { 412 state = HeaderState.fault; 413 } 414 else 415 { 416 import std.conv : parse; 417 value.type = Type.ver; 418 string ver = value.ver; 419 parser.lexer.ver = parse!int(ver); 420 state = HeaderState.tags; 421 } 422 return; // stop if ver found 423 424 case HeaderState.tags: 425 if (parser.isNewLine) 426 { 427 state = HeaderState.ok; 428 } 429 else if (value.type == Type.tags) 430 { 431 if (!value.tags.consume) 432 state = HeaderState.fault; 433 else 434 state = HeaderState.ok; 435 return; 436 } 437 else 438 { 439 value.type = Type.tags; 440 auto tp = NameValueList(*parser); 441 move(tp, value.tags); 442 } 443 return; 444 445 default: 446 break; 447 } 448 } 449 } 450 451 bool consume() 452 { 453 for (; !empty(); popFront()) {} 454 return state == HeaderState.ok; 455 } 456 457 // implementation 458 private: 459 460 this(ref Parser parser) 461 { 462 this.parser = &parser; 463 if (!parser.lexer.empty) 464 popFront(); 465 else 466 state = HeaderState.fault; 467 } 468 @disable { this(); this(this); } 469 470 Parser* parser; 471 Value value = void; 472 enum HeaderState { ver, tags, ok, fault } 473 HeaderState state; 474 475 // parse a grid version 476 bool parseVer() 477 { 478 enum VerState { ver, sep, str, done} 479 VerState state; 480 loop: 481 for (; !parser.lexer.empty(); parser.lexer.popFront()) 482 { 483 if (parser.isSpace()) 484 continue; 485 final switch (state) 486 { 487 case VerState.ver: 488 if (!parser.expectToken(TokenType.id, "ver".tag)) 489 return false; 490 state = VerState.sep; 491 break; 492 493 case VerState.sep: 494 if (!parser.hasChr(':')) 495 return false; 496 state = VerState.str; 497 break; 498 499 case VerState.str: 500 if (!parser.expectToken(TokenType.str)) 501 return false; 502 value.ver = parser.token.value!Str; 503 state = VerState.done; 504 break; 505 506 case VerState.done: 507 break loop; 508 } 509 } 510 return state == VerState.done; 511 } 512 } /// 513 unittest 514 { 515 alias Parser = ZincParser!string; 516 { 517 auto parser = Parser(`ver:"3.0"`); 518 auto header = &parser.front.header(); 519 assert(header.front.type == Parser.Header.Type.ver); 520 assert(header.front.ver == "3.0"); 521 } 522 } 523 524 /////////////////////////////////////////////////////////////////////// 525 // 526 // Column parsing 527 // 528 /////////////////////////////////////////////////////////////////////// 529 530 /// Parser the column portion of a `Grid` 531 struct Columns 532 { 533 enum Type { name, tags } 534 535 struct Value 536 { 537 Type type; 538 union 539 { 540 string name = void; 541 NameValueList tags; 542 } 543 } 544 545 @property bool empty() 546 { 547 return state == ColumnsState.ok 548 || state == ColumnsState.fault; 549 } 550 551 @property ref Value front() 552 { 553 return value; 554 } 555 556 void popFront() 557 { 558 for (; !empty; parser.lexer.popFront()) 559 { 560 start: 561 switch (state) 562 { 563 case ColumnsState.name: 564 if (parser.isSpace() && !parser.lexer.empty) 565 continue; 566 if (!parser.expectToken(TokenType.id)) 567 { 568 state = ColumnsState.fault; 569 } 570 else 571 { 572 value.type = Type.name; 573 value.name = parser.token.value!Str; 574 parser.lexer.popFront(); 575 state = ColumnsState.sep; 576 } 577 return; // stop if ver found 578 579 case ColumnsState.sep: 580 if (parser.hasChr(',')) 581 { 582 state = ColumnsState.name; 583 } 584 else if (parser.hasChr(' ')) 585 { 586 state = ColumnsState.tags; 587 } 588 else if (parser.isNewLine || parser.lexer.empty) 589 { 590 state = ColumnsState.ok; 591 return; 592 } 593 continue; 594 595 case ColumnsState.tags: 596 if (parser.isNewLine) 597 { 598 state = ColumnsState.ok; 599 } 600 else if (value.type == Type.tags) 601 { 602 if (!value.tags.consume) 603 { 604 state = ColumnsState.fault; 605 } 606 else 607 { 608 state = ColumnsState.sep; 609 goto start; 610 } 611 return; 612 } 613 else 614 { 615 value.type = Type.tags; 616 auto nv = NameValueList(*parser); 617 move(nv, value.tags); 618 } 619 return; 620 621 default: 622 break; 623 } 624 } 625 } 626 627 bool consume() 628 { 629 for (; !empty(); popFront()) {} 630 return state == ColumnsState.ok; 631 } 632 633 // implementation 634 private: 635 this(ref Parser parser) 636 { 637 this.parser = &parser; 638 if (!parser.lexer.empty) 639 popFront(); 640 else 641 state = ColumnsState.fault; 642 } 643 @disable { this(); this(this); } 644 645 Parser* parser; 646 Value value = void; 647 enum ColumnsState { name, sep, tags, ok, fault } 648 ColumnsState state; 649 } 650 unittest 651 { 652 alias Parser = ZincParser!string; 653 { 654 auto parser = Parser("col1, col2"); 655 auto cols = Parser.Columns(parser); 656 assert(cols.front.name == "col1"); 657 cols.popFront; 658 assert(cols.front.name == "col2"); 659 } 660 { 661 auto parser = Parser("col1 test,col2 bar:F"); 662 auto cols = Parser.Columns(parser); // pop col name 663 assert(cols.front.name == "col1"); 664 cols.popFront; // pop col tags 665 assert(cols.front.tags.asDict == ["test": marker]); 666 cols.popFront; // pop col name 667 assert(cols.front.name == "col2"); 668 cols.popFront; // po col tags 669 assert(cols.front.tags.asDict == ["bar": false.tag]); 670 } 671 672 } 673 674 /////////////////////////////////////////////////////////////////////// 675 // 676 // Row parsing 677 // 678 /////////////////////////////////////////////////////////////////////// 679 680 /// Parser the rows portion of a $(D Grid) 681 struct Rows 682 { 683 @property bool empty() 684 { 685 return state == RowsState.ok 686 || state == RowsState.fault; 687 } 688 689 @property ref AnyTag front() 690 { 691 return value; 692 } 693 694 void popFront() 695 { 696 for (; !empty; parser.lexer.popFront()) 697 { 698 if (parser.isSpace) 699 { 700 if (!parser.lexer.empty) 701 continue; 702 else 703 state = RowsState.ok; 704 } 705 switch (state) 706 { 707 case RowsState.tag: 708 if (parser.hasChr(',')) 709 { 710 value = AnyTag(Tag.init); 711 if (parser.lexer.empty) 712 state = RowsState.sep; 713 else 714 parser.lexer.popFront(); 715 } 716 else if (parser.isNewLine) 717 { 718 value = AnyTag(Tag.init); 719 state = RowsState.sep; 720 } 721 else 722 { 723 if (parser.hasChr('>')) 724 { 725 state = RowsState.ok; 726 } 727 else 728 { 729 value = AnyTag(*parser); 730 state = RowsState.sep; 731 } 732 } 733 return; 734 735 case RowsState.sep: 736 if (parser.lexer.empty) 737 { 738 state = RowsState.ok; 739 } 740 else if (parser.hasChr(',')) 741 { 742 state = RowsState.tag; 743 if (parser.lexer.empty) 744 state = RowsState.ok; 745 else 746 continue; 747 } 748 else if (parser.isNewLine) 749 { 750 state = RowsState.ok; 751 } 752 else if (value.consume) 753 { 754 state = RowsState.ok; 755 } 756 else 757 { 758 state = RowsState.fault; 759 } 760 return; 761 762 default: 763 break; 764 } 765 } 766 } 767 768 bool consume() 769 { 770 for (; !empty(); popFront()) {} 771 return state == RowsState.ok; 772 } 773 774 // implementation 775 private: 776 this(ref Parser parser) 777 { 778 this.parser = &parser; 779 if (!parser.lexer.empty) 780 popFront(); 781 else 782 state = RowsState.fault; 783 } 784 @disable { this(); this(this); } 785 786 Parser* parser; 787 AnyTag value = void; 788 enum RowsState { tag, sep, ok, fault } 789 RowsState state; 790 } 791 unittest 792 { 793 alias Parser = ZincParser!string; 794 { 795 auto parser = Parser("100,T, , `/a/b/c`, @ref"); 796 auto rows = Parser.Rows(parser); 797 assert(rows.front.asTag == 100.tag); 798 rows.popFront; 799 assert(rows.front.asTag == true.tag); 800 rows.popFront; 801 assert(rows.front.asTag == Tag.init); 802 rows.popFront; 803 assert(rows.front.asTag == Uri(`/a/b/c`)); 804 rows.popFront; 805 assert(rows.front.asTag == Ref("ref")); 806 } 807 } 808 809 ////////////////////////////////////////// 810 /// Components 811 ////////////////////////////////////////// 812 813 /// Decode a list of pair of tags 814 struct NameValueList 815 { 816 import std.algorithm : canFind; 817 818 NameValue value = void; 819 820 @property bool empty() 821 { 822 return state == TagPairListState.ok 823 || state == TagPairListState.fault; 824 } 825 826 @property ref NameValue front() 827 { 828 return value; 829 } 830 831 void popFront() 832 { 833 for (; !empty;) 834 { 835 final switch (state) 836 { 837 case TagPairListState.tag: 838 value = NameValue(*parser); 839 state = TagPairListState.consume; 840 return; 841 842 case TagPairListState.consume: 843 if (!value.consume()) 844 { 845 state = TagPairListState.fault; 846 return; 847 } 848 if (sep.canFind!(s=>parser.hasChr(s))) 849 { 850 state = TagPairListState.tag; 851 parser.lexer.popFront(); 852 continue; 853 } 854 else 855 { 856 state = TagPairListState.ok; 857 } 858 return; 859 860 case TagPairListState.ok: 861 case TagPairListState.fault: 862 assert(false, "Invalid state."); 863 } 864 } 865 } 866 867 bool consume() 868 { 869 for (; !empty; popFront()) {} 870 return state == TagPairListState.ok; 871 } 872 873 void asDict(ref Dict dict) 874 { 875 for (; !empty; popFront()) 876 { 877 foreach (ref kv; front.asDict.byKeyValue) 878 dict[kv.key] = kv.value; 879 } 880 } 881 882 Dict asDict() 883 { 884 Dict dict; 885 asDict(dict); 886 return dict; 887 } 888 889 Tag asTag() 890 { 891 return Tag(asDict()); 892 } 893 894 this(ref Parser parser, char[] sep = [' ']) 895 { 896 this.parser = &parser; 897 this.sep = sep; 898 if (!parser.lexer.empty) 899 popFront(); 900 else 901 state = TagPairListState.ok; 902 } 903 904 private: 905 enum TagPairListState { tag, consume, ok, fault } 906 TagPairListState state; 907 908 @disable this(); 909 @disable this(this); 910 char[] sep; 911 Parser* parser; 912 } 913 unittest 914 { 915 alias Parser = ZincParser!string; 916 { 917 auto parser = Parser("foo bar"); 918 auto list = Parser.NameValueList(parser); 919 assert(list.front.front.name == "foo"); 920 list.popFront(); 921 assert(list.front.front.name == "bar"); 922 } 923 { 924 auto parser = Parser("bool:T num:22"); 925 auto list = Parser.NameValueList(parser); 926 assert(list.front.front.name == "bool"); 927 list.front.consume(); 928 assert(list.front.front.value.asTag == true.tag); 929 list.popFront(); 930 assert(list.front.front.name == "num"); 931 list.front.consume(); 932 assert(list.front.front.value.asTag == 22.tag); 933 } 934 { 935 auto parser = Parser(`str:"text" ref:@ref`); 936 auto list = Parser.NameValueList(parser); 937 assert(list.front.asDict == ["str": "text".tag]); 938 } 939 { 940 auto parser = Parser(`test zum:1% zam`); 941 auto list = Parser.NameValueList(parser); 942 assert(list.asDict == ["test":marker, "zum": 1.tag("%"), "zam": marker]); 943 } 944 945 } 946 947 ////////////////////////////////////////// 948 /// Decode a pair of tags 949 ////////////////////////////////////////// 950 struct NameValue 951 { 952 953 enum Type { marker, nameValue } 954 Type type; 955 956 struct Pair 957 { 958 string name; 959 AnyTag value; 960 } 961 962 @property bool empty() 963 { 964 return state == NameValueState.ok 965 || state == NameValueState.fault; 966 } 967 968 @property ref Pair front() 969 { 970 return pair; 971 } 972 973 void popFront() 974 { 975 for (; !empty; parser.lexer.popFront()) 976 { 977 final switch (state) 978 { 979 case NameValueState.name: 980 if (parser.isSpace) 981 continue; 982 983 if (!parser.expectToken(TokenType.id)) 984 { 985 state = NameValueState.fault; 986 } 987 else 988 { 989 pair.name = parser.token.value!Str; 990 state = NameValueState.sep; 991 parser.lexer.popFront(); 992 } 993 return; 994 995 case NameValueState.sep: 996 if (parser.hasChr(':')) 997 { 998 state = NameValueState.value; 999 continue; 1000 } 1001 else 1002 { 1003 type = Type.marker; 1004 pair.value = AnyTag(marker); 1005 state = NameValueState.ok; 1006 } 1007 return; 1008 1009 case NameValueState.value: 1010 if (parser.isSpace) 1011 continue; 1012 type = Type.nameValue; 1013 pair.value = AnyTag(*parser); 1014 state = NameValueState.consume; 1015 return; 1016 1017 case NameValueState.consume: 1018 if (!pair.value.consume()) 1019 state = NameValueState.fault; 1020 else 1021 state = NameValueState.ok; 1022 return; 1023 1024 case NameValueState.ok: 1025 case NameValueState.fault: 1026 assert(false, "Invalid tag pair state."); 1027 } 1028 } 1029 } 1030 1031 bool consume() 1032 { 1033 for (; !empty; popFront()) {} 1034 return state == NameValueState.ok; 1035 } 1036 1037 void asDict(ref Dict d) 1038 { 1039 if (consume) 1040 d[pair.name] = pair.value.asTag; 1041 } 1042 1043 Dict asDict() 1044 { 1045 if (consume) 1046 return [pair.name: pair.value.asTag]; 1047 else 1048 return Dict.init; 1049 } 1050 1051 private: 1052 this(ref Parser parser) 1053 { 1054 this.parser = &parser; 1055 if (!empty) 1056 popFront(); 1057 else 1058 state = NameValueState.fault; 1059 } 1060 @disable this(); 1061 @disable this(this); 1062 1063 Parser* parser = void; 1064 Pair pair; 1065 enum NameValueState { name, sep, value, consume, ok, fault } 1066 NameValueState state; 1067 } 1068 unittest 1069 { 1070 alias Parser = ZincParser!string; 1071 { 1072 auto parser = Parser("foo"); 1073 auto pair = Parser.NameValue(parser); 1074 assert(pair.front.name == "foo"); 1075 pair.popFront(); 1076 assert(pair.front.value.asTag == marker); 1077 } 1078 { 1079 auto parser = Parser("test:F"); 1080 auto pair = Parser.NameValue(parser); 1081 assert(pair.front.name == "test"); 1082 pair.popFront(); 1083 assert(pair.front.value.asTag == false.tag); 1084 } 1085 { 1086 auto parser = Parser("bar:M"); 1087 auto pair = Parser.NameValue(parser); 1088 assert(pair.front.name == "bar"); 1089 pair.popFront(); 1090 assert(pair.front.value.asTag == marker); 1091 } 1092 } 1093 1094 1095 ////////////////////////////////////////// 1096 /// Tag parsing components 1097 ////////////////////////////////////////// 1098 1099 /// Parsing for any type of $(D Tag)s 1100 struct AnyTag 1101 { 1102 @property bool empty() 1103 { 1104 return state == AnyTagState.ok 1105 || state == AnyTagState.fault; 1106 } 1107 /// Type of tag values 1108 union Val 1109 { 1110 Tag scalar = void; 1111 TagList list; 1112 TagDict dict; 1113 TagGrid grid; 1114 } 1115 1116 @property ref Val front() 1117 { 1118 return val; 1119 } 1120 1121 void popFront() 1122 { 1123 if (state == AnyTagState.scalar) 1124 { 1125 if (!parser.isScalar) 1126 { 1127 state = AnyTagState.list; 1128 } 1129 else 1130 { 1131 val.scalar = cast(Tag) parser.token.tag; 1132 type = Type.scalar; 1133 state = AnyTagState.ok; 1134 return parser.lexer.popFront(); 1135 } 1136 } 1137 1138 if (state == AnyTagState.list) 1139 { 1140 if (!parser.hasChr('[')) 1141 { 1142 state = AnyTagState.dict; 1143 } 1144 else 1145 { 1146 val.list = TagList(*parser); 1147 type = Type.list; 1148 state = AnyTagState.ok; 1149 return; 1150 } 1151 } 1152 1153 if (state == AnyTagState.dict) 1154 { 1155 if (!parser.hasChr('{')) 1156 { 1157 state = AnyTagState.grid; 1158 } 1159 else 1160 { 1161 val.dict = TagDict(*parser); 1162 type = Type.dict; 1163 state = AnyTagState.ok; 1164 return; 1165 } 1166 } 1167 1168 if (state == AnyTagState.grid) 1169 { 1170 if (!parser.hasChr('<')) 1171 { 1172 state = AnyTagState.fault; 1173 } 1174 else 1175 { 1176 val.grid = TagGrid(*parser); 1177 type = Type.grid; 1178 state = !val.grid.empty ? AnyTagState.ok : AnyTagState.fault; 1179 return; 1180 } 1181 } 1182 throw new Exception("Invalid tag type: " ~ to!string(parser.chr)); 1183 } 1184 1185 bool consume() 1186 { 1187 for (; !empty; popFront()) {} 1188 return state == AnyTagState.ok; 1189 } 1190 1191 /// Parse a tag. 1192 Tag asTag() 1193 { 1194 assert(state == AnyTagState.ok); 1195 if (type == Type.scalar) 1196 { 1197 consume(); 1198 return val.scalar; 1199 } 1200 else if (type == Type.list) 1201 { 1202 return val.list.asTag; 1203 } 1204 else if (type == Type.dict) 1205 { 1206 return val.dict.asTag; 1207 } 1208 else if (type == Type.grid) 1209 { 1210 return Tag(val.grid.asTag); 1211 } 1212 else 1213 { 1214 return Tag.init; 1215 } 1216 } 1217 1218 enum Type { scalar, list, dict, grid } 1219 Type type; 1220 1221 this(Tag tag) 1222 { 1223 state = AnyTagState.ok; 1224 val.scalar = tag; 1225 type = Type.scalar; 1226 } 1227 1228 this(ref Parser parser) 1229 { 1230 this.parser = &parser; 1231 if (!empty) 1232 popFront(); 1233 else 1234 state = AnyTagState.fault; 1235 } 1236 @disable this(this); 1237 1238 private: 1239 Parser* parser; 1240 Val val = void; 1241 enum AnyTagState { scalar, list, dict, grid, ok, fault } 1242 AnyTagState state; 1243 } 1244 unittest 1245 { 1246 alias Parser = ZincParser!string; 1247 { 1248 auto parser = Parser("T"); 1249 auto el = Parser.AnyTag(parser); 1250 assert(el.asTag == true.tag); 1251 } 1252 { 1253 auto parser = Parser("99"); 1254 auto el = Parser.AnyTag(parser); 1255 assert(el.asTag == 99.tag); 1256 } 1257 } 1258 1259 /// A list of $(D AnyTag)s 1260 struct TagList 1261 { 1262 @property bool empty() 1263 { 1264 return state == TagListState.ok 1265 || state == TagListState.fault; 1266 } 1267 1268 @property ref AnyTag front() 1269 { 1270 return *val; 1271 } 1272 1273 void popFront() 1274 { 1275 for (; !empty; parser.lexer.popFront()) 1276 { 1277 if (parser.isSpace) 1278 continue; 1279 eval: 1280 switch(state) 1281 { 1282 case TagListState.begin: 1283 if (!parser.hasChr('[')) 1284 state = TagListState.fault; 1285 else 1286 { 1287 state = TagListState.tag; 1288 continue; 1289 } 1290 return; 1291 1292 case TagListState.tag: 1293 if (parser.hasChr(']')) 1294 { 1295 if (val.isNull) 1296 val = Own!AnyTag(Tag()); 1297 state = TagListState.ok; 1298 parser.lexer.popFront(); 1299 } 1300 else if (parser.hasChr(',')) 1301 { 1302 continue; 1303 } 1304 else 1305 { 1306 val = Own!AnyTag(*parser); 1307 state = TagListState.consume; 1308 } 1309 return; 1310 1311 case TagListState.consume: 1312 if (!val.consume()) 1313 state = TagListState.fault; 1314 state = TagListState.tag; 1315 goto eval; 1316 1317 default: 1318 break; 1319 } 1320 } 1321 } 1322 1323 bool consume() 1324 { 1325 for (; !empty; popFront()) {} 1326 return state == TagListState.ok; 1327 } 1328 1329 Tag asTag() 1330 { 1331 import std.array : appender; 1332 auto tagList = appender!(Tag[])(); 1333 for (; !empty; popFront()) 1334 { 1335 tagList.put(front.asTag); 1336 } 1337 return Tag(tagList.data); 1338 } 1339 1340 private: 1341 this(ref Parser parser) 1342 { 1343 this.parser = &parser; 1344 if (!empty) 1345 popFront(); 1346 else 1347 state = TagListState.ok; 1348 } 1349 @disable this(); 1350 @disable this(this); 1351 1352 Parser* parser = void; 1353 Own!AnyTag val; 1354 enum TagListState { begin, tag, consume, ok, fault } 1355 TagListState state; 1356 } 1357 unittest 1358 { 1359 alias Parser = ZincParser!string; 1360 { 1361 auto parser = Parser("[]"); 1362 auto el = Parser.TagList(parser); 1363 assert(el.asTag == [].tag); 1364 } 1365 { 1366 auto parser = Parser("[1]"); 1367 auto el = Parser.TagList(parser); 1368 assert(el.asTag == [1.tag]); 1369 } 1370 { 1371 auto parser = Parser("[1, T]"); 1372 auto el = Parser.TagList(parser); 1373 assert(el.asTag == [1.tag, true.tag]); 1374 } 1375 { 1376 auto parser = Parser("[@foo, M, [T]]"); 1377 auto el = Parser.TagList(parser); 1378 assert(el.asTag == [Ref("foo").tag, marker, [true.tag].tag]); 1379 } 1380 } 1381 1382 /// Dictionary of tags 1383 struct TagDict 1384 { 1385 @property bool empty() 1386 { 1387 return state == TagDictState.ok 1388 || state == TagDictState.fault; 1389 } 1390 1391 @property ref NameValueList front() 1392 { 1393 return *val; 1394 } 1395 1396 void popFront() 1397 { 1398 for (; !empty; parser.lexer.popFront()) 1399 { 1400 if (parser.isSpace) 1401 continue; 1402 switch(state) 1403 { 1404 case TagDictState.begin: 1405 if (!parser.hasChr('{')) 1406 { 1407 state = TagDictState.fault; 1408 } 1409 else 1410 { 1411 state = TagDictState.tag; 1412 continue; 1413 } 1414 return; 1415 1416 case TagDictState.tag: 1417 if (parser.lexer.empty && !parser.hasChr('}')) 1418 { 1419 state = TagDictState.fault; 1420 return; 1421 } 1422 if (parser.hasChr('}')) 1423 { 1424 state = TagDictState.ok; 1425 parser.lexer.popFront(); 1426 return; 1427 } 1428 if (!val.isNull) 1429 { 1430 if (!val.consume()) 1431 state = TagDictState.fault; 1432 } 1433 else 1434 { 1435 val = Own!NameValueList(*parser, [',', ' ']); 1436 } 1437 return; 1438 1439 default: 1440 break; 1441 } 1442 } 1443 } 1444 1445 bool consume() 1446 { 1447 for (; !empty; popFront()) {} 1448 return state == TagDictState.ok; 1449 } 1450 1451 Tag asTag() 1452 { 1453 auto t = val.isNull ? Dict.init.tag : val.asTag; 1454 consume(); 1455 return t; 1456 } 1457 1458 private: 1459 this(ref Parser parser) 1460 { 1461 this.parser = &parser; 1462 if (!empty) 1463 popFront(); 1464 else 1465 state = TagDictState.ok; 1466 } 1467 @disable this(); 1468 @disable this(this); 1469 1470 Parser* parser = void; 1471 Own!NameValueList val; 1472 enum TagDictState { begin, tag, ok, fault } 1473 TagDictState state; 1474 } 1475 unittest 1476 { 1477 alias Parser = ZincParser!string; 1478 { 1479 auto parser = Parser("{}"); 1480 auto el = Parser.TagDict(parser); 1481 assert(el.asTag == Dict.init.tag); 1482 } 1483 { 1484 auto parser = Parser("{age:6"); 1485 auto el = Parser.TagDict(parser); 1486 el.consume(); 1487 assert(el.state == Parser.TagDict.TagDictState.fault); 1488 } 1489 { 1490 auto parser = Parser("{age:6}"); 1491 auto el = Parser.TagDict(parser); 1492 assert(el.asTag == ["age": 6.tag].tag); 1493 } 1494 { 1495 auto parser = Parser(`{foo:"bar", baz}`); 1496 auto el = Parser.TagDict(parser); 1497 assert(el.asTag == ["foo": "bar".tag, "baz": marker].tag); 1498 } 1499 { 1500 auto parser = Parser(`{dict:{dict}`); 1501 auto el = Parser.TagDict(parser); 1502 assert(el.asTag == ["dict": ["dict": marker].tag].tag); 1503 } 1504 { 1505 auto parser = Parser(`{list:[T]}`); 1506 auto el = Parser.TagDict(parser); 1507 assert(el.asTag == ["list": [true.tag].tag].tag); 1508 } 1509 { 1510 auto parser = Parser(`{list:[{dict}]}`); 1511 auto el = Parser.TagDict(parser); 1512 assert(el.asTag == ["list": [["dict":marker].tag].tag].tag); 1513 } 1514 } 1515 1516 /// Grid of tags 1517 struct TagGrid 1518 { 1519 @property bool empty() 1520 { 1521 return state == TagGridState.ok 1522 || state == TagGridState.fault; 1523 } 1524 1525 @property ref Parser front() 1526 { 1527 return *val; 1528 } 1529 1530 void popFront() 1531 { 1532 int markCnt = 0; 1533 for (; !empty; parser.lexer.popFront()) 1534 { 1535 if (parser.isSpace) 1536 continue; 1537 eval: 1538 switch(state) 1539 { 1540 case TagGridState.begin: 1541 if (markCnt < 2) 1542 { 1543 if (!parser.hasChr('<')) 1544 { 1545 state = TagGridState.fault; 1546 return; 1547 } 1548 markCnt++; 1549 continue; 1550 } 1551 else 1552 { 1553 if (parser.isNewLine) 1554 { 1555 parser.lexer.popFront(); 1556 state = TagGridState.grid; 1557 goto eval; 1558 } 1559 } 1560 state = TagGridState.fault; 1561 return; 1562 1563 case TagGridState.grid: 1564 if (parser.hasChr('>') && val.isNull) 1565 { 1566 state = TagGridState.fault; 1567 } 1568 else 1569 { 1570 val = Own!Parser(parser.range); 1571 state = TagGridState.end; 1572 } 1573 return; 1574 1575 case TagGridState.end: 1576 if (val.state != ParserState.ok) 1577 { 1578 state = TagGridState.fault; 1579 return; 1580 } 1581 if (markCnt < 2) 1582 { 1583 // transfer bck the input range 1584 if (markCnt == 0) 1585 parser.range = val.range; 1586 if (!parser.hasChr('>')) 1587 { 1588 state = TagGridState.fault; 1589 return; 1590 } 1591 markCnt++; 1592 continue; 1593 } 1594 else 1595 { 1596 state = TagGridState.ok; 1597 } 1598 return; 1599 1600 default: 1601 break; 1602 } 1603 } 1604 } 1605 1606 bool consume() 1607 { 1608 for (; !empty; popFront()) {} 1609 return state == TagGridState.ok; 1610 } 1611 1612 Tag asTag() 1613 { 1614 Tag t = val.asTag; 1615 consume(); 1616 return t; 1617 } 1618 1619 private: 1620 this(ref Parser parser) 1621 { 1622 this.parser = &parser; 1623 if (!empty) 1624 popFront(); 1625 else 1626 state = TagGridState.ok; 1627 } 1628 @disable this(); 1629 @disable this(this); 1630 1631 Parser* parser = void; 1632 Own!Parser val; 1633 enum TagGridState { begin, grid, end, ok, fault } 1634 TagGridState state; 1635 } 1636 unittest 1637 { 1638 alias Parser = ZincParser!string; 1639 { 1640 auto r =`<< 1641 ver:"3.0" 1642 empty 1643 >>`; 1644 auto parser = Parser(r); 1645 auto el = Parser.TagGrid(parser); 1646 assert(el.asTag == Grid([], [Grid.Col("empty")], Dict.init, "3.0").tag); 1647 } 1648 { 1649 auto r =`<< 1650 ver:"3.0" 1651 name 1652 "foo" 1653 >>`; 1654 auto parser = Parser(r); 1655 auto el = Parser.TagGrid(parser); 1656 assert(el.asTag == Grid([["name": "foo".tag]], [Grid.Col("name")], Dict.init, "3.0").tag); 1657 } 1658 } 1659 1660 // 1661 1662 /////////////////////////////////////////////////// 1663 // 1664 // Parser internals 1665 ////////////////////////////////////////////////// 1666 1667 package(haystack): 1668 Lexer lexer = void; 1669 Element element; 1670 1671 /////////////////////////////////////////////////// 1672 // Implementation for parsing $(D Element)s 1673 /////////////////////////////////////////////////// 1674 1675 ///////////////////////////////////// 1676 // helper functions 1677 ///////////////////////////////////// 1678 1679 @property ref const(Token) token() pure 1680 { 1681 return lexer.front(); 1682 } 1683 1684 bool expectToken() pure 1685 { 1686 return token.isValid; 1687 } 1688 1689 bool expectToken(TokenType type) pure 1690 { 1691 return token.type == type; 1692 } 1693 1694 bool expectToken(TokenType type, Tag value) 1695 { 1696 return token.isOf(type, value); 1697 } 1698 1699 dchar chr() pure 1700 { 1701 return token.curChar; 1702 } 1703 1704 bool hasChr(dchar c) pure 1705 { 1706 return token.hasChr(c); 1707 } 1708 1709 @property bool isSpace() pure 1710 { 1711 return token.isSpace; 1712 } 1713 1714 @property bool isNewLine() pure 1715 { 1716 return token.isNewLine; 1717 } 1718 1719 bool isScalar() pure 1720 { 1721 return token.isScalar; 1722 } 1723 } 1724 /// a string based parser 1725 alias ZincStringParser = ZincParser!string; 1726 unittest 1727 { 1728 import std.exception; 1729 1730 { 1731 auto str = `ver: "3.0" 1732 empty`; 1733 auto empty = ZincStringParser(str).asGrid; 1734 assert(empty.length == 0); 1735 assert(empty.colNames[0] == "empty"); 1736 } 1737 1738 { 1739 auto str = `ver: "3.0" 1740 empty 1741 1742 `; 1743 auto empty = ZincStringParser(str).asGrid; 1744 assert(empty.length == 0); 1745 assert(empty.colNames[0] == "empty"); 1746 } 1747 1748 { 1749 auto str = `ver:"3.0" 1750 id, range 1751 @writePoint, "today" `; 1752 auto grid = ZincStringParser(str).asGrid; 1753 assert(grid.hasCol("id")); 1754 assert(grid.hasCol("range")); 1755 assert(grid.length == 1); 1756 assert(grid[0]["id"] == Ref("writePoint")); 1757 assert(grid[0]["range"] == Str("today")); 1758 } 1759 1760 { 1761 auto str = `ver:"3.0" 1762 id, range 1763 @writePoint, "today" 1764 @readPoint, "yesterday"`; 1765 scope rows = ZincStringParser(str).asRows; 1766 Dict row = rows.front; 1767 assert(!row.empty); 1768 assert(row["id"] == Ref("writePoint")); 1769 assert(row["range"] == Str("today")); 1770 rows.popFront(); 1771 assert(!rows.empty); 1772 row = rows.front; 1773 assert(row["id"] == Ref("readPoint")); 1774 assert(row["range"] == Str("yesterday")); 1775 rows.popFront(); 1776 assert(rows.empty); 1777 1778 size_t count; 1779 foreach(r; ZincStringParser(str).asRows) 1780 { 1781 if (count == 0) 1782 { 1783 assert(r["id"] == Ref("writePoint")); 1784 assert(r["range"] == Str("today")); 1785 } 1786 if (count == 1) 1787 { 1788 assert(r["id"] == Ref("readPoint")); 1789 assert(r["range"] == Str("yesterday")); 1790 } 1791 count++; 1792 } 1793 assert(count == 2); 1794 } 1795 1796 { 1797 auto str = `ver: "3.0" marker num:100 1798 col1, col2 str: "str" 1799 M, 1800 ,"foo"`; 1801 auto grid = ZincStringParser(str).asGrid; 1802 // meta 1803 assert(grid.meta["marker"] == marker); 1804 assert(grid.meta["num"] == 100.tag); 1805 // cols 1806 assert(grid.cols.length == 2); 1807 assert(grid.cols[0].dis == "col1"); 1808 assert(grid.cols[0].meta is null); 1809 assert(grid.cols[1].dis == "col2"); 1810 assert(grid.cols[1].meta["str"] == "str".tag); 1811 1812 // rows 1813 assert(grid.length == 2); 1814 assert(grid[0]["col1"] == marker); 1815 assert(grid[0]["col2"] == Tag.init); 1816 assert(grid[1]["col1"] == Tag.init); 1817 assert(grid[1]["col2"] == "foo".tag); 1818 } 1819 1820 { 1821 auto str = `ver: "3.0" 1822 col1 1823 {foo:T}`; 1824 1825 auto grid = ZincStringParser(str).asGrid; 1826 assert(grid.length == 1); 1827 auto x = grid[0]["col1"].get!Dict; 1828 assert(grid[0]["col1"] == ["foo": true.tag].tag); 1829 } 1830 1831 { 1832 auto str = `ver: "3.0" 1833 somecol 1834 R`; 1835 1836 auto grid = ZincStringParser(str).asGrid; 1837 assert(grid.length == 1); 1838 auto x = grid[0]["somecol"]; 1839 assert(!x.hasValue); 1840 } 1841 1842 { 1843 auto str = `ver:"3.0" 1844 id 1845 @equip 1846 @site 1847 1848 `; 1849 1850 auto grid = ZincStringParser(str).asGrid; 1851 assert(grid.length == 2); 1852 } 1853 1854 { 1855 auto str = `ver: "3.0" 1856 col1 1857 << 1858 ver: "3.0" 1859 empty 1860 >>`; 1861 auto grid = ZincStringParser(str).asGrid; 1862 assert(grid.length == 1); 1863 assert(grid[0]["col1"] == Grid([], [Grid.Col("empty")], Dict.init, "3.0")); 1864 } 1865 { 1866 auto str = `ver: "3.0" 1867 col1, col2 1868 << 1869 ver: "3.0" 1870 empty 1871 >>, T`; 1872 auto grid = ZincStringParser(str).asGrid; 1873 assert(grid.length == 1); 1874 assert(grid.cols.length == 2); 1875 assert(grid[0]["col1"] == Grid([], [Grid.Col("empty")], Dict.init, "3.0")); 1876 assert(grid[0]["col2"] == true.tag); 1877 } 1878 } 1879 1880 void dumpParser(string zinc) 1881 { 1882 import std.stdio : writeln; 1883 alias Parser = ZincStringParser; 1884 auto parser = Parser(zinc); 1885 for (; !parser.empty; parser.popFront) 1886 { 1887 auto el = &parser.front(); 1888 if (el.type == Parser.Element.Type.header) 1889 { 1890 for (; !el.header.empty; el.header.popFront) 1891 { 1892 auto h = &el.header.front(); 1893 writeln("Grid element name: ", el.type); 1894 if (h.type == Parser.Header.Type.ver) 1895 writeln(h.ver); 1896 else if (h.type == Parser.Header.Type.tags) 1897 { 1898 for (; !h.tags.empty; h.tags.popFront) 1899 { 1900 writeln(h.tags.front.asDict()); 1901 } 1902 } 1903 } 1904 } 1905 if (el.type == Parser.Element.Type.columns) 1906 { 1907 for (; !el.columns.empty; el.columns.popFront) 1908 { 1909 auto col = &el.columns.front(); 1910 writeln("Grid element name: ", el.type); 1911 if (col.type == Parser.Columns.Type.name) 1912 writeln(col.name); 1913 else if (col.type == Parser.Columns.Type.tags) 1914 { 1915 for (; !col.tags.empty; col.tags.popFront) 1916 { 1917 writeln(col.tags.front.asDict()); 1918 } 1919 } 1920 } 1921 } 1922 if (el.type == Parser.Element.Type.rows) 1923 { 1924 for (; !el.rows.empty; el.rows.popFront()) 1925 { 1926 auto tag = &el.rows.front(); 1927 writeln("Grid element name: ", el.type); 1928 writeln(tag.asTag.toStr); 1929 } 1930 } 1931 } 1932 writeln("Parser state: ", parser.state); 1933 }