RigsofRods
Soft-body Physics Simulation
GenericFileFormat.cpp
Go to the documentation of this file.
1 /*
2  This source file is part of Rigs of Rods
3  Copyright 2022 Petr Ohlidal
4 
5  For more information, see http://www.rigsofrods.org/
6 
7  Rigs of Rods is free software: you can redistribute it and/or modify
8  it under the terms of the GNU General Public License version 3, as
9  published by the Free Software Foundation.
10 
11  Rigs of Rods is distributed in the hope that it will be useful,
12  but WITHOUT ANY WARRANTY; without even the implied warranty of
13  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  GNU General Public License for more details.
15 
16  You should have received a copy of the GNU General Public License
17  along with Rigs of Rods. If not, see <http://www.gnu.org/licenses/>.
18 */
19 
20 #include "GenericFileFormat.h"
21 
22 #include "Application.h"
23 #include "Console.h"
24 
25 #include <algorithm>
26 
27 using namespace RoR;
28 using namespace Ogre;
29 
30 enum class PartialToken
31 {
32  NONE,
33  COMMENT_SEMICOLON, // Comment starting with ';'
34  COMMENT_SLASH, // Comment starting with '//'
35  COMMENT_HASH,
36  STRING_QUOTED, // String starting/ending with '"'
37  STRING_NAKED, // String without '"' on either end
38  STRING_NAKED_CAPTURING_SPACES, // Only for OPTION_PARENTHESES_CAPTURE_SPACES - A naked string seeking the closing ')'.
39  TITLE_STRING, // A whole-line string, with spaces
40  NUMBER_STUB_MINUS, // Sole '-' character, may start a number or a naked string.
41  NUMBER_INTEGER, // Just digits and optionally leading '-'
42  NUMBER_DECIMAL, // Like INTEGER but already containing '.'
43  NUMBER_SCIENTIFIC_STUB, // Like DECIMAL, already containing 'e' or 'E' but not the exponent value.
44  NUMBER_SCIENTIFIC_STUB_MINUS, // Like SCIENTIFIC_STUB but with only '-' in exponent.
45  NUMBER_SCIENTIFIC, // Valid decimal number in scientific notation.
46  KEYWORD, // Unqoted string at the start of line. Accepted characters: alphanumeric and underscore
47  KEYWORD_BRACED, // Like KEYWORD but starting with '[' and ending with ']'
48  BOOL_TRUE, // Partial 'true'
49  BOOL_FALSE, // Partial 'false'
50  GARBAGE, // Text not fitting any above category, will be discarded
51 };
52 
53 struct DocumentParser
54 {
55  DocumentParser(GenericDocument& d, const BitMask_t opt, Ogre::DataStreamPtr ds)
56  : doc(d), options(opt), datastream(ds) {}
57 
58  // Config
59  GenericDocument& doc;
60  const BitMask_t options;
61  Ogre::DataStreamPtr datastream;
62 
63  // State
64  std::vector<char> tok;
65  size_t line_num = 0;
66  size_t line_pos = 0;
67  PartialToken partial_tok_type = PartialToken::NONE;
68  bool title_found = false; // Only for OPTION_FIRST_LINE_IS_TITLE
69 
70  void ProcessChar(const char c);
71  void ProcessEOF();
72  void ProcessSeparatorWithinBool();
73 
74  void BeginToken(const char c);
75  void UpdateComment(const char c);
76  void UpdateString(const char c);
77  void UpdateNumber(const char c);
78  void UpdateBool(const char c);
79  void UpdateKeyword(const char c);
80  void UpdateTitle(const char c); // Only for OPTION_FIRST_LINE_IS_TITLE
81  void UpdateGarbage(const char c);
82 
83  void DiscontinueBool();
84  void DiscontinueNumber();
85  void DiscontinueKeyword();
86  void FlushStringishToken(RoR::TokenType type);
87  void FlushNumericToken();
88 };
89 
90 void DocumentParser::BeginToken(const char c)
91 {
92  switch (c)
93  {
94  case '\r':
95  break;
96 
97  case ' ':
98  case ',':
99  case '\t':
100  line_pos++;
101  break;
102 
103  case ':':
104  if (options & GenericDocument::OPTION_ALLOW_SEPARATOR_COLON)
105  {
106  line_pos++;
107  }
108  else
109  {
110  if (options & GenericDocument::OPTION_ALLOW_NAKED_STRINGS)
111  partial_tok_type = PartialToken::STRING_NAKED;
112  else
113  partial_tok_type = PartialToken::GARBAGE;
114  tok.push_back(c);
115  line_pos++;
116  }
117  break;
118 
119  case '=':
120  if (options & GenericDocument::OPTION_ALLOW_SEPARATOR_EQUALS)
121  {
122  line_pos++;
123  }
124  else
125  {
126  if (options & GenericDocument::OPTION_ALLOW_NAKED_STRINGS)
127  partial_tok_type = PartialToken::STRING_NAKED;
128  else
129  partial_tok_type = PartialToken::GARBAGE;
130  tok.push_back(c);
131  line_pos++;
132  }
133  break;
134 
135  case '\n':
136  doc.tokens.push_back({ TokenType::LINEBREAK, 0.f });
137  line_num++;
138  line_pos = 0;
139  break;
140 
141  case ';':
142  partial_tok_type = PartialToken::COMMENT_SEMICOLON;
143  line_pos++;
144  break;
145 
146  case '/':
147  if (options & GenericDocument::OPTION_ALLOW_SLASH_COMMENTS)
148  {
149  partial_tok_type = PartialToken::COMMENT_SLASH;
150  }
151  else
152  {
153  if (options & GenericDocument::OPTION_ALLOW_NAKED_STRINGS)
154  partial_tok_type = PartialToken::STRING_NAKED;
155  else
156  partial_tok_type = PartialToken::GARBAGE;
157  tok.push_back(c);
158  }
159  line_pos++;
160  break;
161 
162  case '#':
163  if (options & GenericDocument::OPTION_ALLOW_HASH_COMMENTS)
164  {
165  partial_tok_type = PartialToken::COMMENT_HASH;
166  }
167  else
168  {
169  if (options & GenericDocument::OPTION_ALLOW_NAKED_STRINGS)
170  partial_tok_type = PartialToken::STRING_NAKED;
171  else
172  partial_tok_type = PartialToken::GARBAGE;
173  tok.push_back(c);
174  }
175  line_pos++;
176  break;
177 
178  case '[':
179  if (options & GenericDocument::OPTION_ALLOW_BRACED_KEYWORDS)
180  {
181  partial_tok_type = PartialToken::KEYWORD_BRACED;
182  }
183  else
184  {
185  if (options & GenericDocument::OPTION_ALLOW_NAKED_STRINGS)
186  partial_tok_type = PartialToken::STRING_NAKED;
187  else
188  partial_tok_type = PartialToken::GARBAGE;
189  }
190  tok.push_back(c);
191  line_pos++;
192  break;
193 
194  case '"':
195  partial_tok_type = PartialToken::STRING_QUOTED;
196  line_pos++;
197  break;
198 
199  case '.':
200  tok.push_back(c);
201  partial_tok_type = PartialToken::NUMBER_DECIMAL;
202  line_pos++;
203  break;
204 
205  case 't':
206  tok.push_back(c);
207  partial_tok_type = PartialToken::BOOL_TRUE;
208  line_pos++;
209  break;
210 
211  case 'f':
212  tok.push_back(c);
213  partial_tok_type = PartialToken::BOOL_FALSE;
214  line_pos++;
215  break;
216 
217  case '0':
218  case '1':
219  case '2':
220  case '3':
221  case '4':
222  case '5':
223  case '6':
224  case '7':
225  case '8':
226  case '9':
227  partial_tok_type = PartialToken::NUMBER_INTEGER;
228  tok.push_back(c);
229  line_pos++;
230  break;
231 
232  case '-':
233  partial_tok_type = PartialToken::NUMBER_STUB_MINUS;
234  tok.push_back(c);
235  line_pos++;
236  break;
237 
238  default:
239  if (isalpha(c) &&
240  (doc.tokens.size() == 0 || doc.tokens.back().type == TokenType::LINEBREAK)) // on line start?
241  {
242  tok.push_back(c);
243  partial_tok_type = PartialToken::KEYWORD;
244  }
245  else if (options & GenericDocument::OPTION_ALLOW_NAKED_STRINGS)
246  {
247  tok.push_back(c);
248  partial_tok_type = PartialToken::STRING_NAKED;
249  }
250  else
251  {
252  partial_tok_type = PartialToken::GARBAGE;
253  tok.push_back(c);
254  }
255  line_pos++;
256  break;
257  }
258 
259  if (options & GenericDocument::OPTION_FIRST_LINE_IS_TITLE
260  && !title_found
261  && (doc.tokens.size() == 0 || doc.tokens.back().type == TokenType::LINEBREAK)
262  && partial_tok_type != PartialToken::NONE
263  && partial_tok_type != PartialToken::COMMENT_SEMICOLON
264  && partial_tok_type != PartialToken::COMMENT_SLASH)
265  {
266  title_found = true;
267  partial_tok_type = PartialToken::TITLE_STRING;
268  }
269 
270  if (partial_tok_type == PartialToken::GARBAGE)
271  {
272  App::GetConsole()->putMessage(Console::CONSOLE_MSGTYPE_INFO, Console::CONSOLE_SYSTEM_WARNING,
273  fmt::format("{}, line {}, pos {}: stray character '{}'", datastream->getName(), line_num, line_pos, c));
274  }
275 }
276 
277 void DocumentParser::UpdateComment(const char c)
278 {
279  switch (c)
280  {
281  case '\r':
282  break;
283 
284  case '\n':
285  this->FlushStringishToken(TokenType::COMMENT);
286  // Break line
287  doc.tokens.push_back({ TokenType::LINEBREAK, 0.f });
288  line_num++;
289  line_pos = 0;
290  break;
291 
292  case '/':
293  if (partial_tok_type != PartialToken::COMMENT_SLASH || tok.size() > 0) // With COMMENT_SLASH, skip any number of leading '/'
294  {
295  tok.push_back(c);
296  }
297  line_pos++;
298  break;
299 
300  default:
301  tok.push_back(c);
302  line_pos++;
303  break;
304  }
305 }
306 
307 void DocumentParser::UpdateString(const char c)
308 {
309  switch (c)
310  {
311  case '\r':
312  break;
313 
314  case ' ':
315  if (partial_tok_type == PartialToken::STRING_QUOTED
316  || partial_tok_type == PartialToken::STRING_NAKED_CAPTURING_SPACES)
317  {
318  tok.push_back(c);
319  }
320  else // (partial_tok_type == PartialToken::STRING_NAKED)
321  {
322  this->FlushStringishToken(TokenType::STRING);
323  }
324  line_pos++;
325  break;
326 
327  case ',':
328  case '\t':
329  if (partial_tok_type == PartialToken::STRING_QUOTED)
330  {
331  tok.push_back(c);
332  }
333  else // (partial_tok_type == PartialToken::STRING_NAKED)
334  {
335  this->FlushStringishToken(TokenType::STRING);
336  }
337  line_pos++;
338  break;
339 
340  case '\n':
341  if (partial_tok_type == PartialToken::STRING_QUOTED)
342  {
343  App::GetConsole()->putMessage(Console::CONSOLE_MSGTYPE_INFO, Console::CONSOLE_SYSTEM_WARNING,
344  fmt::format("{}, line {}, pos {}: quoted string interrupted by newline", datastream->getName(), line_num, line_pos));
345  }
346  this->FlushStringishToken(TokenType::STRING);
347  // Break line
348  doc.tokens.push_back({ TokenType::LINEBREAK, 0.f });
349  line_num++;
350  line_pos = 0;
351  break;
352 
353  case ':':
354  if (options & GenericDocument::OPTION_ALLOW_SEPARATOR_COLON
355  && (partial_tok_type == PartialToken::STRING_NAKED || partial_tok_type == PartialToken::STRING_NAKED_CAPTURING_SPACES))
356  {
357  this->FlushStringishToken(TokenType::STRING);
358  }
359  else
360  {
361  tok.push_back(c);
362  }
363  line_pos++;
364  break;
365 
366  case '=':
367  if (options & GenericDocument::OPTION_ALLOW_SEPARATOR_EQUALS
368  && (partial_tok_type == PartialToken::STRING_NAKED || partial_tok_type == PartialToken::STRING_NAKED_CAPTURING_SPACES))
369  {
370  this->FlushStringishToken(TokenType::STRING);
371  }
372  else
373  {
374  tok.push_back(c);
375  }
376  line_pos++;
377  break;
378 
379  case '"':
380  if (partial_tok_type == PartialToken::STRING_QUOTED)
381  {
382  this->FlushStringishToken(TokenType::STRING);
383  }
384  else // (partial_tok_type == PartialToken::STRING_NAKED)
385  {
386  partial_tok_type = PartialToken::GARBAGE;
387  tok.push_back(c);
388  }
389  line_pos++;
390  break;
391 
392  case '(':
393  if (partial_tok_type == PartialToken::STRING_NAKED
394  && options & GenericDocument::OPTION_PARENTHESES_CAPTURE_SPACES)
395  {
396  partial_tok_type = PartialToken::STRING_NAKED_CAPTURING_SPACES;
397  }
398  tok.push_back(c);
399  line_pos++;
400  break;
401 
402  case ')':
403  if (partial_tok_type == PartialToken::STRING_NAKED_CAPTURING_SPACES)
404  {
405  partial_tok_type = PartialToken::STRING_NAKED;
406  }
407  tok.push_back(c);
408  line_pos++;
409  break;
410 
411  default:
412  tok.push_back(c);
413  line_pos++;
414  break;
415  }
416 
417  if (partial_tok_type == PartialToken::GARBAGE)
418  {
419  App::GetConsole()->putMessage(Console::CONSOLE_MSGTYPE_INFO, Console::CONSOLE_SYSTEM_WARNING,
420  fmt::format("{}, line {}, pos {}: stray character '{}' in string", datastream->getName(), line_num, line_pos, c));
421  }
422 }
423 
424 void DocumentParser::UpdateNumber(const char c)
425 {
426  switch (c)
427  {
428  case '\r':
429  break;
430 
431  case ' ':
432  case ',':
433  case '\t':
434  if (partial_tok_type == PartialToken::NUMBER_STUB_MINUS
435  && options & GenericDocument::OPTION_ALLOW_NAKED_STRINGS)
436  {
437  this->FlushStringishToken(TokenType::STRING);
438  }
439  else
440  {
441  this->FlushNumericToken();
442  }
443  line_pos++;
444  break;
445 
446  case '\n':
447  if (partial_tok_type == PartialToken::NUMBER_STUB_MINUS
448  && options & GenericDocument::OPTION_ALLOW_NAKED_STRINGS)
449  {
450  this->FlushStringishToken(TokenType::STRING);
451  }
452  else
453  {
454  this->FlushNumericToken();
455  }
456  // Break line
457  doc.tokens.push_back({ TokenType::LINEBREAK, 0.f });
458  line_num++;
459  line_pos = 0;
460  break;
461 
462  case ':':
463  if (options & GenericDocument::OPTION_ALLOW_SEPARATOR_COLON)
464  {
465  if (partial_tok_type == PartialToken::NUMBER_STUB_MINUS
466  && options & GenericDocument::OPTION_ALLOW_NAKED_STRINGS)
467  {
468  this->FlushStringishToken(TokenType::STRING);
469  }
470  else
471  {
472  this->FlushNumericToken();
473  }
474  }
475  else
476  {
477  this->DiscontinueNumber();
478  tok.push_back(c);
479  }
480  line_pos++;
481  break;
482 
483  case '=':
484  if (options & GenericDocument::OPTION_ALLOW_SEPARATOR_EQUALS)
485  {
486  if (partial_tok_type == PartialToken::NUMBER_STUB_MINUS
487  && options & GenericDocument::OPTION_ALLOW_NAKED_STRINGS)
488  {
489  this->FlushStringishToken(TokenType::STRING);
490  }
491  else
492  {
493  this->FlushNumericToken();
494  }
495  }
496  else
497  {
498  this->DiscontinueNumber();
499  tok.push_back(c);
500  }
501  line_pos++;
502  break;
503 
504  case '.':
505  if (partial_tok_type == PartialToken::NUMBER_INTEGER
506  || partial_tok_type == PartialToken::NUMBER_STUB_MINUS)
507  {
508  partial_tok_type = PartialToken::NUMBER_DECIMAL;
509  }
510  else
511  {
512  this->DiscontinueNumber();
513  }
514  tok.push_back(c);
515  line_pos++;
516  break;
517 
518  case 'e':
519  case 'E':
520  if (partial_tok_type == PartialToken::NUMBER_DECIMAL
521  || partial_tok_type == PartialToken::NUMBER_INTEGER)
522  {
523  partial_tok_type = PartialToken::NUMBER_SCIENTIFIC_STUB;
524  }
525  else
526  {
527  this->DiscontinueNumber();
528  }
529  tok.push_back(c);
530  line_pos++;
531  break;
532 
533  case '-':
534  if (partial_tok_type == PartialToken::NUMBER_SCIENTIFIC_STUB)
535  {
536  partial_tok_type = PartialToken::NUMBER_SCIENTIFIC_STUB_MINUS;
537  }
538  else
539  {
540  this->DiscontinueNumber();
541  }
542  tok.push_back(c);
543  line_pos++;
544  break;
545 
546  case '0':
547  case '1':
548  case '2':
549  case '3':
550  case '4':
551  case '5':
552  case '6':
553  case '7':
554  case '8':
555  case '9':
556  if (partial_tok_type == PartialToken::NUMBER_SCIENTIFIC_STUB
557  || partial_tok_type == PartialToken::NUMBER_SCIENTIFIC_STUB_MINUS)
558  {
559  partial_tok_type = PartialToken::NUMBER_SCIENTIFIC;
560  }
561  else if (partial_tok_type == PartialToken::NUMBER_STUB_MINUS)
562  {
563  partial_tok_type = PartialToken::NUMBER_INTEGER;
564  }
565  tok.push_back(c);
566  line_pos++;
567  break;
568 
569  default:
570  this->DiscontinueNumber();
571  tok.push_back(c);
572  line_pos++;
573  break;
574 
575  }
576 
577  if (partial_tok_type == PartialToken::GARBAGE)
578  {
579  App::GetConsole()->putMessage(Console::CONSOLE_MSGTYPE_INFO, Console::CONSOLE_SYSTEM_WARNING,
580  fmt::format("{}, line {}, pos {}: stray character '{}' in number", datastream->getName(), line_num, line_pos, c));
581  }
582 }
583 
584 void DocumentParser::ProcessSeparatorWithinBool()
585 {
586  this->DiscontinueBool();
587  switch (partial_tok_type)
588  {
589  case PartialToken::KEYWORD:
590  this->FlushStringishToken(TokenType::KEYWORD);
591  break;
592  case PartialToken::STRING_NAKED:
593  this->FlushStringishToken(TokenType::STRING);
594  break;
595  default:
596  // Discard token
597  tok.push_back('\0');
598  App::GetConsole()->putMessage(Console::CONSOLE_MSGTYPE_INFO, Console::CONSOLE_SYSTEM_WARNING,
599  fmt::format("{}, line {}, pos {}: discarding incomplete boolean token '{}'", datastream->getName(), line_num, line_pos, tok.data()));
600  tok.clear();
601  partial_tok_type = PartialToken::NONE;
602  break;
603  }
604 }
605 
606 void DocumentParser::UpdateBool(const char c)
607 {
608  switch (c)
609  {
610  case '\r':
611  break;
612 
613  case ' ':
614  case ',':
615  case '\t':
616  this->ProcessSeparatorWithinBool();
617  line_pos++;
618  break;
619 
620  case '\n':
621  this->ProcessSeparatorWithinBool();
622  // Break line
623  doc.tokens.push_back({ TokenType::LINEBREAK, 0.f });
624  line_num++;
625  line_pos = 0;
626  break;
627 
628  case ':':
629  if (options & GenericDocument::OPTION_ALLOW_SEPARATOR_COLON)
630  {
631  this->ProcessSeparatorWithinBool();
632  }
633  else
634  {
635  this->DiscontinueBool();
636  tok.push_back(c);
637  }
638  line_pos++;
639  break;
640 
641  case '=':
642  if (options & GenericDocument::OPTION_ALLOW_SEPARATOR_EQUALS)
643  {
644  this->ProcessSeparatorWithinBool();
645  }
646  else
647  {
648  this->DiscontinueBool();
649  tok.push_back(c);
650  }
651  line_pos++;
652  break;
653 
654  case 'r':
655  if (partial_tok_type != PartialToken::BOOL_TRUE || tok.size() != 1)
656  {
657  this->DiscontinueBool();
658  }
659  tok.push_back(c);
660  line_pos++;
661  break;
662 
663  case 'u':
664  if (partial_tok_type != PartialToken::BOOL_TRUE || tok.size() != 2)
665  {
666  this->DiscontinueBool();
667  }
668  tok.push_back(c);
669  line_pos++;
670  break;
671 
672  case 'a':
673  if (partial_tok_type != PartialToken::BOOL_FALSE || tok.size() != 1)
674  {
675  this->DiscontinueBool();
676  }
677  tok.push_back(c);
678  line_pos++;
679  break;
680 
681  case 'l':
682  if (partial_tok_type != PartialToken::BOOL_FALSE || tok.size() != 2)
683  {
684  this->DiscontinueBool();
685  }
686  tok.push_back(c);
687  line_pos++;
688  break;
689 
690  case 's':
691  if (partial_tok_type != PartialToken::BOOL_FALSE || tok.size() != 3)
692  {
693  this->DiscontinueBool();
694  }
695  tok.push_back(c);
696  line_pos++;
697  break;
698 
699  case 'e':
700  if (partial_tok_type == PartialToken::BOOL_TRUE && tok.size() == 3)
701  {
702  doc.tokens.push_back({ TokenType::BOOL, 1.f });
703  tok.clear();
704  partial_tok_type = PartialToken::NONE;
705  }
706  else if (partial_tok_type == PartialToken::BOOL_FALSE && tok.size() == 4)
707  {
708  doc.tokens.push_back({ TokenType::BOOL, 0.f });
709  tok.clear();
710  partial_tok_type = PartialToken::NONE;
711  }
712  else
713  {
714  this->DiscontinueBool();
715  tok.push_back(c);
716  }
717  line_pos++;
718  break;
719 
720  default:
721  this->DiscontinueBool();
722  tok.push_back(c);
723  line_pos++;
724  break;
725  }
726 
727  if (partial_tok_type == PartialToken::GARBAGE)
728  {
729  App::GetConsole()->putMessage(Console::CONSOLE_MSGTYPE_INFO, Console::CONSOLE_SYSTEM_WARNING,
730  fmt::format("{}, line {}, pos {}: stray character '{}' in boolean", datastream->getName(), line_num, line_pos, c));
731  }
732 }
733 
734 void DocumentParser::DiscontinueBool()
735 {
736  if (doc.tokens.size() == 0 || doc.tokens.back().type == TokenType::LINEBREAK)
737  partial_tok_type = PartialToken::KEYWORD;
738  else if (options & GenericDocument::OPTION_ALLOW_NAKED_STRINGS)
739  partial_tok_type = PartialToken::STRING_NAKED;
740  else
741  partial_tok_type = PartialToken::GARBAGE;
742 }
743 
744 void DocumentParser::DiscontinueNumber()
745 {
746  if (options & GenericDocument::OPTION_ALLOW_NAKED_STRINGS)
747  partial_tok_type = PartialToken::STRING_NAKED;
748  else
749  partial_tok_type = PartialToken::GARBAGE;
750 }
751 
752 void DocumentParser::DiscontinueKeyword()
753 {
754  if (options & GenericDocument::OPTION_ALLOW_NAKED_STRINGS)
755  partial_tok_type = PartialToken::STRING_NAKED;
756  else
757  partial_tok_type = PartialToken::GARBAGE;
758 }
759 
760 void DocumentParser::UpdateKeyword(const char c)
761 {
762  switch (c)
763  {
764  case '\r':
765  break;
766 
767  case ' ':
768  case ',':
769  case '\t':
770  this->FlushStringishToken(TokenType::KEYWORD);
771  line_pos++;
772  break;
773 
774  case '\n':
775  this->FlushStringishToken(TokenType::KEYWORD);
776  // Break line
777  doc.tokens.push_back({ TokenType::LINEBREAK, 0.f });
778  line_num++;
779  line_pos = 0;
780  break;
781 
782  case ':':
783  if (options & GenericDocument::OPTION_ALLOW_SEPARATOR_COLON)
784  {
785  this->FlushStringishToken(TokenType::KEYWORD);
786  }
787  else
788  {
789  this->DiscontinueKeyword();
790  tok.push_back(c);
791  }
792  line_pos++;
793  break;
794 
795  case '=':
796  if (options & GenericDocument::OPTION_ALLOW_SEPARATOR_EQUALS)
797  {
798  this->FlushStringishToken(TokenType::KEYWORD);
799  }
800  else
801  {
802  this->DiscontinueKeyword();
803  tok.push_back(c);
804  }
805  line_pos++;
806  break;
807 
808  case '_':
809  tok.push_back(c);
810  line_pos++;
811  break;
812 
813  case '(':
814  if (options & GenericDocument::OPTION_ALLOW_NAKED_STRINGS)
815  {
816  if (options & GenericDocument::OPTION_PARENTHESES_CAPTURE_SPACES)
817  partial_tok_type = PartialToken::STRING_NAKED_CAPTURING_SPACES;
818  else
819  partial_tok_type = PartialToken::STRING_NAKED;
820  }
821  else
822  {
823  partial_tok_type = PartialToken::GARBAGE;
824  }
825  tok.push_back(c);
826  line_pos++;
827  break;
828 
829  case ']':
830  if (partial_tok_type == PartialToken::KEYWORD_BRACED)
831  {
832  partial_tok_type = PartialToken::KEYWORD; // Do not allow any more ']'.
833  }
834  else
835  {
836  this->DiscontinueKeyword();
837  }
838  tok.push_back(c);
839  line_pos++;
840  break;
841 
842  default:
843  if (!isalnum(c))
844  {
845  this->DiscontinueKeyword();
846  }
847  tok.push_back(c);
848  line_pos++;
849  break;
850  }
851 
852  if (partial_tok_type == PartialToken::GARBAGE)
853  {
854  App::GetConsole()->putMessage(Console::CONSOLE_MSGTYPE_INFO, Console::CONSOLE_SYSTEM_WARNING,
855  fmt::format("{}, line {}, pos {}: stray character '{}' in keyword", datastream->getName(), line_num, line_pos, c));
856  }
857 }
858 
859 void DocumentParser::UpdateTitle(const char c)
860 {
861  switch (c)
862  {
863  case '\r':
864  break;
865 
866  case '\n':
867  this->FlushStringishToken(TokenType::STRING);
868  // Break line
869  doc.tokens.push_back({ TokenType::LINEBREAK, 0.f });
870  line_num++;
871  line_pos = 0;
872  break;
873 
874  default:
875  tok.push_back(c);
876  line_pos++;
877  break;
878  }
879 }
880 
881 void DocumentParser::UpdateGarbage(const char c)
882 {
883  switch (c)
884  {
885  case '\r':
886  break;
887 
888  case ' ':
889  case ',':
890  case '\t':
891  case '\n':
892  tok.push_back('\0');
893  App::GetConsole()->putMessage(Console::CONSOLE_MSGTYPE_INFO, Console::CONSOLE_SYSTEM_WARNING,
894  fmt::format("{}, line {}, pos {}: discarding garbage token '{}'", datastream->getName(), line_num, line_pos, tok.data()));
895  tok.clear();
896  partial_tok_type = PartialToken::NONE;
897  line_pos++;
898  break;
899 
900  default:
901  tok.push_back(c);
902  line_pos++;
903  break;
904  }
905 }
906 
907 void DocumentParser::FlushStringishToken(RoR::TokenType type)
908 {
909  doc.tokens.push_back({ type, (float)doc.string_pool.size() });
910  tok.push_back('\0');
911  std::copy(tok.begin(), tok.end(), std::back_inserter(doc.string_pool));
912  tok.clear();
913  partial_tok_type = PartialToken::NONE;
914 }
915 
916 void DocumentParser::FlushNumericToken()
917 {
918  tok.push_back('\0');
919  doc.tokens.push_back({ TokenType::NUMBER, (float)Ogre::StringConverter::parseReal(tok.data()) });
920  tok.clear();
921  partial_tok_type = PartialToken::NONE;
922 }
923 
924 void DocumentParser::ProcessChar(const char c)
925 {
926  switch (partial_tok_type)
927  {
928  case PartialToken::NONE:
929  this->BeginToken(c);
930  break;
931 
932  case PartialToken::COMMENT_SEMICOLON:
933  case PartialToken::COMMENT_SLASH:
934  case PartialToken::COMMENT_HASH:
935  this->UpdateComment(c);
936  break;
937 
938  case PartialToken::STRING_QUOTED:
939  case PartialToken::STRING_NAKED:
940  case PartialToken::STRING_NAKED_CAPTURING_SPACES:
941  this->UpdateString(c);
942  break;
943 
944  case PartialToken::NUMBER_INTEGER:
945  case PartialToken::NUMBER_STUB_MINUS:
946  case PartialToken::NUMBER_DECIMAL:
947  case PartialToken::NUMBER_SCIENTIFIC:
948  case PartialToken::NUMBER_SCIENTIFIC_STUB:
949  case PartialToken::NUMBER_SCIENTIFIC_STUB_MINUS:
950  this->UpdateNumber(c);
951  break;
952 
953  case PartialToken::BOOL_TRUE:
954  case PartialToken::BOOL_FALSE:
955  this->UpdateBool(c);
956  break;
957 
958  case PartialToken::KEYWORD:
959  case PartialToken::KEYWORD_BRACED:
960  this->UpdateKeyword(c);
961  break;
962 
963  case PartialToken::TITLE_STRING:
964  this->UpdateTitle(c);
965  break;
966 
967  case PartialToken::GARBAGE:
968  this->UpdateGarbage(c);
969  break;
970  }
971 }
972 
973 void DocumentParser::ProcessEOF()
974 {
975  // Flush any partial token
976  switch (partial_tok_type)
977  {
978  case PartialToken::STRING_QUOTED:
979  case PartialToken::STRING_NAKED_CAPTURING_SPACES:
980  case PartialToken::TITLE_STRING:
981  this->FlushStringishToken(TokenType::STRING);
982  break;
983 
984  case PartialToken::KEYWORD_BRACED:
985  this->FlushStringishToken(TokenType::KEYWORD);
986  break;
987 
988  default:
989  this->ProcessChar(' '); // Pretend processing a separator to flush any partial whitespace-incompatible token.
990  break;
991  }
992 
993  // Ensure newline at end of file
994  if (doc.tokens.size() == 0 || doc.tokens.back().type != TokenType::LINEBREAK)
995  {
996  doc.tokens.push_back({ TokenType::LINEBREAK, 0.f });
997  }
998 }
999 
1000 void GenericDocument::loadFromDataStream(Ogre::DataStreamPtr datastream, const BitMask_t options)
1001 {
1002  // Reset the document
1003  tokens.clear();
1004  string_pool.clear();
1005 
1006  // Prepare context
1007  DocumentParser parser(*this, options, datastream);
1008  const size_t LINE_BUF_MAX = 10 * 1024; // 10Kb
1009  char buf[LINE_BUF_MAX];
1010 
1011  // Parse the text
1012  while (!datastream->eof())
1013  {
1014  size_t buf_len = datastream->read(buf, LINE_BUF_MAX);
1015  for (size_t i = 0; i < buf_len; i++)
1016  {
1017  const char c = buf[i];
1018 
1019  parser.ProcessChar(c);
1020  }
1021  }
1022  parser.ProcessEOF();
1023 }
1024 
1025 #if OGRE_PLATFORM == OGRE_PLATFORM_WIN32
1026  const char* EOL_STR = "\r\n"; // CR+LF
1027 #else
1028  const char* EOL_STR = "\n"; // "LF"
1029 #endif
1030 
1031 void GenericDocument::saveToDataStream(Ogre::DataStreamPtr datastream)
1032 {
1033  std::string separator;
1034  const char* pool_str = nullptr;
1035  const size_t BUF_MAX = 100;
1036  char buf[BUF_MAX];
1037 
1038  for (Token& tok : tokens)
1039  {
1040  switch (tok.type)
1041  {
1042  case TokenType::LINEBREAK:
1043  datastream->write(EOL_STR, strlen(EOL_STR));
1044  separator = "";
1045  break;
1046 
1047  case TokenType::COMMENT:
1048  datastream->write(";", 1);
1049  pool_str = string_pool.data() + (size_t)tok.data;
1050  datastream->write(pool_str, strlen(pool_str));
1051  break;
1052 
1053  case TokenType::STRING:
1054  datastream->write(separator.data(), separator.size());
1055  pool_str = string_pool.data() + (size_t)tok.data;
1056  datastream->write(pool_str, strlen(pool_str));
1057  separator = ",";
1058  break;
1059 
1060  case TokenType::NUMBER:
1061  datastream->write(separator.data(), separator.size());
1062  snprintf(buf, BUF_MAX, "%f", tok.data);
1063  datastream->write(buf, strlen(buf));
1064  separator = ",";
1065  break;
1066 
1067  case TokenType::BOOL:
1068  datastream->write(separator.data(), separator.size());
1069  snprintf(buf, BUF_MAX, "%s", tok.data == 1.f ? "true" : "false");
1070  datastream->write(buf, strlen(buf));
1071  separator = ",";
1072  break;
1073 
1074  case TokenType::KEYWORD:
1075  pool_str = string_pool.data() + (size_t)tok.data;
1076  datastream->write(pool_str, strlen(pool_str));
1077  separator = " ";
1078  break;
1079  }
1080  }
1081 }
1082 
1083 bool GenericDocument::loadFromResource(std::string resource_name, std::string resource_group_name, BitMask_t options/* = 0*/)
1084 {
1085  try
1086  {
1087  Ogre::DataStreamPtr datastream = Ogre::ResourceGroupManager::getSingleton().openResource(resource_name, resource_group_name);
1088  this->loadFromDataStream(datastream, options);
1089  return true;
1090  }
1091  catch (Ogre::Exception& eeh)
1092  {
1093  App::GetConsole()->putMessage(Console::CONSOLE_MSGTYPE_INFO, Console::CONSOLE_SYSTEM_ERROR,
1094  fmt::format("GenericDocument: could not load file '{}' from resource group '{}': {}", resource_name, resource_group_name, eeh.getDescription()));
1095  return false;
1096  }
1097 }
1098 
1099 bool GenericDocument::saveToResource(std::string resource_name, std::string resource_group_name)
1100 {
1101  try
1102  {
1103  Ogre::DataStreamPtr datastream = Ogre::ResourceGroupManager::getSingleton().createResource(resource_name, resource_group_name);
1104  this->saveToDataStream(datastream);
1105  return true;
1106  }
1107  catch (Ogre::Exception& eeh)
1108  {
1109  App::GetConsole()->putMessage(Console::CONSOLE_MSGTYPE_INFO, Console::CONSOLE_SYSTEM_ERROR,
1110  fmt::format("GenericDocument: could not write file '{}' to resource group '{}': {}", resource_name, resource_group_name, eeh.getDescription()));
1111  return false;
1112  }
1113 }
1114 
1115 bool GenericDocContext::seekNextLine()
1116 {
1117  // Skip current line
1118  while (!this->endOfFile() && this->tokenType() != TokenType::LINEBREAK)
1119  {
1120  this->moveNext();
1121  }
1122 
1123  // Skip comments and empty lines
1124  while (!this->endOfFile() && !this->isTokString() && !this->isTokFloat() && !this->isTokBool() && !this->isTokKeyword())
1125  {
1126  this->moveNext();
1127  }
1128 
1129  return this->endOfFile();
1130 }
1131 
1132 int GenericDocContext::countLineArgs()
1133 {
1134  int count = 0;
1135  while (!endOfFile(count) && this->tokenType(count) != TokenType::LINEBREAK)
1136  count++;
1137  return count;
1138 }
1139 
1140 // -----------------
1141 // Editing functions
1142 
1143 bool GenericDocContext::insertToken(int offset)
1144 {
1145  if (endOfFile(offset))
1146  return false;
1147 
1148  doc->tokens.insert(doc->tokens.begin() + token_pos + offset, { TokenType::NONE, 0.f });
1149  return true;
1150 }
1151 
1152 bool GenericDocContext::eraseToken(int offset)
1153 {
1154  if (endOfFile(offset))
1155  return false;
1156 
1157  // Just erase the token.
1158  // We don't care about garbage in `string_pool` - the strings are usually just 1-6 characters long anyway.
1159 
1160  doc->tokens.erase(doc->tokens.begin() + token_pos + offset);
1161  return true;
1162 }
1163 
1164 bool GenericDocContext::setStringData(int offset, TokenType type, const std::string& data)
1165 {
1166  if (endOfFile(offset))
1167  return false;
1168 
1169  // Insert the string at the end of the string_pool
1170  // We don't care about order - updating string offsets in tokens would be complicated and unlikely helpful.
1171 
1172  doc->tokens[token_pos + offset] = { type, (float)doc->string_pool.size() };
1173  std::copy(data.begin(), data.end(), std::back_inserter(doc->string_pool));
1174  doc->string_pool.push_back('\0');
1175  return true;
1176 }
1177 
1178 bool GenericDocContext::setFloatData(int offset, TokenType type, float data)
1179 {
1180  if (endOfFile(offset))
1181  return false;
1182 
1183  doc->tokens[token_pos + offset] = { type, data };
1184  return true;
1185 }
DocumentParser::UpdateKeyword
void UpdateKeyword(const char c)
Definition: GenericFileFormat.cpp:760
DocumentParser::UpdateString
void UpdateString(const char c)
Definition: GenericFileFormat.cpp:307
DocumentParser::ProcessSeparatorWithinBool
void ProcessSeparatorWithinBool()
Definition: GenericFileFormat.cpp:584
RoR::GenericDocContext::setFloatData
bool setFloatData(int offset, TokenType type, float data)
Definition: GenericFileFormat.cpp:1178
RoR::GfxShadowType::NONE
@ NONE
RoR::GenericDocContext::insertToken
bool insertToken(int offset=0)
Inserts TokenType::NONE;.
Definition: GenericFileFormat.cpp:1143
EOL_STR
const char * EOL_STR
Definition: GenericFileFormat.cpp:1026
LINE_BUF_MAX
static const int LINE_BUF_MAX
Definition: GUI_ConsoleView.cpp:43
copy
The MIT free of to any person obtaining a copy of this software and associated documentation to deal in the Software without including without limitation the rights to copy
Definition: LICENSE-ImGui.txt:8
format
Truck file format(technical spec)
RoR::GenericDocument::OPTION_ALLOW_SEPARATOR_COLON
static const BitMask_t OPTION_ALLOW_SEPARATOR_COLON
Allow ':' as separator between tokens.
Definition: GenericFileFormat.h:70
PartialToken::GARBAGE
@ GARBAGE
PartialToken::TITLE_STRING
@ TITLE_STRING
RoR::GenericDocument::OPTION_ALLOW_SLASH_COMMENTS
static const BitMask_t OPTION_ALLOW_SLASH_COMMENTS
Allow comments starting with //.
Definition: GenericFileFormat.h:68
DocumentParser::DocumentParser
DocumentParser(GenericDocument &d, const BitMask_t opt, Ogre::DataStreamPtr ds)
Definition: GenericFileFormat.cpp:55
Console.h
PartialToken::NUMBER_INTEGER
@ NUMBER_INTEGER
RoR::Console::putMessage
void putMessage(MessageArea area, MessageType type, std::string const &msg, std::string icon="")
Definition: Console.cpp:97
RoR::TokenType::NUMBER
@ NUMBER
RoR::GenericDocContext::seekNextLine
bool seekNextLine()
Definition: GenericFileFormat.cpp:1115
DocumentParser::UpdateBool
void UpdateBool(const char c)
Definition: GenericFileFormat.cpp:606
RoR::GenericDocument::loadFromResource
virtual bool loadFromResource(std::string resource_name, std::string resource_group_name, BitMask_t options=0)
Definition: GenericFileFormat.cpp:1083
PartialToken::NUMBER_STUB_MINUS
@ NUMBER_STUB_MINUS
PartialToken::STRING_NAKED_CAPTURING_SPACES
@ STRING_NAKED_CAPTURING_SPACES
RoR::Console::CONSOLE_SYSTEM_ERROR
@ CONSOLE_SYSTEM_ERROR
Definition: Console.h:52
RoR::GenericDocument::saveToResource
virtual bool saveToResource(std::string resource_name, std::string resource_group_name)
Definition: GenericFileFormat.cpp:1099
PartialToken::NONE
@ NONE
DocumentParser::datastream
Ogre::DataStreamPtr datastream
Definition: GenericFileFormat.cpp:61
RoR::GenericDocument::saveToDataStream
virtual void saveToDataStream(Ogre::DataStreamPtr datastream)
Definition: GenericFileFormat.cpp:1031
DocumentParser::DiscontinueNumber
void DiscontinueNumber()
Definition: GenericFileFormat.cpp:744
PartialToken::STRING_NAKED
@ STRING_NAKED
DocumentParser::FlushStringishToken
void FlushStringishToken(RoR::TokenType type)
Definition: GenericFileFormat.cpp:907
RoR::GenericDocContext::countLineArgs
int countLineArgs()
Definition: GenericFileFormat.cpp:1132
RoR::TokenType::KEYWORD
@ KEYWORD
PartialToken::KEYWORD_BRACED
@ KEYWORD_BRACED
DocumentParser::doc
GenericDocument & doc
Definition: GenericFileFormat.cpp:59
PartialToken
PartialToken
Definition: GenericFileFormat.cpp:30
RoR::TokenType::COMMENT
@ COMMENT
PartialToken::COMMENT_SLASH
@ COMMENT_SLASH
PartialToken::BOOL_TRUE
@ BOOL_TRUE
RoR::GenericDocument::OPTION_ALLOW_BRACED_KEYWORDS
static const BitMask_t OPTION_ALLOW_BRACED_KEYWORDS
Allow INI-like '[keyword]' tokens.
Definition: GenericFileFormat.h:72
DocumentParser::ProcessEOF
void ProcessEOF()
Definition: GenericFileFormat.cpp:973
DocumentParser::DiscontinueBool
void DiscontinueBool()
Definition: GenericFileFormat.cpp:734
Application.h
Central state/object manager and communications hub.
RoR::App::GetConsole
Console * GetConsole()
Definition: Application.cpp:270
DocumentParser::UpdateTitle
void UpdateTitle(const char c)
Definition: GenericFileFormat.cpp:859
DocumentParser::UpdateGarbage
void UpdateGarbage(const char c)
Definition: GenericFileFormat.cpp:881
PartialToken::COMMENT_HASH
@ COMMENT_HASH
RoR::Token
Definition: GenericFileFormat.h:59
RoR::GenericDocument::OPTION_FIRST_LINE_IS_TITLE
static const BitMask_t OPTION_FIRST_LINE_IS_TITLE
First non-empty & non-comment line is a naked string with spaces.
Definition: GenericFileFormat.h:69
PartialToken::STRING_QUOTED
@ STRING_QUOTED
DocumentParser::DiscontinueKeyword
void DiscontinueKeyword()
Definition: GenericFileFormat.cpp:752
RoR::GenericDocContext::setStringData
bool setStringData(int offset, TokenType type, const std::string &data)
Definition: GenericFileFormat.cpp:1164
PartialToken::KEYWORD
@ KEYWORD
RoR::GenericDocument::OPTION_ALLOW_SEPARATOR_EQUALS
static const BitMask_t OPTION_ALLOW_SEPARATOR_EQUALS
Allow '=' as separator between tokens.
Definition: GenericFileFormat.h:73
DocumentParser::UpdateComment
void UpdateComment(const char c)
Definition: GenericFileFormat.cpp:277
PartialToken::NUMBER_SCIENTIFIC_STUB
@ NUMBER_SCIENTIFIC_STUB
PartialToken::NUMBER_DECIMAL
@ NUMBER_DECIMAL
DocumentParser::tok
std::vector< char > tok
Definition: GenericFileFormat.cpp:64
PartialToken::NUMBER_SCIENTIFIC
@ NUMBER_SCIENTIFIC
RoR::GenericDocContext::eraseToken
bool eraseToken(int offset=0)
Definition: GenericFileFormat.cpp:1152
RoR::TokenType
TokenType
Definition: GenericFileFormat.h:48
DocumentParser::options
const BitMask_t options
Definition: GenericFileFormat.cpp:60
RoR::GenericDocument
Definition: GenericFileFormat.h:65
DocumentParser::UpdateNumber
void UpdateNumber(const char c)
Definition: GenericFileFormat.cpp:424
DocumentParser::FlushNumericToken
void FlushNumericToken()
Definition: GenericFileFormat.cpp:916
RoR::TokenType::STRING
@ STRING
BitMask_t
uint32_t BitMask_t
Definition: BitFlags.h:7
Ogre
Definition: ExtinguishableFireAffector.cpp:35
RoR::GenericDocument::loadFromDataStream
virtual void loadFromDataStream(Ogre::DataStreamPtr datastream, BitMask_t options=0)
Definition: GenericFileFormat.cpp:1000
RoR::Console::CONSOLE_SYSTEM_WARNING
@ CONSOLE_SYSTEM_WARNING
Definition: Console.h:53
RoR::Console::CONSOLE_MSGTYPE_INFO
@ CONSOLE_MSGTYPE_INFO
Generic message.
Definition: Console.h:60
PartialToken::NUMBER_SCIENTIFIC_STUB_MINUS
@ NUMBER_SCIENTIFIC_STUB_MINUS
RoR::TokenType::LINEBREAK
@ LINEBREAK
GenericFileFormat.h
Generic text file parser.
RoR::GenericDocument::OPTION_ALLOW_NAKED_STRINGS
static const BitMask_t OPTION_ALLOW_NAKED_STRINGS
Allow strings without quotes, for backwards compatibility.
Definition: GenericFileFormat.h:67
RoR
Definition: AppContext.h:36
RoR::GenericDocument::OPTION_ALLOW_HASH_COMMENTS
static const BitMask_t OPTION_ALLOW_HASH_COMMENTS
Allow comments starting with #.
Definition: GenericFileFormat.h:74
DocumentParser
Definition: GenericFileFormat.cpp:53
RoR::GenericDocument::OPTION_PARENTHESES_CAPTURE_SPACES
static const BitMask_t OPTION_PARENTHESES_CAPTURE_SPACES
If non-empty NAKED string encounters '(', following spaces will be captured until matching ')' is fou...
Definition: GenericFileFormat.h:71
DocumentParser::BeginToken
void BeginToken(const char c)
Definition: GenericFileFormat.cpp:90
PartialToken::COMMENT_SEMICOLON
@ COMMENT_SEMICOLON
PartialToken::BOOL_FALSE
@ BOOL_FALSE
DocumentParser::ProcessChar
void ProcessChar(const char c)
Definition: GenericFileFormat.cpp:924
RoR::TokenType::BOOL
@ BOOL
Generated by   doxygen 1.8.17