RigsofRods
Soft-body Physics Simulation
GenericFileFormat.cpp
Go to the documentation of this file.
1 /*
2  This source file is part of Rigs of Rods
3  Copyright 2022 Petr Ohlidal
4 
5  For more information, see http://www.rigsofrods.org/
6 
7  Rigs of Rods is free software: you can redistribute it and/or modify
8  it under the terms of the GNU General Public License version 3, as
9  published by the Free Software Foundation.
10 
11  Rigs of Rods is distributed in the hope that it will be useful,
12  but WITHOUT ANY WARRANTY; without even the implied warranty of
13  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  GNU General Public License for more details.
15 
16  You should have received a copy of the GNU General Public License
17  along with Rigs of Rods. If not, see <http://www.gnu.org/licenses/>.
18 */
19 
20 #include "GenericFileFormat.h"
21 
22 #include "Application.h"
23 #include "Console.h"
24 
25 #include <algorithm>
26 
27 using namespace RoR;
28 using namespace Ogre;
29 
30 enum class PartialToken
31 {
32  NONE,
33  COMMENT_SEMICOLON, // Comment starting with ';'
34  COMMENT_SLASH, // Comment starting with '//'
35  COMMENT_HASH,
36  STRING_QUOTED, // String starting/ending with '"'
37  STRING_NAKED, // String without '"' on either end
38  STRING_NAKED_CAPTURING_SPACES, // Only for OPTION_PARENTHESES_CAPTURE_SPACES - A naked string seeking the closing ')'.
39  TITLE_STRING, // A whole-line string, with spaces
40  NUMBER_STUB_MINUS, // Sole '-' character, may start a number or a naked string.
41  NUMBER_INTEGER, // Just digits and optionally leading '-'
42  NUMBER_DECIMAL, // Like INTEGER but already containing '.'
43  NUMBER_SCIENTIFIC_STUB, // Like DECIMAL, already containing 'e' or 'E' but not the exponent value.
44  NUMBER_SCIENTIFIC_STUB_MINUS, // Like SCIENTIFIC_STUB but with only '-' in exponent.
45  NUMBER_SCIENTIFIC, // Valid decimal number in scientific notation.
46  KEYWORD, // Unqoted string at the start of line. Accepted characters: alphanumeric and underscore
47  KEYWORD_BRACED, // Like KEYWORD but starting with '[' and ending with ']'
48  BOOL_TRUE, // Partial 'true'
49  BOOL_FALSE, // Partial 'false'
50  GARBAGE, // Text not fitting any above category, will be discarded
51 };
52 
53 struct DocumentParser
54 {
55  DocumentParser(GenericDocument& d, const BitMask_t opt, Ogre::DataStreamPtr ds)
56  : doc(d), options(opt), datastream(ds) {}
57 
58  // Config
59  GenericDocument& doc;
60  const BitMask_t options;
61  Ogre::DataStreamPtr datastream;
62 
63  // State
64  std::vector<char> tok;
65  size_t line_num = 0;
66  size_t line_pos = 0;
67  PartialToken partial_tok_type = PartialToken::NONE;
68  bool title_found = false; // Only for OPTION_FIRST_LINE_IS_TITLE
69 
70  void ProcessChar(const char c);
71  void ProcessEOF();
72 
73  void BeginToken(const char c);
74  void UpdateComment(const char c);
75  void UpdateString(const char c);
76  void UpdateNumber(const char c);
77  void UpdateBool(const char c);
78  void UpdateKeyword(const char c);
79  void UpdateTitle(const char c); // Only for OPTION_FIRST_LINE_IS_TITLE
80  void UpdateGarbage(const char c);
81 
82  void DiscontinueBool();
83  void DiscontinueNumber();
84  void DiscontinueKeyword();
85  void FlushStringishToken(RoR::TokenType type);
86  void FlushNumericToken();
87 };
88 
89 void DocumentParser::BeginToken(const char c)
90 {
91  switch (c)
92  {
93  case '\r':
94  break;
95 
96  case ' ':
97  case ',':
98  case '\t':
99  line_pos++;
100  break;
101 
102  case ':':
103  if (options & GenericDocument::OPTION_ALLOW_SEPARATOR_COLON)
104  {
105  line_pos++;
106  }
107  else
108  {
109  if (options & GenericDocument::OPTION_ALLOW_NAKED_STRINGS)
110  partial_tok_type = PartialToken::STRING_NAKED;
111  else
112  partial_tok_type = PartialToken::GARBAGE;
113  tok.push_back(c);
114  line_pos++;
115  }
116  break;
117 
118  case '=':
119  if (options & GenericDocument::OPTION_ALLOW_SEPARATOR_EQUALS)
120  {
121  line_pos++;
122  }
123  else
124  {
125  if (options & GenericDocument::OPTION_ALLOW_NAKED_STRINGS)
126  partial_tok_type = PartialToken::STRING_NAKED;
127  else
128  partial_tok_type = PartialToken::GARBAGE;
129  tok.push_back(c);
130  line_pos++;
131  }
132  break;
133 
134  case '\n':
135  doc.tokens.push_back({ TokenType::LINEBREAK, 0.f });
136  line_num++;
137  line_pos = 0;
138  break;
139 
140  case ';':
141  partial_tok_type = PartialToken::COMMENT_SEMICOLON;
142  line_pos++;
143  break;
144 
145  case '/':
146  if (options & GenericDocument::OPTION_ALLOW_SLASH_COMMENTS)
147  {
148  partial_tok_type = PartialToken::COMMENT_SLASH;
149  }
150  else
151  {
152  if (options & GenericDocument::OPTION_ALLOW_NAKED_STRINGS)
153  partial_tok_type = PartialToken::STRING_NAKED;
154  else
155  partial_tok_type = PartialToken::GARBAGE;
156  tok.push_back(c);
157  }
158  line_pos++;
159  break;
160 
161  case '#':
162  if (options & GenericDocument::OPTION_ALLOW_HASH_COMMENTS)
163  {
164  partial_tok_type = PartialToken::COMMENT_HASH;
165  }
166  else
167  {
168  if (options & GenericDocument::OPTION_ALLOW_NAKED_STRINGS)
169  partial_tok_type = PartialToken::STRING_NAKED;
170  else
171  partial_tok_type = PartialToken::GARBAGE;
172  tok.push_back(c);
173  }
174  line_pos++;
175  break;
176 
177  case '[':
178  if (options & GenericDocument::OPTION_ALLOW_BRACED_KEYWORDS)
179  {
180  partial_tok_type = PartialToken::KEYWORD_BRACED;
181  }
182  else
183  {
184  if (options & GenericDocument::OPTION_ALLOW_NAKED_STRINGS)
185  partial_tok_type = PartialToken::STRING_NAKED;
186  else
187  partial_tok_type = PartialToken::GARBAGE;
188  }
189  tok.push_back(c);
190  line_pos++;
191  break;
192 
193  case '"':
194  partial_tok_type = PartialToken::STRING_QUOTED;
195  line_pos++;
196  break;
197 
198  case '.':
199  tok.push_back(c);
200  partial_tok_type = PartialToken::NUMBER_DECIMAL;
201  line_pos++;
202  break;
203 
204  case 't':
205  tok.push_back(c);
206  partial_tok_type = PartialToken::BOOL_TRUE;
207  line_pos++;
208  break;
209 
210  case 'f':
211  tok.push_back(c);
212  partial_tok_type = PartialToken::BOOL_FALSE;
213  line_pos++;
214  break;
215 
216  case '0':
217  case '1':
218  case '2':
219  case '3':
220  case '4':
221  case '5':
222  case '6':
223  case '7':
224  case '8':
225  case '9':
226  partial_tok_type = PartialToken::NUMBER_INTEGER;
227  tok.push_back(c);
228  line_pos++;
229  break;
230 
231  case '-':
232  partial_tok_type = PartialToken::NUMBER_STUB_MINUS;
233  tok.push_back(c);
234  line_pos++;
235  break;
236 
237  default:
238  if (isalpha(c) &&
239  (doc.tokens.size() == 0 || doc.tokens.back().type == TokenType::LINEBREAK)) // on line start?
240  {
241  tok.push_back(c);
242  partial_tok_type = PartialToken::KEYWORD;
243  }
244  else if (options & GenericDocument::OPTION_ALLOW_NAKED_STRINGS)
245  {
246  tok.push_back(c);
247  partial_tok_type = PartialToken::STRING_NAKED;
248  }
249  else
250  {
251  partial_tok_type = PartialToken::GARBAGE;
252  tok.push_back(c);
253  }
254  line_pos++;
255  break;
256  }
257 
258  if (options & GenericDocument::OPTION_FIRST_LINE_IS_TITLE
259  && !title_found
260  && (doc.tokens.size() == 0 || doc.tokens.back().type == TokenType::LINEBREAK)
261  && partial_tok_type != PartialToken::NONE
262  && partial_tok_type != PartialToken::COMMENT_SEMICOLON
263  && partial_tok_type != PartialToken::COMMENT_SLASH)
264  {
265  title_found = true;
266  partial_tok_type = PartialToken::TITLE_STRING;
267  }
268 
269  if (partial_tok_type == PartialToken::GARBAGE)
270  {
271  App::GetConsole()->putMessage(Console::CONSOLE_MSGTYPE_INFO, Console::CONSOLE_SYSTEM_WARNING,
272  fmt::format("{}, line {}, pos {}: stray character '{}'", datastream->getName(), line_num, line_pos, c));
273  }
274 }
275 
276 void DocumentParser::UpdateComment(const char c)
277 {
278  switch (c)
279  {
280  case '\r':
281  break;
282 
283  case '\n':
284  this->FlushStringishToken(TokenType::COMMENT);
285  // Break line
286  doc.tokens.push_back({ TokenType::LINEBREAK, 0.f });
287  line_num++;
288  line_pos = 0;
289  break;
290 
291  case '/':
292  if (partial_tok_type != PartialToken::COMMENT_SLASH || tok.size() > 0) // With COMMENT_SLASH, skip any number of leading '/'
293  {
294  tok.push_back(c);
295  }
296  line_pos++;
297  break;
298 
299  default:
300  tok.push_back(c);
301  line_pos++;
302  break;
303  }
304 }
305 
306 void DocumentParser::UpdateString(const char c)
307 {
308  switch (c)
309  {
310  case '\r':
311  break;
312 
313  case ' ':
314  if (partial_tok_type == PartialToken::STRING_QUOTED
315  || partial_tok_type == PartialToken::STRING_NAKED_CAPTURING_SPACES)
316  {
317  tok.push_back(c);
318  }
319  else // (partial_tok_type == PartialToken::STRING_NAKED)
320  {
321  this->FlushStringishToken(TokenType::STRING);
322  }
323  line_pos++;
324  break;
325 
326  case ',':
327  case '\t':
328  if (partial_tok_type == PartialToken::STRING_QUOTED)
329  {
330  tok.push_back(c);
331  }
332  else // (partial_tok_type == PartialToken::STRING_NAKED)
333  {
334  this->FlushStringishToken(TokenType::STRING);
335  }
336  line_pos++;
337  break;
338 
339  case '\n':
340  if (partial_tok_type == PartialToken::STRING_QUOTED)
341  {
342  App::GetConsole()->putMessage(Console::CONSOLE_MSGTYPE_INFO, Console::CONSOLE_SYSTEM_WARNING,
343  fmt::format("{}, line {}, pos {}: quoted string interrupted by newline", datastream->getName(), line_num, line_pos));
344  }
345  this->FlushStringishToken(TokenType::STRING);
346  // Break line
347  doc.tokens.push_back({ TokenType::LINEBREAK, 0.f });
348  line_num++;
349  line_pos = 0;
350  break;
351 
352  case ':':
353  if (options & GenericDocument::OPTION_ALLOW_SEPARATOR_COLON
354  && (partial_tok_type == PartialToken::STRING_NAKED || partial_tok_type == PartialToken::STRING_NAKED_CAPTURING_SPACES))
355  {
356  this->FlushStringishToken(TokenType::STRING);
357  }
358  else
359  {
360  tok.push_back(c);
361  }
362  line_pos++;
363  break;
364 
365  case '=':
366  if (options & GenericDocument::OPTION_ALLOW_SEPARATOR_EQUALS
367  && (partial_tok_type == PartialToken::STRING_NAKED || partial_tok_type == PartialToken::STRING_NAKED_CAPTURING_SPACES))
368  {
369  this->FlushStringishToken(TokenType::STRING);
370  }
371  else
372  {
373  tok.push_back(c);
374  }
375  line_pos++;
376  break;
377 
378  case '"':
379  if (partial_tok_type == PartialToken::STRING_QUOTED)
380  {
381  this->FlushStringishToken(TokenType::STRING);
382  }
383  else // (partial_tok_type == PartialToken::STRING_NAKED)
384  {
385  partial_tok_type = PartialToken::GARBAGE;
386  tok.push_back(c);
387  }
388  line_pos++;
389  break;
390 
391  case '(':
392  if (partial_tok_type == PartialToken::STRING_NAKED
393  && options & GenericDocument::OPTION_PARENTHESES_CAPTURE_SPACES)
394  {
395  partial_tok_type = PartialToken::STRING_NAKED_CAPTURING_SPACES;
396  }
397  tok.push_back(c);
398  line_pos++;
399  break;
400 
401  case ')':
402  if (partial_tok_type == PartialToken::STRING_NAKED_CAPTURING_SPACES)
403  {
404  partial_tok_type = PartialToken::STRING_NAKED;
405  }
406  tok.push_back(c);
407  line_pos++;
408  break;
409 
410  default:
411  tok.push_back(c);
412  line_pos++;
413  break;
414  }
415 
416  if (partial_tok_type == PartialToken::GARBAGE)
417  {
418  App::GetConsole()->putMessage(Console::CONSOLE_MSGTYPE_INFO, Console::CONSOLE_SYSTEM_WARNING,
419  fmt::format("{}, line {}, pos {}: stray character '{}' in string", datastream->getName(), line_num, line_pos, c));
420  }
421 }
422 
423 void DocumentParser::UpdateNumber(const char c)
424 {
425  switch (c)
426  {
427  case '\r':
428  break;
429 
430  case ' ':
431  case ',':
432  case '\t':
433  if (partial_tok_type == PartialToken::NUMBER_STUB_MINUS
434  && options & GenericDocument::OPTION_ALLOW_NAKED_STRINGS)
435  {
436  this->FlushStringishToken(TokenType::STRING);
437  }
438  else
439  {
440  this->FlushNumericToken();
441  }
442  line_pos++;
443  break;
444 
445  case '\n':
446  if (partial_tok_type == PartialToken::NUMBER_STUB_MINUS
447  && options & GenericDocument::OPTION_ALLOW_NAKED_STRINGS)
448  {
449  this->FlushStringishToken(TokenType::STRING);
450  }
451  else
452  {
453  this->FlushNumericToken();
454  }
455  // Break line
456  doc.tokens.push_back({ TokenType::LINEBREAK, 0.f });
457  line_num++;
458  line_pos = 0;
459  break;
460 
461  case ':':
462  if (options & GenericDocument::OPTION_ALLOW_SEPARATOR_COLON)
463  {
464  if (partial_tok_type == PartialToken::NUMBER_STUB_MINUS
465  && options & GenericDocument::OPTION_ALLOW_NAKED_STRINGS)
466  {
467  this->FlushStringishToken(TokenType::STRING);
468  }
469  else
470  {
471  this->FlushNumericToken();
472  }
473  }
474  else
475  {
476  this->DiscontinueNumber();
477  tok.push_back(c);
478  }
479  line_pos++;
480  break;
481 
482  case '=':
483  if (options & GenericDocument::OPTION_ALLOW_SEPARATOR_EQUALS)
484  {
485  if (partial_tok_type == PartialToken::NUMBER_STUB_MINUS
486  && options & GenericDocument::OPTION_ALLOW_NAKED_STRINGS)
487  {
488  this->FlushStringishToken(TokenType::STRING);
489  }
490  else
491  {
492  this->FlushNumericToken();
493  }
494  }
495  else
496  {
497  this->DiscontinueNumber();
498  tok.push_back(c);
499  }
500  line_pos++;
501  break;
502 
503  case '.':
504  if (partial_tok_type == PartialToken::NUMBER_INTEGER
505  || partial_tok_type == PartialToken::NUMBER_STUB_MINUS)
506  {
507  partial_tok_type = PartialToken::NUMBER_DECIMAL;
508  }
509  else
510  {
511  this->DiscontinueNumber();
512  }
513  tok.push_back(c);
514  line_pos++;
515  break;
516 
517  case 'e':
518  case 'E':
519  if (partial_tok_type == PartialToken::NUMBER_DECIMAL
520  || partial_tok_type == PartialToken::NUMBER_INTEGER)
521  {
522  partial_tok_type = PartialToken::NUMBER_SCIENTIFIC_STUB;
523  }
524  else
525  {
526  this->DiscontinueNumber();
527  }
528  tok.push_back(c);
529  line_pos++;
530  break;
531 
532  case '-':
533  if (partial_tok_type == PartialToken::NUMBER_SCIENTIFIC_STUB)
534  {
535  partial_tok_type = PartialToken::NUMBER_SCIENTIFIC_STUB_MINUS;
536  }
537  else
538  {
539  this->DiscontinueNumber();
540  }
541  tok.push_back(c);
542  line_pos++;
543  break;
544 
545  case '0':
546  case '1':
547  case '2':
548  case '3':
549  case '4':
550  case '5':
551  case '6':
552  case '7':
553  case '8':
554  case '9':
555  if (partial_tok_type == PartialToken::NUMBER_SCIENTIFIC_STUB
556  || partial_tok_type == PartialToken::NUMBER_SCIENTIFIC_STUB_MINUS)
557  {
558  partial_tok_type = PartialToken::NUMBER_SCIENTIFIC;
559  }
560  else if (partial_tok_type == PartialToken::NUMBER_STUB_MINUS)
561  {
562  partial_tok_type = PartialToken::NUMBER_INTEGER;
563  }
564  tok.push_back(c);
565  line_pos++;
566  break;
567 
568  default:
569  this->DiscontinueNumber();
570  tok.push_back(c);
571  line_pos++;
572  break;
573 
574  }
575 
576  if (partial_tok_type == PartialToken::GARBAGE)
577  {
578  App::GetConsole()->putMessage(Console::CONSOLE_MSGTYPE_INFO, Console::CONSOLE_SYSTEM_WARNING,
579  fmt::format("{}, line {}, pos {}: stray character '{}' in number", datastream->getName(), line_num, line_pos, c));
580  }
581 }
582 
583 void DocumentParser::UpdateBool(const char c)
584 {
585  switch (c)
586  {
587  case '\r':
588  break;
589 
590  case ' ':
591  case ',':
592  case '\t':
593  // Discard token
594  tok.push_back('\0');
595  App::GetConsole()->putMessage(Console::CONSOLE_MSGTYPE_INFO, Console::CONSOLE_SYSTEM_WARNING,
596  fmt::format("{}, line {}, pos {}: discarding incomplete boolean token '{}'", datastream->getName(), line_num, line_pos, tok.data()));
597  tok.clear();
598  partial_tok_type = PartialToken::NONE;
599  line_pos++;
600  break;
601 
602  case '\n':
603  // Discard token
604  tok.push_back('\0');
605  App::GetConsole()->putMessage(Console::CONSOLE_MSGTYPE_INFO, Console::CONSOLE_SYSTEM_WARNING,
606  fmt::format("{}, line {}, pos {}: discarding incomplete boolean token '{}'", datastream->getName(), line_num, line_pos, tok.data()));
607  tok.clear();
608  partial_tok_type = PartialToken::NONE;
609  // Break line
610  doc.tokens.push_back({ TokenType::LINEBREAK, 0.f });
611  line_num++;
612  line_pos = 0;
613  break;
614 
615  case ':':
616  if (options & GenericDocument::OPTION_ALLOW_SEPARATOR_COLON)
617  {
618  // Discard token
619  tok.push_back('\0');
620  App::GetConsole()->putMessage(Console::CONSOLE_MSGTYPE_INFO, Console::CONSOLE_SYSTEM_WARNING,
621  fmt::format("{}, line {}, pos {}: discarding incomplete boolean token '{}'", datastream->getName(), line_num, line_pos, tok.data()));
622  tok.clear();
623  partial_tok_type = PartialToken::NONE;
624  }
625  else
626  {
627  partial_tok_type = PartialToken::GARBAGE;
628  tok.push_back(c);
629  }
630  line_pos++;
631  break;
632 
633  case '=':
634  if (options & GenericDocument::OPTION_ALLOW_SEPARATOR_EQUALS)
635  {
636  // Discard token
637  tok.push_back('\0');
638  App::GetConsole()->putMessage(Console::CONSOLE_MSGTYPE_INFO, Console::CONSOLE_SYSTEM_WARNING,
639  fmt::format("{}, line {}, pos {}: discarding incomplete boolean token '{}'", datastream->getName(), line_num, line_pos, tok.data()));
640  tok.clear();
641  partial_tok_type = PartialToken::NONE;
642  }
643  else
644  {
645  partial_tok_type = PartialToken::GARBAGE;
646  tok.push_back(c);
647  }
648  line_pos++;
649  break;
650 
651  case 'r':
652  if (partial_tok_type != PartialToken::BOOL_TRUE || tok.size() != 1)
653  {
654  this->DiscontinueBool();
655  }
656  tok.push_back(c);
657  line_pos++;
658  break;
659 
660  case 'u':
661  if (partial_tok_type != PartialToken::BOOL_TRUE || tok.size() != 2)
662  {
663  this->DiscontinueBool();
664  }
665  tok.push_back(c);
666  line_pos++;
667  break;
668 
669  case 'a':
670  if (partial_tok_type != PartialToken::BOOL_FALSE || tok.size() != 1)
671  {
672  this->DiscontinueBool();
673  }
674  tok.push_back(c);
675  line_pos++;
676  break;
677 
678  case 'l':
679  if (partial_tok_type != PartialToken::BOOL_FALSE || tok.size() != 2)
680  {
681  this->DiscontinueBool();
682  }
683  tok.push_back(c);
684  line_pos++;
685  break;
686 
687  case 's':
688  if (partial_tok_type != PartialToken::BOOL_FALSE || tok.size() != 3)
689  {
690  this->DiscontinueBool();
691  }
692  tok.push_back(c);
693  line_pos++;
694  break;
695 
696  case 'e':
697  if (partial_tok_type == PartialToken::BOOL_TRUE && tok.size() == 3)
698  {
699  doc.tokens.push_back({ TokenType::BOOL, 1.f });
700  tok.clear();
701  partial_tok_type = PartialToken::NONE;
702  }
703  else if (partial_tok_type == PartialToken::BOOL_FALSE && tok.size() == 4)
704  {
705  doc.tokens.push_back({ TokenType::BOOL, 0.f });
706  tok.clear();
707  partial_tok_type = PartialToken::NONE;
708  }
709  else
710  {
711  this->DiscontinueBool();
712  tok.push_back(c);
713  }
714  line_pos++;
715  break;
716 
717  default:
718  this->DiscontinueBool();
719  tok.push_back(c);
720  line_pos++;
721  break;
722  }
723 
724  if (partial_tok_type == PartialToken::GARBAGE)
725  {
726  App::GetConsole()->putMessage(Console::CONSOLE_MSGTYPE_INFO, Console::CONSOLE_SYSTEM_WARNING,
727  fmt::format("{}, line {}, pos {}: stray character '{}' in boolean", datastream->getName(), line_num, line_pos, c));
728  }
729 }
730 
731 void DocumentParser::DiscontinueBool()
732 {
733  if (doc.tokens.size() == 0 || doc.tokens.back().type == TokenType::LINEBREAK)
734  partial_tok_type = PartialToken::KEYWORD;
735  else if (options & GenericDocument::OPTION_ALLOW_NAKED_STRINGS)
736  partial_tok_type = PartialToken::STRING_NAKED;
737  else
738  partial_tok_type = PartialToken::GARBAGE;
739 }
740 
741 void DocumentParser::DiscontinueNumber()
742 {
743  if (options & GenericDocument::OPTION_ALLOW_NAKED_STRINGS)
744  partial_tok_type = PartialToken::STRING_NAKED;
745  else
746  partial_tok_type = PartialToken::GARBAGE;
747 }
748 
749 void DocumentParser::DiscontinueKeyword()
750 {
751  if (options & GenericDocument::OPTION_ALLOW_NAKED_STRINGS)
752  partial_tok_type = PartialToken::STRING_NAKED;
753  else
754  partial_tok_type = PartialToken::GARBAGE;
755 }
756 
757 void DocumentParser::UpdateKeyword(const char c)
758 {
759  switch (c)
760  {
761  case '\r':
762  break;
763 
764  case ' ':
765  case ',':
766  case '\t':
767  this->FlushStringishToken(TokenType::KEYWORD);
768  line_pos++;
769  break;
770 
771  case '\n':
772  this->FlushStringishToken(TokenType::KEYWORD);
773  // Break line
774  doc.tokens.push_back({ TokenType::LINEBREAK, 0.f });
775  line_num++;
776  line_pos = 0;
777  break;
778 
779  case ':':
780  if (options & GenericDocument::OPTION_ALLOW_SEPARATOR_COLON)
781  {
782  this->FlushStringishToken(TokenType::KEYWORD);
783  }
784  else
785  {
786  this->DiscontinueKeyword();
787  tok.push_back(c);
788  }
789  line_pos++;
790  break;
791 
792  case '=':
793  if (options & GenericDocument::OPTION_ALLOW_SEPARATOR_EQUALS)
794  {
795  this->FlushStringishToken(TokenType::KEYWORD);
796  }
797  else
798  {
799  this->DiscontinueKeyword();
800  tok.push_back(c);
801  }
802  line_pos++;
803  break;
804 
805  case '_':
806  tok.push_back(c);
807  line_pos++;
808  break;
809 
810  case '(':
811  if (options & GenericDocument::OPTION_ALLOW_NAKED_STRINGS)
812  {
813  if (options & GenericDocument::OPTION_PARENTHESES_CAPTURE_SPACES)
814  partial_tok_type = PartialToken::STRING_NAKED_CAPTURING_SPACES;
815  else
816  partial_tok_type = PartialToken::STRING_NAKED;
817  }
818  else
819  {
820  partial_tok_type = PartialToken::GARBAGE;
821  }
822  tok.push_back(c);
823  line_pos++;
824  break;
825 
826  case ']':
827  if (partial_tok_type == PartialToken::KEYWORD_BRACED)
828  {
829  partial_tok_type = PartialToken::KEYWORD; // Do not allow any more ']'.
830  }
831  else
832  {
833  this->DiscontinueKeyword();
834  }
835  tok.push_back(c);
836  line_pos++;
837  break;
838 
839  default:
840  if (!isalnum(c))
841  {
842  this->DiscontinueKeyword();
843  }
844  tok.push_back(c);
845  line_pos++;
846  break;
847  }
848 
849  if (partial_tok_type == PartialToken::GARBAGE)
850  {
851  App::GetConsole()->putMessage(Console::CONSOLE_MSGTYPE_INFO, Console::CONSOLE_SYSTEM_WARNING,
852  fmt::format("{}, line {}, pos {}: stray character '{}' in keyword", datastream->getName(), line_num, line_pos, c));
853  }
854 }
855 
856 void DocumentParser::UpdateTitle(const char c)
857 {
858  switch (c)
859  {
860  case '\r':
861  break;
862 
863  case '\n':
864  this->FlushStringishToken(TokenType::STRING);
865  // Break line
866  doc.tokens.push_back({ TokenType::LINEBREAK, 0.f });
867  line_num++;
868  line_pos = 0;
869  break;
870 
871  default:
872  tok.push_back(c);
873  line_pos++;
874  break;
875  }
876 }
877 
878 void DocumentParser::UpdateGarbage(const char c)
879 {
880  switch (c)
881  {
882  case '\r':
883  break;
884 
885  case ' ':
886  case ',':
887  case '\t':
888  case '\n':
889  tok.push_back('\0');
890  App::GetConsole()->putMessage(Console::CONSOLE_MSGTYPE_INFO, Console::CONSOLE_SYSTEM_WARNING,
891  fmt::format("{}, line {}, pos {}: discarding garbage token '{}'", datastream->getName(), line_num, line_pos, tok.data()));
892  tok.clear();
893  partial_tok_type = PartialToken::NONE;
894  line_pos++;
895  break;
896 
897  default:
898  tok.push_back(c);
899  line_pos++;
900  break;
901  }
902 }
903 
904 void DocumentParser::FlushStringishToken(RoR::TokenType type)
905 {
906  doc.tokens.push_back({ type, (float)doc.string_pool.size() });
907  tok.push_back('\0');
908  std::copy(tok.begin(), tok.end(), std::back_inserter(doc.string_pool));
909  tok.clear();
910  partial_tok_type = PartialToken::NONE;
911 }
912 
913 void DocumentParser::FlushNumericToken()
914 {
915  tok.push_back('\0');
916  doc.tokens.push_back({ TokenType::NUMBER, (float)Ogre::StringConverter::parseReal(tok.data()) });
917  tok.clear();
918  partial_tok_type = PartialToken::NONE;
919 }
920 
921 void DocumentParser::ProcessChar(const char c)
922 {
923  switch (partial_tok_type)
924  {
925  case PartialToken::NONE:
926  this->BeginToken(c);
927  break;
928 
929  case PartialToken::COMMENT_SEMICOLON:
930  case PartialToken::COMMENT_SLASH:
931  case PartialToken::COMMENT_HASH:
932  this->UpdateComment(c);
933  break;
934 
935  case PartialToken::STRING_QUOTED:
936  case PartialToken::STRING_NAKED:
937  case PartialToken::STRING_NAKED_CAPTURING_SPACES:
938  this->UpdateString(c);
939  break;
940 
941  case PartialToken::NUMBER_INTEGER:
942  case PartialToken::NUMBER_STUB_MINUS:
943  case PartialToken::NUMBER_DECIMAL:
944  case PartialToken::NUMBER_SCIENTIFIC:
945  case PartialToken::NUMBER_SCIENTIFIC_STUB:
946  case PartialToken::NUMBER_SCIENTIFIC_STUB_MINUS:
947  this->UpdateNumber(c);
948  break;
949 
950  case PartialToken::BOOL_TRUE:
951  case PartialToken::BOOL_FALSE:
952  this->UpdateBool(c);
953  break;
954 
955  case PartialToken::KEYWORD:
956  case PartialToken::KEYWORD_BRACED:
957  this->UpdateKeyword(c);
958  break;
959 
960  case PartialToken::TITLE_STRING:
961  this->UpdateTitle(c);
962  break;
963 
964  case PartialToken::GARBAGE:
965  this->UpdateGarbage(c);
966  break;
967  }
968 }
969 
970 void DocumentParser::ProcessEOF()
971 {
972  // Flush any partial token
973  switch (partial_tok_type)
974  {
975  case PartialToken::STRING_QUOTED:
976  case PartialToken::STRING_NAKED_CAPTURING_SPACES:
977  case PartialToken::TITLE_STRING:
978  this->FlushStringishToken(TokenType::STRING);
979  break;
980 
981  case PartialToken::KEYWORD_BRACED:
982  this->FlushStringishToken(TokenType::KEYWORD);
983  break;
984 
985  default:
986  this->ProcessChar(' '); // Pretend processing a separator to flush any partial whitespace-incompatible token.
987  break;
988  }
989 
990  // Ensure newline at end of file
991  if (doc.tokens.size() == 0 || doc.tokens.back().type != TokenType::LINEBREAK)
992  {
993  doc.tokens.push_back({ TokenType::LINEBREAK, 0.f });
994  }
995 }
996 
997 void GenericDocument::loadFromDataStream(Ogre::DataStreamPtr datastream, const BitMask_t options)
998 {
999  // Reset the document
1000  tokens.clear();
1001  string_pool.clear();
1002 
1003  // Prepare context
1004  DocumentParser parser(*this, options, datastream);
1005  const size_t LINE_BUF_MAX = 10 * 1024; // 10Kb
1006  char buf[LINE_BUF_MAX];
1007 
1008  // Parse the text
1009  while (!datastream->eof())
1010  {
1011  size_t buf_len = datastream->read(buf, LINE_BUF_MAX);
1012  for (size_t i = 0; i < buf_len; i++)
1013  {
1014  const char c = buf[i];
1015 
1016  parser.ProcessChar(c);
1017  }
1018  }
1019  parser.ProcessEOF();
1020 }
1021 
1022 #if OGRE_PLATFORM == OGRE_PLATFORM_WIN32
1023  const char* EOL_STR = "\r\n"; // CR+LF
1024 #else
1025  const char* EOL_STR = "\n"; // "LF"
1026 #endif
1027 
1028 void GenericDocument::saveToDataStream(Ogre::DataStreamPtr datastream)
1029 {
1030  std::string separator;
1031  const char* pool_str = nullptr;
1032  const size_t BUF_MAX = 100;
1033  char buf[BUF_MAX];
1034 
1035  for (Token& tok : tokens)
1036  {
1037  switch (tok.type)
1038  {
1039  case TokenType::LINEBREAK:
1040  datastream->write(EOL_STR, strlen(EOL_STR));
1041  separator = "";
1042  break;
1043 
1044  case TokenType::COMMENT:
1045  datastream->write(";", 1);
1046  pool_str = string_pool.data() + (size_t)tok.data;
1047  datastream->write(pool_str, strlen(pool_str));
1048  break;
1049 
1050  case TokenType::STRING:
1051  datastream->write(separator.data(), separator.size());
1052  pool_str = string_pool.data() + (size_t)tok.data;
1053  datastream->write(pool_str, strlen(pool_str));
1054  separator = ",";
1055  break;
1056 
1057  case TokenType::NUMBER:
1058  datastream->write(separator.data(), separator.size());
1059  snprintf(buf, BUF_MAX, "%f", tok.data);
1060  datastream->write(buf, strlen(buf));
1061  separator = ",";
1062  break;
1063 
1064  case TokenType::BOOL:
1065  datastream->write(separator.data(), separator.size());
1066  snprintf(buf, BUF_MAX, "%s", tok.data == 1.f ? "true" : "false");
1067  datastream->write(buf, strlen(buf));
1068  separator = ",";
1069  break;
1070 
1071  case TokenType::KEYWORD:
1072  pool_str = string_pool.data() + (size_t)tok.data;
1073  datastream->write(pool_str, strlen(pool_str));
1074  separator = " ";
1075  break;
1076  }
1077  }
1078 }
1079 
1080 bool GenericDocument::loadFromResource(std::string resource_name, std::string resource_group_name, BitMask_t options/* = 0*/)
1081 {
1082  try
1083  {
1084  Ogre::DataStreamPtr datastream = Ogre::ResourceGroupManager::getSingleton().openResource(resource_name, resource_group_name);
1085  this->loadFromDataStream(datastream, options);
1086  return true;
1087  }
1088  catch (Ogre::Exception& eeh)
1089  {
1090  App::GetConsole()->putMessage(Console::CONSOLE_MSGTYPE_INFO, Console::CONSOLE_SYSTEM_ERROR,
1091  fmt::format("GenericDocument: could not load file '{}' from resource group '{}': {}", resource_name, resource_group_name, eeh.getDescription()));
1092  return false;
1093  }
1094 }
1095 
1096 bool GenericDocument::saveToResource(std::string resource_name, std::string resource_group_name)
1097 {
1098  try
1099  {
1100  Ogre::DataStreamPtr datastream = Ogre::ResourceGroupManager::getSingleton().createResource(resource_name, resource_group_name);
1101  this->saveToDataStream(datastream);
1102  return true;
1103  }
1104  catch (Ogre::Exception& eeh)
1105  {
1106  App::GetConsole()->putMessage(Console::CONSOLE_MSGTYPE_INFO, Console::CONSOLE_SYSTEM_ERROR,
1107  fmt::format("GenericDocument: could not write file '{}' to resource group '{}': {}", resource_name, resource_group_name, eeh.getDescription()));
1108  return false;
1109  }
1110 }
1111 
1112 bool GenericDocContext::seekNextLine()
1113 {
1114  // Skip current line
1115  while (!this->endOfFile() && this->tokenType() != TokenType::LINEBREAK)
1116  {
1117  this->moveNext();
1118  }
1119 
1120  // Skip comments and empty lines
1121  while (!this->endOfFile() && !this->isTokString() && !this->isTokFloat() && !this->isTokBool() && !this->isTokKeyword())
1122  {
1123  this->moveNext();
1124  }
1125 
1126  return this->endOfFile();
1127 }
1128 
1129 int GenericDocContext::countLineArgs()
1130 {
1131  int count = 0;
1132  while (!endOfFile(count) && this->tokenType(count) != TokenType::LINEBREAK)
1133  count++;
1134  return count;
1135 }
1136 
1137 // -----------------
1138 // Editing functions
1139 
1140 bool GenericDocContext::insertToken(int offset)
1141 {
1142  if (endOfFile(offset))
1143  return false;
1144 
1145  doc->tokens.insert(doc->tokens.begin() + token_pos + offset, { TokenType::NONE, 0.f });
1146  return true;
1147 }
1148 
1149 bool GenericDocContext::eraseToken(int offset)
1150 {
1151  if (endOfFile(offset))
1152  return false;
1153 
1154  // Just erase the token.
1155  // We don't care about garbage in `string_pool` - the strings are usually just 1-6 characters long anyway.
1156 
1157  doc->tokens.erase(doc->tokens.begin() + token_pos + offset);
1158  return true;
1159 }
1160 
1161 bool GenericDocContext::setStringData(int offset, TokenType type, const std::string& data)
1162 {
1163  if (endOfFile(offset))
1164  return false;
1165 
1166  // Insert the string at the end of the string_pool
1167  // We don't care about order - updating string offsets in tokens would be complicated and unlikely helpful.
1168 
1169  doc->tokens[token_pos + offset] = { type, (float)doc->string_pool.size() };
1170  std::copy(data.begin(), data.end(), std::back_inserter(doc->string_pool));
1171  doc->string_pool.push_back('\0');
1172  return true;
1173 }
1174 
1175 bool GenericDocContext::setFloatData(int offset, TokenType type, float data)
1176 {
1177  if (endOfFile(offset))
1178  return false;
1179 
1180  doc->tokens[token_pos + offset] = { type, data };
1181  return true;
1182 }
DocumentParser::UpdateKeyword
void UpdateKeyword(const char c)
Definition: GenericFileFormat.cpp:757
DocumentParser::UpdateString
void UpdateString(const char c)
Definition: GenericFileFormat.cpp:306
RoR::GenericDocContext::setFloatData
bool setFloatData(int offset, TokenType type, float data)
Definition: GenericFileFormat.cpp:1175
RoR::GfxShadowType::NONE
@ NONE
RoR::GenericDocContext::insertToken
bool insertToken(int offset=0)
Inserts TokenType::NONE;.
Definition: GenericFileFormat.cpp:1140
EOL_STR
const char * EOL_STR
Definition: GenericFileFormat.cpp:1023
LINE_BUF_MAX
static const int LINE_BUF_MAX
Definition: GUI_ConsoleView.cpp:43
copy
The MIT free of to any person obtaining a copy of this software and associated documentation to deal in the Software without including without limitation the rights to copy
Definition: LICENSE-ImGui.txt:8
format
Truck file format(technical spec)
RoR::GenericDocument::OPTION_ALLOW_SEPARATOR_COLON
static const BitMask_t OPTION_ALLOW_SEPARATOR_COLON
Allow ':' as separator between tokens.
Definition: GenericFileFormat.h:70
PartialToken::GARBAGE
@ GARBAGE
PartialToken::TITLE_STRING
@ TITLE_STRING
RoR::GenericDocument::OPTION_ALLOW_SLASH_COMMENTS
static const BitMask_t OPTION_ALLOW_SLASH_COMMENTS
Allow comments starting with //.
Definition: GenericFileFormat.h:68
DocumentParser::DocumentParser
DocumentParser(GenericDocument &d, const BitMask_t opt, Ogre::DataStreamPtr ds)
Definition: GenericFileFormat.cpp:55
Console.h
PartialToken::NUMBER_INTEGER
@ NUMBER_INTEGER
RoR::Console::putMessage
void putMessage(MessageArea area, MessageType type, std::string const &msg, std::string icon="")
Definition: Console.cpp:97
RoR::TokenType::NUMBER
@ NUMBER
RoR::GenericDocContext::seekNextLine
bool seekNextLine()
Definition: GenericFileFormat.cpp:1112
DocumentParser::UpdateBool
void UpdateBool(const char c)
Definition: GenericFileFormat.cpp:583
RoR::GenericDocument::loadFromResource
virtual bool loadFromResource(std::string resource_name, std::string resource_group_name, BitMask_t options=0)
Definition: GenericFileFormat.cpp:1080
PartialToken::NUMBER_STUB_MINUS
@ NUMBER_STUB_MINUS
PartialToken::STRING_NAKED_CAPTURING_SPACES
@ STRING_NAKED_CAPTURING_SPACES
RoR::Console::CONSOLE_SYSTEM_ERROR
@ CONSOLE_SYSTEM_ERROR
Definition: Console.h:52
RoR::GenericDocument::saveToResource
virtual bool saveToResource(std::string resource_name, std::string resource_group_name)
Definition: GenericFileFormat.cpp:1096
PartialToken::NONE
@ NONE
DocumentParser::datastream
Ogre::DataStreamPtr datastream
Definition: GenericFileFormat.cpp:61
RoR::GenericDocument::saveToDataStream
virtual void saveToDataStream(Ogre::DataStreamPtr datastream)
Definition: GenericFileFormat.cpp:1028
DocumentParser::DiscontinueNumber
void DiscontinueNumber()
Definition: GenericFileFormat.cpp:741
PartialToken::STRING_NAKED
@ STRING_NAKED
DocumentParser::FlushStringishToken
void FlushStringishToken(RoR::TokenType type)
Definition: GenericFileFormat.cpp:904
RoR::GenericDocContext::countLineArgs
int countLineArgs()
Definition: GenericFileFormat.cpp:1129
RoR::TokenType::KEYWORD
@ KEYWORD
PartialToken::KEYWORD_BRACED
@ KEYWORD_BRACED
DocumentParser::doc
GenericDocument & doc
Definition: GenericFileFormat.cpp:59
PartialToken
PartialToken
Definition: GenericFileFormat.cpp:30
RoR::TokenType::COMMENT
@ COMMENT
PartialToken::COMMENT_SLASH
@ COMMENT_SLASH
PartialToken::BOOL_TRUE
@ BOOL_TRUE
RoR::GenericDocument::OPTION_ALLOW_BRACED_KEYWORDS
static const BitMask_t OPTION_ALLOW_BRACED_KEYWORDS
Allow INI-like '[keyword]' tokens.
Definition: GenericFileFormat.h:72
DocumentParser::ProcessEOF
void ProcessEOF()
Definition: GenericFileFormat.cpp:970
DocumentParser::DiscontinueBool
void DiscontinueBool()
Definition: GenericFileFormat.cpp:731
Application.h
Central state/object manager and communications hub.
RoR::App::GetConsole
Console * GetConsole()
Definition: Application.cpp:269
DocumentParser::UpdateTitle
void UpdateTitle(const char c)
Definition: GenericFileFormat.cpp:856
DocumentParser::UpdateGarbage
void UpdateGarbage(const char c)
Definition: GenericFileFormat.cpp:878
PartialToken::COMMENT_HASH
@ COMMENT_HASH
RoR::Token
Definition: GenericFileFormat.h:59
RoR::GenericDocument::OPTION_FIRST_LINE_IS_TITLE
static const BitMask_t OPTION_FIRST_LINE_IS_TITLE
First non-empty & non-comment line is a naked string with spaces.
Definition: GenericFileFormat.h:69
PartialToken::STRING_QUOTED
@ STRING_QUOTED
DocumentParser::DiscontinueKeyword
void DiscontinueKeyword()
Definition: GenericFileFormat.cpp:749
RoR::GenericDocContext::setStringData
bool setStringData(int offset, TokenType type, const std::string &data)
Definition: GenericFileFormat.cpp:1161
PartialToken::KEYWORD
@ KEYWORD
RoR::GenericDocument::OPTION_ALLOW_SEPARATOR_EQUALS
static const BitMask_t OPTION_ALLOW_SEPARATOR_EQUALS
Allow '=' as separator between tokens.
Definition: GenericFileFormat.h:73
DocumentParser::UpdateComment
void UpdateComment(const char c)
Definition: GenericFileFormat.cpp:276
PartialToken::NUMBER_SCIENTIFIC_STUB
@ NUMBER_SCIENTIFIC_STUB
PartialToken::NUMBER_DECIMAL
@ NUMBER_DECIMAL
DocumentParser::tok
std::vector< char > tok
Definition: GenericFileFormat.cpp:64
PartialToken::NUMBER_SCIENTIFIC
@ NUMBER_SCIENTIFIC
RoR::GenericDocContext::eraseToken
bool eraseToken(int offset=0)
Definition: GenericFileFormat.cpp:1149
RoR::TokenType
TokenType
Definition: GenericFileFormat.h:48
DocumentParser::options
const BitMask_t options
Definition: GenericFileFormat.cpp:60
RoR::GenericDocument
Definition: GenericFileFormat.h:65
DocumentParser::UpdateNumber
void UpdateNumber(const char c)
Definition: GenericFileFormat.cpp:423
DocumentParser::FlushNumericToken
void FlushNumericToken()
Definition: GenericFileFormat.cpp:913
RoR::TokenType::STRING
@ STRING
BitMask_t
uint32_t BitMask_t
Definition: BitFlags.h:7
Ogre
Definition: ExtinguishableFireAffector.cpp:35
RoR::GenericDocument::loadFromDataStream
virtual void loadFromDataStream(Ogre::DataStreamPtr datastream, BitMask_t options=0)
Definition: GenericFileFormat.cpp:997
RoR::Console::CONSOLE_SYSTEM_WARNING
@ CONSOLE_SYSTEM_WARNING
Definition: Console.h:53
RoR::Console::CONSOLE_MSGTYPE_INFO
@ CONSOLE_MSGTYPE_INFO
Generic message.
Definition: Console.h:60
PartialToken::NUMBER_SCIENTIFIC_STUB_MINUS
@ NUMBER_SCIENTIFIC_STUB_MINUS
RoR::TokenType::LINEBREAK
@ LINEBREAK
GenericFileFormat.h
Generic text file parser.
RoR::GenericDocument::OPTION_ALLOW_NAKED_STRINGS
static const BitMask_t OPTION_ALLOW_NAKED_STRINGS
Allow strings without quotes, for backwards compatibility.
Definition: GenericFileFormat.h:67
RoR
Definition: AppContext.h:36
RoR::GenericDocument::OPTION_ALLOW_HASH_COMMENTS
static const BitMask_t OPTION_ALLOW_HASH_COMMENTS
Allow comments starting with #.
Definition: GenericFileFormat.h:74
DocumentParser
Definition: GenericFileFormat.cpp:53
RoR::GenericDocument::OPTION_PARENTHESES_CAPTURE_SPACES
static const BitMask_t OPTION_PARENTHESES_CAPTURE_SPACES
If non-empty NAKED string encounters '(', following spaces will be captured until matching ')' is fou...
Definition: GenericFileFormat.h:71
DocumentParser::BeginToken
void BeginToken(const char c)
Definition: GenericFileFormat.cpp:89
PartialToken::COMMENT_SEMICOLON
@ COMMENT_SEMICOLON
PartialToken::BOOL_FALSE
@ BOOL_FALSE
DocumentParser::ProcessChar
void ProcessChar(const char c)
Definition: GenericFileFormat.cpp:921
RoR::TokenType::BOOL
@ BOOL
Generated by   doxygen 1.8.17