mirror of
https://github.com/RPCS3/rpcs3.git
synced 2025-04-28 13:28:01 +03:00
Moved the three scalar token scanning functions back to scantoken.cpp, so scanscalar.cpp now only has the main scalar scanning function.
Renamed ScanScalarInfo to ScanScalarParams.
This commit is contained in:
Jesse Beder
parent
0683cbf859
commit
2b8628922f
4 changed files with 252 additions and 161 deletions
174
scanscalar.cpp
174
scanscalar.cpp
|
|
@ -7,39 +7,48 @@
|
|||
namespace YAML
|
||||
{
|
||||
// ScanScalar
|
||||
std::string ScanScalar(Stream& INPUT, ScanScalarInfo& info)
|
||||
// . This is where the scalar magic happens.
|
||||
//
|
||||
// . We do the scanning in three phases:
|
||||
// 1. Scan until newline
|
||||
// 2. Eat newline
|
||||
// 3. Scan leading blanks.
|
||||
//
|
||||
// . Depending on the parameters given, we store or stop
|
||||
// and different places in the above flow.
|
||||
std::string ScanScalar(Stream& INPUT, ScanScalarParams& params)
|
||||
{
|
||||
bool foundNonEmptyLine = false;
|
||||
bool emptyLine = false, moreIndented = false;
|
||||
std::string scalar;
|
||||
info.leadingSpaces = false;
|
||||
params.leadingSpaces = false;
|
||||
|
||||
while(INPUT) {
|
||||
// ********************************
|
||||
// Phase #1: scan until line ending
|
||||
while(!info.end.Matches(INPUT) && !Exp::Break.Matches(INPUT)) {
|
||||
while(!params.end.Matches(INPUT) && !Exp::Break.Matches(INPUT)) {
|
||||
if(INPUT.peek() == EOF)
|
||||
break;
|
||||
|
||||
// document indicator?
|
||||
if(INPUT.column == 0 && Exp::DocIndicator.Matches(INPUT)) {
|
||||
if(info.onDocIndicator == BREAK)
|
||||
if(params.onDocIndicator == BREAK)
|
||||
break;
|
||||
else if(info.onDocIndicator == THROW)
|
||||
else if(params.onDocIndicator == THROW)
|
||||
throw IllegalDocIndicator();
|
||||
}
|
||||
|
||||
foundNonEmptyLine = true;
|
||||
|
||||
// escaped newline? (only if we're escaping on slash)
|
||||
if(info.escape == '\\' && Exp::EscBreak.Matches(INPUT)) {
|
||||
if(params.escape == '\\' && Exp::EscBreak.Matches(INPUT)) {
|
||||
int n = Exp::EscBreak.Match(INPUT);
|
||||
INPUT.Eat(n);
|
||||
continue;
|
||||
}
|
||||
|
||||
// escape this?
|
||||
if(INPUT.peek() == info.escape) {
|
||||
if(INPUT.peek() == params.escape) {
|
||||
scalar += Exp::Escape(INPUT);
|
||||
continue;
|
||||
}
|
||||
|
|
@ -50,19 +59,19 @@ namespace YAML
|
|||
|
||||
// eof? if we're looking to eat something, then we throw
|
||||
if(INPUT.peek() == EOF) {
|
||||
if(info.eatEnd)
|
||||
if(params.eatEnd)
|
||||
throw IllegalEOF();
|
||||
break;
|
||||
}
|
||||
|
||||
// doc indicator?
|
||||
if(info.onDocIndicator == BREAK && INPUT.column == 0 && Exp::DocIndicator.Matches(INPUT))
|
||||
if(params.onDocIndicator == BREAK && INPUT.column == 0 && Exp::DocIndicator.Matches(INPUT))
|
||||
break;
|
||||
|
||||
// are we done via character match?
|
||||
int n = info.end.Match(INPUT);
|
||||
int n = params.end.Match(INPUT);
|
||||
if(n >= 0) {
|
||||
if(info.eatEnd)
|
||||
if(params.eatEnd)
|
||||
INPUT.Eat(n);
|
||||
break;
|
||||
}
|
||||
|
|
@ -76,20 +85,20 @@ namespace YAML
|
|||
// Phase #3: scan initial spaces
|
||||
|
||||
// first the required indentation
|
||||
while(INPUT.peek() == ' ' && (INPUT.column < info.indent || (info.detectIndent && !foundNonEmptyLine)))
|
||||
while(INPUT.peek() == ' ' && (INPUT.column < params.indent || (params.detectIndent && !foundNonEmptyLine)))
|
||||
INPUT.Eat(1);
|
||||
|
||||
// update indent if we're auto-detecting
|
||||
if(info.detectIndent && !foundNonEmptyLine)
|
||||
info.indent = std::max(info.indent, INPUT.column);
|
||||
if(params.detectIndent && !foundNonEmptyLine)
|
||||
params.indent = std::max(params.indent, INPUT.column);
|
||||
|
||||
// and then the rest of the whitespace
|
||||
while(Exp::Blank.Matches(INPUT)) {
|
||||
// we check for tabs that masquerade as indentation
|
||||
if(INPUT.peek() == '\t'&& INPUT.column < info.indent && info.onTabInIndentation == THROW)
|
||||
if(INPUT.peek() == '\t'&& INPUT.column < params.indent && params.onTabInIndentation == THROW)
|
||||
throw IllegalTabInIndentation();
|
||||
|
||||
if(!info.eatLeadingWhitespace)
|
||||
if(!params.eatLeadingWhitespace)
|
||||
break;
|
||||
|
||||
INPUT.Eat(1);
|
||||
|
|
@ -101,7 +110,7 @@ namespace YAML
|
|||
|
||||
// TODO: for block scalars, we always start with a newline, so we should fold OR keep that
|
||||
|
||||
if(info.fold && !emptyLine && !nextEmptyLine && !moreIndented && !nextMoreIndented)
|
||||
if(params.fold && !emptyLine && !nextEmptyLine && !moreIndented && !nextMoreIndented)
|
||||
scalar += " ";
|
||||
else
|
||||
scalar += "\n";
|
||||
|
|
@ -110,146 +119,27 @@ namespace YAML
|
|||
moreIndented = nextMoreIndented;
|
||||
|
||||
// are we done via indentation?
|
||||
if(!emptyLine && INPUT.column < info.indent) {
|
||||
info.leadingSpaces = true;
|
||||
if(!emptyLine && INPUT.column < params.indent) {
|
||||
params.leadingSpaces = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// post-processing
|
||||
if(info.trimTrailingSpaces) {
|
||||
if(params.trimTrailingSpaces) {
|
||||
unsigned pos = scalar.find_last_not_of(' ');
|
||||
if(pos < scalar.size())
|
||||
scalar.erase(pos + 1);
|
||||
}
|
||||
|
||||
if(info.chomp <= 0) {
|
||||
if(params.chomp <= 0) {
|
||||
unsigned pos = scalar.find_last_not_of('\n');
|
||||
if(info.chomp == 0 && pos + 1 < scalar.size())
|
||||
if(params.chomp == 0 && pos + 1 < scalar.size())
|
||||
scalar.erase(pos + 2);
|
||||
else if(info.chomp == -1 && pos < scalar.size())
|
||||
else if(params.chomp == -1 && pos < scalar.size())
|
||||
scalar.erase(pos + 1);
|
||||
}
|
||||
|
||||
return scalar;
|
||||
}
|
||||
|
||||
// PlainScalarToken
|
||||
template <> PlainScalarToken *Scanner::ScanToken(PlainScalarToken *pToken)
|
||||
{
|
||||
// set up the scanning parameters
|
||||
ScanScalarInfo info;
|
||||
info.end = (m_flowLevel > 0 ? Exp::EndScalarInFlow : Exp::EndScalar) || (RegEx(' ') + Exp::Comment);
|
||||
info.eatEnd = false;
|
||||
info.indent = (m_flowLevel > 0 ? 0 : m_indents.top() + 1);
|
||||
info.fold = true;
|
||||
info.eatLeadingWhitespace = true;
|
||||
info.trimTrailingSpaces = true;
|
||||
info.chomp = CLIP;
|
||||
info.onDocIndicator = BREAK;
|
||||
info.onTabInIndentation = THROW;
|
||||
|
||||
// insert a potential simple key
|
||||
if(m_simpleKeyAllowed)
|
||||
InsertSimpleKey();
|
||||
|
||||
pToken->value = ScanScalar(INPUT, info);
|
||||
|
||||
// can have a simple key only if we ended the scalar by starting a new line
|
||||
m_simpleKeyAllowed = info.leadingSpaces;
|
||||
|
||||
// finally, we can't have any colons in a scalar, so if we ended on a colon, there
|
||||
// had better be a break after it
|
||||
if(Exp::IllegalColonInScalar.Matches(INPUT))
|
||||
throw IllegalScalar();
|
||||
|
||||
return pToken;
|
||||
}
|
||||
|
||||
// QuotedScalarToken
|
||||
template <> QuotedScalarToken *Scanner::ScanToken(QuotedScalarToken *pToken)
|
||||
{
|
||||
// eat single or double quote
|
||||
char quote = INPUT.GetChar();
|
||||
pToken->single = (quote == '\'');
|
||||
|
||||
// setup the scanning parameters
|
||||
ScanScalarInfo info;
|
||||
info.end = (pToken->single ? RegEx(quote) && !Exp::EscSingleQuote : RegEx(quote));
|
||||
info.eatEnd = true;
|
||||
info.escape = (pToken->single ? '\'' : '\\');
|
||||
info.indent = 0;
|
||||
info.fold = true;
|
||||
info.eatLeadingWhitespace = true;
|
||||
info.trimTrailingSpaces = false;
|
||||
info.chomp = CLIP;
|
||||
info.onDocIndicator = THROW;
|
||||
|
||||
// insert a potential simple key
|
||||
if(m_simpleKeyAllowed)
|
||||
InsertSimpleKey();
|
||||
|
||||
pToken->value = ScanScalar(INPUT, info);
|
||||
m_simpleKeyAllowed = false;
|
||||
|
||||
return pToken;
|
||||
}
|
||||
|
||||
// BlockScalarToken
|
||||
// . These need a little extra processing beforehand.
|
||||
// . We need to scan the line where the indicator is (this doesn't count as part of the scalar),
|
||||
// and then we need to figure out what level of indentation we'll be using.
|
||||
template <> BlockScalarToken *Scanner::ScanToken(BlockScalarToken *pToken)
|
||||
{
|
||||
ScanScalarInfo info;
|
||||
info.indent = 1;
|
||||
info.detectIndent = true;
|
||||
|
||||
// eat block indicator ('|' or '>')
|
||||
char indicator = INPUT.GetChar();
|
||||
info.fold = (indicator == Keys::FoldedScalar);
|
||||
|
||||
// eat chomping/indentation indicators
|
||||
int n = Exp::Chomp.Match(INPUT);
|
||||
for(int i=0;i<n;i++) {
|
||||
char ch = INPUT.GetChar();
|
||||
if(ch == '+')
|
||||
info.chomp = KEEP;
|
||||
else if(ch == '-')
|
||||
info.chomp = STRIP;
|
||||
else if(Exp::Digit.Matches(ch)) {
|
||||
info.indent = ch - '0';
|
||||
info.detectIndent = false;
|
||||
if(info.indent == 0)
|
||||
throw ZeroIndentationInBlockScalar();
|
||||
}
|
||||
}
|
||||
|
||||
// now eat whitespace
|
||||
while(Exp::Blank.Matches(INPUT))
|
||||
INPUT.Eat(1);
|
||||
|
||||
// and comments to the end of the line
|
||||
if(Exp::Comment.Matches(INPUT))
|
||||
while(INPUT && !Exp::Break.Matches(INPUT))
|
||||
INPUT.Eat(1);
|
||||
|
||||
// if it's not a line break, then we ran into a bad character inline
|
||||
if(INPUT && !Exp::Break.Matches(INPUT))
|
||||
throw UnexpectedCharacterInBlockScalar();
|
||||
|
||||
// set the initial indentation
|
||||
if(m_indents.top() >= 0)
|
||||
info.indent += m_indents.top();
|
||||
|
||||
info.eatLeadingWhitespace = false;
|
||||
info.trimTrailingSpaces = false;
|
||||
info.onTabInIndentation = THROW;
|
||||
|
||||
pToken->value = ScanScalar(INPUT, info);
|
||||
|
||||
// simple keys always ok after block scalars (since we're gonna start a new line anyways)
|
||||
m_simpleKeyAllowed = true;
|
||||
return pToken;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ namespace YAML
|
|||
enum CHOMP { STRIP = -1, CLIP, KEEP };
|
||||
enum ACTION { NONE, BREAK, THROW };
|
||||
|
||||
struct ScanScalarInfo {
|
||||
ScanScalarInfo(): eatEnd(false), indent(0), detectIndent(false), eatLeadingWhitespace(0), escape(0), fold(false),
|
||||
struct ScanScalarParams {
|
||||
ScanScalarParams(): eatEnd(false), indent(0), detectIndent(false), eatLeadingWhitespace(0), escape(0), fold(false),
|
||||
trimTrailingSpaces(0), chomp(CLIP), onDocIndicator(NONE), onTabInIndentation(NONE), leadingSpaces(false) {}
|
||||
|
||||
// input:
|
||||
|
|
@ -31,5 +31,5 @@ namespace YAML
|
|||
bool leadingSpaces;
|
||||
};
|
||||
|
||||
std::string ScanScalar(Stream& INPUT, ScanScalarInfo& info);
|
||||
std::string ScanScalar(Stream& INPUT, ScanScalarParams& info);
|
||||
}
|
||||
|
|
|
|||
121
scantoken.cpp
121
scantoken.cpp
|
|
@ -2,6 +2,7 @@
|
|||
#include "token.h"
|
||||
#include "exceptions.h"
|
||||
#include "exp.h"
|
||||
#include "scanscalar.h"
|
||||
|
||||
namespace YAML
|
||||
{
|
||||
|
|
@ -220,4 +221,124 @@ namespace YAML
|
|||
pToken->value = tag;
|
||||
return pToken;
|
||||
}
|
||||
|
||||
// PlainScalarToken
|
||||
template <> PlainScalarToken *Scanner::ScanToken(PlainScalarToken *pToken)
|
||||
{
|
||||
// set up the scanning parameters
|
||||
ScanScalarParams params;
|
||||
params.end = (m_flowLevel > 0 ? Exp::EndScalarInFlow : Exp::EndScalar) || (RegEx(' ') + Exp::Comment);
|
||||
params.eatEnd = false;
|
||||
params.indent = (m_flowLevel > 0 ? 0 : m_indents.top() + 1);
|
||||
params.fold = true;
|
||||
params.eatLeadingWhitespace = true;
|
||||
params.trimTrailingSpaces = true;
|
||||
params.chomp = CLIP;
|
||||
params.onDocIndicator = BREAK;
|
||||
params.onTabInIndentation = THROW;
|
||||
|
||||
// insert a potential simple key
|
||||
if(m_simpleKeyAllowed)
|
||||
InsertSimpleKey();
|
||||
|
||||
pToken->value = ScanScalar(INPUT, params);
|
||||
|
||||
// can have a simple key only if we ended the scalar by starting a new line
|
||||
m_simpleKeyAllowed = params.leadingSpaces;
|
||||
|
||||
// finally, we can't have any colons in a scalar, so if we ended on a colon, there
|
||||
// had better be a break after it
|
||||
if(Exp::IllegalColonInScalar.Matches(INPUT))
|
||||
throw IllegalScalar();
|
||||
|
||||
return pToken;
|
||||
}
|
||||
|
||||
// QuotedScalarToken
|
||||
template <> QuotedScalarToken *Scanner::ScanToken(QuotedScalarToken *pToken)
|
||||
{
|
||||
// eat single or double quote
|
||||
char quote = INPUT.GetChar();
|
||||
pToken->single = (quote == '\'');
|
||||
|
||||
// setup the scanning parameters
|
||||
ScanScalarParams params;
|
||||
params.end = (pToken->single ? RegEx(quote) && !Exp::EscSingleQuote : RegEx(quote));
|
||||
params.eatEnd = true;
|
||||
params.escape = (pToken->single ? '\'' : '\\');
|
||||
params.indent = 0;
|
||||
params.fold = true;
|
||||
params.eatLeadingWhitespace = true;
|
||||
params.trimTrailingSpaces = false;
|
||||
params.chomp = CLIP;
|
||||
params.onDocIndicator = THROW;
|
||||
|
||||
// insert a potential simple key
|
||||
if(m_simpleKeyAllowed)
|
||||
InsertSimpleKey();
|
||||
|
||||
pToken->value = ScanScalar(INPUT, params);
|
||||
m_simpleKeyAllowed = false;
|
||||
|
||||
return pToken;
|
||||
}
|
||||
|
||||
// BlockScalarToken
|
||||
// . These need a little extra processing beforehand.
|
||||
// . We need to scan the line where the indicator is (this doesn't count as part of the scalar),
|
||||
// and then we need to figure out what level of indentation we'll be using.
|
||||
template <> BlockScalarToken *Scanner::ScanToken(BlockScalarToken *pToken)
|
||||
{
|
||||
ScanScalarParams params;
|
||||
params.indent = 1;
|
||||
params.detectIndent = true;
|
||||
|
||||
// eat block indicator ('|' or '>')
|
||||
char indicator = INPUT.GetChar();
|
||||
params.fold = (indicator == Keys::FoldedScalar);
|
||||
|
||||
// eat chomping/indentation indicators
|
||||
int n = Exp::Chomp.Match(INPUT);
|
||||
for(int i=0;i<n;i++) {
|
||||
char ch = INPUT.GetChar();
|
||||
if(ch == '+')
|
||||
params.chomp = KEEP;
|
||||
else if(ch == '-')
|
||||
params.chomp = STRIP;
|
||||
else if(Exp::Digit.Matches(ch)) {
|
||||
if(ch == '0')
|
||||
throw ZeroIndentationInBlockScalar();
|
||||
|
||||
params.indent = ch - '0';
|
||||
params.detectIndent = false;
|
||||
}
|
||||
}
|
||||
|
||||
// now eat whitespace
|
||||
while(Exp::Blank.Matches(INPUT))
|
||||
INPUT.Eat(1);
|
||||
|
||||
// and comments to the end of the line
|
||||
if(Exp::Comment.Matches(INPUT))
|
||||
while(INPUT && !Exp::Break.Matches(INPUT))
|
||||
INPUT.Eat(1);
|
||||
|
||||
// if it's not a line break, then we ran into a bad character inline
|
||||
if(INPUT && !Exp::Break.Matches(INPUT))
|
||||
throw UnexpectedCharacterInBlockScalar();
|
||||
|
||||
// set the initial indentation
|
||||
if(m_indents.top() >= 0)
|
||||
params.indent += m_indents.top();
|
||||
|
||||
params.eatLeadingWhitespace = false;
|
||||
params.trimTrailingSpaces = false;
|
||||
params.onTabInIndentation = THROW;
|
||||
|
||||
pToken->value = ScanScalar(INPUT, params);
|
||||
|
||||
// simple keys always ok after block scalars (since we're gonna start a new line anyways)
|
||||
m_simpleKeyAllowed = true;
|
||||
return pToken;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
112
test.yaml
112
test.yaml
|
|
@ -1,17 +1,97 @@
|
|||
---
|
||||
- here's a key: value
|
||||
here's the first block: |
|
||||
after the block: value
|
||||
and here's a block: |-
|
||||
What's going on?
|
||||
How are you doing?
|
||||
Here's some code:
|
||||
|
||||
#include <iostream>
|
||||
int main()
|
||||
{
|
||||
std::cout << "Hello World!\n";
|
||||
}
|
||||
|
||||
I'm doing fine!
|
||||
and last key: value
|
||||
model:
|
||||
file: data/models/compound.model
|
||||
textures: data/materials/compound
|
||||
rooms:
|
||||
- name: "Room #1"
|
||||
pos: [0, 0, 0]
|
||||
size: [1000, 1000, 500]
|
||||
height: 500
|
||||
stairtype: none
|
||||
display: []
|
||||
pathfinding:
|
||||
tilesize: 50
|
||||
size: [24, 24]
|
||||
map: |
|
||||
-----------------------
|
||||
-+++++++++++++++++++++-
|
||||
-+-------------------+-
|
||||
-+-------------------+-
|
||||
-+-------------------+-
|
||||
-+-------------------+-
|
||||
-+-------------------+-
|
||||
-+-------------------+-
|
||||
-+-------------------+-
|
||||
-+-------------------+-
|
||||
-+---------------------
|
||||
-+---------------------
|
||||
-+---------------------
|
||||
-+---------------------
|
||||
-+-------------------+-
|
||||
-+-------------------+-
|
||||
-+-------------------+-
|
||||
-+-------------------+-
|
||||
-+-------------------+-
|
||||
-+-------------------+-
|
||||
-+-------------------+-
|
||||
-+-------------------+-
|
||||
-+++++++++++++++++++++-
|
||||
-----------------------
|
||||
- name: Doorway
|
||||
pos: [1000, 400, 0]
|
||||
size: [50, 200, 500]
|
||||
height: 500
|
||||
stairtype: none
|
||||
display: []
|
||||
pathfinding:
|
||||
tilesize: 50
|
||||
size: [5, 9]
|
||||
map: |
|
||||
-----
|
||||
-+++-
|
||||
-----
|
||||
-----
|
||||
-----
|
||||
-----
|
||||
-----
|
||||
-+++-
|
||||
-----
|
||||
- name: "Room #2"
|
||||
pos: [1050, 0, 0]
|
||||
size: [1000, 1000, 500]
|
||||
height: 500
|
||||
stairtype: none
|
||||
display: []
|
||||
pathfinding:
|
||||
tilesize: 50
|
||||
size: [24, 24]
|
||||
map: |
|
||||
-----------------------
|
||||
-+++++++++++++++++++++-
|
||||
-+-------------------+-
|
||||
-+-------------------+-
|
||||
-+-------------------+-
|
||||
-+-------------------+-
|
||||
-+-------------------+-
|
||||
-+-------------------+-
|
||||
-+-------------------+-
|
||||
-+-------------------+-
|
||||
---------------------+-
|
||||
---------------------+-
|
||||
---------------------+-
|
||||
---------------------+-
|
||||
-+-------------------+-
|
||||
-+-------------------+-
|
||||
-+-------------------+-
|
||||
-+-------------------+-
|
||||
-+-------------------+-
|
||||
-+-------------------+-
|
||||
-+-------------------+-
|
||||
-+-------------------+-
|
||||
-+++++++++++++++++++++-
|
||||
-----------------------
|
||||
exits:
|
||||
- room1: "Room #1"
|
||||
room2: "Room #2"
|
||||
dir: e
|
||||
pos: [400, 600]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue