[scn_base.h] Basic Token Separation

contents



#include "symbols.h"
#include "scn_io.h"



The module [scn_base] implements the scan stream interface based on a given STYX-konform scanner definition. ( see [scn_gen] )

The scan stream is reentrant and allows multiple scan operations at a time.

Macros


Token definition flags ( see Scn_dfnToken ) 

#define SCN_FLG_IgnoreToken  1    /* token has to be ignored          */
#define SCN_FLG_IgnoreCase   2    /* case insensitive token           */
#define SCN_FLG_EofToken     4    /* eof token in embedded language   */
#define SCN_FLG_SwitchToken  8    /* switch to/from embedded language */
#define SCN_FLG_CfgToken     16   /* embedded language token          */
#define SCN_FLG_IndentToken  32   /* (de)indent token                 */
#define SCN_FLG_PatternToken 64   /* (start or end) pattern token     */
#define SCN_FLG_GroupStack   128  /* push/pop token group             */

Token preprocessing result values 

#define SCN_FLG_RescanNone     0
#define SCN_FLG_RescanString   1
#define SCN_FLG_RescanBString  2
#define SCN_FLG_RescanFile     3
#define SCN_FLG_RescanBFile    4
#define SCN_FLG_RescanExtFile  5
#define SCN_FLG_RescanExtBFile 6
#define SCN_FLG_RescanChar     7
#define SCN_FLG_RescanChrItr   8
#define SCN_FLG_RescanBChrItr  9

Stream of tokens


Scn_Stream Stream onto a scanner

Scn_Stream_Itr Character iterator onto a scan stream 

   Type of token preprocessing function
   RC = SCN_FLG_RescanChrItr   & valid cMacVal --> rescan character iterator cMacVal
                                                   ( frees character iterator )
   RC = SCN_FLG_RescanBChrItr  & valid cMacVal --> rescan binary char iterator cMacVal
                                                   ( frees character iterator )
   RC = SCN_FLG_RescanString   & valid cMacVal --> rescan string cMacVal
   RC = SCN_FLG_RescanFile     & valid cMacVal --> rescan file cMacVal
   RC = SCN_FLG_RescanBString  & valid cMacVal --> rescan binary string cMacVal
   RC = SCN_FLG_RescanBFile    & valid cMacVal --> rescan binary file cMacVal
   RC = SCN_FLG_RescanExtFile  & valid cMacVal --> rescan file cMacVal,
                                                   with main stream read function if possible
   RC = SCN_FLG_RescanExtBFile & valid cMacVal --> rescan binary file cMacVal,
                                                   with main stream read function if possible
   RC = SCN_FLG_RescanChar     & valid cMacVal --> rescan character cMacVal
                                                   ( single byte or ucs4 )
   RC = SCN_FLG_RescanNone     & valid cMacVal --> replace cTokVal with cMacVal
                                                   ( single byte or utf-8 )
   ELSE                                        --> ignore cTokVal



typedef int (*Scn_PreMacFun)
            (
              Scn_Stream pStream, c_string cTokNam,
              c_string cTokVal, symbol* cMacVal
            );

Type of the embedded language token constructor 
typedef Any_T (*Scn_eTerm)
              (
                Abs_T CurPTCfg, c_string language, c_string StartSymbol
              );

Type of the embedded language accept token recognizer 
typedef c_bool (*Scn_eAccept)(Abs_T CurPTCfg);

Accessing scanner definition 

c_string Scn_id(Scn_T scn)
 name of the scanner (group) 'scn'; allocs memory

short Scn_check_Token(Scn_T scn, c_string s)
 whether string 's' is a token in scanner 'scn';
result = token number ( > 0 ) or 0
assertion: Scn_groups(scn) = 0

short Scn_check_WCToken(Scn_T scn, wc_string s)
 whether wide string 's' is a token in scanner 'scn';
result = token number ( > 0 ) or 0
assertion: Scn_groups(scn) = 0

int Scn_tokens(Scn_T scn)
 number of tokens defined in scanner ( group ) 'scn' 
c_string Scn_tokid(Scn_T scn, int i)
 name of the i-th token definied in scanner 'scn'; allocs memory
assertion: Scn_groups(scn) = 0

c_byte Scn_tokFlags(Scn_T scn, int i)
 flags of the i-th token definied in scanner 'scn'
assertion: Scn_groups(scn) = 0

int Scn_dycks(Scn_T scn)
 number of dyck token defined in scanner ( group ) 'scn' 
Scn_T Scn_dyck(Scn_T scn, int i)
 i-th dyck scanner definied in scanner group 'scn'
assertion: Scn_dycks(scn) > 0

int Scn_dycktoken(Scn_T scn, int i)
 i-th dyck token definied in scanner group 'scn'
assertion: Scn_dycks(scn) > 0

int Scn_groups(Scn_T scn)
 number of groups defined in scanner ( group ) 'scn' 
Scn_T Scn_group(Scn_T scn, int i)
 i-th scanner definied in scanner group 'scn'
assertion: Scn_groups(scn) > 0

int Scn_SwitchGroup(Scn_T scn, int i)
 switch group of the i-th token definied in scanner group 'scn' or -1
assertion: Scn_groups(scn) = 0

long Scn_check_GroupToken(Scn_T scn, c_string s)
 whether string 's' is a token in scanner group 'scn';
result: high = group index , low = token number ( > 0 ) or 0
assertion: Scn_groups(scn) > 0

long Scn_check_GroupWCToken(Scn_T scn, wc_string s)
 whether wide string 's' is a token in scanner group 'scn';
result: high = group index , low = token number ( > 0 ) or 0
assertion: Scn_groups(scn) > 0


Character iterator definition 

Scn_Stream_Itr Stream_Itr_new
               (
                 int      f_getc (StdCPtr file),
                 void     f_close(StdCPtr file),
                 c_string   f_wc2mb(wc_string wc),
                 LONG_INT f_seek (StdCPtr file, long offset, int origin),
                 StdCPtr  file,
                 c_string   fileid
               )
 creates and initializes a character iterator
onto the open character source 'file'
'f_getc' : next character
'f_close': closes character source
'f_wc2mb': converts wide to multibyte character
'f_seek' : positions character source
'fileid' : character source identifier

void Stream_Itr_free(Scn_Stream_Itr itr)
frees character iterator 'itr'

Scan stream definition 

Scn_Stream Stream_bgn(Scn_T scn, Scn_Stream_Itr itr)
 creates and initializes a scan stream
onto scanner 'scn' and character iterator 'itr'

void Stream_close(Scn_Stream t)
 closes character source i.e. file of scan stream 't' 
void Stream_free(Scn_Stream t)
frees scan stream 't' 
void Stream_reset(Scn_Stream t)
 re-initializes scan process in current scan stream 't' 
void Stream_premac_set(Scn_Stream t, Scn_PreMacFun cPreMac)
 adds 'cPreMac' to scan stream 't'
for token preprocessing & macro expansion

void Stream_eterm_set
     (
       Scn_Stream t, Abs_T CurPTCfg, Scn_eTerm eTerm, Scn_eAccept eAccept
     )
 adds 'eTerm', 'eAccept' and 'CurPTCfg' to scan stream 't'
for embedded language token construction
( deactivates other token concatenation )

void Stream_add_ctxval(Scn_Stream t, Any_T id, Any_T val)
 adds the context pair 'id' |--> 'val' to scan stream 't';
updates existing entries

void Stream_concat_other(Scn_Stream t)
 activates other token concatenation in scan stream 't' 
void Stream_skip_character(Scn_Stream t, int cC)
 skips character 'cC' ( but tracks position ) 
void Stream_enable_lookahead(Scn_Stream t)
 activates n character lookahead in scan stream 't' 
void Stream_binmode_set(Scn_Stream t)
 activates binary mode in scan stream 't'
( deactivates other token concatenation )

Scn_Stream Stream_line
           (
             Scn_T scn, StdCPtr line, int (*cGet)(StdCPtr line), c_string id
           )
#define STREAM_STDIN(scn) \
	Stream_line(scn,stdin,(int (*)(StdCPtr f))fgetc,"<stdin>")
 creates a scan stream
onto scanner 'scn' and character stream 'line'
'cGet': next character
'id' : character stream identifier

Scn_Stream Stream_file(Scn_T scn, c_string EnvVar, c_string FileName, c_string Ext)
 creates a scan stream
onto scanner 'scn' and file [$'EnvVar'/]'FileName''Ext'

Scn_Stream Stream_extfile
           (
             Scn_T scn, c_string EnvVar, c_string FileName, c_string Ext,
             int (*cGet)(FILE* f)
           )
 creates a scan stream
onto scanner 'scn' and file [$'EnvVar'/]'FileName''Ext'
with external next character function 'cGet'

Scn_Stream Stream_bstring(Scn_T scn, c_bstring bstr)
 creates a scan stream onto scanner 'scn' and binary string 'bstr' 
void Stream_bstring_set(Scn_Stream stream, c_bstring bstr)
 re-initializes scan stream 'stream' with binary string 'bstr' 
Scn_Stream Stream_string(Scn_T scn, c_string Text)
 creates a scan stream onto scanner 'scn' and string 'Text' 
void Stream_string_set(Scn_Stream stream, c_string Text)
 re-initializes scan stream 'stream' with string 'Text' 
void Stream_string_insert(Scn_Stream stream, c_string Text)
 inserts string 'Text' at current position in scan stream 'stream'

Defining token numbers 

void Stream_defEofId(Scn_Stream t, short tid)
 defines 'tid' as EOF token number in scan stream 't' 
void Stream_defErrId(Scn_Stream t, short tid)
 defines 'tid' as error token number in scan stream 't' 
void Stream_defTokId(Scn_Stream t, c_string Name, short tid)
 defines 'tid' as number for token 'Name' in scan stream 't' 
c_bool Stream_hasTokId(Scn_Stream t, c_string Name)
 whether 'Name' is a token in scan stream 't' 
c_bool Stream_defDynKeyId(Scn_Stream t, symbol keysym, short tid)
 defines 'tid' as number for dynamic keyword 'keysym' in scan stream 't' 
void Stream_defKeyId(Scn_Stream t, c_string Text, short tid)
 defines 'tid' as number for keyword 'Text' in scan stream 't' 
void Stream_defWCKeyId(Scn_Stream t, wc_string Text, short tid)
 defines 'tid' as number for wide keyword 'Text' in scan stream 't'

Scan stream operations 

void Stream_next(Scn_Stream t)
 separates the next token in scan stream 't';
That's the workhorse operation.


Accessing current token 

symbol Stream_csym(Scn_Stream t)
 symbol of the current token in scan stream 't' 
short Stream_ctid(Scn_Stream t)
 token number of the current token in scan stream 't' 
short Stream_ctid_nokey(Scn_Stream t)
 token number of the current token in scan stream 't'
(non-indent token --> ignore keyword)

c_string Stream_ctnam(Scn_Stream t)
 name of the current token in scan stream 't' or NULL (EOF) 
c_bool Stream_cica(Scn_Stream t)
 whether the current token in scan stream 't'
has the attribute 'ignore case'

c_string Stream_cbuf(Scn_Stream t)
 string of the current token in scan stream 't' or NULL
( not valid for an embedded language token )

wc_string Stream_wcbuf(Scn_Stream t)
 wide string of the current token in scan stream 't' or NULL
( not valid for an embedded language token )

long Stream_clen(Scn_Stream t)
 length of the current token in scan stream 't'
( not valid for an embedded language and the follow token )

symbol Stream_cfil(Scn_Stream t)
 source identifier of the current token in scan stream 't' 
long  Stream_clin(Scn_Stream t)
 first line number of the current token in scan stream 't' 
long  Stream_ccol(Scn_Stream t)
 first column number of the current token in scan stream 't'

Accessing scan stream definition 

Scn_T Stream_scn(Scn_Stream t)
scanner definition of scan stream 't' 
Scn_PreMacFun Stream_premac(Scn_Stream t)
 preprocessor of scan stream 't' 
Scn_eTerm Stream_etermFun(Scn_Stream t)
 embedded language token constructor of scan stream 't' 
Scn_eAccept Stream_etermEof(Scn_Stream t)
 embedded language accept token recognizer of scan stream 't' 
Abs_T Stream_etermCfg(Scn_Stream t)
 current parse term configuration of scan stream 't' 
c_bool Stream_binmode(Scn_Stream t)
 whether current scan stream 't' has binary mode 
c_bool Stream_lookahead(Scn_Stream t)
 whether current scan stream 't' has n character lookahead enabled 
c_bool Stream_unicode(Scn_Stream t)
 whether current scan stream 't' is unicode-based 
StdCPtr Stream_cstream(Scn_Stream t)
 character stream of current scan stream 't' 
Any_T Stream_get_ctxval(Scn_Stream t, Any_T id)
 value of context variable 'id' in scan stream 't' or NULL