#ifndef _TOKENIZER_H_INCLUDED_
#define _TOKENIZER_H_INCLUDED_
///////////////////////////////////////////////////////////////////////////////
//                                                         
// Tokenizer.h
// -----------
// Tokenizer class definition
//                                                         
// Design and Implementation by Bjoern Lemke               
//                                                         
// (C)opyright 2000-2016 Bjoern Lemke
//                                                         
// INTERFACE MODULE
//
// Class: Tokenizer
// 
// Description: Splitting a string into tokens
//
// Status: CLEAN
//
///////////////////////////////////////////////////////////////////////////////

// INCLUDES

#include "Chain.h"

// CLASS DEFINITION

class Tokenizer {

 public:
    
    Tokenizer(const Chain& tokenStream, const Chain& sep, char esc = '\\');
    Tokenizer(char* buf, int len, const Chain& sep, char esc = '\\');
    ~Tokenizer();
    
    bool nextToken(Chain& token);
    bool getTail(Chain& token);

    void skip(int n);
    bool rawRead(Chain& token, int n);
    
    int numAhead() const;    

    void reset(char* buf, int len);

    Tokenizer& operator = (const Tokenizer& t);
 
 private:

    bool checkSeparator();

    bool _isLocal;
    char *_streamBuf;
    int _bufLen;
    
    Chain _tokenStream;
    Chain _sep;
    char _esc;

    int _lower;
    int _upper;
};
#endif



