1 files changed, 48 insertions, 0 deletions
diff --git a/src/T2DLL/CTokenizer.cpp b/src/T2DLL/CTokenizer.cpp
new file mode 100644
index 0000000..e205c0b
--- /dev/null
+++ b/src/T2DLL/CTokenizer.cpp
@@ -0,0 +1,48 @@
+#include "CTokenizer.h"
+
+CTokenizer::CTokenizer(char *str, const char *newLineChars, const char *spaceChars)
+	: mNext(str)
+	, mCurrent(NULL)
+	, mNewLineChars(newLineChars)
+	, mSpaceChars(spaceChars)
+{
+#line 31
+	_ASSERT(mNext);
+
+	if (!mNewLineChars)
+		mNewLineChars = "\r\n\t";
+	if (!mSpaceChars)
+		mSpaceChars = " \r\n\t";
+}
+
+/*virtual*/ CTokenizer::~CTokenizer() {
+}
+
+CTokenizer& CTokenizer::operator=(char *str) {
+	mNext = str;
+	mCurrent = NULL;
+
+#line 49
+	_ASSERT(mNext);
+	return *this;
+}
+
+const char* CTokenizer::NextToken(const char *delim) {
+#line 58
+	_ASSERT(delim);
+
+	mCurrent = mNext;
+
+	if (mNext) {
+		mNext = (char *) _mbspbrk((unsigned char *) mNext, (const unsigned char *) delim);
+		if (mNext) {
+			mNext[0] = 0;
+			mNext++;
+			mNext += _mbsspn((unsigned char *) mNext, (const unsigned char *) delim);
+			if (mNext[0] == 0)
+				mNext = NULL;
+		}
+	}
+
+	return mCurrent;
+}