#define C_KINO_ANALYZER
#define C_KINO_TOKEN
#include "KinoSearch/Util/ToolSet.h"
#include "KinoSearch/Analysis/Analyzer.h"
#include "KinoSearch/Analysis/Token.h"
#include "KinoSearch/Analysis/Inversion.h"
Analyzer*
Analyzer_init(Analyzer *self)
{
ABSTRACT_CLASS_CHECK(self, ANALYZER);
return
self;
}
Inversion*
Analyzer_transform_text(Analyzer *self, CharBuf *text)
{
size_t
token_len = CB_Get_Size(text);
Token *seed = Token_new((
char
*)CB_Get_Ptr8(text), token_len, 0,
token_len, 1.0, 1);
Inversion *starter = Inversion_new(seed);
Inversion *retval = Analyzer_Transform(self, starter);
DECREF(seed);
DECREF(starter);
return
retval;
}
VArray*
Analyzer_split(Analyzer *self, CharBuf *text)
{
Inversion *inversion = Analyzer_Transform_Text(self, text);
VArray *out = VA_new(0);
Token *token;
while
((token = Inversion_Next(inversion)) != NULL) {
VA_Push(out, (Obj*)CB_new_from_trusted_utf8(token->text, token->len));
}
DECREF(inversion);
return
out;
}