using JiebaNet.Analyser;
using JiebaNet.Segmenter;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace LDA.LdaModel
{
    public static class ChinesePreprocessing
    {
		public static string[] ChineseWordSegmentation(List<String> chinesetext)
		{
			var segmenter = new JiebaSegmenter();
			List<string> filtertext = new List<string>();
			foreach (string onetext in chinesetext)
            {
				var segments = segmenter.Cut(onetext);  // 默认为精确模式
				string[] tokens = segments.ToArray();
				string filter = "";
				for (int i = 0; i < tokens.Length; i++)
                {
					if (tokens[i].Trim().Length > 0 && !CnStopWordsHandler.IsStopword(tokens[i])) filter +=tokens[i] + "\r\t";
				}
				filtertext.Add(filter);
			}
			return filtertext.ToArray();


		}
	}
}
