using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Diagnostics;
using LemmaSharp.Classes;
using System.IO;
using System.Text.RegularExpressions;
using System.Reflection;

namespace LDA.LdaModel
{
	public static class LemmatizeEn
	{
		public static string[] lemmatizeEnglish(List<String> englishtext)
		{
			Regex r = new Regex("([ \\t{}():;. \n])");
			List<string> filtertext = new List<string>();
			using (var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream("LDA.LdaModel.Resources.full7z-mlteast-en-modified.lem"))
			{
				var lemmatizer = new Lemmatizer(stream);
				foreach (string onetext in englishtext)
				{
					string input = onetext.ToLower();
					input = Regex.Replace(input, "[ \\[ \\] \\^ \\-_*×――(^)（^）$%~!@#$…&%￥—+=<>《》!！??？:：•`·、。，；,.;\"‘’“”-]", " ");
					String[] tokens = r.Split(input);
					string filter = "";

					for (int i = 0; i < tokens.Length; i++)
					{
						MatchCollection mc = r.Matches(tokens[i]);
						if (mc.Count <= 0 && tokens[i].Trim().Length > 0 && !StopWordsHandler.IsStopword(tokens[i])) filter += lemmatizer.Lemmatize(tokens[i]) + "\r\t";
					}
					filtertext.Add(filter);

				}
			}
			return filtertext.ToArray();
		}
	}
}
