using Kivii;
using Kivii.Linq;
using Kivii.Web;
using LDA.LdaModel;
using Njust.Pdf.Analysis.Entities;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;

namespace Njust.Pdf.Analysis.Tranforms
{
    [Api(Description = "作者分布")]
    public class DisAuthorAnalysing : RestfulExecution<DisAuthor>
    {
        public override object OnExecution(IRequest req, IResponse res)
        {
            var rtns = new RestfulQueryResponse<DisAuthor>();
            rtns.Results = new List<DisAuthor>();

            var conn = KiviiContext.GetOpenedDbConnection<Author>();

            var query = conn.From<Author>();
            query.OrderBy(o => o.Name);
            var allAuthors = conn.Select(query);

            var group = allAuthors.GroupBy(o => new { o.Name, o.DomainName, o.Language });

            conn.InitEntityType<DisAuthor>();
            conn.Delete<DisAuthor>(o => o.Kvid == o.Kvid);
            //var queryExistDisAuthors = conn.From<DisAuthor>();
            //queryExistDisAuthors.Where(o => Sql.In(o.Author, group.ToList().ConvertAll(p => p.Key.Name)));

            //var existDisAuthors = conn.Select(queryExistDisAuthors);

            foreach (var kv in group)
            {
                //var exist = existDisAuthors.FirstOrDefault(o => o.Author == kv.Key.Name & o.DomainName == kv.Key.DomainName & o.Language == kv.Key.Language);
                //if (exist == null)
                //{
                var disAuthor = new DisAuthor();
                disAuthor.Language = kv.Key.Language;
                disAuthor.DomainName = kv.Key.DomainName;
                disAuthor.Author = kv.Key.Name;
                disAuthor.Count = kv.Count();
                conn.Insert(disAuthor);
                disAuthor.RemoveAllOnlyProperties();
                rtns.Results.Add(disAuthor);
                //    continue;
                //}
                //exist.Count = kv.Count();
                //exist.AddOnlyProperties(o => o.Count);
                //conn.UpdateOnly(exist);
                //rtns.Results.Add(exist);
            }
            rtns.Results.OrderBy(o => o.Count).Take(20);
            rtns.Total = rtns.Results.Count();
            return rtns;
        }
    }

    [Api(Description = "作者合作分布")]
    public class DisCoauthorAnalysing : RestfulExecution<DisCoauthor>
    {
        public override object OnExecution(IRequest req, IResponse res)
        {
            var rtns = new RestfulQueryResponse<DisCoauthor>();
            rtns.Results = new List<DisCoauthor>();

            var conn = KiviiContext.GetOpenedDbConnection<Author>();

            var query = conn.From<Author>();
            query.OrderBy(o => o.Name);
            var allAuthors = conn.Select(query);

            conn.InitEntityType<DisCoauthor>();
            conn.Delete<DisCoauthor>(o => o.Kvid == o.Kvid);

            var group = allAuthors.GroupBy(o => o.AnalyseKvid);

            var disCoauthors = new List<DisCoauthor>();

            foreach (var kv in group)
            {
                if (kv.Count() < 2) continue;
                var items = kv.ToList();
                items = items.OrderBy(o => o.Name).ToList();
                for (var i = 0; i < items.Count() - 1; i++)
                {
                    for (var j = 1; j < items.Count() - i; j++)
                    {
                        var disCoauthor = new DisCoauthor();
                        disCoauthor.DomainName = items[i].DomainName;
                        disCoauthor.Language = items[i].Language;
                        disCoauthor.Author1 = items[i].Name;
                        disCoauthor.Author2 = items[i + j].Name;
                        disCoauthors.Add(disCoauthor);
                    }
                }

            }

            var groupDisCoauthor = disCoauthors.GroupBy(o => new { o.DomainName, o.Language, o.Author1, o.Author2 });
            foreach (var kv in groupDisCoauthor)
            {
                var disCoauthor = new DisCoauthor();
                disCoauthor.DomainName = kv.Key.DomainName;
                disCoauthor.Language = kv.Key.Language;
                disCoauthor.Author1 = kv.Key.Author1;
                disCoauthor.Author2 = kv.Key.Author2;
                disCoauthor.Count = kv.Count();
                conn.Insert(disCoauthor);
                disCoauthor.RemoveAllOnlyProperties();
                rtns.Results.Add(disCoauthor);
            }
            rtns.Results.OrderBy(o => o.Count).Take(20);
            rtns.Total = rtns.Results.Count();
            return rtns;
        }
    }

    [Api(Description = "年份分布")]
    public class DisTrendAnalysing : RestfulExecution<DisTrend>
    {
        public override object OnExecution(IRequest req, IResponse res)
        {
            var rtns = new RestfulQueryResponse<DisTrend>();
            rtns.Results = new List<DisTrend>();

            var conn = KiviiContext.GetOpenedDbConnection<Analyse>();
            var query = conn.From<Analyse>();
            query.OrderBy(o => o.PublishTime);
            query.Where(o => o.IsChecked == true);
            var allTrends = conn.Select(query);

            var group = allTrends.GroupBy(o => new { o.PublishTime, o.DomainName, o.Language });

            conn.InitEntityType<DisTrend>();
            conn.Delete<DisTrend>(o => o.Kvid == o.Kvid);
            foreach (var kv in group)
            {

                var disTrend = new DisTrend();
                disTrend.Language = kv.Key.Language;
                disTrend.DomainName = kv.Key.DomainName;
                disTrend.Year = kv.Key.PublishTime;
                disTrend.Count = kv.Count();
                conn.Insert(disTrend);
                disTrend.RemoveAllOnlyProperties();
                rtns.Results.Add(disTrend);

            }
            rtns.Results.OrderBy(o => o.Count).Take(20);
            rtns.Total = rtns.Results.Count();
            return rtns;
        }
    }

    [Api(Description = "机构分布")]
    public class DisOrganAnalysing : RestfulExecution<DisOrgan>
    {
        public override object OnExecution(IRequest req, IResponse res)
        {
            var rtns = new RestfulQueryResponse<DisOrgan>();
            rtns.Results = new List<DisOrgan>();

            var conn = KiviiContext.GetOpenedDbConnection<Organ>();

            var query = conn.From<Organ>();
            query.OrderBy(o => o.Title);
            var allOrgans = conn.Select(query);

            var group = allOrgans.GroupBy(o => new { o.Title, o.DomainName, o.Language });

            conn.InitEntityType<DisOrgan>();
            conn.Delete<DisOrgan>(o => o.Kvid == o.Kvid);
            foreach (var kv in group)
            {

                var disOrgan = new DisOrgan();
                disOrgan.Language = kv.Key.Language;
                disOrgan.DomainName = kv.Key.DomainName;
                disOrgan.Organ = kv.Key.Title;
                disOrgan.Count = kv.Count();
                conn.Insert(disOrgan);
                disOrgan.RemoveAllOnlyProperties();
                rtns.Results.Add(disOrgan);
            }
            rtns.Results.OrderBy(o => o.Count).Take(20);
            rtns.Total = rtns.Results.Count();
            return rtns;
        }
    }

    [Api(Description = "机构合作分布")]
    public class DisCoOrganAnalysing : RestfulExecution<DisCoOrgan>
    {
        public override object OnExecution(IRequest req, IResponse res)
        {
            var rtns = new RestfulQueryResponse<DisCoOrgan>();
            rtns.Results = new List<DisCoOrgan>();

            var conn = KiviiContext.GetOpenedDbConnection<Organ>();

            var query = conn.From<Organ>();
            query.OrderBy(o => o.Title);
            var allCoOrgans = conn.Select(query);

            conn.InitEntityType<DisCoOrgan>();
            conn.Delete<DisCoOrgan>(o => o.Kvid == o.Kvid);

            var group = allCoOrgans.GroupBy(o => o.AnalyseKvid);

            var disCoOrgrans = new List<DisCoOrgan>();

            foreach (var kv in group)
            {
                if (kv.Count() < 2) continue;
                var items = kv.ToList();
                items = items.OrderBy(o => o.Title).ToList();
                for (var i = 0; i < items.Count() - 1; i++)
                {
                    for (var j = 1; j < items.Count() - i; j++)
                    {
                        var disCoOrgran = new DisCoOrgan();
                        disCoOrgran.DomainName = items[i].DomainName;
                        disCoOrgran.Language = items[i].Language;
                        disCoOrgran.Organ1 = items[i].Title;
                        disCoOrgran.Organ2 = items[i + j].Title;
                        disCoOrgrans.Add(disCoOrgran);
                    }
                }
            }

            var groupDisCoOrgran = disCoOrgrans.GroupBy(o => new { o.DomainName, o.Language, o.Organ1, o.Organ2 });
            foreach (var kv in groupDisCoOrgran)
            {
                var disCoOrgran = new DisCoOrgan();
                disCoOrgran.DomainName = kv.Key.DomainName;
                disCoOrgran.Language = kv.Key.Language;
                disCoOrgran.Organ1 = kv.Key.Organ1;
                disCoOrgran.Organ2 = kv.Key.Organ2;
                disCoOrgran.Count = kv.Count();
                conn.Insert(disCoOrgran);
                disCoOrgran.RemoveAllOnlyProperties();
                rtns.Results.Add(disCoOrgran);
            }
            rtns.Results.OrderBy(o => o.Count).Take(20);
            rtns.Total = rtns.Results.Count();
            return rtns;
        }
    }

    [Api(Description = "国家分布")]
    public class DisCountryAnalysing : RestfulExecution<DisCountry>
    {
        public override object OnExecution(IRequest req, IResponse res)
        {
            var rtns = new RestfulQueryResponse<DisCountry>();
            rtns.Results = new List<DisCountry>();

            var conn = KiviiContext.GetOpenedDbConnection<Analyse>();
            var query = conn.From<Analyse>();
            query.OrderBy(o => o.Country);
            query.Where(o => o.IsChecked == true);
            var allCountrys = conn.Select(query);

            var group = allCountrys.GroupBy(o => new { o.Country, o.DomainName, o.Language });

            conn.InitEntityType<DisCountry>();
            conn.Delete<DisCountry>(o => o.Kvid == o.Kvid);

            string line = "";
            //List<string> EnCountry = new List<string>();
            //List<string> CnCountry = new List<string>();
            var country = new Dictionary<string, string>();

            using (StreamReader sr = new StreamReader(Assembly.GetExecutingAssembly().GetManifestResourceStream("Njust.Pdf.Analysis.Resources.countrytranslation.txt")))
            {
                while ((line = sr.ReadLine()) != null)
                {
                    var sArray = line.Split(':').ToList();
                    country[sArray[1].ToLower()] = sArray[0];
                    //EnCountry.Add(sArray[0]);
                    //CnCountry.Add(sArray[1]);
                }
            }
            foreach (var kv in group)
            {
                var disCountry = new DisCountry();
                disCountry.Language = kv.Key.Language;
                disCountry.DomainName = kv.Key.DomainName;
                disCountry.Country = kv.Key.Country;
                try
                {
                    disCountry.EnCountry = disCountry.Country.IsNullOrEmpty() ? "" : country[disCountry.Country];
                }
                catch
                {
                    disCountry.EnCountry = disCountry.Country;
                }
                //disCountry.EnCountry = EnCountry[CnCountry.IndexOf(disCountry.Country)];
                disCountry.Count = kv.Count();
                conn.Insert(disCountry);
                disCountry.RemoveAllOnlyProperties();
                rtns.Results.Add(disCountry);

            }
            rtns.Results.OrderBy(o => o.Count).Take(20);
            rtns.Total = rtns.Results.Count();
            return rtns;
        }
    }

    [Api(Description = "来源分布")]
    public class DisSourceAnalysing : RestfulExecution<DisSource>
    {
        public override object OnExecution(IRequest req, IResponse res)
        {
            var rtns = new RestfulQueryResponse<DisSource>();
            rtns.Results = new List<DisSource>();

            var conn = KiviiContext.GetOpenedDbConnection<Analyse>();
            var query = conn.From<Analyse>();
            query.OrderBy(o => o.Source);
            query.Where(o => o.IsChecked == true);
            var allSources = conn.Select(query);

            var group = allSources.GroupBy(o => new { o.Source, o.DomainName, o.Language });

            conn.InitEntityType<DisSource>();
            conn.Delete<DisSource>(o => o.Kvid == o.Kvid);
            foreach (var kv in group)
            {

                var disSource = new DisSource();
                disSource.Language = kv.Key.Language;
                disSource.DomainName = kv.Key.DomainName;
                disSource.Source = kv.Key.Source;
                disSource.Count = kv.Count();
                conn.Insert(disSource);
                disSource.RemoveAllOnlyProperties();
                rtns.Results.Add(disSource);

            }
            rtns.Results.OrderBy(o => o.Count).Take(20);
            rtns.Total = rtns.Results.Count();
            return rtns;
        }
    }

    [Api(Description = "关键词共现")]
    public class DisCoKeywordAnalysing : RestfulExecution<DisCoKeyword>
    {
        public override object OnExecution(IRequest req, IResponse res)
        {
            var rtns = new RestfulQueryResponse<DisCoKeyword>();
            rtns.Results = new List<DisCoKeyword>();

            var conn = KiviiContext.GetOpenedDbConnection<Keyword>();

            var query = conn.From<Keyword>();
            query.OrderBy(o => o.Title);
            var allCoKeywords = conn.Select(query);

            conn.InitEntityType<DisCoKeyword>();
            conn.Delete<DisCoKeyword>(o => o.Kvid == o.Kvid);

            var group = allCoKeywords.GroupBy(o => o.AnalyseKvid);

            var disCoKeywords = new List<DisCoKeyword>();

            foreach (var kv in group)
            {
                if (kv.Count() < 2) continue;
                var items = kv.ToList();
                items = items.OrderBy(o => o.Title).ToList();
                for (var i = 0; i < items.Count() - 1; i++)
                {
                    for (var j = 1; j < items.Count() - i; j++)
                    {
                        var disCoKeyword = new DisCoKeyword();
                        disCoKeyword.DomainName = items[i].DomainName;
                        disCoKeyword.Language = items[i].Language;
                        disCoKeyword.KeyWord1 = items[i].Title;
                        disCoKeyword.KeyWord2 = items[i + j].Title;
                        disCoKeywords.Add(disCoKeyword);
                    }
                }
            }

            var groupDisCoKeyword = disCoKeywords.GroupBy(o => new { o.DomainName, o.Language, o.KeyWord1, o.KeyWord2 });
            foreach (var kv in groupDisCoKeyword)
            {
                var disCoKeyword = new DisCoKeyword();
                disCoKeyword.DomainName = kv.Key.DomainName;
                disCoKeyword.Language = kv.Key.Language;
                disCoKeyword.KeyWord1 = kv.Key.KeyWord1;
                disCoKeyword.KeyWord2 = kv.Key.KeyWord2;
                disCoKeyword.Count = kv.Count();
                conn.Insert(disCoKeyword);
                disCoKeyword.RemoveAllOnlyProperties();
                rtns.Results.Add(disCoKeyword);
            }
            rtns.Results.OrderBy(o => o.Count).Take(20);
            rtns.Total = rtns.Results.Count();
            return rtns;
        }
    }

    [Api(Description = "时序主题分布")]
    public class DisTimeTopicAnalysing : RestfulExecution<DisTimeTopic>
    {
        public override object OnExecution(IRequest req, IResponse res)
        {
            var rtns = new RestfulQueryResponse<DisTimeTopic>();
            rtns.Results = new List<DisTimeTopic>();

            var conn = KiviiContext.GetOpenedDbConnection<Analyse>();
            var query = conn.From<Analyse>();
            query.OrderBy(o => o.PublishTime);
            query.Where(o => o.IsChecked == true);
            var allDisTimeTopics = conn.Select(query);

            var group = allDisTimeTopics.GroupBy(o => new { o.PublishTime, o.DomainName, o.Language });

            conn.InitEntityType<DisTimeTopic>();
            conn.Delete<DisTimeTopic>(o => o.Kvid == o.Kvid);

            foreach (var kv in group)
            {
                if (kv.Key.Language.IsNullOrEmpty() || kv.Key.DomainName.IsNullOrEmpty() || kv.Key.PublishTime.IsNullOrEmpty()) continue;
                var items = kv.ToList();
                string[] lemmatizeText;
                if (kv.Key.Language == "英文")
                {
                var englishtext = new List<string>();
                    foreach (var item in items)
                    {
                        englishtext.Add(item.Title + ". " + item.Abstract);
                    }
                    lemmatizeText = LemmatizeEn.lemmatizeEnglish(englishtext);
                }
                else if (kv.Key.Language == "中文")
                {
                var chinesetext = new List<string>();
                    foreach (var item in items)
                    {
                        chinesetext.Add(item.Title + "。" + item.Abstract);
                    }
                    lemmatizeText = ChinesePreprocessing.ChineseWordSegmentation(chinesetext);
                }
                else continue;

                CommandLineOption ldaoption = new CommandLineOption();
                ldaoption.topics = (int)Math.Ceiling(System.Math.Sqrt(lemmatizeText.Length));
                double normolK = 200;
                ldaoption.alpha = ldaoption.topics / normolK;
                ldaoption.beta = 0.5;
                ldaoption.savestep = 500;
                ldaoption.niters = 500;
                ldaoption.twords = 10;
                try
                {
                    LDAGibbsSampling model = new LDAGibbsSampling();
                    Corpora cor = new Corpora();
                    cor.LoadDataFile(lemmatizeText);
                    var lda = model.TrainNewModel(cor, ldaoption);
                    for (int k = 0; k < lda.K; k++)
                    {
                        var wordsProbsList = new Dictionary<int, double>();

                        for (int w = 0; w < lda.V; w++)
                        {
                            wordsProbsList.Add(w, lda.phi[k][w]);
                        }
                        var wordsProbsListOrdereds = wordsProbsList.OrderBy(e => -e.Value).ToList();
                        List<string> words = new List<string>();
                        List<double> TopicWordsProbability = new List<double>();
                        foreach (var wordsProbsListOrdered in wordsProbsListOrdereds)
                        {
                            if (words.Count >= 10) break;
                            var word = cor.GetStringByID(wordsProbsListOrdered.Key);
                            if (word.IsNullOrEmpty()) continue;
                            words.Add(cor.GetStringByID(wordsProbsListOrdered.Key));
                            TopicWordsProbability.Add(wordsProbsListOrdered.Value);
                        }
                        var disTimeTopic = new DisTimeTopic();
                        disTimeTopic.Language = kv.Key.Language;
                        disTimeTopic.DomainName = kv.Key.DomainName;
                        var year= kv.Key.PublishTime.Replace(" ", "");
                        //提取多个数字，该方式会分别提取字符串中的数字，如："ABC#123@AS456测试789"就会分别提取123、456、789
                        var reg = new Regex("[0-9]+", RegexOptions.IgnoreCase | RegexOptions.Singleline, TimeSpan.FromSeconds(2));
                        var mc = reg.Matches(year);
                        if (mc.Count <= 0) continue;
                        disTimeTopic.Year = mc[0].Value;
                        disTimeTopic.Topic = "Topic " + k.ToString();
                        //DisTimeTopic.TopicWord = word;
                        disTimeTopic.TopicWord = string.Join(";", words);
                        disTimeTopic.TopicWordProbability = TopicWordsProbability;
                        disTimeTopic.Documentlist = lda.theta.ToList();
                        disTimeTopic.Parameter = "Aplha: " + lda.alpha.ToString() + ";" + "Beta: " + lda.beta.ToString() + ";" + "文档数: " + lda.M.ToString() + ";" + "主题数: " + lda.K.ToString() + ";" + "词袋数: " + lda.V.ToString() + ";" + "迭代次数:" + ldaoption.niters.ToString() + ";" + "LogLikelihood= " + lda.logLikelihood.ToString();
                        conn.Insert(disTimeTopic);
                        disTimeTopic.RemoveAllOnlyProperties();
                        rtns.Results.Add(disTimeTopic);
                    }
                }
                catch (Exception ex)
                {
                    Console.WriteLine(ex.StackTrace);
                    Console.WriteLine(ex.Message);
                }
            }
            rtns.Total = rtns.Results.Count();
            return rtns;
        }
    }

    [Api(Description = "主题演化分布")]
    public class DisTopicEvolutionAnalysing : RestfulExecution<DisTopicEvolution>
    {
        public override object OnExecution(IRequest req, IResponse res)
        {
            var rtns = new RestfulQueryResponse<DisTopicEvolution>();
            rtns.Results = new List<DisTopicEvolution>();

            var conn = KiviiContext.GetOpenedDbConnection<DisTimeTopic>();
            var query = conn.From<DisTimeTopic>();
            query.OrderBy(o => o.Year);
            var allDisTimeTopics = conn.Select(query);

            var group = allDisTimeTopics.GroupBy(o => new { o.Year, o.DomainName, o.Language }).ToList();

            conn.InitEntityType<DisTopicEvolution>();
            conn.Delete<DisTopicEvolution>(o => o.Kvid == o.Kvid);

            foreach(var kv in group)
            {
                var index = group.IndexOf(kv);
                if (index + 1 == group.Count()) continue;
                foreach(var item in kv)
                {
                    var nextGroup = group.FirstOrDefault(o => o.Key.Language == item.Language & o.Key.DomainName == item.DomainName & o.Key.Year.ToInt() > item.Year.ToInt());
                    if (nextGroup == null) continue;

                    var sourceTopicWords = item.TopicWord.Split(';').ToList();

                    var disTopicEvolution = new DisTopicEvolution();
                    disTopicEvolution.DomainName = item.DomainName;
                    disTopicEvolution.Language = item.Language;
                    disTopicEvolution.SourceYear = item.Year;
                    disTopicEvolution.SourceTopic = item.Topic;
                    disTopicEvolution.SourceTopicWord = item.TopicWord;
                    foreach (var next in nextGroup)
                    {
                        var targetTopicWords = next.TopicWord.Split(';').ToList();
                        var cosSim = TopicSimilarityCos.SimilarityCos(sourceTopicWords, targetTopicWords);
                        if (disTopicEvolution.CosSim <= cosSim)
                        {
                            disTopicEvolution.CosSim = cosSim;
                            disTopicEvolution.TargetTopic = next.Topic;
                            disTopicEvolution.TargetYear = next.Year;
                            disTopicEvolution.TargetTopicWord = next.TopicWord;
                        }
                    }
                    conn.Insert(disTopicEvolution);
                    disTopicEvolution.RemoveAllOnlyProperties();
                    rtns.Results.Add(disTopicEvolution);
                }
            }
            rtns.Total = rtns.Results.Count();
            return rtns;
        }
    }
}
