上周末,幫朋友處理了一個關於大數據的查詢與導出問題,整理一下,在此記錄一下用以備忘,同時也為有類似需要的朋友提供一個參考.
背景:
改造步驟:
/// <summary> /// 執行導出操作 /// </summary> /// <param name="p"></param> private void ExecExport(string fileName) { plProcessStatus.Visible = true; SetControlStatus(true); IExport rpter = new ExcelExporter(); var formater = BuildExportFormater(); bool isQueryEnd = false; //當前是否查詢結束 var templateFieName = Path.Combine(Application.StartupPath, "Template", "Rpt_CustomerList.xls"); //創建供導出的隊列 Queue<List<OrderDetail>> exportQueue = new Queue<List<OrderDetail>>(); #region 查詢線程 //處理後台查詢 Thread thQuery = new Thread(() => { int tempTotal = 0; int tempPgIndex = 0; int queryPageSize = 3000; //每次查詢3k var tempList = ExecQuery(tempPgIndex, queryPageSize, out tempTotal); if (tempList != null && tempList.Count > 0) { lock (locker) { exportQueue.Enqueue(tempList); Monitor.PulseAll(locker); } tempPgIndex += 1; //循環查詢直至查詢結束 while (tempPgIndex * _pageSize < tempTotal) { var temp_tempList = ExecQuery(tempPgIndex, queryPageSize, out tempTotal); if (temp_tempList != null && temp_tempList.Count > 0) { lock (locker) { exportQueue.Enqueue(temp_tempList); //將查詢結果加入到隊列 Monitor.PulseAll(locker); } } tempPgIndex += 1; } } isQueryEnd = true; }); #endregion #region 導出excel線程 //處理將查詢的結果寫入到文件中 Thread thExport = new Thread(() => { rpter.Export(templateFieName, fileName, formater);//讀取模板,並創建新文件, int tempRowIndex = 0; while (!isQueryEnd || exportQueue.Count > 0) //未查詢結束及隊列不為空,執行導出 { if (exportQueue.Count > 0) { List<OrderDetail> tempExpotLst = null; lock (locker) { tempExpotLst = exportQueue.Dequeue(); //取隊列數據,導出excel操作 } if (tempExpotLst != null && tempExpotLst.Count > 0) { formater.DetailRowBeginIndex += tempRowIndex; rpter.ExportByAppend(fileName, formater, tempExpotLst); //執行導出操作(追加形式) tempRowIndex = tempExpotLst.Count; } } else { Thread.Sleep(200); } } //導出貼圖片 var imgRow = formater.DetailRowBeginIndex + tempRowIndex + 8; formater.ImageCellFormaters.Add(new ImageCellFormater(imgRow, 2, Resources.ywz)); rpter.ExportByAppend(fileName, formater, null); //導出結束 恢復按鈕可用狀態 btnExport.Invoke(new Action(() => { plProcessStatus.Visible = false; //隱藏進度欄 SetControlStatus(false); if (MessageBox.Show(this, "數據已成功導出至[" + fileName + "],是否立即打開導出文件?", "提示", MessageBoxButtons.YesNo, MessageBoxIcon.Question) == DialogResult.Yes) { Process.Start(fileName); } })); }); #endregion thQuery.IsBackground = true; thExport.IsBackground = true; thQuery.Start(); //開始查詢線程 thExport.Start(); //開始導出線程 }
/// <summary> /// 獲取物流明細記錄 /// </summary> /// <param name="queryParam">查詢條件</param> /// <param name="total">返回符合條件的總記錄數量</param> /// <returns></returns> public List<OrderDetail> GetOrderDetailList(EFQueryParam<OrderDetail> queryParam, out int total) { total = 0; var lst = GetRepository<OrderDetail, Int64>().Get(queryParam, out total).ToList(); //組織其他冗余數據 if (lst != null && lst.Count > 0) { //冗余公司信息,供前台UI使用 var companyList = GetCompanyList(); if (companyList != null && companyList.Count > 0) { var companyDic = companyList.ToDictionary(p => p.Id); //轉化為字典,提高效率 var tempbgIndex = queryParam.PageIndex * queryParam.PageSize + 1; //生成排序 lst.ForEach(t => { t.Index = tempbgIndex; //寄件公司 if (companyDic.ContainsKey(t.FromCompanyId)) { t.FromComoany = companyDic[t.FromCompanyId]; } //收件公司 if (companyDic.ContainsKey(t.ToCompanyId)) { t.ToCompany = companyDic[t.ToCompanyId]; } //付款公司 if (companyDic.ContainsKey(t.PaymentCompanyId)) { t.PaymentCompany = companyDic[t.PaymentCompanyId]; } tempbgIndex += 1; }); } } return lst; } /// <summary> /// 獲取公司信息 /// </summary> /// <returns></returns> public List<Company> GetCompanyList() { //從緩存中獲取 var lst = ApplicationRuntime.Instance.CurrentCache.Get<List<Company>>(KYEConsts.CachesKey.Company, () => this.GetRepository<Company>().Get().ToList()); return lst; }
結語: 經過這一系列改造後, 性能上大大改進了,查詢響應耗時<=1s, 導出<=8s, 每次符合條件的數據大概在2W條左右.
至於本文中提到的
1. EF構建查詢條件表達式,及查詢數據庫方式,可參見之前文章.http://www.cnblogs.com/xie-zhonglai/archive/2012/04/07/2435903.html
2. 導出Excel.本文使用了NPOI 這個組件,詳情可參見文章: http://www.cnblogs.com/xie-zhonglai/p/3979771.html