我有一個將 Ebcdic 轉換為 Ascii 的 C# 程式。程式的輸入檔案大小約為 250MB 到 300MB,當我通過放置單個檔案進行處理時,在共享路徑中檔案處理正在發生沒有任何問題,但是當輸入檔案位置中的檔案超過 1 個時,當第二個檔案處理到 50% 時,我得到System.OutOfMemoryException,我粘貼了示例代碼,我使用垃圾收集(處理單個檔案后)來釋放記憶體。但它沒有按預期作業,同時我也將決議器物件設為空,然后嘗試垃圾收集但仍然得到同樣的錯誤。
using System;
using System.Text;
using System.IO;
using System.Linq;
namespace Ebcdic2Ascii
{
class Program
{
static void Main(string[] args)
{
DirectoryInfo d = new DirectoryInfo(@"C:\SampleFiles\input\");
FileInfo[] Files = d.GetFiles("*.dat");
StreamWriter[] writer = new StreamWriter[Files.Count()];
LineTemplate lineTemplate = new LineTemplate(73, "ReservationsData");
lineTemplate.AddFieldTemplate(new FieldTemplate("RESERVATION-NUMBER", FieldType.String, 0, 11));
lineTemplate.AddFieldTemplate(new FieldTemplate("CHECKIN-DATE", FieldType.DateString, 11, 6));
lineTemplate.AddFieldTemplate(new FieldTemplate("CALC-NET-AMOUNT", FieldType.BinaryNum, 17, 4, 2));
lineTemplate.AddFieldTemplate(new FieldTemplate("CUSTOMER-NAME", FieldType.String, 21, 30));
lineTemplate.AddFieldTemplate(new FieldTemplate("RUNDATE", FieldType.DateStringMMDDYY, 51, 6));
lineTemplate.AddFieldTemplate(new FieldTemplate("CURRENCY-CONV-RATE", FieldType.Packed, 57, 6, 6));
lineTemplate.AddFieldTemplate(new FieldTemplate("US-DOLLAR-AMOUNT-DUE", FieldType.Packed, 63, 6, 2));
lineTemplate.AddFieldTemplate(new FieldTemplate("DATE-OF-BIRTH", FieldType.PackedDate, 69, 4));
int i = 0;
foreach (FileInfo File in Files)
{
EbcdicParser parser = new EbcdicParser(File.FullName, lineTemplate);
string Outputpath = "C:\\output\\" File.Name ".txt";
writer[i] = new StreamWriter(Outputpath);
foreach (ParsedLine line in parser.ParsedLines)
{
writer[i].WriteLine(line["RESERVATION-NUMBER"].ToString() "\t" line["CHECKIN-DATE"].ToString() "\t" line["CALC-NET-AMOUNT"].ToString() "\t" line["CALC-NET-AMOUNT"].ToString() "\t" line["CUSTOMER-NAME"].ToString() "\t" line["RUNDATE"].ToString() "\t" line["CURRENCY-CONV-RATE"].ToString() "\t" line["US-DOLLAR-AMOUNT-DUE"].ToString() "\t" line["DATE - OF - BIRTH"].ToString());
}
i = i 1;
GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced);
GC.Collect();
GC.WaitForPendingFinalizers();
}
}
}
}
決議器的代碼這也有垃圾收集方法
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
namespace Ebcdic2Ascii
{
public class EbcdicParser
{
public ParsedLine[] Lines { get; private set; }
public EbcdicParser()
{
//Empty constructor
}
public EbcdicParser(byte[] allBytes, LineTemplate lineTemplate)
{
double expectedRows = (double)allBytes.Length / lineTemplate.LineSize;
Console.WriteLine("{0}: Parsing started", DateTime.Now);
Console.WriteLine("{1}: Line count est {0:#,###.00}", expectedRows, DateTime.Now);
this.Lines = this.ParseAllLines(lineTemplate, allBytes);
//Collect garbage
GC.Collect();
GC.WaitForPendingFinalizers();
Console.WriteLine("{1}: {0} line(s) have been parsed", this.Lines.Count(), DateTime.Now);
}
public EbcdicParser(string sourceFilePath, LineTemplate lineTemplate)
: this(File.ReadAllBytes(sourceFilePath), lineTemplate)
{
//Constructor with the file path
}
public ParsedLine[] ParseAllLines(LineTemplate lineTemplate, byte[] allBytes)
{
bool isSingleLine = false;
this.ValidateInputParameters(lineTemplate, allBytes, isSingleLine);
List<ParsedLine> parsedLines = new List<ParsedLine>();
byte[] lineBytes = new byte[lineTemplate.LineSize];
ParsedLine parsedLine;
for (int i = 0; i < allBytes.Length; i = lineTemplate.LineSize)
{
if (i % 1000 == 0)
{
//Print progress
Console.Write(i "\r");
}
Array.Copy(allBytes, i, lineBytes, 0, lineTemplate.LineSize);
parsedLine = this.ParseSingleLine(lineTemplate, lineBytes);
parsedLines.Add(parsedLine);
}
return parsedLines.ToArray();
}
public ParsedLine[] ParseAllLines(LineTemplate lineTemplate, string sourceFilePath)
{
return this.ParseAllLines(lineTemplate, File.ReadAllBytes(sourceFilePath));
}
public ParsedLine ParseSingleLine(LineTemplate lineTemplate, byte[] lineBytes)
{
bool isSingleLine = true;
this.ValidateInputParameters(lineTemplate, lineBytes, isSingleLine);
ParsedLine parsedLine = new ParsedLine(lineTemplate, lineBytes);
return parsedLine;
}
private bool ValidateInputParameters(LineTemplate lineTemplate, byte[] allBytes, bool isSingleLine)
{
if (allBytes == null)
{
throw new ArgumentNullException("Ebcdic data is not provided");
}
if (lineTemplate == null)
{
throw new ArgumentNullException("Line template is not provided");
}
if (lineTemplate.FieldsCount == 0)
{
throw new Exception("Line template must contain at least one field");
}
if (allBytes.Length < lineTemplate.LineSize)
{
throw new Exception("Data length is shorter than the line size");
}
if (isSingleLine && allBytes.Length != lineTemplate.LineSize)
{
throw new Exception("Bytes count doesn't equal to line size");
}
double expectedRows = (double)allBytes.Length / lineTemplate.LineSize;
if (expectedRows % 1 != 0) //Expected number of rows is not a whole number
{
throw new Exception("Expected number of rows is not a whole number. Check line template.");
}
return true;
}
public void CreateCsvFile(string outputFilePath, bool includeColumnNames, bool addQuotes)
{
if (this.Lines == null || this.Lines.Length == 0)
{
throw new Exception("No lines have been parsed");
}
ParserUtilities.ConvertLineArrayToCsv(this.Lines, outputFilePath, includeColumnNames, addQuotes);
}
}
}
ParsedLine 類檔案
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace Ebcdic2Ascii
{
public class ParsedLine
{
public LineTemplate Line_Template { get; private set; }
public Dictionary<string, ParsedField> FieldDictionary
{ get; private set; } //= new Dictionary<string, ParsedField>();
public string this[string fieldName]
{
get
{
return this.FieldDictionary[fieldName].Value.Trim();
}
}
//Constructor
public ParsedLine(LineTemplate lineTemplate, byte[] lineBytes)
{
this.Line_Template = lineTemplate;
this.ParseLine(lineBytes, lineTemplate);
}
private void ParseLine(byte[] lineBytes, LineTemplate lineTemplate)
{
this.ValidateInputParameters(lineBytes, lineTemplate);
foreach (var fieldTemplate in lineTemplate.FieldTemplateDictionary)
{
FieldDictionary.Add(fieldTemplate.Key,
new ParsedField(lineBytes, lineTemplate.FieldTemplateDictionary[fieldTemplate.Key]));
}
}
private void ValidateInputParameters(byte[] lineBytes, LineTemplate template)
{
if (lineBytes == null)
{
throw new ArgumentNullException("Line bytes required");
}
if (lineBytes.Length < template.LineSize)
{
throw new Exception(String.Format(
"Bytes provided: {0}, line size: {1}", lineBytes.Length, template.LineSize));
}
if (template == null)
{
throw new ArgumentNullException("line template is required");
}
if (template.FieldsCount == 0)
{
throw new Exception("Field templates have not been defined in the line template");
}
}
public string GetParsedFieldValuesCSV(bool addQuotes)
{
StringBuilder sb = new StringBuilder();
int count = 0;
foreach (ParsedField parsedField in this.FieldDictionary.Values)
{
sb.Append(addQuotes ? "\"" : "");
sb.Append(parsedField.Value);
sb.Append(addQuotes ? "\"" : "");
sb.Append(this.FieldDictionary.Count < count ? "," : "");
count ;
}
return sb.ToString();
}
}
}
類 ParsedField
using System;
using System.Globalization;
using System.Text;
using System.Text.RegularExpressions;
namespace Ebcdic2Ascii
{
public class ParsedField
{
public FieldTemplate Field_Template { get; private set; }
public string Value { get; private set; }
public byte[] OriginalBytes { get; private set; }
public string OriginalBytesInHex
{
get
{
return BitConverter.ToString(this.OriginalBytes);
}
}
public string OriginalBytesInDec
{
get
{
return ParserUtilities.ConvertBytesToDec(this.OriginalBytes);
}
}
public bool ParsedSuccessfully { get; private set; }
//Constructor
public ParsedField(byte[] lineBytes, FieldTemplate fieldTemplate)
{
this.ParsedSuccessfully = true;
this.Field_Template = fieldTemplate;
this.Value = ParseField(lineBytes, fieldTemplate);
}
private string ParseField(byte[] lineBytes, FieldTemplate template)
{
if (lineBytes == null || lineBytes.Length == 0)
{
ParserUtilities.PrintError("Line bytes is null or empty");
this.ParsedSuccessfully = false;
return null;
}
if (lineBytes.Length < (template.StartPosition template.FieldSize))
{
this.ParsedSuccessfully = false;
throw new Exception(String.Format(
"Field \"{0}\" length falls outside the line length", template.FieldName));
}
byte[] fieldBytes = new byte[template.FieldSize];
Array.Copy(lineBytes, template.StartPosition, fieldBytes, 0, template.FieldSize);
this.OriginalBytes = fieldBytes;
if (this.Field_Template.Type == FieldType.AlphaNum)
{
return this.ConvertAlphaNumEbcdic(fieldBytes);
}
else if (this.Field_Template.Type == FieldType.Numeric)
{
return this.ConvertNumericEbcdic(fieldBytes, template.DecimalPlaces);
}
else if (this.Field_Template.Type == FieldType.Packed)
{
return this.Unpack(fieldBytes, template.DecimalPlaces);
}
else if (this.Field_Template.Type == FieldType.Binary)
{
return ConvertBinaryEbcdic(fieldBytes, template.DecimalPlaces);
}
else if (this.Field_Template.Type == FieldType.Date)
{
return ConvertDateStrEbcdic(fieldBytes);
}
else if (this.Field_Template.Type == FieldType.PackedDate)
{
return ConvertPackedDateStrEbcdic(fieldBytes);
}
else if (this.Field_Template.Type == FieldType.SourceBytesInHex)
{
return this.OriginalBytesInHex;
}
else if (this.Field_Template.Type == FieldType.SourceBytesInDec)
{
return this.OriginalBytesInDec;
}
else
{
this.ParsedSuccessfully = false;
throw new Exception(String.Format(
"Unable to parse field \"{0}\". Unknown field type: {1}",
template.FieldName, template.Type.ToString()));
}
}
private string ConvertAlphaNumEbcdic(byte[] ebcdicBytes)
{
if (this.ByteArrayIsFullOf_0xFF(ebcdicBytes))
{
return "";
}
//Encoding asciiEnc = Encoding.ASCII;
//Encoding ebcdicEnc = Encoding.GetEncoding("IBM037");
//string result = Encoding.ASCII.GetString(Encoding.Convert(ebcdicEnc, asciiEnc, ebcdicBytes));
//Thank you sx2008
Encoding ebcdicEnc = Encoding.GetEncoding("IBM037");
string result = ebcdicEnc.GetString(ebcdicBytes); // convert EBCDIC Bytes -> Unicode string
return result;
}
private string ConvertNumericEbcdic(byte[] ebcdicBytes, int decimalPlaces)
{
string tempNumStr = this.ConvertAlphaNumEbcdic(ebcdicBytes).Trim();
if (tempNumStr == null || tempNumStr.Length == 0)
{
return "";
}
if (Regex.IsMatch(tempNumStr, @"^\d $")) //Unsigned integer
{
string result = this.AdjustDecimalValues(Int64.Parse(tempNumStr), decimalPlaces);
return result;
}
else if (Regex.IsMatch(tempNumStr, @"^\d [A-R{}]$")) //Signed integer
{
string lastChar = tempNumStr.Substring(tempNumStr.Length - 1);
switch (lastChar)
{
case "{":
tempNumStr = tempNumStr.Replace("{", "0");
break;
case "A":
tempNumStr = tempNumStr.Replace("A", "1");
break;
case "B":
tempNumStr = tempNumStr.Replace("B", "2");
break;
case "C":
tempNumStr = tempNumStr.Replace("C", "3");
break;
case "D":
tempNumStr = tempNumStr.Replace("D", "4");
break;
case "E":
tempNumStr = tempNumStr.Replace("E", "5");
break;
case "F":
tempNumStr = tempNumStr.Replace("F", "6");
break;
case "G":
tempNumStr = tempNumStr.Replace("G", "7");
break;
case "H":
tempNumStr = tempNumStr.Replace("H", "8");
break;
case "I":
tempNumStr = tempNumStr.Replace("I", "9");
break;
case "}":
tempNumStr = "-" tempNumStr.Replace("}", "0");//Fixed
break;
case "J":
tempNumStr = "-" tempNumStr.Replace("J", "1");
break;
case "K":
tempNumStr = "-" tempNumStr.Replace("K", "2");
break;
case "L":
tempNumStr = "-" tempNumStr.Replace("L", "3");
break;
case "M":
tempNumStr = "-" tempNumStr.Replace("M", "4");
break;
case "N":
tempNumStr = "-" tempNumStr.Replace("N", "5");
break;
case "O":
tempNumStr = "-" tempNumStr.Replace("O", "6");
break;
case "P":
tempNumStr = "-" tempNumStr.Replace("P", "7");
break;
case "Q":
tempNumStr = "-" tempNumStr.Replace("Q", "8");
break;
case "R":
tempNumStr = "-" tempNumStr.Replace("R", "9");
break;
}
string result = this.AdjustDecimalValues(Int64.Parse(tempNumStr), decimalPlaces);
return result;
}
else
{
this.ParsedSuccessfully = false;
return tempNumStr;
}
}
private string ConvertBinaryEbcdic(byte[] ebcdicBytes, int decimalPlaces)
{
if (this.ByteArrayIsFullOf_0xFF(ebcdicBytes))
{
return "";
}
//BitConverter requires low order bytes goes first, followed by the higher order bytes.
//Bytes are stored in the file in the opposite order, thus need to reverse bytes
Array.Reverse(ebcdicBytes);
long tempNum;
if (ebcdicBytes.Length == 2)
{
//If 2 bytes are provided -- assume it's a short
tempNum = BitConverter.ToUInt16(ebcdicBytes, 0);
}
else if (ebcdicBytes.Length == 4)
{
//If 4 bytes are provided -- assume it's an int
tempNum = BitConverter.ToInt32(ebcdicBytes, 0);
}
else
{
//Just in case
throw new Exception(String.Format(
"Incorrect number of bytes provided for a binary field: {1}", decimalPlaces));
}
string result = this.AdjustDecimalValues(tempNum, decimalPlaces);
return result;
}
private string AdjustDecimalValues(long numericValue, int decimalPlaces)
{
if (decimalPlaces == 0)
{
return numericValue.ToString();
}
double result = numericValue / Math.Pow(10, decimalPlaces);
return result.ToString();
}
private string ConvertDateStrEbcdic(byte[] ebcdicBytes)
{
string dateStr = this.ConvertAlphaNumEbcdic(ebcdicBytes).Trim();
string result = this.ConvertDateStr(dateStr);
return result;
}
private string ConvertPackedDateStrEbcdic(byte[] ebcdicBytes)
{
string dateStr = this.Unpack(ebcdicBytes, 0);
string result = this.ConvertDateStr(dateStr);
return result;
}
private string ConvertDateStr(string dateStr)
{
dateStr = dateStr.Trim();
if (dateStr.Trim() == "" || dateStr == "0" ||
dateStr == "0000000" || dateStr == "9999999")
{
return "";
}
if (Regex.IsMatch(dateStr, @"^\d{3,5}$"))
{
dateStr = dateStr.PadLeft(6, '0');
}
Match match = Regex.Match(dateStr, @"^(?<Year>\d{3})(?<Month>\d{2})
(?<Day>\d{2})$"); //E.g.: 0801232 = 1980-12-31; 1811231 = 2080-12-31
if (match.Success)
{
int year = Int32.Parse(match.Groups["Year"].Value) 1900; //013 => 1913, 113 => 2013...
int month = Int32.Parse(match.Groups["Month"].Value);
int day = Int32.Parse(match.Groups["Day"].Value);
try
{
DateTime tempDate = new DateTime(year, month, day);
return tempDate.ToString("yyyy-MM-dd");
}
catch { }
}
if (Regex.IsMatch(dateStr, @"^\d{6}$"))
{
DateTime tempDate;
if (DateTime.TryParseExact(dateStr, "yyMMdd",
CultureInfo.InvariantCulture, DateTimeStyles.None, out tempDate))
{
return tempDate.ToString("yyyy-MM-dd");
}
}
this.ParsedSuccessfully = false;
return dateStr;
}
private string Unpack(byte[] ebcdicBytes, int decimalPlaces)
{
if (ByteArrayIsFullOf_0xFF(ebcdicBytes))
{
return "";
}
long lo = 0;
long mid = 0;
long hi = 0;
bool isNegative;
// this nybble stores only the sign, not a digit.
// "C" hex is positive, "D" hex is negative, and "F" hex is unsigned.
switch (Nibble(ebcdicBytes, 0))
{
case 0x0D:
isNegative = true;
break;
case 0x0F:
case 0x0C:
isNegative = false;
break;
default:
//throw new Exception("Bad sign nibble");
this.ParsedSuccessfully = false;
return this.ConvertAlphaNumEbcdic(ebcdicBytes);
}
long intermediate;
long carry;
long digit;
for (int j = ebcdicBytes.Length * 2 - 1; j > 0; j--)
{
// multiply by 10
intermediate = lo * 10;
lo = intermediate & 0xffffffff;
carry = intermediate >> 32;
intermediate = mid * 10 carry;
mid = intermediate & 0xffffffff;
carry = intermediate >> 32;
intermediate = hi * 10 carry;
hi = intermediate & 0xffffffff;
carry = intermediate >> 32;
// By limiting input length to 14, we ensure overflow will never occur
digit = Nibble(ebcdicBytes, j);
if (digit > 9)
{
//throw new Exception("Bad digit");
this.ParsedSuccessfully = false;
return this.ConvertAlphaNumEbcdic(ebcdicBytes);
}
intermediate = lo digit;
lo = intermediate & 0xffffffff;
carry = intermediate >> 32;
if (carry > 0)
{
intermediate = mid carry;
mid = intermediate & 0xffffffff;
carry = intermediate >> 32;
if (carry > 0)
{
intermediate = hi carry;
hi = intermediate & 0xffffffff;
carry = intermediate >> 32;
// carry should never be non-zero. Back up with validation
}
}
}
decimal result = new Decimal((int)lo, (int)mid, (int)hi, isNegative, (byte)decimalPlaces);
return result.ToString();
}
private int Nibble(byte[] ebcdicBytes, int nibbleNo)
{
int b = ebcdicBytes[ebcdicBytes.Length - 1 - nibbleNo / 2];
return (nibbleNo % 2 == 0) ? (b & 0x0000000F) : (b >> 4);
}
private bool ByteArrayIsFullOf_0xFF(byte[] ebcdicBytes)
{
if (ebcdicBytes == null || ebcdicBytes.Length == 0)
{
return false;
}
foreach (byte b in ebcdicBytes)
{
if (b != 0xFF)
{
return false;
}
}
return true;
}
}
}
LineTemplate class,FieldTemplate class,ParserUtilities class are present in page
Click 
uj5u.com熱心網友回復:
這很可能System.OutOfMemoryException是因為您用完了檔案句柄,而不是實際記憶體。StreamWriter您正在為您處理的每個檔案創建一個新檔案而不對其進行處理。
請參閱System.OutOfMemoryException的檔案。它具體說:
盡管垃圾收集器能夠釋放分配給托管型別的記憶體,但它不管理分配給非托管資源的記憶體,例如作業系統句柄(包括檔案句柄、記憶體映射檔案、管道、注冊表項和等待句柄)和記憶體由 Windows API 呼叫或通過呼叫諸如 malloc 之類的記憶體分配函式直接分配的塊。消耗非托管資源的型別實作 IDisposable 介面。
如果您正在使用使用非托管資源的型別,則應確保在使用完畢后呼叫其 IDisposable.Dispose 方法。(某些型別還實作了與 Dispose 方法功能相同的 Close 方法。)有關詳細資訊,請參閱使用實作 IDisposable 的物件主題。
重要的是要注意垃圾收集器不會要求Dispose()您。
您必須writer[i].Dispose()在完成每個流的寫入后呼叫。
uj5u.com熱心網友回復:
我同意 John Glenn 的觀點,嘗試使用單個 StreamWriter 將您的決議作業放入一個方法和 Enigmativity 的建議中。
static void Parse(.....)
{
EbcdicParser parser = new EbcdicParser(File.FullName, lineTemplate);
string Outputpath = "C:\\output\\" File.Name ".txt";
using(var writer = new StreamWriter(Outputpath))
{
foreach (ParsedLine line in parser.ParsedLines)
{
.....
}
}
}
在范圍之外運行 GC。
int i = 0;
foreach (FileInfo File in Files)
{
Parse(......);
GC.Collect();
}
uj5u.com熱心網友回復:
您的垃圾收集呼叫沒有您想要的效果。在您的 foreach 回圈中,EbcdicParser parser仍然可以訪問,因為它在當前執行的函式的范圍內,因此不考慮進行垃圾回收。
同樣,在您的 EbcdicParser 中,位元組陣列仍然可以訪問,因此您的垃圾回收呼叫也不會清除它。
轉載請註明出處,本文鏈接:https://www.uj5u.com/qiye/420772.html
標籤:
