diff --git a/CHCSVParser/CHCSVParser/CHCSVParser.m b/CHCSVParser/CHCSVParser/CHCSVParser.m index eff27a1..ec87e68 100644 --- a/CHCSVParser/CHCSVParser/CHCSVParser.m +++ b/CHCSVParser/CHCSVParser/CHCSVParser.m @@ -32,12 +32,17 @@ of this software and associated documentation files (the "Software"), to deal NSString *const CHCSVErrorDomain = @"com.davedelong.csv"; #define CHUNK_SIZE 512 +#define DELIMITER_SNIFF_SIZE 16384 #define DOUBLE_QUOTE '"' #define COMMA ',' +#define TAB '\t' +#define SEMICOLON ';' #define OCTOTHORPE '#' #define EQUAL '=' #define BACKSLASH '\\' #define NULLCHAR '\0' +#define NEWLINE '\n' // newline can be used as a dummy method parameter to signify + // that we wish to use automatic delimiter sniffing @interface CHCSVParser () @property (assign) NSUInteger totalBytesRead; @@ -85,7 +90,6 @@ - (instancetype)initWithContentsOfDelimitedURL:(NSURL *)URL delimiter:(unichar)d - (id)initWithInputStream:(NSInputStream *)stream usedEncoding:(NSStringEncoding *)encoding delimiter:(unichar)delimiter { NSParameterAssert(stream); NSParameterAssert(delimiter); - NSAssert([[NSCharacterSet newlineCharacterSet] characterIsMember:delimiter] == NO, @"The field delimiter may not be a newline"); NSAssert(delimiter != DOUBLE_QUOTE, @"The field delimiter may not be a double quote"); self = [super init]; @@ -96,8 +100,6 @@ - (id)initWithInputStream:(NSInputStream *)stream usedEncoding:(NSStringEncoding _stringBuffer = [[NSMutableData alloc] init]; _string = [[NSMutableString alloc] init]; - _delimiter = delimiter; - _nextIndex = 0; _recognizesComments = NO; _recognizesBackslashesAsEscapes = NO; @@ -120,6 +122,15 @@ - (id)initWithInputStream:(NSInputStream *)stream usedEncoding:(NSStringEncoding } else { _streamEncoding = *encoding; } + + // newline is now used as a dummy parameter to signify that we wish + // to use automatic delimiter sniffing + if ([[NSCharacterSet newlineCharacterSet] characterIsMember:delimiter]) { + // we need to determine the delimiter + [self _sniffDelimiter]; + } else { + _delimiter = delimiter; + } } return self; } @@ -207,6 +218,45 @@ - (void)_sniffEncoding { _streamEncoding = encoding; } +- (void)_sniffDelimiter { + NSInteger appendLength = DELIMITER_SNIFF_SIZE - [self totalBytesRead]; + if (appendLength > 0) { + uint8_t bytes[appendLength]; + NSInteger readLength = [_stream read:bytes maxLength:appendLength]; + if (readLength > 0 && readLength <= appendLength) { + [_stringBuffer appendBytes:bytes length:readLength]; + [self setTotalBytesRead:[self totalBytesRead] + readLength]; + } + } + + NSString *csv = [[NSString alloc] initWithData:_stringBuffer encoding:_streamEncoding]; + + NSError *error = NULL; + NSRegularExpression *regex = [NSRegularExpression regularExpressionWithPattern:@"[^\"]+" options:NSRegularExpressionCaseInsensitive error:&error]; + + const int delimiterCount = 3; + unichar delimiters[delimiterCount] = {TAB, COMMA, SEMICOLON}; + long maxOccurences = 0; + int delimiterIndex = 0; + + for (int i=0; i maxOccurences) { + maxOccurences = currentOccurences; + delimiterIndex = i; + } + } + + _delimiter = delimiters[delimiterIndex]; +} + - (void)_loadMoreIfNecessary { NSUInteger stringLength = [_string length]; NSUInteger reloadPortion = stringLength / 3; @@ -844,11 +894,11 @@ - (NSString *)CSVString { @implementation NSString (CHCSVAdditions) - (NSArray *)CSVComponents { - return [self componentsSeparatedByDelimiter:COMMA options:0 error:nil]; + return [self componentsSeparatedByDelimiter:'\n' options:0 error:nil]; } - (NSArray *)CSVComponentsWithOptions:(CHCSVParserOptions)options { - return [self componentsSeparatedByDelimiter:COMMA options:options error:nil]; + return [self componentsSeparatedByDelimiter:'\n' options:options error:nil]; } - (NSArray *)componentsSeparatedByDelimiter:(unichar)delimiter { @@ -975,7 +1025,7 @@ - (BOOL)stripsLeadingAndTrailingWhitespace { @implementation NSArray (CHCSVAdditions_Deprecated) + (instancetype)arrayWithContentsOfCSVFile:(NSString *)csvFilePath { - return [self arrayWithContentsOfDelimitedURL:[NSURL fileURLWithPath:csvFilePath] options:0 delimiter:COMMA error:nil]; + return [self arrayWithContentsOfDelimitedURL:[NSURL fileURLWithPath:csvFilePath] options:0 delimiter:'\n' error:nil]; } + (instancetype)arrayWithContentsOfCSVFile:(NSString *)csvFilePath delimiter:(unichar)delimiter { @@ -983,7 +1033,7 @@ + (instancetype)arrayWithContentsOfCSVFile:(NSString *)csvFilePath delimiter:(un } + (instancetype)arrayWithContentsOfCSVFile:(NSString *)csvFilePath options:(CHCSVParserOptions)options { - return [self arrayWithContentsOfDelimitedURL:[NSURL fileURLWithPath:csvFilePath] options:options delimiter:COMMA error:nil]; + return [self arrayWithContentsOfDelimitedURL:[NSURL fileURLWithPath:csvFilePath] options:options delimiter:'\n' error:nil]; } + (instancetype)arrayWithContentsOfCSVFile:(NSString *)csvFilePath options:(CHCSVParserOptions)options delimiter:(unichar)delimiter { diff --git a/Unit Tests/UnitTests.m b/Unit Tests/UnitTests.m index 44e0191..7e79cfc 100644 --- a/Unit Tests/UnitTests.m +++ b/Unit Tests/UnitTests.m @@ -129,6 +129,16 @@ - (void)testSimpleUTF8 { TEST(csv, expected); } +- (void)testDelimiterSniffing { + NSString *csv = FIELD1 COMMA FIELD2 COMMA FIELD3 COMMA UTF8FIELD4 NEWLINE FIELD1 COMMA FIELD2 COMMA FIELD3 COMMA UTF8FIELD4; + NSArray *expected = @[@[FIELD1, FIELD2, FIELD3, UTF8FIELD4], @[FIELD1, FIELD2, FIELD3, UTF8FIELD4]]; + TEST(csv, expected); + csv = FIELD1 SEMICOLON FIELD2 SEMICOLON FIELD3 SEMICOLON UTF8FIELD4 NEWLINE FIELD1 SEMICOLON FIELD2 SEMICOLON FIELD3 SEMICOLON UTF8FIELD4; + TEST(csv, expected); + csv = FIELD1 TAB FIELD2 TAB FIELD3 TAB UTF8FIELD4 NEWLINE FIELD1 TAB FIELD2 TAB FIELD3 TAB UTF8FIELD4; + TEST(csv, expected); +} + - (void)testGithubIssue35 { NSString *tsv = @"1,a" TAB @"1,b" TAB @"1,c" TAB @"1,\"d\"" NEWLINE @"2,a" TAB @"2,b" TAB @"2,c" TAB @"2,d" NEWLINE