Repository navigation
Expand file tree
/
Copy pathcomparecontent.py
More file actions
executable file
·219 lines (176 loc) · 7.37 KB
/
Copy pathcomparecontent.py
File metadata and controls
executable file
·219 lines (176 loc) · 7.37 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# comparecontent.py
# Sep 2024 (updated Feb 2026)
#
# Compare two content list CSV files and report differences.
# - Validates header match (or match after dropping date columns).
# - Can ignore creationTimeStamp and modifiedTimeStamp.
# - Compares order-insensitive (so different row ordering does not count as a difference).
#
# usage:
# python comparecontent.py --file1 <path> --file2 <path> [--file1label <label>] [--file2label <label>] [-d|--debug] [--ignore-dates] [--ignore-ownership] [--ignore-id]
#
import argparse
import csv
import io
import os
import sys
from collections import Counter
def read_lines(path):
"""Read file as a list of lines with normalized newlines, skipping empty lines."""
with open(path, "r", newline="") as f:
# splitlines() removes \n and \r\n cleanly
lines = f.read().splitlines()
# drop completely empty lines
return [ln for ln in lines if ln.strip()]
def parse_header(line):
"""Parse a CSV header line into a list of column names (raw + stripped)."""
cols = next(csv.reader([line]))
stripped = [c.strip() for c in cols]
return cols, stripped
def row_to_line(row):
"""Render a CSV row consistently (stable quoting + stable newline)."""
sio = io.StringIO()
csv.writer(sio, lineterminator="\n").writerow(row)
return sio.getvalue()
def normalize_body(lines, drop_cols=None):
"""
Normalize body lines:
- Parse CSV rows
- Drop drop_cols indices (if provided)
- Render back to consistent CSV lines
Returns:
norm_lines: list[str] normalized rows
norm_to_orig: dict[str, list[str]] mapping normalized line -> original line(s)
"""
if drop_cols is None:
drop_cols = set()
else:
drop_cols = set(drop_cols)
norm_lines = []
norm_to_orig = {}
for orig in lines:
# parse a row safely
row = next(csv.reader([orig]))
kept = [v for i, v in enumerate(row) if i not in drop_cols]
norm = row_to_line(kept)
norm_lines.append(norm)
norm_to_orig.setdefault(norm, []).append(orig)
return norm_lines, norm_to_orig
def multiset_diff(norm1, map1, norm2, map2):
"""
Order-insensitive comparison with duplicate support.
Returns lists of original lines only in file1 and only in file2.
"""
c1 = Counter(norm1)
c2 = Counter(norm2)
only1 = []
only2 = []
# For deterministic output, iterate sorted keys
for norm in sorted((c1 - c2).keys()):
needed = (c1 - c2)[norm]
only1.extend(map1.get(norm, [norm])[:needed])
for norm in sorted((c2 - c1).keys()):
needed = (c2 - c1)[norm]
only2.extend(map2.get(norm, [norm])[:needed])
return only1, only2
def main():
parser = argparse.ArgumentParser(description="Compare two content lists.")
parser.add_argument("-f1", "--file1", required=True, help="Full path to the first content file.")
parser.add_argument("-f2", "--file2", required=True, help="Full path to the second content file.")
parser.add_argument("-f1l", "--file1label", default="First file.", help="Optional label for file1.")
parser.add_argument("-fl2", "--file2label", default="Second file.", help="Optional label for file2.")
parser.add_argument("-d", "--debug", action="store_true", help="Debug output")
parser.add_argument("--ignore-dates", action="store_true",
help="Ignore creationTimeStamp and modifiedTimeStamp columns when comparing")
parser.add_argument("--ignore-ownership", action="store_true",
help="Ignore createdByand modifiedBy columns when comparing")
parser.add_argument("--ignore-id", action="store_true",
help="Ignore id columns when comparing, some content gets a new id on import")
args = parser.parse_args()
file1 = args.file1
file2 = args.file2
label1 = args.file1label
label2 = args.file2label
if not os.path.exists(file1):
print(f"ERROR: The file '{file1}' does not exist.")
sys.exit(2)
if not os.path.exists(file2):
print(f"ERROR: The file '{file2}' does not exist.")
sys.exit(2)
lines1 = read_lines(file1)
lines2 = read_lines(file2)
if not lines1 or not lines2:
print("ERROR: One or both files are empty (or contain only blank lines).")
sys.exit(2)
# Parse headers
header1_raw, header1 = parse_header(lines1[0])
header2_raw, header2 = parse_header(lines2[0])
# Decide which columns to drop (by name)
drop_cols = []
if args.ignore_dates:
ignore_fields = {"creationTimeStamp", "modifiedTimeStamp"}
if args.ignore_ownership:
ignore_fields.update({"createdBy", "modifiedBy"})
if args.ignore_id:
ignore_fields.add("id")
drop_cols = [i for i, name in enumerate(header1) if name in ignore_fields]
# Validate header equality AFTER dropping date columns
h1_dropped = [c for i, c in enumerate(header1) if i not in drop_cols]
h2_dropped = [c for i, c in enumerate(header2) if i not in drop_cols]
if h1_dropped != h2_dropped:
print("ERROR: Headers do not match after dropping date columns.")
if args.debug:
print("DEBUG: header1_dropped:", h1_dropped)
print("DEBUG: header2_dropped:", h2_dropped)
sys.exit(2)
else:
# Validate header equality EXACTLY (original behavior)
if lines1[0] != lines2[0]:
print("ERROR: Files must have a matching header line in the first line.")
if args.debug:
print("DEBUG: file1 header:", lines1[0])
print("DEBUG: file2 header:", lines2[0])
sys.exit(2)
# Normalize bodies
body1 = lines1[1:]
body2 = lines2[1:]
if args.ignore_dates:
norm1, map1 = normalize_body(body1, drop_cols=drop_cols)
norm2, map2 = normalize_body(body2, drop_cols=drop_cols)
else:
# Still normalize line endings & CSV formatting to reduce false diffs
norm1, map1 = normalize_body(body1, drop_cols=[])
norm2, map2 = normalize_body(body2, drop_cols=[])
# Compare order-insensitive
only_in_file1, only_in_file2 = multiset_diff(norm1, map1, norm2, map2)
print(f"NOTE: Compare the content of file1 ({label1}) and file2 ({label2})")
print("NOTE: SUMMARY")
if only_in_file1:
print(f"NOTE: There is content in file1 ({label1}) that is not in file2 ({label2}).")
if only_in_file2:
print(f"NOTE: There is content in file2 ({label2}) that is not in file1 ({label1}).")
if only_in_file1 or only_in_file2:
print("NOTE: DETAILS")
else:
print("NOTE: Files are the same.")
# Print header line for context
if only_in_file1:
print(f"NOTE: The content listed below is in file1 ({label1}) but not in file2 ({label2}):")
print(lines1[0])
for line in only_in_file1:
print(line)
if only_in_file2:
print(f"NOTE: The content listed below is in file2 ({label2}) but not in file1 ({label1}):")
print(lines2[0])
for line in only_in_file2:
print(line)
if args.debug:
print("DEBUG: rows file1:", len(body1))
print("DEBUG: rows file2:", len(body2))
print("DEBUG: ignore_dates:", args.ignore_dates)
print("DEBUG: drop_cols:", drop_cols)
if __name__ == "__main__":
main()