Datathon-NLP/xml_parser.py at main · nickrwu/Datathon-NLP · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
# -*- coding: utf-8 -*-
"""XML Parser.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1_afNuMx0ALGpVhix1jV8_7Sv2i9f4X-q
"""

from zipfile import ZipFile
from io import BytesIO
import re
from bs4 import BeautifulSoup as bs

import pandas as pd
import csv

def parse_zip(zip_file):
  ad_list = list()

  with ZipFile(zip_file, 'r') as zfile:
      for name in zfile.namelist():
          if re.search(r'\.zip$', name) is not None:  # we have a zip within a zip
              print(name)
              zfiledata = BytesIO(zfile.read(name))
              with ZipFile(zfiledata) as zfile2:
                  counter = 0
                  for xml_name in zfile2.namelist():
                    counter += 1

                    if counter % 100 == 0: print(f'{len(ad_list)} ads out of {counter} files')

                    if counter > 1000: break # -> Uncomment for smaller output

                    # check if xml file contains an advertisement; skip if does not
                    try:
                      indexDict[xml_name.split('.')[0]]
                    except KeyError:
                      continue

                    ad_obj = {}

                    # open xml file
                    xml_file = zfile2.open(xml_name)
                    content = xml_file.read()

                    # parse xml with BeautifulSoup
                    bs_content = bs(content, "xml")

                    # get important fields from xml contents
                    # Record ID
                    recordid = bs_content.find('RecordID')
                    ad_obj['RecordID']=recordid.get_text()

                    # Full Text
                    text = bs_content.find('FullText')
                    try:
                      ad_obj['text']=text.get_text()
                    except:
                      ad_obj['text']=None

                    # Publishing Date
                    pub_date = bs_content.find('NumericPubDate')
                    ad_obj['pubDate']=pub_date.get_text()

                    # Publisher
                    publisher = bs_content.find('Publisher')
                    ad_obj['publisher']=publisher.get_text()

                    ad_list.append(ad_obj)

  return pd.json_normalize(ad_list)

if (__name__ == '__main__'):
  # files = ['./ProQuest-Datathon2021-Challenge-20210205T233945Z-011.zip'] -> from download
  files = ['./ProQuest-Datathon2021-Challenge-20210205T233945Z-010.zip', './ProQuest-Datathon2021-Challenge-20210205T233945Z-011.zip']

  # dictionary with structure: { RecordID <string> : True <bool> }
  # throws 'KeyError' if you try to access a RecordID that doesn't exist (meaning it's not in the ad index)
  indexDict = {}
  # NOTE: I deleted last 10 lines of downloaded file because it has unrelated (meta?)data
  indexFilePath = './NewYorkTimes-Advertisement-List.csv'
  with open(indexFilePath, mode='r') as indexFile:
      reader = csv.reader(indexFile)
      indexDict = {rows[0]:True for rows in reader}
      del indexDict['RecordID']

  data_frames = []
  for file in files:
    data_frames.append(parse_zip(file))

  pd.concat(data_frames).to_csv('./out/AdData.csv', index=False, encoding='utf-8')