FortiOS_AppControlID/get_id.py at master · Connie-Wild/FortiOS_AppControlID · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
import requests
import csv
import re
import time
from bs4 import BeautifulSoup

s = requests.Session()

url = 'https://fortiguard.fortinet.com/appcontrol'

user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/155.0.0.0 Safari/537.36'
headers = {
  'User-Agent': user_agent
}

while True:
  try:
    get_url_info = s.get(url, headers=headers, timeout=5)
    get_url_info.raise_for_status()
    bs4Obj = BeautifulSoup(get_url_info.content, 'lxml')
    result = bs4Obj.find_all('div', class_='col-md-auto')
    appids = int(re.findall('[0-9]+', result[2].find('p').get_text().replace(',', ''))[0])
    result = bs4Obj.find('section', class_='table-body')
    perPage = int(len(result.find_all('div', class_='row')))
    break
  except Exception as e:
    print(e)
    time.sleep(1)
    continue

if appids % perPage == 0:
  pages = int(appids // perPage)
else:
  pages = int(appids // perPage) + 1

i = 1
datas = []
url_base = 'https://fortiguard.fortinet.com/appcontrol?page='

while pages + 1 > i:
  print('page:', i, '/', pages)
  url = url_base + str(i)
  try:
    time.sleep(1)
    get_url_info = s.get(url, headers=headers, timeout=5)
    get_url_info.raise_for_status()
    bs4Obj = BeautifulSoup(get_url_info.content, 'lxml')
    result = bs4Obj.find('section', class_='table-body')
    app_list = result.find_all('div', class_='row')
    app_nums = dict(re.findall(r"getElementById\('([^']+)'\).*?'/appcontrol/(\d+)'", get_url_info.text))
    if len(app_list) != perPage:
      if i == pages:
        pass
      else:
        continue
  except Exception as e:
    print(e)
    continue
  j = 0
  while True:
    try:
      app_name = str(app_list[j].find('div', class_='col-md-3').get_text()).split(' (')[0].replace('\n', '')
      app_num = int(app_nums[app_list[j].get('id')])
      data = [app_num, app_name]
      datas.append(data)
      j += 1
    except:
      break
  i += 1

print('result:', len(datas), '/', appids)

with open('appid.csv', 'w') as file:
  writer = csv.writer(file, lineterminator='\n')
  writer.writerows(sorted(datas))