JobScraper/true_remote_filter.py at main · Harsh-C7/JobScraper · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
import pandas as pd
import re
import requests
from bs4 import BeautifulSoup
import concurrent.futures
from tqdm import tqdm

RED_FLAGS = [
    r'\bus\b only',
    r'\bunited states\b only',
    r'must reside in (?:the )?us',
    r'no visa sponsorship',
    r'us citizen',
    r'w2 only',
    r'w-2 only',
    r'citizenship required',
    r'cleared candidates',
    r'security clearance',
    r'uk only',
    r'europe only',
    r'must live in (?:the )?(?:us|uk|eu)'
]

GREEN_FLAGS = [
    r'work from anywhere',
    r'anywhere in the world',
    r'global team',
    r'worldwide',
    r'remote india',
    r'remote - india',
    r'overlap with',
    r'distributed team',
    r'digital nomad'
]

def analyze_description(text):
    if not text or not isinstance(text, str):
        return 0, 'Unknown'

    text_lower = text.lower()
    score = 0
    status = 'Open Worldwide'

    for flag in RED_FLAGS:
        if re.search(flag, text_lower):
            score -= 10
            status = 'Likely Restricted (US/EU Only)'

    for flag in GREEN_FLAGS:
        if re.search(flag, text_lower):
            score += 5
            status = 'Verified Worldwide'

    return score, status

def fetch_text_from_url(url):
    try:
        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'}
        response = requests.get(url, headers=headers, timeout=5)
        soup = BeautifulSoup(response.text, 'html.parser')
        # naive text extraction
        return soup.get_text(separator=' ', strip=True)
    except Exception:
        return ""

def filter_true_remote(df):
    print("\n--- Starting Phase 3: True Remote Filtering Engine ---")
    if df.empty:
        return df

    # We need a Description column
    if 'Description' not in df.columns:
        df['Description'] = ""

    # Ensure Description is text
    df['Description'] = df['Description'].fillna("").astype(str)

    # For those missing descriptions, try a quick scrape
    missing_desc_mask = df['Description'].str.strip() == ""
    urls_to_scrape = df.loc[missing_desc_mask, 'Job URL'].tolist()

    if urls_to_scrape:
        print(f"Fetching deeper job descriptions for {len(urls_to_scrape)} raw links...")
        with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
            fetched_texts = list(tqdm(executor.map(fetch_text_from_url, urls_to_scrape), total=len(urls_to_scrape), desc="Downloading Missing JDs", unit="page"))

        df.loc[missing_desc_mask, 'Description'] = fetched_texts

    print("Analyzing Job Descriptions for Location Restrictions (US Only vs Global)...")

    scores = []
    statuses = []
    for desc in tqdm(df['Description'], desc="NLP Parsing", unit="job"):
        score, status = analyze_description(desc)
        scores.append(score)
        statuses.append(status)

    df['Remote Score'] = scores
    df['Remote Status'] = statuses

    # Filter out strongly restricted "Fake Remote" jobs (US/EU specific)
    original_count = len(df)
    df = df[df['Remote Score'] > -5]
    filtered_count = len(df)

    print(f"Phase 3 Complete: Vaporized {original_count - filtered_count} 'Fake Remote' jobs.")

    # Clean up massive text column before exporting to excel
    if 'Description' in df.columns:
        df = df.drop(columns=['Description'])

    # Sort so 'Verified Worldwide' is at the top
    df = df.sort_values(by=['Remote Score', 'Company Name'], ascending=[False, True])

    # Reset index cleanly
    df = df.reset_index(drop=True)

    return df