Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 14 additions & 14 deletions src/urlscan/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -380,7 +380,7 @@ def _map_http_status_error(self, exc: httpx.HTTPStatusError) -> APIError:
# fallback to HTTP status code if "status" is missing
status: int = data.get("status") or exc.response.status_code

# ref. https://urlscan.io/docs/api/#ratelimit
# ref. https://docs.urlscan.io/pages/api-rate-limits
if status == 429:
rate_limit_reset_after = float(
exc.response.headers.get("X-Rate-Limit-Reset-After", 0)
Expand Down Expand Up @@ -474,7 +474,7 @@ def get_result(self, uuid: str) -> dict:
Dict: Scan result.

Reference:
https://urlscan.io/docs/api/#result
https://docs.urlscan.io/apis/urlscan-openapi/scanning/getresult

"""
return self.get_json(f"/api/v1/result/{uuid}/")
Expand All @@ -489,7 +489,7 @@ def get_screenshot(self, uuid: str) -> BytesIO:
BytesIO: Screenshot (img/png).

Reference:
https://urlscan.io/docs/api/#screenshot
https://docs.urlscan.io/apis/urlscan-openapi/scanning/getscreenshot

"""
res = self.get_content(f"/screenshots/{uuid}.png")
Expand All @@ -507,7 +507,7 @@ def get_dom(self, uuid: str) -> str:
str: DOM as a string.

Reference:
https://urlscan.io/docs/api/#dom
https://docs.urlscan.io/apis/urlscan-openapi/scanning/getdom

"""
return self.get_text(f"/dom/{uuid}/")
Expand All @@ -522,7 +522,7 @@ def get_response(self, file_hash: str) -> str:
str: Response content as a string.

Reference:
https://docs.urlscan.io/apis/urlscan-openapi/scanning/response
https://docs.urlscan.io/apis/urlscan-openapi/scanning/getresponse

"""
return self.get_text(f"/responses/{file_hash}/")
Expand All @@ -543,14 +543,14 @@ def search(
size (int, optional): Number of results returned in a search. Defaults to 100.
limit (int | None, optional): Maximum number of results that will be returned by the iterator. Defaults to None.
search_after (str | None, optional): Search after to retrieve next results. Defaults to None.
datasource (SearchDataSource | None, optional): Datasources to search: scans (urlscan.io), hostnames, incidents, notifications, certificates (urlscan Pro). Defaults to None.
datasource (SearchDataSource | None, optional): Datasources to search: scans, files (urlscan.io), hostnames, incidents, notifications, certificates (urlscan Pro). Defaults to None.
collapse (str | None, optional): Field to collapse results on. Only works on current page of results. Defaults to None.

Returns:
SearchIterator: Search iterator.

Reference:
https://urlscan.io/docs/api/#search
https://docs.urlscan.io/apis/urlscan-openapi/search/searchdatasource

"""
return SearchIterator(
Expand Down Expand Up @@ -590,7 +590,7 @@ def scan(
dict: Scan response.

Reference:
https://urlscan.io/docs/api/#scan
https://docs.urlscan.io/apis/urlscan-openapi/scanning/submitscan

"""
data = _compact(
Expand Down Expand Up @@ -644,7 +644,7 @@ def bulk_scan(
list[tuple[str, dict | Exception]]: A list of tuples of (url, scan response or error).

Reference:
https://urlscan.io/docs/api/#scan
https://docs.urlscan.io/apis/urlscan-openapi/scanning/submitscan

"""

Expand Down Expand Up @@ -732,7 +732,7 @@ def scan_and_get_result(
dict: Scan result.

Reference:
https://urlscan.io/docs/api/#scan
https://docs.urlscan.io/apis/urlscan-openapi/scanning/submitscan

"""
res = self.scan(
Expand Down Expand Up @@ -781,7 +781,7 @@ def bulk_scan_and_get_results(
list[tuple[str, dict | Exception]]: A list of tuples of (url, result or error).

Reference:
https://urlscan.io/docs/api/#scan
https://docs.urlscan.io/apis/urlscan-openapi/scanning/submitscan

"""
responses = self.bulk_scan(
Expand Down Expand Up @@ -813,7 +813,7 @@ def get_available_countries(self) -> dict:
dict: Available countries.

Reference:
https://docs.urlscan.io/apis/urlscan-openapi/scanning/availablecountries
https://docs.urlscan.io/apis/urlscan-openapi/scanning/getavailablecountries

"""
return self.get_json("/api/v1/availableCountries")
Expand All @@ -825,7 +825,7 @@ def get_user_agents(self) -> dict:
dict: Available user agents.

Reference:
https://docs.urlscan.io/apis/urlscan-openapi/scanning/useragents
https://docs.urlscan.io/apis/urlscan-openapi/scanning/getuseragents

"""
return self.get_json("/api/v1/userAgents")
Expand All @@ -837,7 +837,7 @@ def get_quotas(self) -> dict:
dict: API quotas.

Reference:
https://docs.urlscan.io/apis/urlscan-openapi/generic/quotas
https://docs.urlscan.io/apis/urlscan-openapi/generic/getquotas

"""
return self.get_json("/api/v1/quotas")
4 changes: 2 additions & 2 deletions src/urlscan/pro/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,7 @@ def get_user(self) -> dict:
dict: User information.

Reference:
https://docs.urlscan.io/apis/urlscan-openapi/generic/prousername
https://docs.urlscan.io/apis/urlscan-openapi/generic/getusername

"""
return self.get_json("/api/v1/pro/username")
Expand All @@ -290,7 +290,7 @@ def lookup_malicious_observable(
dict: Malicious observable lookup result.

Reference:
https://docs.urlscan.io/apis/urlscan-openapi/malicious/maliciouslookup
https://docs.urlscan.io/apis/urlscan-openapi/malicious/lookupmaliciousobservable

"""
path = f"/api/v1/malicious/{type_}/{quote_plus(value)}"
Expand Down
4 changes: 2 additions & 2 deletions src/urlscan/pro/brand.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ def get_available_brands(self) -> dict:
dict: Response containing a list of brand objects.

Reference:
https://docs.urlscan.io/apis/urlscan-openapi/brands/availablebrands
https://docs.urlscan.io/apis/urlscan-openapi/brands/getavailablebrands

"""
return self.get_json("/api/v1/pro/availableBrands")
Expand All @@ -27,7 +27,7 @@ def get_brands(self) -> dict:
dict: Response containing a list of brand object with detection statistics.

Reference:
https://docs.urlscan.io/apis/urlscan-openapi/brands/brandsummary
https://docs.urlscan.io/apis/urlscan-openapi/brands/getbrandsummary

"""
return self.get_json("/api/v1/pro/brands")
8 changes: 4 additions & 4 deletions src/urlscan/pro/channel.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ def get_channels(self) -> dict:
dict: Object containing an array of channels.

Reference:
https://docs.urlscan.io/apis/urlscan-openapi/channels/channels
https://docs.urlscan.io/apis/urlscan-openapi/channels/listchannels

"""
return self.get_json("/api/v1/user/channels/")
Expand Down Expand Up @@ -63,7 +63,7 @@ def create(
dict: Object containing the created channel.

Reference:
https://docs.urlscan.io/apis/urlscan-openapi/channels/channelscreate
https://docs.urlscan.io/apis/urlscan-openapi/channels/createchannel

"""
channel: dict[str, Any] = _compact(
Expand Down Expand Up @@ -99,7 +99,7 @@ def get(self, channel_id: str) -> dict:
dict: Object containing the channel.

Reference:
https://docs.urlscan.io/apis/urlscan-openapi/channels/channelsget
https://docs.urlscan.io/apis/urlscan-openapi/channels/getchannel

"""
return self.get_json(f"/api/v1/user/channels/{channel_id}/")
Expand Down Expand Up @@ -142,7 +142,7 @@ def update(
dict: Object containing the updated channel.

Reference:
https://docs.urlscan.io/apis/urlscan-openapi/channels/channelsupdate
https://docs.urlscan.io/apis/urlscan-openapi/channels/updatechannel

"""
channel: dict[str, Any] = _compact(
Expand Down
12 changes: 6 additions & 6 deletions src/urlscan/pro/livescan.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ def get_scanners(self) -> dict:
dict: List of available scanners with metadata.

Reference:
https://docs.urlscan.io/apis/urlscan-openapi/live-scanning/livescanscanners
https://docs.urlscan.io/apis/urlscan-openapi/live-scanning/getlivescanscanners

"""
return self.get_json("/api/v1/livescan/scanners/")
Expand Down Expand Up @@ -94,7 +94,7 @@ def task(
dict: Response containing the scan UUID.

Reference:
https://docs.urlscan.io/apis/urlscan-openapi/live-scanning/livescantask
https://docs.urlscan.io/apis/urlscan-openapi/live-scanning/createlivescantask

"""
data = self._build_scan_payload(
Expand Down Expand Up @@ -140,7 +140,7 @@ def scan(
dict: Response containing the scan UUID.

Reference:
https://docs.urlscan.io/apis/urlscan-openapi/live-scanning/livescanscan
https://docs.urlscan.io/apis/urlscan-openapi/live-scanning/createlivescanscan

"""
data = self._build_scan_payload(
Expand Down Expand Up @@ -174,7 +174,7 @@ def get_resource(
Any: Resource content. Returns dict for "result", str for "dom", bytes for binary resources.

Reference:
https://docs.urlscan.io/apis/urlscan-openapi/live-scanning/livescangetresource
https://docs.urlscan.io/apis/urlscan-openapi/live-scanning/getlivescanresource

"""
path = f"/api/v1/livescan/{scanner_id}/{resource_type}/{resource_id}"
Expand Down Expand Up @@ -205,7 +205,7 @@ def store(
visibility (VisibilityType): Visibility for the stored scan ("public", "private", or "unlisted").

Reference:
https://docs.urlscan.io/apis/urlscan-openapi/live-scanning/livescanstore
https://docs.urlscan.io/apis/urlscan-openapi/live-scanning/storelivescanresult

"""
data = {"task": {"visibility": visibility}}
Expand All @@ -225,7 +225,7 @@ def purge(
scan_id (str): Scan UUID.

Reference:
https://docs.urlscan.io/apis/urlscan-openapi/live-scanning/livescandiscard
https://docs.urlscan.io/apis/urlscan-openapi/live-scanning/discardlivescanresult

"""
res = self._delete(f"/api/v1/livescan/{scanner_id}/{scan_id}/")
Expand Down
10 changes: 5 additions & 5 deletions src/urlscan/pro/saved_search.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ def get_list(self) -> dict:
dict: Response containing an array of Saved Search objects with their properties.

Reference:
https://docs.urlscan.io/apis/urlscan-openapi/saved-searches/savedsearches-get
https://docs.urlscan.io/apis/urlscan-openapi/saved-searches/listsavedsearches

"""
return self.get_json("/api/v1/user/searches/")
Expand Down Expand Up @@ -61,7 +61,7 @@ def create(
dict: Created Saved Search object containing the search properties and unique _id.

Reference:
https://docs.urlscan.io/apis/urlscan-openapi/saved-searches/savedsearches-post
https://docs.urlscan.io/apis/urlscan-openapi/saved-searches/createsavedsearch

"""
search: dict[str, Any] = _compact(
Expand Down Expand Up @@ -125,7 +125,7 @@ def update(
dict: Updated Saved Search object containing the search properties and unique _id.

Reference:
https://docs.urlscan.io/apis/urlscan-openapi/saved-searches/savedsearches-put
https://docs.urlscan.io/apis/urlscan-openapi/saved-searches/updatesavedsearch

"""
search: dict[str, Any] = _compact(
Expand Down Expand Up @@ -158,7 +158,7 @@ def remove(self, search_id: str) -> dict:
dict: Empty JSON object on success.

Reference:
https://docs.urlscan.io/apis/urlscan-openapi/saved-searches/savedsearches-delete
https://docs.urlscan.io/apis/urlscan-openapi/saved-searches/deletesavedsearch

"""
res = super()._delete(f"/api/v1/user/searches/{search_id}/")
Expand All @@ -175,7 +175,7 @@ def get_results(self, search_id: str) -> dict:
datasource (hostnames or scans) specified in the saved search.

Reference:
https://docs.urlscan.io/apis/urlscan-openapi/saved-searches/savedsearches-results
https://docs.urlscan.io/apis/urlscan-openapi/saved-searches/getsavedsearchresults

"""
return self.get_json(f"/api/v1/user/searches/{search_id}/results/")
10 changes: 5 additions & 5 deletions src/urlscan/pro/subscription.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ def get_subscriptions(self) -> dict:
dict: List of subscriptions.

Reference:
https://docs.urlscan.io/apis/urlscan-openapi/subscriptions/subscriptions
https://docs.urlscan.io/apis/urlscan-openapi/subscriptions/listsubscriptions

"""
return self.get_json("/api/v1/user/subscriptions/")
Expand Down Expand Up @@ -73,7 +73,7 @@ def create(
dict: Response containing the created subscription with an '_id' field.

Reference:
https://docs.urlscan.io/apis/urlscan-openapi/subscriptions/subscriptionscreate
https://docs.urlscan.io/apis/urlscan-openapi/subscriptions/createsubscription

"""
subscription: dict[str, Any] = _compact(
Expand Down Expand Up @@ -149,7 +149,7 @@ def update(
dict: Response containing the updated subscription with an '_id' field.

Reference:
https://docs.urlscan.io/apis/urlscan-openapi/subscriptions/subscriptionsget
https://docs.urlscan.io/apis/urlscan-openapi/subscriptions/updatesubscription

"""
subscription: dict[str, Any] = _compact(
Expand Down Expand Up @@ -189,7 +189,7 @@ def delete_subscription(self, *, subscription_id: str) -> dict:
dict: Empty response object confirming deletion.

Reference:
https://docs.urlscan.io/apis/urlscan-openapi/subscriptions/subscriptionsdelete
https://docs.urlscan.io/apis/urlscan-openapi/subscriptions/deletesubscription

"""
res = self._delete(f"/api/v1/user/subscriptions/{subscription_id}/")
Expand All @@ -206,7 +206,7 @@ def get_results(self, *, subscription_id: str, datasource: str) -> dict:
dict: Search results.

Reference:
https://docs.urlscan.io/apis/urlscan-openapi/subscriptions/subscriptionsresults
https://docs.urlscan.io/apis/urlscan-openapi/subscriptions/getsubscriptionresults

"""
return self.get_json(
Expand Down
2 changes: 1 addition & 1 deletion tests/integration/pro/test_structure_search.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from urlscan import Pro

# Reference UUID from urlscan.io documentation
# https://docs.urlscan.io/apis/urlscan-openapi/search/similarsearch
# https://docs.urlscan.io/apis/urlscan-openapi/search/getsimilarresults
REFERENCE_UUID = "68e26c59-2eae-437b-aeb1-cf750fafe7d7"


Expand Down