Skip to content

Provide user ids in team + time entries #146

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mlahp7 wants to merge 21 commits into
base: main
Choose a base branch
from
Open

Provide user ids in team + time entries #146

Show file tree
Hide file tree
Changes from 18 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
b785339
WIP, pass down user_ids to the children of Team
mlahp7 Mar 15, 2024
de82323
Remove if for now
mlahp7 Mar 22, 2024
d39ad16
Fix time entry schema
mlahp7 Mar 22, 2024
e59ec0f
Another schema fix
mlahp7 Mar 22, 2024
437335e
Formatting
mlahp7 Mar 22, 2024
5661eab
Super weird, but when we don't have access to the task related to the…
mlahp7 Mar 22, 2024
0e39e9c
task custom type can be a number
mlahp7 Mar 22, 2024
ae8d960
list ids are nullable
mlahp7 Mar 22, 2024
b208d68
Folder id nullable
mlahp7 Mar 22, 2024
c0d3783
Another null field. It seems this may be a bug with the clickup api. …
mlahp7 Mar 22, 2024
cd3a7fe
Pass assignees via setting / conf
mlahp7 Mar 22, 2024
bef9070
Allow for time_entry start date to know when to start fetching time e…
mlahp7 Mar 22, 2024
efdf2fe
Add new conf to meltano.yml
mlahp7 Mar 22, 2024
e18b632
Fix conversion issue
mlahp7 Mar 22, 2024
f1ab1ce
Bad import
mlahp7 Mar 22, 2024
061b55c
Take 59 with format
mlahp7 Mar 22, 2024
9aa2d07
Attempt to use 'at' field from time entry as replication key
mlahp7 Mar 22, 2024
581e52b
Merge branch 'AutoIDM:main' into provide-user-ids-in-team
mcarriere May 17, 2024
cc47a89
Feedback tweaks (#1)
mlahp7 May 17, 2024
527af20
Update current status of Time Entries data in the readme
mlahp7 May 17, 2024
372c098
Merge branch 'main' into provide-user-ids-in-team
mlahp7 May 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions meltano.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@ plugins:
settings:
- name: api_token
kind: password
- name: time_entry_assignees
kind: string
- name: time_entry_start_date
kind: string
# config:
select:
- '!shared_hierarchy.*'
Expand Down
134 changes: 73 additions & 61 deletions tap_clickup/schemas/time_entries.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"type": "string"
},
"task": {
"type": "object",
"type": ["object", "string"],
"properties": {
"id": {
"type": "string"
Expand All @@ -20,84 +20,96 @@
"type": "string"
},
"color": {
"type": "string"
"type": [
"string",
"null"
]
},
"type": {
"type": "string"
},
"orderindex": {
"type": "integer"
}
} },
"custom_type": {
"type": "null"
}
} },
"wid": {
"type": "string"
},
"user": {
"type": "object",
"properties": {
"id": {
"type": "integer"
},
"username": {
"type": "string"
},
"email": {
"type": "string"
},
"color": {
"type": "string"
},
"initials": {
"type": "string"
},
"profilePicture": {
"type": "null"
}
} },
"billable": {
"type": "boolean"
},
"start": {
"type": "string"
},
"end": {
"type": "string"
"custom_type": {
"type": ["null", "integer"]
}
}
},
"wid": {
"type": "string"
},
"user": {
"type": "object",
"properties": {
"id": {
"type": "integer"
},
"duration": {
"username": {
"type": "string"
},
"description": {
"email": {
"type": "string"
},
"tags": {
"type": "array"
"color": {
"type": [
"string",
"null"
]
},
"source": {
"initials": {
"type": "string"
},
"at": {
"type": "string"
"profilePicture": {
"type": [
"string",
"null"
]
}
}
},
"billable": {
"type": "boolean"
},
"start": {
"type": "string"
},
"end": {
"type": "string"
},
"duration": {
"type": "string"
},
"description": {
"type": "string"
},
"tags": {
"type": "array"
},
"source": {
"type": "string"
},
"at": {
"type": "string"
},
"task_location": {
"type": "object",
"properties": {
"list_id": {
"type": ["string", "null"]
},
"task_location": {
"type": "object",
"properties": {
"list_id": {
"type": "string"
},
"folder_id": {
"type": "string"
},
"space_id": {
"type": "string"
}
}
"folder_id": {
"type": ["string", "null"]
},
"task_url": {
"type": "string"
"space_id": {
"type": ["string", "null"]
}
}
},
"task_url": {
"type": "string"
}
}
}
}
22 changes: 21 additions & 1 deletion tap_clickup/streams.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
"""Stream type classes for tap-clickup."""
from datetime import datetime
from pathlib import Path
from typing import Optional, Any, Dict
import requests
Expand All @@ -20,8 +21,11 @@ class TeamsStream(ClickUpStream):

def get_child_context(self, record: dict, context: Optional[dict]) -> dict:
"""Return a context dictionary for child streams."""
user_ids = [str(member.get("user", {}).get("id")) for member in record.get("members", []) if isinstance(member, dict)]

return {
"team_id": record["id"],
"user_ids": user_ids
}


Expand All @@ -31,12 +35,28 @@ class TimeEntries(ClickUpStream):
name = "time_entries"
path = "/team/{team_id}/time_entries"
primary_keys = ["id"]
replication_key = None
replication_key = "at"
schema_filepath = SCHEMAS_DIR / "time_entries.json"
records_jsonpath = "$.data[*]"
parent_stream_type = TeamsStream
# TODO not clear why this is needed
partitions = None
def get_url_params(
self, context: Optional[dict], next_page_token: Optional[Any]
) -> Dict[str, Any]:
"""Return a dictionary of values to be used in URL parameterization."""
params = super().get_url_params(context, next_page_token)

if "time_entry_start_date" in self.config:
# Formatted in ISO 8601, it must now be converted to milliseconds
start_date = datetime.strptime(self.config["time_entry_start_date"], "%Y-%m-%dT%H:%M:%SZ")
# Convert the datetime object to milliseconds
params["start_date"] = int(start_date.timestamp() * 1000)

@visch visch Mar 25, 2024

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just noticed this, we should really be using the at replication key here not just the start_date.

See https://sdk.meltano.com/en/latest/incremental_replication.html#incremental-replication , specefically self.get_starting_timestamp(context) , and instead of having a seperate start date for time_entry we should probably just drop the time_entry_start_date configuration and just document that start_date does what time_entry_start_date is currently documented to do

@mlahp7 mlahp7 May 17, 2024

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've added an if statement to fallback to the time_entry_start_date in the case that the state is empty on a user's first execution. Otherwise it would use the "at" field as the replication key as suggested.

@visch visch Jul 1, 2024

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why not use self.get_starting_timestamp(context) here? the start_date config variable is already use in this function so we don't need time_entry_start_date either, unless you think we need two seperate start date windows?

if "time_entry_assignees" in self.config:
params["assignee"] = self.config["time_entry_assignees"]
else:
params["assignee"] = ",".join(context["user_ids"])
return params


class SpacesStream(ClickUpStream):
Expand Down
16 changes: 16 additions & 0 deletions tap_clickup/tap.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,22 @@ class TapClickUp(Tap):
th.Property(
"api_token", th.StringType, required=True, description="Example: 'pk_12345"
),
th.Property(
"time_entry_assignees",
th.StringType,
required=False,
description="""By default, the extractor will get all user ids from your
team and use them when fetching time entries. If you want to fetch time entries
assigned to specific users, provide a comma-separated list of user IDs here. Ex. '420230,452346,784219'"""
),
th.Property(
"time_entry_start_date",
Comment thread
mlahp7 marked this conversation as resolved.
th.StringType,
required=False,
description="""The start date that determines how far back in time the extractor gets time entries.
Without this, only the last thirty days of time entries will be fetched.
Ex. '2023-01-01T00:00:00Z' to follow singer date format."""
),
# Removing "official" start_date support re https://github.com/AutoIDM/tap-clickup/issues/118
# th.Property(
# "start_date",
Expand Down