diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..2a10c44 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +**/*.pyc +**/*__pycache__ diff --git a/.github/workflows/publish-docker.yaml b/.github/workflows/publish-docker.yaml new file mode 100644 index 0000000..75f9cb4 --- /dev/null +++ b/.github/workflows/publish-docker.yaml @@ -0,0 +1,96 @@ +name: Publish Docker Image + +on: + push: + branches: + - main + - mvp # TODO: remove after testing + # Publish semver tags as releases. + tags: + - '*.*.*' + pull_request: + branches: + - main + +env: + # Use docker.io for Docker Hub if empty + REGISTRY: ghcr.io + # github.repository as / + IMAGE_NAME: ${{ github.repository }} + + +jobs: + build: + + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + # This is used to complete the identity challenge + # with sigstore/fulcio when running outside of PRs. + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Install the cosign tool except on PR + # https://github.com/sigstore/cosign-installer + - name: Install cosign + if: github.event_name != 'pull_request' + uses: sigstore/cosign-installer@1e95c1de343b5b0c23352d6417ee3e48d5bcd422 + with: + cosign-release: 'v1.4.0' + + + # Workaround: https://github.com/docker/build-push-action/issues/461 + - name: Setup Docker buildx + uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf + + # Login against a Docker registry except on PR + # https://github.com/docker/login-action + - name: Log into registry ${{ env.REGISTRY }} + if: github.event_name != 'pull_request' + uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Extract metadata (tags, labels) for Docker + # https://github.com/docker/metadata-action + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=schedule + type=ref,event=branch + type=ref,event=tag + type=ref,event=pr + type=raw,value=latest,enable=${{ endsWith(github.ref, github.event.repository.default_branch) }} + + # Build and push Docker image with Buildx (don't push on PR) + # https://github.com/docker/build-push-action + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + # Sign the resulting Docker image digest except on PRs. + # This will only write to the public Rekor transparency log when the Docker + # repository is public to avoid leaking data. If you would like to publish + # transparency data even for private images, pass --force to cosign below. + # https://github.com/sigstore/cosign + - name: Sign the published Docker image + if: ${{ github.event_name != 'pull_request' }} + env: + COSIGN_EXPERIMENTAL: "true" + # This step uses the identity token to provision an ephemeral certificate + # against the sigstore community Fulcio instance. + run: cosign sign ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.build-and-push.outputs.digest }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2a10c44 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +**/*.pyc +**/*__pycache__ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e0daf6c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.10-slim + +SHELL ["/bin/bash", "-c"] + +RUN pip install poetry + +WORKDIR /src + +COPY pyproject.toml poetry.lock README.rst ./ + +RUN pip install -r <(poetry export) + +COPY ./virtual_site.py ./virtual_site.py + +RUN pip install . + +ENV PYTHONUNBUFFERED=true + +ENTRYPOINT ["virtual-site"] diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..f747041 --- /dev/null +++ b/README.rst @@ -0,0 +1,31 @@ +virtual-site +#### + +Example application showcasing OCS API interactions needed to fulfill +observation requests at a telescope site. + +You might find this helpful if you're looking to integrate your telescope's +control system with the Observatory Control System. Also, see +https://observatorycontrolsystem.github.io/integration/tcs/. + +Usage +---- + +You can spin up the client straight from docker:: + + $ docker run ghcr.io/observatorycontrolsystem/virtual-site --help + $ docker run ghcr.io/observatorycontrolsystem/virtual-site \ + --name ogg \ + --api-url http://localhost:8000/api/ \ + --api-token 'sutoken1234abcd' + +Development +---- + +Install dependencies:: + + $ poetry install + +Run:: + + $ poetry run virtual-site --name ogg --api-url http://localhost:8000/api/ --api-token 'sutoken1234abcd' diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..f64b692 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,223 @@ +[[package]] +name = "anyio" +version = "3.5.0" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +category = "main" +optional = false +python-versions = ">=3.6.2" + +[package.dependencies] +idna = ">=2.8" +sniffio = ">=1.1" + +[package.extras] +doc = ["packaging", "sphinx-rtd-theme", "sphinx-autodoc-typehints (>=1.2.0)"] +test = ["coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "pytest (>=6.0)", "pytest-mock (>=3.6.1)", "trustme", "contextlib2", "uvloop (<0.15)", "mock (>=4)", "uvloop (>=0.15)"] +trio = ["trio (>=0.16)"] + +[[package]] +name = "arrow" +version = "1.2.1" +description = "Better dates & times for Python" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +python-dateutil = ">=2.7.0" + +[[package]] +name = "certifi" +version = "2021.10.8" +description = "Python package for providing Mozilla's CA Bundle." +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "charset-normalizer" +version = "2.0.10" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" +optional = false +python-versions = ">=3.5.0" + +[package.extras] +unicode_backport = ["unicodedata2"] + +[[package]] +name = "click" +version = "8.0.3" +description = "Composable command line interface toolkit" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.4" +description = "Cross-platform colored terminal text." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "h11" +version = "0.12.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "httpcore" +version = "0.14.5" +description = "A minimal low-level HTTP client." +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +anyio = ">=3.0.0,<4.0.0" +certifi = "*" +h11 = ">=0.11,<0.13" +sniffio = ">=1.0.0,<2.0.0" + +[package.extras] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (>=1.0.0,<2.0.0)"] + +[[package]] +name = "httpx" +version = "0.21.3" +description = "The next generation HTTP client." +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +certifi = "*" +charset-normalizer = "*" +httpcore = ">=0.14.0,<0.15.0" +rfc3986 = {version = ">=1.3,<2", extras = ["idna2008"]} +sniffio = "*" + +[package.extras] +brotli = ["brotlicffi", "brotli"] +cli = ["click (>=8.0.0,<9.0.0)", "rich (>=10.0.0,<11.0.0)", "pygments (>=2.0.0,<3.0.0)"] +http2 = ["h2 (>=3,<5)"] + +[[package]] +name = "idna" +version = "3.3" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" +optional = false +python-versions = ">=3.5" + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "rfc3986" +version = "1.5.0" +description = "Validating URI References per RFC 3986" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +idna = {version = "*", optional = true, markers = "extra == \"idna2008\""} + +[package.extras] +idna2008 = ["idna"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "sniffio" +version = "1.2.0" +description = "Sniff out which async library your code is running under" +category = "main" +optional = false +python-versions = ">=3.5" + +[metadata] +lock-version = "1.1" +python-versions = "^3.10" +content-hash = "7da8f8eddbf76f22978afd3623653d05dfbd5807bcc749c9847283496b7e8ef6" + +[metadata.files] +anyio = [ + {file = "anyio-3.5.0-py3-none-any.whl", hash = "sha256:b5fa16c5ff93fa1046f2eeb5bbff2dad4d3514d6cda61d02816dba34fa8c3c2e"}, + {file = "anyio-3.5.0.tar.gz", hash = "sha256:a0aeffe2fb1fdf374a8e4b471444f0f3ac4fb9f5a5b542b48824475e0042a5a6"}, +] +arrow = [ + {file = "arrow-1.2.1-py3-none-any.whl", hash = "sha256:6b2914ef3997d1fd7b37a71ce9dd61a6e329d09e1c7b44f4d3099ca4a5c0933e"}, + {file = "arrow-1.2.1.tar.gz", hash = "sha256:c2dde3c382d9f7e6922ce636bf0b318a7a853df40ecb383b29192e6c5cc82840"}, +] +certifi = [ + {file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"}, + {file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"}, +] +charset-normalizer = [ + {file = "charset-normalizer-2.0.10.tar.gz", hash = "sha256:876d180e9d7432c5d1dfd4c5d26b72f099d503e8fcc0feb7532c9289be60fcbd"}, + {file = "charset_normalizer-2.0.10-py3-none-any.whl", hash = "sha256:cb957888737fc0bbcd78e3df769addb41fd1ff8cf950dc9e7ad7793f1bf44455"}, +] +click = [ + {file = "click-8.0.3-py3-none-any.whl", hash = "sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3"}, + {file = "click-8.0.3.tar.gz", hash = "sha256:410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b"}, +] +colorama = [ + {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, + {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, +] +h11 = [ + {file = "h11-0.12.0-py3-none-any.whl", hash = "sha256:36a3cb8c0a032f56e2da7084577878a035d3b61d104230d4bd49c0c6b555a9c6"}, + {file = "h11-0.12.0.tar.gz", hash = "sha256:47222cb6067e4a307d535814917cd98fd0a57b6788ce715755fa2b6c28b56042"}, +] +httpcore = [ + {file = "httpcore-0.14.5-py3-none-any.whl", hash = "sha256:2621ee769d0236574df51b305c5f4c69ca8f0c7b215221ad247b1ee42a9a9de1"}, + {file = "httpcore-0.14.5.tar.gz", hash = "sha256:435ab519628a6e2393f67812dea3ca5c6ad23b457412cd119295d9f906d96a2b"}, +] +httpx = [ + {file = "httpx-0.21.3-py3-none-any.whl", hash = "sha256:df9a0fd43fa79dbab411d83eb1ea6f7a525c96ad92e60c2d7f40388971b25777"}, + {file = "httpx-0.21.3.tar.gz", hash = "sha256:7a3eb67ef0b8abbd6d9402248ef2f84a76080fa1c839f8662e6eb385640e445a"}, +] +idna = [ + {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, + {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, +] +python-dateutil = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] +rfc3986 = [ + {file = "rfc3986-1.5.0-py2.py3-none-any.whl", hash = "sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97"}, + {file = "rfc3986-1.5.0.tar.gz", hash = "sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835"}, +] +six = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] +sniffio = [ + {file = "sniffio-1.2.0-py3-none-any.whl", hash = "sha256:471b71698eac1c2112a40ce2752bb2f4a4814c22a54a3eed3676bc0f5ca9f663"}, + {file = "sniffio-1.2.0.tar.gz", hash = "sha256:c4666eecec1d3f50960c6bdf61ab7bc350648da6c126e3cf6898d8cd4ddcd3de"}, +] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..a7e3729 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,18 @@ +[tool.poetry] +name = "virtual-site" +version = "0.1.0" +description = "Virtual OCS Site" +authors = ["Jashandeep Sohi "] + +[tool.poetry.dependencies] +python = "^3.10" +click = "^8.0.3" +httpx = "^0.21.3" +arrow = "^1.2.1" + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry.scripts] +virtual-site = "virtual_site:cli" diff --git a/virtual_site.py b/virtual_site.py new file mode 100644 index 0000000..36868a3 --- /dev/null +++ b/virtual_site.py @@ -0,0 +1,242 @@ +import logging +import asyncio + +import click +import httpx +import arrow + + +log = logging.getLogger(__name__) + + +@click.command() +@click.option( + "--name", + help="name of this site (e.g. 'ogg')", + required=True, + default="ogg" +) +@click.option( + "--api-url", + help="OCS Portal API root URL (e.g. 'http://localhost:8000/api/'", + required=True, +) +@click.option("--api-token", help="OCS Portal API auth token", required=True) +@click.option( + "--log-level", + help="log level", + type=click.Choice(["critical", "error", "warning", "info", "debug"]), + default="info", +) +def cli(name, api_url, api_token, log_level): + """ + A virtual controller that fulfills observation requests for a site. + """ + # Just setting up logging. + logging.basicConfig(level=log_level.upper()) + + s = Site(name, api_url, api_token) + + # We're using asyncio just to make concurrent programming a bit + # more legible. You can ignore the async/await syntax for the most part. + asyncio.run(s.run()) + + +class Site: + + name: str + + # Arrow is just like a datetime object but w/ full ISO 8601 + last_sync_time: arrow.Arrow + + observation_tasks: list["ObservationTask"] + + api_client: httpx.AsyncClient + + + def __init__(self, name: str, api_url: str, api_token: str): + self.name = name + + # setup a HTTP client we'll use to talk with API + self.api_client = httpx.AsyncClient( + base_url=api_url, + headers={ + "Authorization": f"Token {api_token}" + }, + follow_redirects=True + ) + + # this datetime is used to keep track of when to poll for a new schedule + # set to "0" (epoch start) initially to always sync the schedule on + # start-up + self.last_sync_time = arrow.get(0) + + # a list of running or scheduled observation tasks + self.observation_tasks = [] + + async def run(self) -> None: + """ + Run site tasks. + """ + try: + # we start polling the schedule in a coroutine + await self.poll_schedule() + except asyncio.CancelledError: # raised on CTRL-C + # gracefully shut down the HTTP client + await self.api_client.aclose() + + async def poll_schedule(self) -> None: + """ + Poll the schedule every 5 seconds and schedule observations locally + whenever a new schedule is detected. + """ + while True: + log.info("checking for new schedule") + + r = await self.api_client.get("/last_scheduled/") + last_scheduled_time = arrow.get(r.json()["last_schedule_time"]) + + if self.last_sync_time <= last_scheduled_time: + log.info("found new schedule") + + await self.sync_schedule() + + self.last_sync_time = last_scheduled_time + + await asyncio.sleep(5) + + async def sync_schedule(self) -> None: + """ + Retrieve the schedule from the API and create a ObservationTask for each + observation. + """ + log.info("synchronizing schedule") + + now = arrow.utcnow() + schedule = (await self.api_client.get( + f"/schedule/", + params={ + "site": self.name, + "start_after": now.isoformat(), + "start_before": now.shift(days=7).isoformat(), + "state": "PENDING", + } + )).json() + + log.info("cancelling previously scheduled tasks") + for ot in self.observation_tasks: + ot.cancel() + + log.info("scheduling new tasks") + observation_tasks = [] + + for obs in schedule["results"]: + ot = ObservationTask(spec=obs, api_client=self.api_client) + observation_tasks.append(ot) + + self.observation_tasks = observation_tasks + + log.info("schedule synchronized") + + +class ObservationTask: + """ + A ObservationTask carries out the observation on the telescope/instruments + (although in this case we just sleep). + + On instantiation, it creates a asyncio.Task to run in the background. + """ + + def __init__(self, spec: dict, api_client: httpx.AsyncClient): + self.spec = spec + self.api_client = api_client + self.started = False + + self.log = logging.getLogger(repr(self)) + self.task = asyncio.create_task(self.run()) + + async def run(self): + """ + Perform an "observation". + + This task sits idle until the observation start time and then mocks + the observation. + """ + start_time = arrow.get(self.spec["start"]) + + self.log.info( + f"waiting until observation start time: " + f"{start_time} ({start_time.humanize()})" + ) + await asyncio.sleep((start_time - arrow.utcnow()).total_seconds()) + self.started = True + self.log.info("observation started") + + for config in self.spec["request"]["configurations"]: + config_id = config["id"] + status_id = config["configuration_status"] + run_time = sum( + ic["exposure_time"] * ic["exposure_count"] + for ic in config["instrument_configs"] + ) + + self.log.info( + f"notify OCS that we're about to start config {config_id}" + ) + r = await self.api_client.patch( + f"/configurationstatus/{status_id}/", + json={"state": "ATTEMPTED"} + ) + self.log.info(f"OCS response: {r.content}") + if not r.is_success: + return + + self.log.info(f"beep bop taking some pictures, wait {run_time} sec") + config_start = arrow.utcnow() + await asyncio.sleep(run_time) + config_end = arrow.utcnow() + + time_completed = (config_end - config_start).total_seconds() + + self.log.info( + f"notify OCS that we're done with config {config_id}" + ) + r = await self.api_client.patch( + f"/configurationstatus/{status_id}/", + json={ + "state": "COMPLETED", + "summary": { + "state": "COMPLETED", + "start": str(config_start), + "end": str(config_end), + "time_completed": time_completed, + "reason": "", + "events": { + "msg": "completed by virtual-site", + "events": ["payload", "can", "be whatever you like"] + } + } + } + ) + self.log.info(f"OCS response: {r.content}") + if not r.is_success: + return + + self.log.info("observation finished") + + def cancel(self) -> None: + """ + Cancel this observation if it hasn't started yet. + """ + if not self.started: + self.log.info( + "skip cancelling because observation has already started" + ) + return + + self.log.info("cancelling") + self.task.cancel() + + def __repr__(self): + fqn = f"{self.__module__}.{self.__class__.__qualname__}" + return f"<{fqn} id={self.spec['id']!r} name={self.spec['name']!r}>"