Skip to content

Commit 285e464

Browse files
denismakogondenismakogon
authored
Fn-powered truly-serverless apps with functions (#7)
Fn-powered truly-serverless apps with functions
1 parent 443de21 commit 285e464

9 files changed

Lines changed: 397 additions & 23 deletions

File tree

‎README.md‎

Lines changed: 63 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,8 @@ FDK is not only about developing functions, but providing necessary API to build
130130
that look like nothing but classes with methods powered by Fn.
131131

132132
```python
133+
import requests
134+
133135
from fdk.application import decorators
134136

135137

@@ -139,15 +141,28 @@ class Application(object):
139141
def __init__(self, *args, **kwargs):
140142
pass
141143

142-
@decorators.fn_route(fn_image="denismakogon/os.environ:latest")
144+
@decorators.with_fn(fn_image="denismakogon/os.environ:latest")
143145
def env(self, fn_data=None):
144146
return fn_data
145147

146-
@decorators.fn_route(fn_image="denismakogon/py-traceback-test:0.0.1",
147-
fn_format="http")
148+
@decorators.with_fn(fn_image="denismakogon/py-traceback-test:0.0.1",
149+
fn_format="http")
148150
def traceback(self, fn_data=None):
149151
return fn_data
150152

153+
@decorators.fn(fn_type="sync")
154+
def square(self, x, y, *args, **kwargs):
155+
return x * y
156+
157+
@decorators.fn(fn_type="sync", dependencies={
158+
"requests_get": requests.get
159+
})
160+
def request(self, *args, **kwargs):
161+
requests_get = kwargs["dependencies"].get("requests_get")
162+
r = requests_get('https://api.github.com/events')
163+
r.raise_for_status()
164+
return r.text
165+
151166
if __name__ == "__main__":
152167
app = Application(config={})
153168

@@ -161,6 +176,16 @@ if __name__ == "__main__":
161176
raise err
162177
print(res)
163178

179+
res, err = app.square(10, 20)
180+
if err:
181+
raise err
182+
print(res)
183+
184+
res, err = app.request()
185+
if err:
186+
raise err
187+
print(res)
188+
164189
```
165190
In order to identify to which Fn instance code needs to talk set following env var:
166191

@@ -191,12 +216,45 @@ Applications powered by Fn: working with function's result
191216

192217
In order to work with result from function you just need to read key-value argument `fn_data`:
193218
```python
194-
@decorators.fn_route(fn_image="denismakogon/py-traceback-test:0.0.1",
195-
fn_format="http")
219+
@decorators.with_fn(fn_image="denismakogon/py-traceback-test:0.0.1",
220+
fn_format="http")
196221
def traceback(self, fn_data=None):
197222
return fn_data
198223
```
199224

225+
Applications powered by Fn: advanced serverless functions
226+
---------------------------------------------------------
227+
228+
Since release v0.0.3 developer can consume new API to build truly serverless functions
229+
without taking care of Docker images, application, etc.
230+
231+
```python
232+
@decorators.fn(fn_type="sync")
233+
def square(self, x, y, *args, **kwargs):
234+
return x * y
235+
236+
@decorators.fn(fn_type="sync", dependencies={
237+
"requests_get": requests.get
238+
})
239+
def request(self, *args, **kwargs):
240+
requests_get = kwargs["dependencies"].get("requests_get")
241+
r = requests_get('https://api.github.com/events')
242+
r.raise_for_status()
243+
return r.text
244+
```
245+
246+
Each function decorated with `@decorator.fn` will become truly serverless and distributed.
247+
So, how it works?
248+
249+
* A developer writes function
250+
* FDK (Fn-powered app) creates a recursive Pickle v4.0 with 3rd-party dependencies
251+
* FDK (Fn-powered app) transfers pickled object to a function based on Python3 GPI (general purpose image)
252+
* FDK unpickles function and its 3rd-party dependencies and runs it
253+
* Function sends response back to Fn-powered application function caller
254+
255+
So, each CPU-intensive functions can be sent to Fn with the only load on networking (given example creates 7kB of traffic between app's host and Fn).
256+
257+
200258
Applications powered by Fn: exceptions
201259
--------------------------------------
202260

‎fdk/application/decorators.py‎

Lines changed: 125 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,11 @@
1212
# License for the specific language governing permissions and limitations
1313
# under the License.
1414

15+
import dill
1516
import functools
1617
import os
1718
import requests
18-
19+
import types
1920

2021
from fdk.application import errors
2122

@@ -69,10 +70,124 @@ def wrapper(*args, **kwargs):
6970
return wrapper
7071

7172

72-
def fn_route(fn_image=None, fn_type=None,
73-
fn_memory=256, fn_format=None,
74-
fn_timeout=60, fn_idle_timeout=200,
75-
fn_method="GET"):
73+
def fn(fn_type=None, fn_timeout=60, fn_idle_timeout=200, fn_memory=256,
74+
dependencies=None):
75+
"""
76+
Runs Python's function on general purpose Fn function
77+
78+
79+
What it does?
80+
81+
Decorator does following:
82+
- collects dat for Fn route and creates it
83+
- when function called, that function transforms
84+
into byte array (Pickle) then gets sent to general
85+
purpose Fn Python3 function
86+
- each external dependency (3rd-party libs)
87+
that are required for func gets transformed
88+
into byte array (Pickle)
89+
90+
It means that functions does't run locally but on Fn.
91+
92+
How is it different from other Python FDK functions?
93+
94+
- This function works with serialized Python callable objects via wire.
95+
Each function supplied with set of external dependencies that are
96+
represented as serialized functions, no matter if they are module-level,
97+
class-level Python objects
98+
99+
:param fn_type: Fn function call type
100+
:type fn_type: str
101+
:param fn_timeout: Fn function call timeout
102+
:type fn_timeout: int
103+
:param fn_idle_timeout: Fn function call idle timeout
104+
:type fn_idle_timeout: int
105+
:param fn_memory: Fn function memory limit
106+
:type fn_memory: int
107+
:param dependencies: Python's function 3rd-party callable dependencies
108+
:type dependencies: dict
109+
:return:
110+
"""
111+
fn_method = "POST"
112+
fn_image = "denismakogon/python3-fn-gpi:0.0.1"
113+
fn_format = "http"
114+
dependencies = dependencies if dependencies else {}
115+
116+
def ext_wrapper(action):
117+
@functools.wraps(action)
118+
def inner_wrapper(*f_args, **f_kwargs):
119+
fn_api_url = os.environ.get("API_URL")
120+
requests.get(fn_api_url).raise_for_status()
121+
self = f_args[0]
122+
fn_path = action.__name__.lower()
123+
if not hasattr(action, "__path_created"):
124+
fn_routes_url = "{}/v1/apps/{}/routes".format(
125+
fn_api_url, self.__class__.__name__.lower())
126+
resp = requests.post(fn_routes_url, json={
127+
"route": {
128+
"path": "/{}".format(fn_path),
129+
"image": fn_image,
130+
"memory": fn_memory if fn_memory else 256,
131+
"type": fn_type if fn_type else "sync",
132+
"format": fn_format if fn_format else "default",
133+
"timeout": fn_timeout if fn_timeout else 60,
134+
"idle_timeout": (fn_idle_timeout if
135+
fn_idle_timeout else 120),
136+
},
137+
})
138+
139+
try:
140+
resp.raise_for_status()
141+
except requests.HTTPError:
142+
resp.close()
143+
return Exception(resp.content)
144+
145+
setattr(action, "__path_created", True)
146+
147+
fn_path = action.__name__.lower()
148+
fn_exec_url = "{}/r/{}/{}".format(
149+
fn_api_url, self.__class__.__name__.lower(), fn_path)
150+
151+
action_in_bytes = dill.dumps(action, recurse=True)
152+
self_in_bytes = dill.dumps(self, recurse=True)
153+
154+
for name, method in dependencies.items():
155+
dependencies[name] = list(dill.dumps(method, recurse=True))
156+
157+
f_kwargs.update(dependencies=dependencies)
158+
req = requests.Request(
159+
method=fn_method, url=fn_exec_url,
160+
json={
161+
"is_coroutine": isinstance(action, types.CoroutineType),
162+
"action": list(action_in_bytes),
163+
"self": list(self_in_bytes),
164+
"args": f_args[1:],
165+
"kwargs": f_kwargs,
166+
}
167+
)
168+
session = requests.Session()
169+
resp = session.send(req.prepare())
170+
171+
try:
172+
resp.raise_for_status()
173+
except requests.HTTPError:
174+
resp.close()
175+
return None, errors.FnError(
176+
"{}/{}".format(self.__class__.__name__.lower(), fn_path),
177+
resp.content)
178+
179+
resp.close()
180+
return dill.loads(resp.content), None
181+
182+
return inner_wrapper
183+
184+
return ext_wrapper
185+
186+
187+
def with_fn(fn_image=None, fn_type=None,
188+
fn_memory=256, fn_format=None,
189+
fn_timeout=60, fn_idle_timeout=200,
190+
fn_method="GET"):
76191
"""
77192
Sets up Fn app route based on parameters given above
78193
:param fn_image: Docker image
@@ -99,6 +214,9 @@ def inner_wrapper(*f_args, **f_kwargs):
99214
requests.get(fn_api_url).raise_for_status()
100215
self = f_args[0]
101216
fn_path = action.__name__.lower()
217+
# TODO(xxx): hate code duplicates but extracting common function
218+
# to create a route breaks dill routine somehow
219+
# need to figure out how and fix that!
102220
if not hasattr(action, "__path_created"):
103221
fn_routes_url = "{}/v1/apps/{}/routes".format(
104222
fn_api_url, self.__class__.__name__.lower())
@@ -119,10 +237,11 @@ def inner_wrapper(*f_args, **f_kwargs):
119237
resp.raise_for_status()
120238
except requests.HTTPError:
121239
resp.close()
122-
return None, Exception(resp.content)
240+
return Exception(resp.content)
123241

124242
setattr(action, "__path_created", True)
125243

244+
fn_path = action.__name__.lower()
126245
fn_exec_url = "{}/r/{}/{}".format(
127246
fn_api_url, self.__class__.__name__.lower(), fn_path)
128247
req = requests.Request(method=fn_method,

‎fdk/tests/test_application.py‎

Lines changed: 74 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,12 @@
1212
# License for the specific language governing permissions and limitations
1313
# under the License.
1414

15+
import os
16+
import requests
17+
import testtools
18+
1519
from fdk.application import decorators
20+
from fdk.application import errors
1621

1722

1823
@decorators.fn_app
@@ -21,25 +26,82 @@ class Application(object):
2126
def __init__(self, *args, **kwargs):
2227
pass
2328

24-
@decorators.fn_route(fn_image="denismakogon/os.environ:latest")
29+
@decorators.with_fn(fn_image="denismakogon/os.environ:latest")
2530
def env(self, fn_data=None):
2631
return fn_data
2732

28-
@decorators.fn_route(fn_image="denismakogon/py-traceback-test:0.0.1",
29-
fn_format="http")
33+
@decorators.with_fn(fn_image="denismakogon/py-traceback-test:0.0.1",
34+
fn_format="http")
3035
def traceback(self, fn_data=None):
3136
return fn_data
3237

38+
@decorators.fn(fn_type="sync")
39+
def square(self, x, y, *args, **kwargs):
40+
return x * y
41+
42+
@decorators.fn(fn_type="sync", dependencies={
43+
"requests_get": requests.get
44+
})
45+
def request(self, *args, **kwargs):
46+
requests_get = kwargs["dependencies"].get("requests_get")
47+
r = requests_get('https://api.github.com/events')
48+
r.raise_for_status()
49+
return r.text
50+
51+
52+
class TestApplication(testtools.TestCase):
53+
54+
@testtools.skipIf(os.getenv("API_URL") is None,
55+
"API_URL is not set")
56+
def test_can_call_with_fn(self):
57+
app = Application(config={})
58+
res, err = app.env()
59+
self.assertIsNone(err, message=str(err))
3360

61+
@testtools.skipIf(os.getenv("API_URL") is None,
62+
"API_URL is not set")
63+
def test_can_get_traceback(self):
64+
app = Application(config={})
65+
res, err = app.traceback()
66+
self.assertIsNotNone(err, message=str(err))
67+
self.assertIsInstance(err, errors.FnError)
68+
69+
# @testtools.skipIf(os.getenv("API_URL") is None,
70+
# "API_URL is not set")
71+
# def test_can_call_fn_simple(self):
72+
# app = Application(config={})
73+
# res, err = app.square(10, 20)
74+
# self.assertIsNone(err, message=str(err))
75+
# self.assertEqual(int(res), 10 * 20)
76+
#
77+
# @testtools.skipIf(os.getenv("API_URL") is None,
78+
# "API_URL is not set")
79+
# def test_can_call_fn_with_deps(self):
80+
# app = Application(config={})
81+
# res, err = app.request()
82+
# self.assertIsNone(err, message=str(err))
83+
# self.assertNotNone(res)
84+
85+
86+
# NOTE: somehow dill pickler doesn't work fine in tests,
87+
# it gives pretty weird error:
88+
# _pickle.PicklingError: Can't pickle
89+
# <class 'fdk.tests.test_application.Application'>:
90+
# it's not the same object as fdk.tests.test_application.Application
91+
#
92+
# That why this module should be tested separately as CLI script
3493
if __name__ == "__main__":
35-
app = Application(config={})
94+
if os.getenv("API_URL") is not None:
95+
app = Application(config={})
3696

37-
res, err = app.env()
38-
if err:
39-
raise err
40-
print(res)
97+
res, err = app.square(10, 20)
98+
if err:
99+
raise err
100+
print(res)
41101

42-
res, err = app.traceback()
43-
if err:
44-
raise err
45-
print(res)
102+
res, err = app.request()
103+
if err:
104+
raise err
105+
print(res)
106+
else:
107+
print("API_URL not test, skipping...")

‎requirements.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
pbr!=2.1.0,>=2.0.0 # Apache-2.0
22
ujson==1.35
33
requests==2.18.4
4+
dill==0.2.7.1

‎samples/python3-fn-gpi/Dockerfile‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
FROM python:3.6.2
2+
3+
RUN mkdir /code
4+
ADD . /code/
5+
WORKDIR /code/
6+
RUN pip install -r requirements.txt
7+
8+
ENTRYPOINT ["python3", "func.py"]

0 commit comments

Comments
 (0)