improve: generalize transformations and scripts of runner and preloads into TemplateTransformer (#4487)
Bowen Liang
2 years ago
committed by
GitHub
parent
c255a20d7c
commit
5f4df34829
@ -1,58 +1,25 @@
|
||||
import json
|
||||
import re
|
||||
from typing import Optional
|
||||
from textwrap import dedent
|
||||
|
||||
from core.helper.code_executor.entities import CodeDependency
|
||||
from core.helper.code_executor.template_transformer import TemplateTransformer
|
||||
|
||||
NODEJS_RUNNER = """// declare main function here
|
||||
{{code}}
|
||||
|
||||
// execute main function, and return the result
|
||||
// inputs is a dict, unstructured inputs
|
||||
output = main({{inputs}})
|
||||
|
||||
// convert output to json and print
|
||||
output = JSON.stringify(output)
|
||||
|
||||
result = `<<RESULT>>${output}<<RESULT>>`
|
||||
|
||||
console.log(result)
|
||||
"""
|
||||
|
||||
NODEJS_PRELOAD = """"""
|
||||
|
||||
|
||||
class NodeJsTemplateTransformer(TemplateTransformer):
|
||||
@classmethod
|
||||
def transform_caller(cls, code: str, inputs: dict,
|
||||
dependencies: Optional[list[CodeDependency]] = None) -> tuple[str, str, list[CodeDependency]]:
|
||||
"""
|
||||
Transform code to python runner
|
||||
:param code: code
|
||||
:param inputs: inputs
|
||||
:return:
|
||||
"""
|
||||
|
||||
# transform inputs to json string
|
||||
inputs_str = json.dumps(inputs, indent=4, ensure_ascii=False)
|
||||
|
||||
# replace code and inputs
|
||||
runner = NODEJS_RUNNER.replace('{{code}}', code)
|
||||
runner = runner.replace('{{inputs}}', inputs_str)
|
||||
|
||||
return runner, NODEJS_PRELOAD, []
|
||||
|
||||
@classmethod
|
||||
def transform_response(cls, response: str) -> dict:
|
||||
"""
|
||||
Transform response to dict
|
||||
:param response: response
|
||||
:return:
|
||||
"""
|
||||
# extract result
|
||||
result = re.search(r'<<RESULT>>(.*)<<RESULT>>', response, re.DOTALL)
|
||||
if not result:
|
||||
raise ValueError('Failed to parse result')
|
||||
result = result.group(1)
|
||||
return json.loads(result)
|
||||
def get_runner_script(cls) -> str:
|
||||
runner_script = dedent(
|
||||
f"""
|
||||
// declare main function
|
||||
{cls._code_placeholder}
|
||||
|
||||
// decode and prepare input object
|
||||
var inputs_obj = JSON.parse(atob('{cls._inputs_placeholder}'))
|
||||
|
||||
// execute main function
|
||||
var output_obj = main(inputs_obj)
|
||||
|
||||
// convert output to json and print
|
||||
var output_json = JSON.stringify(output_obj)
|
||||
var result = `<<RESULT>>${{output_json}}<<RESULT>>`
|
||||
console.log(result)
|
||||
""")
|
||||
return runner_script
|
||||
|
||||
@ -1,83 +1,51 @@
|
||||
import json
|
||||
import re
|
||||
from base64 import b64encode
|
||||
from textwrap import dedent
|
||||
from typing import Optional
|
||||
|
||||
from core.helper.code_executor.entities import CodeDependency
|
||||
from core.helper.code_executor.template_transformer import TemplateTransformer
|
||||
|
||||
PYTHON_RUNNER = dedent("""
|
||||
# declare main function here
|
||||
{{code}}
|
||||
|
||||
from json import loads, dumps
|
||||
from base64 import b64decode
|
||||
|
||||
# execute main function, and return the result
|
||||
# inputs is a dict, and it
|
||||
inputs = b64decode('{{inputs}}').decode('utf-8')
|
||||
output = main(**json.loads(inputs))
|
||||
|
||||
# convert output to json and print
|
||||
output = dumps(output, indent=4)
|
||||
|
||||
result = f'''<<RESULT>>
|
||||
{output}
|
||||
<<RESULT>>'''
|
||||
|
||||
print(result)
|
||||
""")
|
||||
|
||||
PYTHON_PRELOAD = """"""
|
||||
|
||||
PYTHON_STANDARD_PACKAGES = {
|
||||
'json', 'datetime', 'math', 'random', 're', 'string', 'sys', 'time', 'traceback', 'uuid', 'os', 'base64',
|
||||
'hashlib', 'hmac', 'binascii', 'collections', 'functools', 'operator', 'itertools', 'uuid',
|
||||
}
|
||||
|
||||
|
||||
class Python3TemplateTransformer(TemplateTransformer):
|
||||
@classmethod
|
||||
def transform_caller(cls, code: str, inputs: dict,
|
||||
dependencies: Optional[list[CodeDependency]] = None) -> tuple[str, str, list[CodeDependency]]:
|
||||
"""
|
||||
Transform code to python runner
|
||||
:param code: code
|
||||
:param inputs: inputs
|
||||
:return:
|
||||
"""
|
||||
|
||||
# transform inputs to json string
|
||||
inputs_str = b64encode(json.dumps(inputs, ensure_ascii=False).encode()).decode('utf-8')
|
||||
|
||||
# replace code and inputs
|
||||
runner = PYTHON_RUNNER.replace('{{code}}', code)
|
||||
runner = runner.replace('{{inputs}}', inputs_str)
|
||||
|
||||
# add standard packages
|
||||
if dependencies is None:
|
||||
dependencies = []
|
||||
|
||||
for package in PYTHON_STANDARD_PACKAGES:
|
||||
if package not in dependencies:
|
||||
dependencies.append(CodeDependency(name=package, version=''))
|
||||
|
||||
# deduplicate
|
||||
dependencies = list({dep.name: dep for dep in dependencies if dep.name}.values())
|
||||
def get_standard_packages(cls) -> set[str]:
|
||||
return {
|
||||
'base64',
|
||||
'binascii',
|
||||
'collections',
|
||||
'datetime',
|
||||
'functools',
|
||||
'hashlib',
|
||||
'hmac',
|
||||
'itertools',
|
||||
'json',
|
||||
'math',
|
||||
'operator',
|
||||
'os',
|
||||
'random',
|
||||
're',
|
||||
'string',
|
||||
'sys',
|
||||
'time',
|
||||
'traceback',
|
||||
'uuid',
|
||||
}
|
||||
|
||||
return runner, PYTHON_PRELOAD, dependencies
|
||||
|
||||
@classmethod
|
||||
def transform_response(cls, response: str) -> dict:
|
||||
"""
|
||||
Transform response to dict
|
||||
:param response: response
|
||||
:return:
|
||||
"""
|
||||
# extract result
|
||||
result = re.search(r'<<RESULT>>(.*?)<<RESULT>>', response, re.DOTALL)
|
||||
if not result:
|
||||
raise ValueError('Failed to parse result')
|
||||
result = result.group(1)
|
||||
return json.loads(result)
|
||||
def get_runner_script(cls) -> str:
|
||||
runner_script = dedent(f"""
|
||||
# declare main function
|
||||
{cls._code_placeholder}
|
||||
|
||||
import json
|
||||
from base64 import b64decode
|
||||
|
||||
# decode and prepare input dict
|
||||
inputs_obj = json.loads(b64decode('{cls._inputs_placeholder}').decode('utf-8'))
|
||||
|
||||
# execute main function
|
||||
output_obj = main(**inputs_obj)
|
||||
|
||||
# convert output to json and print
|
||||
output_json = json.dumps(output_obj, indent=4)
|
||||
result = f'''<<RESULT>>{{output_json}}<<RESULT>>'''
|
||||
print(result)
|
||||
""")
|
||||
return runner_script
|
||||
|
||||
Loading…
Reference in New Issue