Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
45 commits
Select commit Hold shift + click to select a range
18c2a0d
chore: remove jinja2
0hsn Oct 5, 2023
ef601cd
chore: add better debugging tool
0hsn Oct 5, 2023
11349f0
chore: add pydentic pkg
0hsn Oct 5, 2023
96d5a30
feat: reusable test fixtures
0hsn Oct 7, 2023
082fa75
feat: workflow document with test
0hsn Oct 7, 2023
430e65d
feat: workflow driver with tests
0hsn Oct 7, 2023
32adb54
refactor: update file name parse logic
0hsn Oct 8, 2023
fdcb43f
feat: entity for internal chkware tasks
0hsn Oct 8, 2023
30f94c3
feat: entity for internal chkware validate task
0hsn Oct 8, 2023
d96fd89
feat: parsing tasks in workflow
0hsn Oct 8, 2023
cb39224
refactor: remove unused
0hsn Oct 8, 2023
8fec50d
chore: package version update
0hsn Oct 12, 2023
6010981
refactor: versioned document v2 moved to core document
0hsn Oct 14, 2023
a85517a
feat: support versioned document v2 in VariableTableManager.handle
0hsn Oct 14, 2023
9465cfb
feat: process task signature
0hsn Oct 14, 2023
ceb62ed
chore: update packages
0hsn Oct 29, 2023
bcf35ad
feat: disallow extra data in input yaml
0hsn Oct 29, 2023
3376eff
feat: move ParsedTask to entities
0hsn Oct 29, 2023
ff79531
refactor: use __future__.annotations
0hsn Oct 29, 2023
52f423f
fix: add missing field
0hsn Oct 29, 2023
4780e52
feat: use parsed task instead of dict
0hsn Oct 29, 2023
820e3f0
refactor: move all domain code to main module
0hsn Nov 1, 2023
e41d2aa
feat: add ExecResponse to capture generic response
0hsn Nov 2, 2023
03e83d3
feat: add call api to fetch module
0hsn Nov 2, 2023
8b943b4
refactor: type isolation on replace_value
0hsn Nov 2, 2023
0ae2c0b
feat: tests for fetch.call
0hsn Nov 2, 2023
12afec3
refactor: convert static method to class method for VariableTableManager
0hsn Nov 3, 2023
0d025a9
refactor: move loading envvars in handle_environment
0hsn Nov 3, 2023
032b508
refactor: reflect new api change from platform
0hsn Nov 3, 2023
8f17e22
feat: add slugify helper method
0hsn Nov 3, 2023
0f4d6f5
feat: type of workflow supported stirngs
0hsn Nov 3, 2023
d52677f
feat: support name and id for workflow
0hsn Nov 3, 2023
571592f
feat: add name and id to workflow doc
0hsn Nov 3, 2023
b4b1686
feat: initial workflow run for fetch done
0hsn Nov 3, 2023
e53c24c
fix: tests for WorkflowDocument
0hsn Nov 5, 2023
6de58ae
fix: remove load env test for combine_initial_variables
0hsn Nov 5, 2023
7cb8031
feat: remove unused param doc
0hsn Nov 5, 2023
0f66a3e
fix: update json data struct as the API changed
0hsn Nov 5, 2023
c44a6e4
feat: use class signature from return
0hsn Nov 5, 2023
ca0be62
feat: add extra data field for ExecResponse
0hsn Nov 5, 2023
d5e3245
feat: call function in validate module
0hsn Nov 5, 2023
93e7b19
feat: simple data validation spec
0hsn Nov 5, 2023
b36545b
feat: dumping output for workflow
0hsn Nov 5, 2023
85cc90d
chore: update packages
0hsn Nov 15, 2023
4d95d39
refactor: update to avoid slow regular expressions is security-sensitive
0hsn Nov 15, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Pipfile
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,18 @@ requests = "*"
cerberus = "*"
defusedxml = "*"
xmltodict = "*"
jinja2 = "*"
python-dotenv = "*"
pydantic = "*"

[dev-packages]
pytest = "*"
mypy = "*"
pylint = "*"
flake8 = "*"
black = {extras = ["d"], version = "*"}
var-dump = "*"
types-pyyaml = "*"
types-requests = "*"
icecream = "*"

[requires]
python_version = "3.11"
249 changes: 173 additions & 76 deletions Pipfile.lock

Large diffs are not rendered by default.

6 changes: 1 addition & 5 deletions chk/console/main.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
"""Commands"""

import typing
from os import environ

import click
from dotenv import load_dotenv

import chk.modules.fetch as fetch_executor
import chk.modules.validate as validate_executor
Expand All @@ -30,9 +28,7 @@ def combine_initial_variables(external_vars: str, **kwargs: typing.Any) -> dict:
"""Reads a json string and converts to dict, and combines with env and dotenv
variables"""

load_dotenv()

return load_variables_as_dict(external_vars, **kwargs) | {"_ENV": dict(environ)}
return load_variables_as_dict(external_vars, **kwargs)


def after_hook(resp: object) -> None:
Expand Down
10 changes: 10 additions & 0 deletions chk/infrastructure/document.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import dataclasses

import cerberus
from pydantic import BaseModel, Field

from chk.infrastructure.file_loader import FileContext

Expand All @@ -18,6 +19,15 @@ class VersionedDocument:
version: str = dataclasses.field(default_factory=str)


class VersionedDocumentV2(BaseModel):
"""
versioned document entity
"""

context: tuple = Field(default_factory=tuple)
version: str = Field(default_factory=str)


class VersionedDocumentSupport:
"""DocumentVersionSupport"""

Expand Down
3 changes: 2 additions & 1 deletion chk/infrastructure/file_loader.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""
File loader utility
"""
from __future__ import annotations
import hashlib
from typing import NamedTuple
from pathlib import Path
Expand Down Expand Up @@ -81,7 +82,7 @@ class FileContext(NamedTuple):
filepath_hash: str = ""

@staticmethod
def from_file(file: str, **kwarg: dict) -> "FileContext":
def from_file(file: str, **kwarg: dict) -> FileContext:
FileLoader.is_file_ok(file)
absolute_path = str(Path(file).absolute())
fpath_hash = hashlib.sha256(absolute_path.encode("utf-8")).hexdigest()
Expand Down
14 changes: 14 additions & 0 deletions chk/infrastructure/helper.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
Helper functions module
"""
import ast
import re
from collections.abc import Callable
from typing import Any

Expand Down Expand Up @@ -175,3 +176,16 @@ def formatter(message: object, cb: Callable = str, dump: bool = True) -> str:
click.echo(printable)

return printable


def slugify(string: str) -> str:
"""Make slug out of string"""

if not isinstance(string, str):
raise TypeError("slugify: only string value supported.")

string = string.lower().strip()
string = re.sub(r"[^\w\s-]", "", string)
string = re.sub(r"[\s_-]+", "-", string)
string = re.sub(r"-{2,}", "", string)
return string
67 changes: 55 additions & 12 deletions chk/infrastructure/symbol_table.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,15 @@
Symbol and variable management
"""
import enum
import os
import typing
from collections import UserDict
from collections.abc import Callable

from chk.infrastructure.document import VersionedDocument
from dotenv import load_dotenv
from pydantic import BaseModel, ConfigDict, Field

from chk.infrastructure.document import VersionedDocument, VersionedDocumentV2
from chk.infrastructure.file_loader import FileContext, ExecuteContext
from chk.infrastructure.helper import data_get
from chk.infrastructure.templating import StrTemplate
Expand Down Expand Up @@ -59,6 +64,28 @@ class ExposableVariables(UserDict):
"""Holds data for a expose data"""


class ExecResponse(BaseModel):
"""ExecResponse"""

model_config = ConfigDict(arbitrary_types_allowed=True)

file_ctx: FileContext
exec_ctx: ExecuteContext
variables_exec: ExposableVariables
variables: Variables
extra: object = Field(default=None)


@typing.overload
def replace_value(doc: dict, var_s: dict) -> dict:
...


@typing.overload
def replace_value(doc: list, var_s: dict) -> list:
...


def replace_value(doc: dict | list, var_s: dict) -> dict | list:
"""
replace variables with values
Expand All @@ -79,9 +106,12 @@ def replace_value(doc: dict | list, var_s: dict) -> dict | list:
class VariableTableManager:
"""VariableTableManager"""

@staticmethod
@classmethod
def handle(
variable_doc: Variables, document: VersionedDocument, exec_ctx: ExecuteContext
cls,
variable_doc: Variables,
document: VersionedDocument | VersionedDocumentV2,
exec_ctx: ExecuteContext,
) -> None:
"""Handles variable handling

Expand All @@ -90,20 +120,22 @@ def handle(
document: VersionedDocument of document data
exec_ctx: ExecuteContext; passed ExecuteContext
"""
# load environment variables
cls.handle_environment(variable_doc)

# make file contexts out od tuple
# make file contexts out of tuple
file_ctx = FileContext(*document.context)

if variables := data_get(file_ctx.document, VariableConfigNode.VARIABLES):
VariableTableManager.handle_absolute(variable_doc, variables)
cls.handle_absolute(variable_doc, variables)

VariableTableManager.handle_execute_context(variable_doc, exec_ctx)
cls.handle_execute_context(variable_doc, exec_ctx)

if variables:
VariableTableManager.handle_composite(variable_doc, variables)
cls.handle_composite(variable_doc, variables)

@staticmethod
def handle_absolute(variable_doc: Variables, document: dict) -> None:
@classmethod
def handle_absolute(cls, variable_doc: Variables, document: dict) -> None:
"""Handles absolute variables and values from document

Args:
Expand All @@ -117,8 +149,9 @@ def handle_absolute(variable_doc: Variables, document: dict) -> None:

variable_doc[key] = val

@staticmethod
@classmethod
def handle_composite(
cls,
variable_doc: Variables,
document: dict,
replace_callback: Callable = replace_value,
Expand All @@ -143,9 +176,9 @@ def handle_composite(
for key, val in replaced_values.items():
variable_doc[key] = val

@staticmethod
@classmethod
def handle_execute_context(
variable_doc: Variables, exec_ctx: ExecuteContext
cls, variable_doc: Variables, exec_ctx: ExecuteContext
) -> None:
"""Handle variables passed from external context

Expand All @@ -159,6 +192,16 @@ def handle_execute_context(
for key, val in ext_vars.items():
variable_doc[key] = val

@classmethod
def handle_environment(cls, variable_doc: Variables) -> None:
"""Handle variables passed from external context

Args:
variable_doc: Variables; Variable store
"""
load_dotenv()
variable_doc["_ENV"] = dict(os.environ)


class ExposeManager:
"""ExposeManager handles all expose related functionality"""
Expand Down
4 changes: 2 additions & 2 deletions chk/infrastructure/third_party/http_fetcher.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
"""
Driver to handle requests library (https://pypi.org/project/requests/)
"""

from __future__ import annotations
from collections import UserDict

import requests
Expand Down Expand Up @@ -37,7 +37,7 @@ def as_fmt_str(self) -> str:
return presentation

@staticmethod
def from_response(response: requests.Response) -> "ApiResponse":
def from_response(response: requests.Response) -> ApiResponse:
"""Create a ApiResponse object from requests.Response object

Args:
Expand Down
47 changes: 32 additions & 15 deletions chk/modules/fetch.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""
Fetch module
"""
from __future__ import annotations
import dataclasses
import enum
import json
Expand All @@ -25,6 +26,7 @@
EXPOSE_SCHEMA as EXP_SCHEMA,
ExposeManager,
ExposableVariables,
ExecResponse,
)

from chk.infrastructure.third_party.http_fetcher import (
Expand Down Expand Up @@ -317,7 +319,7 @@ class HttpDocument(VersionedDocument):
request: dict = dataclasses.field(default_factory=dict)

@staticmethod
def from_file_context(ctx: FileContext) -> "HttpDocument":
def from_file_context(ctx: FileContext) -> HttpDocument:
"""Create a HttpDocument from FileContext
:param ctx: FileContext to create the HttpDocument from
"""
Expand Down Expand Up @@ -348,7 +350,7 @@ class ApiResponseDict(UserDict):
body_as_dict: dict

@staticmethod
def from_api_response(resp: ApiResponse) -> "ApiResponseDict":
def from_api_response(resp: ApiResponse) -> ApiResponseDict:
"""Create JsonApiResponse from ApiResponse

Args:
Expand Down Expand Up @@ -508,18 +510,10 @@ def display(expose_list: list, exec_ctx: ExecuteContext) -> None:
formatter(_to_display, dump=exec_ctx.options["dump"])


def execute(
ctx: FileContext, exec_ctx: ExecuteContext, cb: abc.Callable = lambda *args: ...
) -> None:
"""Run a http document

Args:
ctx: FileContext object to handle
exec_ctx: ExecuteContext
cb: Callable
"""
def call(file_ctx: FileContext, exec_ctx: ExecuteContext) -> ExecResponse:
"""Call a http document"""

http_doc = HttpDocument.from_file_context(ctx)
http_doc = HttpDocument.from_file_context(file_ctx)

DocumentVersionMaker.verify_if_allowed(
DocumentVersionMaker.from_dict(http_doc.as_dict), VERSION_SCOPE
Expand All @@ -536,9 +530,32 @@ def execute(
response = HttpDocumentSupport.execute_request(http_doc)
output_data = ExposableVariables({"_response": response.data})

return ExecResponse(
file_ctx=file_ctx,
exec_ctx=exec_ctx,
variables_exec=output_data,
variables=variable_doc,
)


def execute(
ctx: FileContext, exec_ctx: ExecuteContext, cb: abc.Callable = lambda *args: ...
) -> None:
"""Call with a http document

Args:
ctx: FileContext object to handle
exec_ctx: ExecuteContext
cb: Callable
"""

exec_response = call(file_ctx=ctx, exec_ctx=exec_ctx)

http_doc = HttpDocument.from_file_context(exec_response.file_ctx)
exposed_data = ExposeManager.get_exposed_replaced_data(
http_doc, {**variable_doc.data, **output_data.data}
http_doc,
{**exec_response.variables.data, **exec_response.variables_exec.data},
)

cb({ctx.filepath_hash: output_data.data})
cb({ctx.filepath_hash: exec_response.variables_exec.data})
HttpDocumentSupport.display(exposed_data, exec_ctx)
Loading