Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 0 additions & 4 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -89,10 +89,6 @@ jobs:
working-directory: ${{ matrix.package }}
run: uv sync --frozen

- name: Building first to generate files
working-directory: ${{ matrix.package }}
run: uv build

- name: Type checking
if: matrix.python-version == env.TARGET_PYTHON_VERSION
working-directory: ${{ matrix.package }}
Expand Down
6 changes: 1 addition & 5 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,9 @@ To install uv, follow the [installation guide](https://docs.astral.sh/uv/getting

You will also need to setup the `pre-commit` hooks. Run `pre-commit install` in the root directory of the repository. If you don't have `pre-commit` installed, you can install it with `pip install pre-commit`.

> **Note**
> Currently our protobuf files will be generated during `hatch build`
> Please run this command once, to generate all necessary files.

### Testing

Run tests by entering the package directory and running `hatch test`.
Run tests by entering the package directory and running `poe test`. Packages that generate files from a submodule -- the flagd provider's protobuf modules, the flagd testkit's feature files -- generate them as the first step of that task, so there is nothing to run beforehand.

We use `pytest` for our unit testing, making use of `parametrized` to inject cases at scale.

Expand Down
30 changes: 30 additions & 0 deletions providers/openfeature-provider-flagd/hatch_build.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
"""Hatch build hook to ship the generated protobuf modules in the distribution.

Generating them is the job of hatch_build_protoc.py, which `poe generate-protos`
also runs; this hook only makes sure a build of a checkout is generated from the
submodule pin and that the gitignored results end up inside the wheel and sdist.
"""

import sys
from pathlib import Path

from hatchling.builders.hooks.plugin.interface import BuildHookInterface

# Hatchling loads this file by path rather than importing it as part of a
# package, so its directory is not on sys.path and the sibling generation module
# -- the single definition of how protoc is invoked -- would not be importable.
sys.path.insert(0, str(Path(__file__).parent))

from hatch_build_protoc import generate


class ProtobufGenerateHook(BuildHookInterface):
PLUGIN_NAME = "protobuf-generate"

def initialize(self, version: str, build_data: dict) -> None:
# Building from a checkout: regenerate, so what ships always matches the
# revision the submodule pin names. Building a wheel from an sdist: there
# is no submodule, `generate` finds nothing, and the modules are already
# in the tree because the sdist carries them.
outputs = generate()
build_data["artifacts"] += [path.as_posix() for path in outputs]
107 changes: 107 additions & 0 deletions providers/openfeature-provider-flagd/hatch_build_protoc.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
"""Generate the gRPC and protobuf modules from the flagd schemas submodule.

Used by `poe generate-protos` before the tests and the type check, and by the
hatch build hook (hatch_build.py) when a distribution is built. Generation is a
test *input*, not an artifact of the build, so it has to be runnable on its own:
requiring `uv build` to get importable code is what forced CI to build every
package on every Python version just to obtain these modules.

The .proto files are owned by open-feature/schemas and reach this package
through a git submodule, so nothing here can fork their definition. The
generated modules are gitignored and regenerated from whatever revision the
submodule pin names, which is what keeps the two from drifting apart unnoticed.
"""

import shutil
import subprocess
import sys
from pathlib import Path

ROOT = Path(__file__).parent
# protoc resolves imports and package paths against this, so the generated
# modules land under src/ mirroring their path here: a .proto at
# openfeature/schemas/protobuf/flagd/sync/v1/sync.proto becomes
# src/openfeature/schemas/protobuf/flagd/sync/v1/sync_pb2.py.
PROTO_PATH = "."
OUTPUT_PATH = "src"

# (protoc generator, generated-file suffix). `python` and `grpc_python` are
# built into grpc_tools.protoc; the two mypy_* ones are separate executables that
# mypy-protobuf installs, which is why it is a dev dependency and not only a
# build-time one.
GENERATORS = [
("python", "_pb2.py"),
("grpc_python", "_pb2_grpc.py"),
("mypy", "_pb2.pyi"),
("mypy_grpc", "_pb2_grpc.pyi"),
]
EXTERNAL_PLUGINS = ("mypy", "mypy_grpc")


def find_plugin(name: str) -> str | None:
"""Locate a protoc plugin executable next to the running interpreter.

protoc would find it on PATH, but only if the environment happens to be
activated. Resolving it against sys.executable means the plugin that ships
with *this* interpreter is used -- the dev environment when poe runs the
script, the isolated build environment when the build hook does.
"""
return shutil.which(f"protoc-gen-{name}", path=str(Path(sys.executable).parent))


def find_protos() -> list[Path]:
"""Every .proto under the package root, relative to it.

Dot-directories are skipped so a virtualenv inside the package directory
cannot drag site-packages' own .proto files into the generation.
"""
return sorted(
proto.relative_to(ROOT)
for proto in (ROOT / PROTO_PATH).glob("**/*.proto")
if not any(part.startswith(".") for part in proto.relative_to(ROOT).parts)
)


def outputs_for(protos: list[Path]) -> list[Path]:
"""The files `generate` writes for those inputs, relative to the root."""
return [
Path(OUTPUT_PATH) / proto.parent / f"{proto.stem}{suffix}"
for proto in protos
for _, suffix in GENERATORS
]


def generate() -> list[Path]:
"""Run protoc; return what it generated, relative to the package root.

Returns an empty list when there is no .proto to compile. That is the normal
case when building a wheel from an sdist: the submodule is not in the sdist
but the generated modules are, so there is nothing to do and nothing wrong.
"""
protos = find_protos()
if not protos:
return []

args = [sys.executable, "-m", "grpc_tools.protoc", "--proto_path", PROTO_PATH]
for name, _ in GENERATORS:
if name in EXTERNAL_PLUGINS:
plugin = find_plugin(name)
if plugin:
args.append(f"--plugin=protoc-gen-{name}={plugin}")
args.append(f"--{name}_out={OUTPUT_PATH}")
args += [str(proto) for proto in protos]

# Every argument is built here from the checked-in configuration above and
# from paths found inside this package, so there is no untrusted input.
subprocess.run(args, cwd=ROOT, check=True) # noqa: S603
return outputs_for(protos)


if __name__ == "__main__":
if not generate():
msg = (
f"No .proto files found under {ROOT}. Make sure submodules are "
"initialized: `git submodule update --init "
"providers/openfeature-provider-flagd/openfeature/schemas`."
)
raise SystemExit(msg)
30 changes: 12 additions & 18 deletions providers/openfeature-provider-flagd/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,11 @@ dev = [
"asserts>=0.14.0,<0.15.0",
"coverage[toml]>=7.10.0,<8.0.0",
"grpcio-health-checking>=1.82.1,<2.0.0",
# Needed to generate the protobuf modules before the tests and the type check;
# kept in step with the pin in the build hook's own dependencies.
"grpcio-tools==1.82.1",
"mypy>=1.18.0,<2.0.0",
"mypy-protobuf~=3.0",
"poethepoet>=0.37.0",
"pytest>=9.0.0,<10.0.0",
"pytest-bdd>=8.1.0,<9.0.0",
Expand All @@ -44,28 +48,17 @@ dev = [
"types-pyyaml>=6.0.0,<7.0.0",
]

[tool.hatch.build.hooks.protobuf]
generate_pyi = false
# The hook only ships the generated modules (hatch_build.py). Generating them is
# hatch_build_protoc.py, which `poe generate-protos` runs before the tests: they
# are a test input, not something a build should have to be run to obtain.
[tool.hatch.build.hooks.custom]
dependencies = [
# grpcio-tools >=1.82 requires protobuf>=7.35.1,<8.0.0
"protobuf>=7.35.1,<8.0.0",
"hatch-protobuf",
"mypy-protobuf~=3.0",
# Pin grpcio-tools to the same version as the runtime `grpcio` floor
"grpcio-tools==1.82.1",
]
proto_paths = [
".",
]
output_path = "src/"

[[tool.hatch.build.hooks.protobuf.generators]]
name = "mypy"
outputs = ["{proto_path}/{proto_name}_pb2.pyi"]

[[tool.hatch.build.hooks.protobuf.generators]]
name = "mypy_grpc"
outputs = ["{proto_path}/{proto_name}_pb2_grpc.pyi"]

[tool.hatch.build.targets.sdist]
exclude = [
Expand Down Expand Up @@ -117,11 +110,12 @@ openfeature-flagd-core = { workspace = true }
strict = true

[tool.poe.tasks]
test = "pytest tests"
test-cov = "coverage run -m pytest tests"
generate-protos = "python hatch_build_protoc.py"
test = ["generate-protos", {cmd = "pytest tests"}]
test-cov = ["generate-protos", {cmd = "coverage run -m pytest tests"}]
cov-report = "coverage xml"
cov = [
"test-cov",
"cov-report"
]
mypy = "mypy"
mypy = ["generate-protos", {cmd = "mypy"}]
Loading
Loading