Change engine/.env to be committed and have .env.local override (#6150)

# Description of Changes
We keep adding stuff to `engine/config/.env.example` and have to
manually update `.env` because of it, which is really clunky, especially
when working on multiple worktrees at once. This PR changes it so that
we just have a committed `.env` file and have an `.env.local` override
to put the actual private keys into, which should make it a bit easier
to manage.

> [!warning]
>
> After this goes in, be very careful for a little while not to
accidentally commit any keys that you've got inside your `.env` file!

.dockerignore

@@ -65,6 +65,7 @@ README*
 .env
 .env.*
 !.env.example
+!engine/.env
 
 # Misc
 *.swp

.gitignore

@@ -165,6 +165,7 @@ __pycache__/
 # Virtual environments
 .env*
 !.env*.example
+!engine/.env
 .venv*
 env*/
 venv*/

.taskfiles/engine.yml

@@ -20,9 +20,8 @@ tasks:
       - uv run scripts/setup_env.py
     sources:
       - scripts/setup_env.py
-      - config/.env.example
     generates:
-      - .env
+      - .env.local
 
   run:
     desc: "Run engine server"

engine/.env

@@ -1,3 +1,9 @@
+###############################################################################
+# Environment variables used within the AI Engine.
+# Values can be overridden in the uncommitted sibling `.env.local` file.
+# Note: This file is committed to Git, so should not contain any private keys.
+###############################################################################
+
 # Configure the model strings passed to pydantic-ai. Provider credentials are handled by
 # pydantic-ai and should be set using the provider's native environment variables, for example
 # ANTHROPIC_API_KEY or OPENAI_API_KEY.

engine/.gitignore

@@ -19,7 +19,6 @@ yarn-error.log*
 .vite/
 
 # Environment
-.env
 .env.local
 
 # LaTeX outputs

engine/Dockerfile

@@ -12,9 +12,8 @@ RUN apt-get update \
 
 WORKDIR /app
 
-COPY pyproject.toml uv.lock Taskfile.yml ./
+COPY pyproject.toml uv.lock Taskfile.yml .env ./
 COPY .taskfiles/ ./.taskfiles/
-COPY config/ ./config/
 COPY scripts/ ./scripts/
 RUN --mount=type=cache,target=/root/.cache/uv \
     uv sync --frozen --no-dev

engine/scripts/setup_env.py

@@ -1,48 +1,24 @@
 """
-Copies .env from .env.example if missing, and errors if any keys from the example
+Ensures `.env.local` exists so developers have a place to put overrides
-are absent from the actual .env file.
+(API keys, local model choices, etc.) without touching the committed `.env`.
 
 Usage:
     uv run scripts/setup_env.py
 """
 
-import os
-import shutil
-import sys
 from pathlib import Path
 
-from dotenv import dotenv_values
-
 ROOT = Path(__file__).parent.parent
-EXAMPLE_FILE = ROOT / "config" / ".env.example"
+ENV_LOCAL_FILE = ROOT / ".env.local"
-ENV_FILE = ROOT / ".env"
 
-print("setup-env: see engine/config/.env.example for documentation")
+TEMPLATE = """\
+###############################################################################
+# Local overrides for `engine/.env`
+# Put API keys and machine-specific settings here. Any variable defined here
+# takes precedence over the committed `.env`
+###############################################################################
+"""
 
-if not EXAMPLE_FILE.exists():
+if not ENV_LOCAL_FILE.exists():
-    print(f"setup-env: {EXAMPLE_FILE.name} not found, skipping", file=sys.stderr)
+    ENV_LOCAL_FILE.write_text(TEMPLATE)
-    sys.exit(0)
+    print("setup-env: created empty .env.local for local overrides")
-
-if not ENV_FILE.exists():
-    shutil.copy(EXAMPLE_FILE, ENV_FILE)
-    print("setup-env: created .env from .env.example")
-
-env_keys = set(dotenv_values(ENV_FILE).keys()) | set(os.environ.keys())
-example_keys = set(dotenv_values(EXAMPLE_FILE).keys())
-missing = sorted(example_keys - env_keys)
-
-if missing:
-    sys.exit(
-        "setup-env: .env is missing keys from .env.example:\n"
-        + "\n".join(f"  {k}" for k in missing)
-        + "\n  Add them manually or delete your local .env to re-copy from config/.env.example."
-    )
-
-extra = sorted(k for k in dotenv_values(ENV_FILE) if k.startswith("STIRLING_") and k not in example_keys)
-if extra:
-    print(
-        "setup-env: .env contains STIRLING_ keys not in config/.env.example:\n"
-        + "\n".join(f"  {k}" for k in extra)
-        + "\n  Add them to config/.env.example if they are intentional.",
-        file=sys.stderr,
-    )

engine/src/stirling/config/settings.py

@@ -12,6 +12,7 @@ from pydantic_settings import BaseSettings, SettingsConfigDict
 
 ENGINE_ROOT = Path(__file__).resolve().parents[3]
 ENV_FILE = ENGINE_ROOT / ".env"
+ENV_LOCAL_FILE = ENGINE_ROOT / ".env.local"
 
 
 class RagBackend(StrEnum):
@@ -20,7 +21,7 @@ class RagBackend(StrEnum):
 
 
 class AppSettings(BaseSettings):
-    model_config = SettingsConfigDict(env_file=ENV_FILE, extra="ignore", populate_by_name=True)
+    model_config = SettingsConfigDict(env_file=(ENV_FILE, ENV_LOCAL_FILE), extra="ignore", populate_by_name=True)
 
     smart_model_name: str = Field(validation_alias="STIRLING_SMART_MODEL")
     fast_model_name: str = Field(validation_alias="STIRLING_FAST_MODEL")
@@ -74,6 +75,7 @@ def _configure_logging(level_name: str, log_file: str) -> None:
 @lru_cache(maxsize=1)
 def load_settings() -> AppSettings:
     load_dotenv(ENV_FILE)
+    load_dotenv(ENV_LOCAL_FILE, override=True)
     settings = AppSettings.model_validate({})
     _configure_logging(settings.log_level, settings.log_file)
     return settings

engine/src/stirling/rag/README.md

@@ -37,7 +37,9 @@ multi = RagCapability(runtime.rag_service, collections=["company-docs", "product
 everything = RagCapability(runtime.rag_service)
 ```
 
-## Config (.env)
+## Config
+
+Non-secret defaults live in the committed `engine/.env`:
 
 ```
 STIRLING_RAG_BACKEND=sqlite              # or "pgvector"
@@ -47,6 +49,11 @@ STIRLING_RAG_PGVECTOR_DSN=               # used when backend=pgvector
 STIRLING_RAG_CHUNK_SIZE=512
 STIRLING_RAG_CHUNK_OVERLAP=64
 STIRLING_RAG_TOP_K=5
+```
+
+Provider credentials (and any local overrides) go in the uncommitted `engine/.env.local`:
+
+```
 VOYAGE_API_KEY=your-key
 ```