NNO Provisioning Service

┌─── Trigger paths ────────────────────────────────────────────────┐
│                                                                   │
│  1. Registration (automatic)                                      │
│     services/iam sign-up ──► platform.registered (CF Queue)      │
│                                                                   │
│  2. Feature activation (self-service)                             │
│     services/registry PATCH /features ──► feature.activating     │
│                              (CF Queue)                           │
│                                                                   │
│  3. Operator override (manual)                                    │
│     NNO Zero UI ──► services/gateway ──► POST /provision/*       │
│                                                                   │
└──────────────────────────────┬────────────────────────────────────┘
                               │
                               ▼
                  NNO Provisioning Service
                  (CF Queue consumer)
                               │
               ┌───────────────┼───────────────┐
               ▼               ▼               ▼
         Cloudflare API   NNO Registry    services/billing
         (CF Workers,     (platform,      (Stripe customer
          D1, R2, KV,      resources,      creation)
          Pages)           feature status)

Client sign-up
  │
  ▼
services/iam
  ├── Create user record in IAM D1
  ├── Create platform record in Registry
  └── Enqueue: platform.registered → CF Queue
                    │
                    ▼
         services/provisioning (queue consumer)
           ├── Deploy auth Worker        [always]
           ├── Create auth D1 + migrate  [always]
           └── Create Stripe customer    [always]

Client activates "Analytics" feature
  │
  ▼
services/registry
  ├── Quota check via services/billing   ← plan gate
  ├── PATCH feature_activation status → 'activating'
  └── Enqueue: feature.activating → CF Queue
                    │
                    ▼
         services/provisioning (queue consumer)
           ├── Read FeatureManifest.resources
           │   { d1: true, worker: true, minimumPlan: 'growth' }
           ├── Create analytics D1 + run migrations
           ├── Deploy analytics Worker
           └── PATCH feature_activation status → 'active'

Platform admin activates "[email protected]" from portal UI
  │
  ▼
services/registry
  ├── Fetch StackDefinition from services/stack-registry (for template-based stacks)
  │   OR use inline definition (for platform-local stacks)
  ├── Validate all featureIds exist in feature catalogue
  ├── Quota check via services/billing (minimumPlan gate)
  ├── Create stacks record (status: PENDING)
  └── Enqueue: stack.activating → CF Queue
                    │
                    ▼
         services/provisioning (queue consumer)
           ├── Fetch StackDefinition from stack-registry (step 1)
           ├── Provision shared resources (D1, R2, KV as declared)
           └── Activate each feature with shared resource bindings

{
  platformId: string;
  planTier: string;              // 'starter' | 'growth' | 'scale'
  billingEmail: string;          // billing contact email
  defaultEntityId?: string;      // optional pre-generated tenant ID for the default tenant
  environment?: 'dev' | 'stg' | 'prod';  // default: 'prod'
}

{
  platformId: string;
  entityId: string; // required — ID of the entity (tenant/sub-tenant) activating the feature
  featureId: string; // e.g. 'analytics'
  featureVersion: string; // e.g. '1.2.0'
  environment: "dev" | "stg" | "prod";
  resources: FeatureResourceRequirements; // read from FeatureManifest.resources at activation time
}

interface FeatureResourceRequirements {
  worker?: boolean; // Deploy a CF Worker for this feature's backend API
  pages?: boolean; // Create a CF Pages project (SPA portal)
  d1?: boolean; // Create a D1 SQLite database
  r2?: boolean; // Create an R2 object storage bucket
  kv?: boolean; // Create a KV namespace
  queue?: boolean; // Create a Cloudflare Queue
  minimumPlan?: "starter" | "growth" | "scale"; // Plan gate — checked before job is queued
}

{
  platformId: string;
  entityId: string;                         // required — ID of the entity whose feature resources are being removed
  featureId: string;
  environment: string;
  deleteData?: boolean;      // default: false — preserve D1 data
}

{
  platformId: string;
  stackInstanceId: string; // pre-created stacks record
  stackDefinition: StackDefinition; // resolved from stack-registry or inline (local)
  environment: "dev" | "stg" | "prod";
}

// Bindings set on each feature Worker in the stack (at activation time)
{
  STACK_DB:      sharedD1?.cfId ?? undefined,
  STACK_STORAGE: sharedR2?.bucketName ?? undefined,
  STACK_KV:      sharedKv?.namespaceId ?? undefined,
  STACK_ID:      stackInstanceId,
}

D1:     {platformId}-{stackId}-db-{env}              e.g. k3m9p2xw7q-saas-starter-db-prod
R2:     {platformId}-{stackId}-storage-{env}         e.g. k3m9p2xw7q-saas-starter-storage-prod
KV:     {platformId}-{stackId}-kv-{env}              e.g. k3m9p2xw7q-saas-starter-kv-prod
Worker: {platformId}-{stackId}-orchestrator-{env}    e.g. k3m9p2xw7q-saas-starter-orchestrator-prod

Client submits custom domain (e.g. app.acmecorp.com) for dashboard.app.x7y8z9w0q1.a1b2c3d4e5.nno.app
  │
  ▼
Registry: create dns_records row (status: pending_validation)
  │
  ▼
Provisioning: POST CF4SaaS custom hostname API
  → CF returns validation TXT record details
  │
  ▼
Registry: update dns_records row with validation details
Client: add CNAME + TXT records in their DNS registrar
  │
  ▼
Provisioning: poll CF4SaaS until SSL status = active (or timeout)
  → CF validates ownership and provisions TLS certificate automatically
  │
  ▼
Registry: update dns_records row (status: active)

                     ┌──────────┐
                     │  PENDING │  ← job created in Registry
                     └────┬─────┘
                          │ worker picks up job
                     ┌────▼─────┐
                     │ RUNNING  │  ← steps executing
                     └────┬─────┘
              ┌───────────┼───────────┐
              │           │           │
         ┌────▼────┐  ┌───▼───┐  ┌───▼──────┐
         │COMPLETED│  │FAILED │  │ TIMED_OUT│
         └─────────┘  └───┬───┘  └────┬─────┘
                          │            │
                    ┌──── ▼────────────▼──────┐
                    │  retry_count < max_retries│
                    └────┬──────────────────────┘
                         │ yes                no
                    ┌────▼──────┐       ┌────────────┐
                    │  PENDING  │       │ ROLLING_BACK│
                    │(re-queued)│       └─────┬───────┘
                    └───────────┘             │
                                        ┌─────▼──────┐
                                        │ ROLLED_BACK│
                                        └────────────┘

// provision_jobs.steps (after partial completion)
[
  {
    "step": 1,
    "action": "create_d1",
    "status": "completed",
    "output": { "cf_id": "fa098e4d-...", "resource_id": "res_abc123" },
    "completedAt": 1740000001000
  },
  {
    "step": 2,
    "action": "deploy_worker",
    "status": "completed",
    "output": {
      "script_name": "k3m9p2xw7q-r8n4t6y1z5-auth-prod",
      "resource_id": "res_def456"
    },
    "completedAt": 1740000008000
  },
  {
    "step": 3,
    "action": "set_secrets",
    "status": "failed",
    "error": "CF API 429: rate limited",
    "failedAt": 1740000012000
  }
]

async function withRetry<T>(
  fn: () => Promise<T>,
  job: ProvisionJob,
  step: number,
): Promise<T> {
  const maxRetries = 3;
  let attempt = 0;

  while (attempt <= maxRetries) {
    try {
      return await fn();
    } catch (err) {
      if (err instanceof RateLimitError) {
        // Honour Cloudflare's Retry-After header
        await sleep(err.retryAfterSeconds * 1000);
        attempt++;
        continue;
      }

      if (err instanceof CloudflareApiError && err.isTransient()) {
        // 500, 502, 503, 504 — exponential backoff
        const delay = Math.min(1000 * Math.pow(2, attempt), 30_000);
        await sleep(delay);
        attempt++;
        continue;
      }

      // Non-transient error (400, 409, etc.) — fail immediately
      throw err;
    }
  }

  throw new MaxRetriesExceededError(
    `Step ${step} failed after ${maxRetries} retries`,
  );
}

async function ensureD1Exists(
  cf: CloudflareClient,
  name: string,
): Promise<{ cfId: string; wasCreated: boolean }> {
  // Check Registry first (fastest path)
  const existing = await registry.resources.lookupByCfName(name);
  if (existing?.status === "active") {
    return { cfId: existing.cfId, wasCreated: false };
  }

  // Check CF API (in case Registry is stale)
  const cfExisting = await cf.d1.findByName(name);
  if (cfExisting) {
    return { cfId: cfExisting.uuid, wasCreated: false };
  }

  // Create
  const created = await cf.d1.create({ name });
  return { cfId: created.uuid, wasCreated: true };
}

GET  /api/v1/provision/dlq                  List DLQ messages (paginated)
GET  /api/v1/provision/dlq/:messageId       Get DLQ message details + failed steps
POST /api/v1/provision/dlq/:messageId/retry Re-enqueue a DLQ message for retry
POST /api/v1/provision/dlq/:messageId/dismiss Mark as resolved without retry

POST   /api/v1/provision/platform             Trigger BOOTSTRAP_PLATFORM
POST   /api/v1/provision/feature/activate     Trigger ACTIVATE_FEATURE
POST   /api/v1/provision/feature/deactivate   Trigger DEACTIVATE_FEATURE
POST   /api/v1/provision/stack/activate       Trigger PROVISION_STACK
POST   /api/v1/provision/stack/deactivate     Trigger DEACTIVATE_STACK
POST   /api/v1/provision/platform/deprovision Trigger DEPROVISION_PLATFORM (requires confirmation token)
GET    /api/v1/provision/jobs/:jobId          Get job status + step details
GET    /api/v1/provision/jobs                 List jobs for a platform (query: ?platformId=&limit=25&cursor=)

{
  "platformId": "k3m9p2xw7q",
  "entityId": "r8n4t6y1z5",
  "featureId": "analytics",
  "featureVersion": "1.2.0",
  "environment": "prod"
}

202 Accepted
{ "jobId": "job_n3r8t5w2y6", "status": "PENDING" }

{
  "id": "job_n3r8t5w2y6",
  "type": "ACTIVATE_FEATURE",
  "status": "RUNNING",
  "platformId": "k3m9p2xw7q",
  "entityId": "r8n4t6y1z5",
  "featureId": "analytics",
  "environment": "prod",
  "steps": [
    {
      "name": "create_d1",
      "status": "COMPLETED",
      "result": { "cfId": "fa098e4d-...", "message": "D1 database created" },
      "startedAt": 1740230460000,
      "completedAt": 1740230462000
    },
    {
      "name": "deploy_worker",
      "status": "RUNNING",
      "result": null,
      "startedAt": 1740230465000,
      "completedAt": null
    }
  ],
  "error": null,
  "createdAt": 1740230460000,
  "startedAt": 1740230461000,
  "completedAt": null
}

// Every step logs at start and end
console.log(
  JSON.stringify({
    level: "info",
    event: "provision_step",
    jobId: job.id,
    step: step.number,
    action: step.action,
    status: "started" | "completed" | "failed",
    durationMs: elapsed,
    cfResource: cfName,
    error: err?.message ?? null,
  }),
);

DEACTIVATE_FEATURE job
  ├─ Step 1: Mark feature → status: "deactivating" (Registry PATCH)
  ├─ Step 2: Delete feature Worker via cf.workers.delete()  ← only CF resource removed
  ├─ Step 3: Mark feature → status: "inactive" (Registry PATCH)
  └─ Step 4: Trigger shell rebuild (CLI Service)

DEACTIVATE_STACK job
  ├─ Step 1: Resolve stack instance (Registry GET)
  ├─ Step 2: Mark stack → status: "deactivating" (Registry PATCH)
  ├─ Step 3: Enqueue DEACTIVATE_FEATURE sub-job for each feature activation
  ├─ Step 4: Delete shared CF resources — ONLY if deleteData: true
  │           ├─ cf.workers.delete(workerName)   if sharedResources.workerName
  │           ├─ cf.d1.delete(d1Id)              if sharedResources.d1Id
  │           ├─ cf.r2.delete(r2Name)            if sharedResources.r2Name
  │           └─ cf.kv.delete(kvId)              if sharedResources.kvId
  ├─ Step 5: Mark stack → status: "deactivated" (Registry PATCH)
  └─ Step 6: Trigger shell rebuild (CLI Service)

# services/provisioning/wrangler.toml

name = "nno-k3m9p2xw7q-provisioning"
main = "src/index.ts"
compatibility_date = "2024-09-13"
compatibility_flags = ["nodejs_compat"]

[triggers]
crons = ["*/15 * * * *"]   # ssl-poller: checks pending CF4SaaS SSL issuance

# Production (default — no --env flag)
[[d1_databases]]
binding = "DB"
database_name = "nno-k3m9p2xw7q-provisioning-db"
database_id = "db158316-1fa6-4fbf-a758-bfff40fb0e46"
migrations_dir = "migrations"

[[queues.producers]]
binding = "PROVISION_QUEUE"
queue = "nno-k3m9p2xw7q-provision-queue"

[[queues.consumers]]
queue = "nno-k3m9p2xw7q-provision-queue"
max_batch_size = 1
max_retries = 3
dead_letter_queue = "nno-k3m9p2xw7q-provision-dlq"

[[queues.producers]]
binding = "PROVISION_DLQ"
queue = "nno-k3m9p2xw7q-provision-dlq"

[[analytics_engine_datasets]]
binding = "NNO_METRICS"
dataset = "nno_metrics"

[[kv_namespaces]]
binding = "NNO_PLATFORM_STATUS_KV"
id = "63deeb49457946c6a68a49b23ea4fc5c"

[env.stg]
name = "nno-k3m9p2xw7q-provisioning-stg"
[[env.stg.d1_databases]]
binding = "DB"
database_name = "nno-k3m9p2xw7q-provisioning-db-stg"
database_id = "74b91da9-0e38-4373-bd74-e95ed67dd210"
migrations_dir = "migrations"
[[env.stg.queues.producers]]
binding = "PROVISION_QUEUE"
queue = "nno-k3m9p2xw7q-provision-queue-stg"
[[env.stg.queues.consumers]]
queue = "nno-k3m9p2xw7q-provision-queue-stg"
max_batch_size = 1
max_retries = 3
dead_letter_queue = "nno-k3m9p2xw7q-provision-dlq-stg"
[[env.stg.queues.producers]]
binding = "PROVISION_DLQ"
queue = "nno-k3m9p2xw7q-provision-dlq-stg"
[[env.stg.analytics_engine_datasets]]
binding = "NNO_METRICS"
dataset = "nno_metrics"
[[env.stg.kv_namespaces]]
binding = "NNO_PLATFORM_STATUS_KV"
id = "db49667ca1904af7ba128a02806e7c7e"